npm - @biggora/claude-plugins - Versions diffs - 1.2.0 → 1.2.2 - Mend

@biggora/claude-plugins 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (185) hide show

package/gemini-cli-workspace/iteration-1/review.html DELETED Viewed

@@ -1,1325 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Eval Review</title>
-  <link rel="preconnect" href="https://fonts.googleapis.com">
-  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-  <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
-  <script src="https://cdn.sheetjs.com/xlsx-0.20.3/package/dist/xlsx.full.min.js" integrity="sha384-EnyY0/GSHQGSxSgMwaIPzSESbqoOLSexfnSMN2AP+39Ckmn92stwABZynq1JyzdT" crossorigin="anonymous"></script>
-  <style>
-    :root {
-      --bg: #faf9f5;
-      --surface: #ffffff;
-      --border: #e8e6dc;
-      --text: #141413;
-      --text-muted: #b0aea5;
-      --accent: #d97757;
-      --accent-hover: #c4613f;
-      --green: #788c5d;
-      --green-bg: #eef2e8;
-      --red: #c44;
-      --red-bg: #fceaea;
-      --header-bg: #141413;
-      --header-text: #faf9f5;
-      --radius: 6px;
-    }
-    * { box-sizing: border-box; margin: 0; padding: 0; }
-    body {
-      font-family: 'Lora', Georgia, serif;
-      background: var(--bg);
-      color: var(--text);
-      height: 100vh;
-      display: flex;
-      flex-direction: column;
-    }
-    /* ---- Header ---- */
-    .header {
-      background: var(--header-bg);
-      color: var(--header-text);
-      padding: 1rem 2rem;
-      display: flex;
-      justify-content: space-between;
-      align-items: center;
-      flex-shrink: 0;
-    }
-    .header h1 {
-      font-family: 'Poppins', sans-serif;
-      font-size: 1.25rem;
-      font-weight: 600;
-    }
-    .header .instructions {
-      font-size: 0.8rem;
-      opacity: 0.7;
-      margin-top: 0.25rem;
-    }
-    .header .progress {
-      font-size: 0.875rem;
-      opacity: 0.8;
-      text-align: right;
-    }
-    /* ---- Main content ---- */
-    .main {
-      flex: 1;
-      overflow-y: auto;
-      padding: 1.5rem 2rem;
-      display: flex;
-      flex-direction: column;
-      gap: 1.25rem;
-    }
-    /* ---- Sections ---- */
-    .section {
-      background: var(--surface);
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      flex-shrink: 0;
-    }
-    .section-header {
-      font-family: 'Poppins', sans-serif;
-      padding: 0.75rem 1rem;
-      font-size: 0.75rem;
-      font-weight: 500;
-      text-transform: uppercase;
-      letter-spacing: 0.05em;
-      color: var(--text-muted);
-      border-bottom: 1px solid var(--border);
-      background: var(--bg);
-    }
-    .section-body {
-      padding: 1rem;
-    }
-    /* ---- Config badge ---- */
-    .config-badge {
-      display: inline-block;
-      padding: 0.2rem 0.625rem;
-      border-radius: 9999px;
-      font-family: 'Poppins', sans-serif;
-      font-size: 0.6875rem;
-      font-weight: 600;
-      text-transform: uppercase;
-      letter-spacing: 0.03em;
-      margin-left: 0.75rem;
-      vertical-align: middle;
-    }
-    .config-badge.config-primary {
-      background: rgba(33, 150, 243, 0.12);
-      color: #1976d2;
-    }
-    .config-badge.config-baseline {
-      background: rgba(255, 193, 7, 0.15);
-      color: #f57f17;
-    }
-    /* ---- Prompt ---- */
-    .prompt-text {
-      white-space: pre-wrap;
-      font-size: 0.9375rem;
-      line-height: 1.6;
-    }
-    /* ---- Outputs ---- */
-    .output-file {
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      overflow: hidden;
-    }
-    .output-file + .output-file {
-      margin-top: 1rem;
-    }
-    .output-file-header {
-      padding: 0.5rem 0.75rem;
-      font-size: 0.8rem;
-      font-weight: 600;
-      color: var(--text-muted);
-      background: var(--bg);
-      border-bottom: 1px solid var(--border);
-      font-family: 'SF Mono', SFMono-Regular, Consolas, 'Liberation Mono', Menlo, monospace;
-      display: flex;
-      justify-content: space-between;
-      align-items: center;
-    }
-    .output-file-header .dl-btn {
-      font-size: 0.7rem;
-      color: var(--accent);
-      text-decoration: none;
-      cursor: pointer;
-      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
-      font-weight: 500;
-      opacity: 0.8;
-    }
-    .output-file-header .dl-btn:hover {
-      opacity: 1;
-      text-decoration: underline;
-    }
-    .output-file-content {
-      padding: 0.75rem;
-      overflow-x: auto;
-    }
-    .output-file-content pre {
-      font-size: 0.8125rem;
-      line-height: 1.5;
-      white-space: pre-wrap;
-      word-break: break-word;
-      font-family: 'SF Mono', SFMono-Regular, Consolas, 'Liberation Mono', Menlo, monospace;
-    }
-    .output-file-content img {
-      max-width: 100%;
-      height: auto;
-      border-radius: 4px;
-    }
-    .output-file-content iframe {
-      width: 100%;
-      height: 600px;
-      border: none;
-    }
-    .output-file-content table {
-      border-collapse: collapse;
-      font-size: 0.8125rem;
-      width: 100%;
-    }
-    .output-file-content table td,
-    .output-file-content table th {
-      border: 1px solid var(--border);
-      padding: 0.375rem 0.5rem;
-      text-align: left;
-    }
-    .output-file-content table th {
-      background: var(--bg);
-      font-weight: 600;
-    }
-    .output-file-content .download-link {
-      display: inline-flex;
-      align-items: center;
-      gap: 0.5rem;
-      padding: 0.5rem 1rem;
-      background: var(--bg);
-      border: 1px solid var(--border);
-      border-radius: 4px;
-      color: var(--accent);
-      text-decoration: none;
-      font-size: 0.875rem;
-      cursor: pointer;
-    }
-    .output-file-content .download-link:hover {
-      background: var(--border);
-    }
-    .empty-state {
-      color: var(--text-muted);
-      font-style: italic;
-      padding: 2rem;
-      text-align: center;
-    }
-    /* ---- Feedback ---- */
-    .prev-feedback {
-      background: var(--bg);
-      border: 1px solid var(--border);
-      border-radius: 4px;
-      padding: 0.625rem 0.75rem;
-      margin-top: 0.75rem;
-      font-size: 0.8125rem;
-      color: var(--text-muted);
-      line-height: 1.5;
-    }
-    .prev-feedback-label {
-      font-size: 0.7rem;
-      font-weight: 600;
-      text-transform: uppercase;
-      letter-spacing: 0.04em;
-      margin-bottom: 0.25rem;
-      color: var(--text-muted);
-    }
-    .feedback-textarea {
-      width: 100%;
-      min-height: 100px;
-      padding: 0.75rem;
-      border: 1px solid var(--border);
-      border-radius: 4px;
-      font-family: inherit;
-      font-size: 0.9375rem;
-      line-height: 1.5;
-      resize: vertical;
-      color: var(--text);
-    }
-    .feedback-textarea:focus {
-      outline: none;
-      border-color: var(--accent);
-      box-shadow: 0 0 0 3px rgba(37, 99, 235, 0.1);
-    }
-    .feedback-status {
-      font-size: 0.75rem;
-      color: var(--text-muted);
-      margin-top: 0.5rem;
-      min-height: 1.1em;
-    }
-    /* ---- Grades (collapsible) ---- */
-    .grades-toggle {
-      display: flex;
-      align-items: center;
-      cursor: pointer;
-      user-select: none;
-    }
-    .grades-toggle:hover {
-      color: var(--accent);
-    }
-    .grades-toggle .arrow {
-      margin-right: 0.5rem;
-      transition: transform 0.15s;
-      font-size: 0.75rem;
-    }
-    .grades-toggle .arrow.open {
-      transform: rotate(90deg);
-    }
-    .grades-content {
-      display: none;
-      margin-top: 0.75rem;
-    }
-    .grades-content.open {
-      display: block;
-    }
-    .grades-summary {
-      font-size: 0.875rem;
-      margin-bottom: 0.75rem;
-      display: flex;
-      align-items: center;
-      gap: 0.5rem;
-    }
-    .grade-badge {
-      display: inline-block;
-      padding: 0.125rem 0.5rem;
-      border-radius: 9999px;
-      font-size: 0.75rem;
-      font-weight: 600;
-    }
-    .grade-pass { background: var(--green-bg); color: var(--green); }
-    .grade-fail { background: var(--red-bg); color: var(--red); }
-    .assertion-list {
-      list-style: none;
-    }
-    .assertion-item {
-      padding: 0.625rem 0;
-      border-bottom: 1px solid var(--border);
-      font-size: 0.8125rem;
-    }
-    .assertion-item:last-child { border-bottom: none; }
-    .assertion-status {
-      font-weight: 600;
-      margin-right: 0.5rem;
-    }
-    .assertion-status.pass { color: var(--green); }
-    .assertion-status.fail { color: var(--red); }
-    .assertion-evidence {
-      color: var(--text-muted);
-      font-size: 0.75rem;
-      margin-top: 0.25rem;
-      padding-left: 1.5rem;
-    }
-    /* ---- View tabs ---- */
-    .view-tabs {
-      display: flex;
-      gap: 0;
-      padding: 0 2rem;
-      background: var(--bg);
-      border-bottom: 1px solid var(--border);
-      flex-shrink: 0;
-    }
-    .view-tab {
-      font-family: 'Poppins', sans-serif;
-      padding: 0.625rem 1.25rem;
-      font-size: 0.8125rem;
-      font-weight: 500;
-      cursor: pointer;
-      border: none;
-      background: none;
-      color: var(--text-muted);
-      border-bottom: 2px solid transparent;
-      transition: all 0.15s;
-    }
-    .view-tab:hover { color: var(--text); }
-    .view-tab.active {
-      color: var(--accent);
-      border-bottom-color: var(--accent);
-    }
-    .view-panel { display: none; }
-    .view-panel.active { display: flex; flex-direction: column; flex: 1; overflow: hidden; }
-    /* ---- Benchmark view ---- */
-    .benchmark-view {
-      padding: 1.5rem 2rem;
-      overflow-y: auto;
-      flex: 1;
-    }
-    .benchmark-table {
-      border-collapse: collapse;
-      background: var(--surface);
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      font-size: 0.8125rem;
-      width: 100%;
-      margin-bottom: 1.5rem;
-    }
-    .benchmark-table th, .benchmark-table td {
-      padding: 0.625rem 0.75rem;
-      text-align: left;
-      border: 1px solid var(--border);
-    }
-    .benchmark-table th {
-      font-family: 'Poppins', sans-serif;
-      background: var(--header-bg);
-      color: var(--header-text);
-      font-weight: 500;
-      font-size: 0.75rem;
-      text-transform: uppercase;
-      letter-spacing: 0.04em;
-    }
-    .benchmark-table tr:hover { background: var(--bg); }
-    .benchmark-table tr.benchmark-row-with { background: rgba(33, 150, 243, 0.06); }
-    .benchmark-table tr.benchmark-row-without { background: rgba(255, 193, 7, 0.06); }
-    .benchmark-table tr.benchmark-row-with:hover { background: rgba(33, 150, 243, 0.12); }
-    .benchmark-table tr.benchmark-row-without:hover { background: rgba(255, 193, 7, 0.12); }
-    .benchmark-table tr.benchmark-row-avg { font-weight: 600; border-top: 2px solid var(--border); }
-    .benchmark-table tr.benchmark-row-avg.benchmark-row-with { background: rgba(33, 150, 243, 0.12); }
-    .benchmark-table tr.benchmark-row-avg.benchmark-row-without { background: rgba(255, 193, 7, 0.12); }
-    .benchmark-delta-positive { color: var(--green); font-weight: 600; }
-    .benchmark-delta-negative { color: var(--red); font-weight: 600; }
-    .benchmark-notes {
-      background: var(--surface);
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      padding: 1rem;
-    }
-    .benchmark-notes h3 {
-      font-family: 'Poppins', sans-serif;
-      font-size: 0.875rem;
-      margin-bottom: 0.75rem;
-    }
-    .benchmark-notes ul {
-      list-style: disc;
-      padding-left: 1.25rem;
-    }
-    .benchmark-notes li {
-      font-size: 0.8125rem;
-      line-height: 1.6;
-      margin-bottom: 0.375rem;
-    }
-    .benchmark-empty {
-      color: var(--text-muted);
-      font-style: italic;
-      text-align: center;
-      padding: 3rem;
-    }
-    /* ---- Navigation ---- */
-    .nav {
-      display: flex;
-      justify-content: space-between;
-      align-items: center;
-      padding: 1rem 2rem;
-      border-top: 1px solid var(--border);
-      background: var(--surface);
-      flex-shrink: 0;
-    }
-    .nav-btn {
-      font-family: 'Poppins', sans-serif;
-      padding: 0.5rem 1.25rem;
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      background: var(--surface);
-      cursor: pointer;
-      font-size: 0.875rem;
-      font-weight: 500;
-      color: var(--text);
-      transition: all 0.15s;
-    }
-    .nav-btn:hover:not(:disabled) {
-      background: var(--bg);
-      border-color: var(--text-muted);
-    }
-    .nav-btn:disabled {
-      opacity: 0.4;
-      cursor: not-allowed;
-    }
-    .done-btn {
-      font-family: 'Poppins', sans-serif;
-      padding: 0.5rem 1.5rem;
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      background: var(--surface);
-      color: var(--text);
-      cursor: pointer;
-      font-size: 0.875rem;
-      font-weight: 500;
-      transition: all 0.15s;
-    }
-    .done-btn:hover {
-      background: var(--bg);
-      border-color: var(--text-muted);
-    }
-    .done-btn.ready {
-      border: none;
-      background: var(--accent);
-      color: white;
-      font-weight: 600;
-    }
-    .done-btn.ready:hover {
-      background: var(--accent-hover);
-    }
-    /* ---- Done overlay ---- */
-    .done-overlay {
-      display: none;
-      position: fixed;
-      inset: 0;
-      background: rgba(0, 0, 0, 0.5);
-      z-index: 100;
-      justify-content: center;
-      align-items: center;
-    }
-    .done-overlay.visible {
-      display: flex;
-    }
-    .done-card {
-      background: var(--surface);
-      border-radius: 12px;
-      padding: 2rem 3rem;
-      text-align: center;
-      box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
-      max-width: 500px;
-    }
-    .done-card h2 {
-      font-size: 1.5rem;
-      margin-bottom: 0.5rem;
-    }
-    .done-card p {
-      color: var(--text-muted);
-      margin-bottom: 1.5rem;
-      line-height: 1.5;
-    }
-    .done-card .btn-row {
-      display: flex;
-      gap: 0.5rem;
-      justify-content: center;
-    }
-    .done-card button {
-      padding: 0.5rem 1.25rem;
-      border: 1px solid var(--border);
-      border-radius: var(--radius);
-      background: var(--surface);
-      cursor: pointer;
-      font-size: 0.875rem;
-    }
-    .done-card button:hover {
-      background: var(--bg);
-    }
-    /* ---- Toast ---- */
-    .toast {
-      position: fixed;
-      bottom: 5rem;
-      left: 50%;
-      transform: translateX(-50%);
-      background: var(--header-bg);
-      color: var(--header-text);
-      padding: 0.625rem 1.25rem;
-      border-radius: var(--radius);
-      font-size: 0.875rem;
-      opacity: 0;
-      transition: opacity 0.3s;
-      pointer-events: none;
-      z-index: 200;
-    }
-    .toast.visible {
-      opacity: 1;
-    }
-  </style>
-</head>
-<body>
-  <div id="app" style="height:100vh; display:flex; flex-direction:column;">
-    <div class="header">
-      <div>
-        <h1>Eval Review: <span id="skill-name"></span></h1>
-        <div class="instructions">Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into Claude Code.</div>
-      </div>
-      <div class="progress" id="progress"></div>
-    </div>
-    <!-- View tabs (only shown when benchmark data exists) -->
-    <div class="view-tabs" id="view-tabs" style="display:none;">
-      <button class="view-tab active" onclick="switchView('outputs')">Outputs</button>
-      <button class="view-tab" onclick="switchView('benchmark')">Benchmark</button>
-    </div>
-    <!-- Outputs panel (qualitative review) -->
-    <div class="view-panel active" id="panel-outputs">
-    <div class="main">
-      <!-- Prompt -->
-      <div class="section">
-        <div class="section-header">Prompt <span class="config-badge" id="config-badge" style="display:none;"></span></div>
-        <div class="section-body">
-          <div class="prompt-text" id="prompt-text"></div>
-        </div>
-      </div>
-      <!-- Outputs -->
-      <div class="section">
-        <div class="section-header">Output</div>
-        <div class="section-body" id="outputs-body">
-          <div class="empty-state">No output files found</div>
-        </div>
-      </div>
-      <!-- Previous Output (collapsible) -->
-      <div class="section" id="prev-outputs-section" style="display:none;">
-        <div class="section-header">
-          <div class="grades-toggle" onclick="togglePrevOutputs()">
-            <span class="arrow" id="prev-outputs-arrow">&#9654;</span>
-            Previous Output
-          </div>
-        </div>
-        <div class="grades-content" id="prev-outputs-content"></div>
-      </div>
-      <!-- Grades (collapsible) -->
-      <div class="section" id="grades-section" style="display:none;">
-        <div class="section-header">
-          <div class="grades-toggle" onclick="toggleGrades()">
-            <span class="arrow" id="grades-arrow">&#9654;</span>
-            Formal Grades
-          </div>
-        </div>
-        <div class="grades-content" id="grades-content"></div>
-      </div>
-      <!-- Feedback -->
-      <div class="section">
-        <div class="section-header">Your Feedback</div>
-        <div class="section-body">
-          <textarea
-            class="feedback-textarea"
-            id="feedback"
-            placeholder="What do you think of this output? Any issues, suggestions, or things that look great?"
-          ></textarea>
-          <div class="feedback-status" id="feedback-status"></div>
-          <div class="prev-feedback" id="prev-feedback" style="display:none;">
-            <div class="prev-feedback-label">Previous feedback</div>
-            <div id="prev-feedback-text"></div>
-          </div>
-        </div>
-      </div>
-    </div>
-    <div class="nav" id="outputs-nav">
-      <button class="nav-btn" id="prev-btn" onclick="navigate(-1)">&#8592; Previous</button>
-      <button class="done-btn" id="done-btn" onclick="showDoneDialog()">Submit All Reviews</button>
-      <button class="nav-btn" id="next-btn" onclick="navigate(1)">Next &#8594;</button>
-    </div>
-    </div><!-- end panel-outputs -->
-    <!-- Benchmark panel (quantitative stats) -->
-    <div class="view-panel" id="panel-benchmark">
-      <div class="benchmark-view" id="benchmark-content">
-        <div class="benchmark-empty">No benchmark data available. Run a benchmark to see quantitative results here.</div>
-      </div>
-    </div>
-  </div>
-  <!-- Done overlay -->
-  <div class="done-overlay" id="done-overlay">
-    <div class="done-card">
-      <h2>Review Complete</h2>
-      <p>Your feedback has been saved. Go back to your Claude Code session and tell Claude you're done reviewing.</p>
-      <div class="btn-row">
-        <button onclick="closeDoneDialog()">OK</button>
-      </div>
-    </div>
-  </div>
-  <!-- Toast -->
-  <div class="toast" id="toast"></div>
-  <script>
-    // ---- Embedded data (injected by generate_review.py) ----
-    const EMBEDDED_DATA = {"skill_name": "gemini-cli", "runs": [{"id": "eval-1-cicd-setup-with_skill", "prompt": "I want to set up gemini cli in my CI/CD pipeline on GitHub Actions so it can automatically review PRs. We use an API key stored in GitHub secrets. How do I set this up with structured JSON output so I can parse the review?", "eval_id": 1, "outputs": [{"name": "response.md", "type": "text", "content": "# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions\n\nHere is a complete guide to configuring Gemini CLI in your CI/CD pipeline to automatically review pull requests with structured JSON output.\n\n---\n\n## Prerequisites\n\n1. A Gemini API key (get one at https://aistudio.google.com/apikey).\n2. The API key stored as a GitHub Actions secret named `GEMINI_API_KEY`.\n\n---\n\n## Step 1: Basic GitHub Actions Workflow\n\nCreate `.github/workflows/gemini-pr-review.yml`:\n\n```yaml\nname: Gemini PR Review\n\non:\n  pull_request:\n    types: [opened, synchronize, reopened]\n\npermissions:\n  pull-requests: write\n  contents: read\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout code\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0  # Full history so we can diff against base\n\n      - name: Set up Node.js\n        uses: actions/setup-node@v4\n        with:\n          node-version: '20'\n\n      - name: Install Gemini CLI\n        run: npm install -g @google/gemini-cli\n\n      - name: Get PR diff\n        id: diff\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          git diff origin/${{ github.base_ref }}...HEAD > /tmp/pr-diff.txt\n          echo \"diff_size=$(wc -c < /tmp/pr-diff.txt)\" >> \"$GITHUB_OUTPUT\"\n\n      - name: Run Gemini PR Review\n        id: review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          REVIEW_JSON=$(cat /tmp/pr-diff.txt | gemini -p \"You are a senior code reviewer. Review the following pull request diff for:\n          1. Bugs and logical errors\n          2. Security vulnerabilities\n          3. Performance issues\n          4. Code style and best practices\n\n          Provide your review as structured text with sections for each category.\n          If no issues are found in a category, say 'No issues found.'\n          End with an overall summary and a verdict: APPROVE, REQUEST_CHANGES, or COMMENT.\" \\\n            --output-format json)\n\n          # Extract the response text from the JSON envelope\n          REVIEW_TEXT=$(echo \"$REVIEW_JSON\" | jq -r '.response')\n\n          # Extract token stats for logging\n          TOTAL_TOKENS=$(echo \"$REVIEW_JSON\" | jq '.stats.models | to_entries[0].value.tokens.total // 0')\n          echo \"Tokens used: $TOTAL_TOKENS\"\n\n          # Save review to file (handles multi-line safely)\n          echo \"$REVIEW_TEXT\" > /tmp/review-output.txt\n\n          # Save for the summary\n          echo \"$REVIEW_TEXT\" >> \"$GITHUB_STEP_SUMMARY\"\n\n      - name: Post review as PR comment\n        uses: actions/github-script@v7\n        with:\n          script: |\n            const fs = require('fs');\n            const review = fs.readFileSync('/tmp/review-output.txt', 'utf8');\n            await github.rest.issues.createComment({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              issue_number: context.issue.number,\n              body: `## Gemini Code Review\\n\\n${review}\\n\\n---\\n*Automated review by Gemini CLI*`\n            });\n```\n\nThis workflow triggers on every PR event, runs the diff through Gemini CLI with `--output-format json`, extracts the response with `jq`, and posts it as a PR comment.\n\n---\n\n## Step 2: Advanced Workflow with Full JSON Parsing\n\nIf you want to parse the structured JSON output more thoroughly -- for example, to extract token usage, check for errors, or conditionally fail the build -- use this expanded version:\n\n```yaml\nname: Gemini PR Review (Advanced)\n\non:\n  pull_request:\n    types: [opened, synchronize]\n\npermissions:\n  pull-requests: write\n  contents: read\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n\n      - name: Setup Node.js\n        uses: actions/setup-node@v4\n        with:\n          node-version: '20'\n\n      - name: Install Gemini CLI and jq\n        run: |\n          npm install -g @google/gemini-cli\n          # jq is pre-installed on ubuntu-latest, but just in case:\n          which jq || sudo apt-get install -y jq\n\n      - name: Run structured review\n        id: gemini\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)\n\n          # Skip review if diff is empty\n          if [ -z \"$DIFF\" ]; then\n            echo \"No changes to review.\"\n            echo \"skip=true\" >> \"$GITHUB_OUTPUT\"\n            exit 0\n          fi\n\n          # Run Gemini with JSON output\n          RESULT=$(echo \"$DIFF\" | gemini -p \"You are a code review bot. Analyze this PR diff and respond with ONLY a valid JSON object (no markdown fences) using this exact schema:\n          {\n            \\\"verdict\\\": \\\"APPROVE\\\" | \\\"REQUEST_CHANGES\\\" | \\\"COMMENT\\\",\n            \\\"summary\\\": \\\"Brief overall summary\\\",\n            \\\"bugs\\\": [{ \\\"file\\\": \\\"path\\\", \\\"line\\\": number, \\\"description\\\": \\\"...\\\" }],\n            \\\"security\\\": [{ \\\"file\\\": \\\"path\\\", \\\"severity\\\": \\\"low|medium|high|critical\\\", \\\"description\\\": \\\"...\\\" }],\n            \\\"performance\\\": [{ \\\"file\\\": \\\"path\\\", \\\"description\\\": \\\"...\\\" }],\n            \\\"suggestions\\\": [{ \\\"file\\\": \\\"path\\\", \\\"description\\\": \\\"...\\\" }]\n          }\n          Return empty arrays if no issues found in a category.\" \\\n            --output-format json)\n\n          # The outer JSON has a .response field containing the model's text.\n          # The model's text itself is a JSON string we need to parse.\n          REVIEW_RESPONSE=$(echo \"$RESULT\" | jq -r '.response')\n\n          # Parse the inner JSON (the actual review)\n          echo \"$REVIEW_RESPONSE\" | jq '.' > /tmp/review.json\n\n          # Extract fields for downstream steps\n          VERDICT=$(cat /tmp/review.json | jq -r '.verdict')\n          SUMMARY=$(cat /tmp/review.json | jq -r '.summary')\n          BUG_COUNT=$(cat /tmp/review.json | jq '.bugs | length')\n          SECURITY_COUNT=$(cat /tmp/review.json | jq '.security | length')\n          CRITICAL_COUNT=$(cat /tmp/review.json | jq '[.security[] | select(.severity == \"critical\")] | length')\n\n          echo \"verdict=$VERDICT\" >> \"$GITHUB_OUTPUT\"\n          echo \"bug_count=$BUG_COUNT\" >> \"$GITHUB_OUTPUT\"\n          echo \"security_count=$SECURITY_COUNT\" >> \"$GITHUB_OUTPUT\"\n          echo \"critical_count=$CRITICAL_COUNT\" >> \"$GITHUB_OUTPUT\"\n          echo \"skip=false\" >> \"$GITHUB_OUTPUT\"\n\n          # Log token usage from the Gemini stats envelope\n          TOKENS=$(echo \"$RESULT\" | jq '.stats.models | to_entries[0].value.tokens.total // 0')\n          LATENCY=$(echo \"$RESULT\" | jq '.stats.models | to_entries[0].value.api.totalLatencyMs // 0')\n          echo \"Tokens used: $TOKENS, Latency: ${LATENCY}ms\"\n\n      - name: Format and post review comment\n        if: steps.gemini.outputs.skip != 'true'\n        uses: actions/github-script@v7\n        with:\n          script: |\n            const fs = require('fs');\n            const review = JSON.parse(fs.readFileSync('/tmp/review.json', 'utf8'));\n\n            let body = `## Gemini Code Review\\n\\n`;\n            body += `**Verdict:** ${review.verdict}\\n\\n`;\n            body += `**Summary:** ${review.summary}\\n\\n`;\n\n            if (review.bugs.length > 0) {\n              body += `### Bugs (${review.bugs.length})\\n`;\n              for (const bug of review.bugs) {\n                body += `- **${bug.file}** (line ${bug.line}): ${bug.description}\\n`;\n              }\n              body += '\\n';\n            }\n\n            if (review.security.length > 0) {\n              body += `### Security Issues (${review.security.length})\\n`;\n              for (const issue of review.security) {\n                const emoji = issue.severity === 'critical' ? '\u00f0\u0178\u201d\u00b4' :\n                              issue.severity === 'high' ? '\u00f0\u0178\u0178\u00a0' :\n                              issue.severity === 'medium' ? '\u00f0\u0178\u0178\u00a1' : '\u00f0\u0178\u0178\u00a2';\n                body += `- ${emoji} **[${issue.severity.toUpperCase()}]** ${issue.file}: ${issue.description}\\n`;\n              }\n              body += '\\n';\n            }\n\n            if (review.performance.length > 0) {\n              body += `### Performance (${review.performance.length})\\n`;\n              for (const perf of review.performance) {\n                body += `- **${perf.file}**: ${perf.description}\\n`;\n              }\n              body += '\\n';\n            }\n\n            if (review.suggestions.length > 0) {\n              body += `### Suggestions (${review.suggestions.length})\\n`;\n              for (const sug of review.suggestions) {\n                body += `- **${sug.file}**: ${sug.description}\\n`;\n              }\n              body += '\\n';\n            }\n\n            body += `---\\n*Automated review by Gemini CLI*`;\n\n            await github.rest.issues.createComment({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              issue_number: context.issue.number,\n              body\n            });\n\n      - name: Fail on critical security issues\n        if: steps.gemini.outputs.critical_count != '0' && steps.gemini.outputs.skip != 'true'\n        run: |\n          echo \"::error::Gemini found ${{ steps.gemini.outputs.critical_count }} critical security issue(s). Review required.\"\n          exit 1\n```\n\n---\n\n## Understanding the JSON Output\n\nGemini CLI's `--output-format json` wraps the model's response in an envelope with this schema:\n\n```json\n{\n  \"response\": \"The model's text response (string)\",\n  \"stats\": {\n    \"models\": {\n      \"gemini-2.5-pro\": {\n        \"api\": {\n          \"totalRequests\": 2,\n          \"totalErrors\": 0,\n          \"totalLatencyMs\": 5053\n        },\n        \"tokens\": {\n          \"prompt\": 24939,\n          \"candidates\": 20,\n          \"total\": 25113,\n          \"cached\": 21263,\n          \"thoughts\": 154,\n          \"tool\": 0\n        }\n      }\n    },\n    \"tools\": {\n      \"totalCalls\": 1,\n      \"totalSuccess\": 1,\n      \"totalFail\": 0,\n      \"totalDurationMs\": 1881\n    },\n    \"files\": {\n      \"totalLinesAdded\": 5,\n      \"totalLinesRemoved\": 2\n    }\n  }\n}\n```\n\nTo parse this in your pipeline:\n\n```bash\n# Get the model's text response\necho \"$RESULT\" | jq -r '.response'\n\n# Get total tokens used\necho \"$RESULT\" | jq '.stats.models | to_entries[0].value.tokens.total'\n\n# Get API latency\necho \"$RESULT\" | jq '.stats.models | to_entries[0].value.api.totalLatencyMs'\n\n# Check for API errors\necho \"$RESULT\" | jq '.stats.models | to_entries[0].value.api.totalErrors'\n```\n\n---\n\n## Exit Codes for CI Logic\n\nGemini CLI returns meaningful exit codes you can use for conditional logic:\n\n| Exit Code | Meaning |\n|-----------|---------|\n| 0 | Success |\n| 1 | General error |\n| 2 | Authentication error (bad or missing API key) |\n| 3 | Rate limit / quota exceeded |\n\nExample usage:\n\n```bash\ngemini -p \"Review this code\" --output-format json > /tmp/result.json 2>/dev/null\nEXIT_CODE=$?\n\nif [ $EXIT_CODE -eq 2 ]; then\n  echo \"::error::Gemini API key is invalid or missing. Check your GEMINI_API_KEY secret.\"\n  exit 1\nelif [ $EXIT_CODE -eq 3 ]; then\n  echo \"::warning::Gemini rate limit reached. Skipping review.\"\n  exit 0\nelif [ $EXIT_CODE -ne 0 ]; then\n  echo \"::warning::Gemini CLI failed with exit code $EXIT_CODE. Skipping review.\"\n  exit 0\nfi\n```\n\n---\n\n## Tips and Best Practices\n\n1. **Use `--output-format json` (not `stream-json`)** for CI. The `json` format gives you a single complete JSON object after all processing is done. The `stream-json` format emits newline-delimited JSON events in real-time, which is harder to parse in a script.\n\n2. **Authentication**: Set `GEMINI_API_KEY` as an environment variable. Gemini CLI picks it up automatically. No `settings.json` or `.env` file is needed in CI.\n\n3. **Model selection**: Add `-m gemini-2.5-flash` for faster, cheaper reviews or `-m gemini-2.5-pro` for deeper analysis:\n   ```bash\n   gemini -m gemini-2.5-flash -p \"Review this diff\" --output-format json\n   ```\n\n4. **Large diffs**: Gemini models support up to 1M tokens of context. For very large PRs, consider filtering the diff to only changed files that matter:\n   ```bash\n   git diff origin/main...HEAD -- '*.py' '*.js' '*.ts' | gemini -p \"Review\" --output-format json\n   ```\n\n5. **File references with `@` syntax**: You can reference specific files directly in your prompt:\n   ```bash\n   gemini -p \"Review @./src/auth.py and @./src/middleware.py for security issues\" --output-format json\n   ```\n\n6. **Do NOT use `--yolo` for reviews**. The `--yolo` flag auto-accepts all tool actions (file writes, shell commands). For a read-only review, you do not need it. Only use `--yolo` if you want Gemini to make actual code changes.\n\n7. **Rate limits on the free tier**: 60 requests/minute, 1,000 requests/day. For high-volume repos, consider using a paid API key or adding concurrency limits to your workflow.\n\n8. **Debug logging**: If reviews are producing unexpected output, add `--debug-log` to capture full API traces:\n   ```bash\n   gemini -p \"Review this code\" --output-format json --debug-log ./gemini-debug.jsonl\n   ```\n   Then upload `gemini-debug.jsonl` as a workflow artifact for inspection.\n\n---\n\n## Minimal Quick-Start Example\n\nIf you just want the simplest possible setup to get started:\n\n```yaml\nname: Gemini Review\non: [pull_request]\npermissions:\n  pull-requests: write\n  contents: read\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n      - run: npm install -g @google/gemini-cli\n      - name: Review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          REVIEW=$(git diff origin/${{ github.base_ref }}...HEAD | gemini -p \"Review these changes for bugs and issues\" --output-format json | jq -r '.response')\n          echo \"$REVIEW\" >> $GITHUB_STEP_SUMMARY\n```\n\nThis pipes the PR diff into Gemini CLI, extracts the review text from the JSON output with `jq`, and writes it to the GitHub Actions step summary where it is visible directly on the workflow run page.\n"}], "grading": {"eval_id": 1, "eval_name": "cicd-setup", "configuration": "with_skill", "expectations": [{"text": "Uses -p or --prompt flag for headless/non-interactive mode", "passed": true, "evidence": "Uses `-p` flag throughout: `gemini -p \"...\" --output-format json`"}, {"text": "Includes --output-format json flag for structured output", "passed": true, "evidence": "Correctly uses `--output-format json` in all examples"}, {"text": "Shows how to set GEMINI_API_KEY from GitHub secrets", "passed": true, "evidence": "Shows `GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}` in workflow env block"}, {"text": "Provides a GitHub Actions workflow YAML snippet", "passed": true, "evidence": "Provides 3 complete workflows: basic, advanced, and minimal quick-start"}, {"text": "Shows how to parse JSON response (e.g., with jq .response)", "passed": true, "evidence": "Shows `jq -r '.response'` and detailed stats extraction with jq"}, {"text": "Does not invent non-existent CLI flags or options", "passed": true, "evidence": "All flags used (-p, --output-format json, -m, --yolo, --debug-log) are real Gemini CLI flags"}]}}, {"id": "eval-1-cicd-setup-without_skill", "prompt": "I want to set up gemini cli in my CI/CD pipeline on GitHub Actions so it can automatically review PRs. We use an API key stored in GitHub secrets. How do I set this up with structured JSON output so I can parse the review?", "eval_id": 1, "outputs": [{"name": "response.md", "type": "text", "content": "# Setting Up Gemini CLI for Automated PR Reviews in GitHub Actions\n\nThis guide covers how to configure Gemini CLI in a GitHub Actions CI/CD pipeline to automatically review pull requests, using an API key from GitHub Secrets and structured JSON output for parsing.\n\n## Prerequisites\n\n- A Google AI API key (from [Google AI Studio](https://aistudio.google.com/apikey))\n- The API key stored as a GitHub repository secret (e.g., `GEMINI_API_KEY`)\n- A GitHub repository with pull request workflows\n\n## Step 1: Store Your API Key in GitHub Secrets\n\n1. Go to your repository on GitHub.\n2. Navigate to **Settings > Secrets and variables > Actions**.\n3. Click **New repository secret**.\n4. Name it `GEMINI_API_KEY` and paste your API key as the value.\n\n## Step 2: Create the GitHub Actions Workflow\n\nCreate a file at `.github/workflows/gemini-pr-review.yml`:\n\n```yaml\nname: Gemini PR Review\n\non:\n  pull_request:\n    types: [opened, synchronize, reopened]\n\npermissions:\n  contents: read\n  pull-requests: write\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n\n      - name: Set up Node.js\n        uses: actions/setup-node@v4\n        with:\n          node-version: '20'\n\n      - name: Install Gemini CLI\n        run: npm install -g @anthropic-ai/gemini-cli || npm install -g gemini-cli\n\n      - name: Get PR diff\n        id: diff\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)\n          # Write diff to a file to avoid shell escaping issues\n          echo \"$DIFF\" > /tmp/pr_diff.txt\n\n      - name: Get changed files list\n        id: files\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)\n          echo \"$FILES\" > /tmp/changed_files.txt\n\n      - name: Run Gemini CLI review\n        id: gemini-review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          DIFF=$(cat /tmp/pr_diff.txt)\n          FILES=$(cat /tmp/changed_files.txt)\n\n          PROMPT=$(cat <<'PROMPT_EOF'\n          You are a senior code reviewer. Analyze the following pull request diff and provide a structured review.\n\n          Changed files:\n          FILE_LIST_PLACEHOLDER\n\n          Diff:\n          DIFF_PLACEHOLDER\n\n          Respond ONLY with valid JSON in this exact format (no markdown fencing, no extra text):\n          {\n            \"summary\": \"Brief summary of the changes\",\n            \"risk_level\": \"low|medium|high\",\n            \"issues\": [\n              {\n                \"file\": \"path/to/file\",\n                \"line\": 42,\n                \"severity\": \"error|warning|suggestion\",\n                \"message\": \"Description of the issue\",\n                \"suggestion\": \"How to fix it\"\n              }\n            ],\n            \"positive_aspects\": [\"List of things done well\"],\n            \"approval_recommendation\": \"approve|request_changes|comment\"\n          }\n          PROMPT_EOF\n          )\n\n          # Replace placeholders\n          PROMPT=\"${PROMPT/FILE_LIST_PLACEHOLDER/$FILES}\"\n          PROMPT=\"${PROMPT/DIFF_PLACEHOLDER/$DIFF}\"\n\n          # Run Gemini CLI with the prompt\n          # Using the --json flag if available, or parsing output directly\n          RESPONSE=$(echo \"$PROMPT\" | gemini \\\n            --model gemini-2.0-flash \\\n            --api-key \"$GEMINI_API_KEY\" \\\n            2>/dev/null) || true\n\n          # If gemini CLI is not available, fall back to direct API call\n          if [ -z \"$RESPONSE\" ]; then\n            RESPONSE=$(curl -s \"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_API_KEY}\" \\\n              -H 'Content-Type: application/json' \\\n              -d \"$(jq -n --arg prompt \"$PROMPT\" '{\n                \"contents\": [{\"parts\": [{\"text\": $prompt}]}],\n                \"generationConfig\": {\n                  \"responseMimeType\": \"application/json\",\n                  \"temperature\": 0.2\n                }\n              }')\" | jq -r '.candidates[0].content.parts[0].text')\n          fi\n\n          echo \"$RESPONSE\" > /tmp/review_result.json\n\n          # Validate JSON\n          if jq empty /tmp/review_result.json 2>/dev/null; then\n            echo \"valid_json=true\" >> $GITHUB_OUTPUT\n          else\n            echo \"valid_json=false\" >> $GITHUB_OUTPUT\n            echo \"Raw response was not valid JSON, attempting extraction...\"\n            # Try to extract JSON from markdown fenced blocks\n            sed -n '/^```json/,/^```$/p' /tmp/review_result.json | sed '1d;$d' > /tmp/review_cleaned.json\n            if jq empty /tmp/review_cleaned.json 2>/dev/null; then\n              mv /tmp/review_cleaned.json /tmp/review_result.json\n              echo \"valid_json=true\" >> $GITHUB_OUTPUT\n            fi\n          fi\n\n      - name: Parse and post review\n        if: success()\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          REVIEW_FILE=\"/tmp/review_result.json\"\n\n          if ! jq empty \"$REVIEW_FILE\" 2>/dev/null; then\n            echo \"Failed to get valid JSON review. Posting raw output as comment.\"\n            gh pr comment ${{ github.event.pull_request.number }} \\\n              --body \"## Gemini Review\n\n          > Could not parse structured output. Raw response attached.\n\n          \\`\\`\\`\n          $(cat \"$REVIEW_FILE\")\n          \\`\\`\\`\"\n            exit 0\n          fi\n\n          # Extract fields from JSON\n          SUMMARY=$(jq -r '.summary' \"$REVIEW_FILE\")\n          RISK=$(jq -r '.risk_level' \"$REVIEW_FILE\")\n          APPROVAL=$(jq -r '.approval_recommendation' \"$REVIEW_FILE\")\n          POSITIVES=$(jq -r '.positive_aspects // [] | map(\"- \" + .) | join(\"\\n\")' \"$REVIEW_FILE\")\n          ISSUE_COUNT=$(jq -r '.issues | length' \"$REVIEW_FILE\")\n\n          # Build issue table\n          ISSUES_MD=\"\"\n          if [ \"$ISSUE_COUNT\" -gt 0 ]; then\n            ISSUES_MD=\"### Issues Found ($ISSUE_COUNT)\n\n          | Severity | File | Line | Message | Suggestion |\n          |----------|------|------|---------|------------|\n          \"\n            ISSUES_MD+=$(jq -r '.issues[] | \"| \\(.severity) | `\\(.file)` | \\(.line) | \\(.message) | \\(.suggestion) |\"' \"$REVIEW_FILE\")\n          else\n            ISSUES_MD=\"### No issues found\"\n          fi\n\n          # Build risk badge\n          case \"$RISK\" in\n            low)    RISK_BADGE=\"![Risk: Low](https://img.shields.io/badge/Risk-Low-green)\" ;;\n            medium) RISK_BADGE=\"![Risk: Medium](https://img.shields.io/badge/Risk-Medium-yellow)\" ;;\n            high)   RISK_BADGE=\"![Risk: High](https://img.shields.io/badge/Risk-High-red)\" ;;\n            *)      RISK_BADGE=\"Risk: $RISK\" ;;\n          esac\n\n          # Compose comment\n          COMMENT=$(cat <<EOF\n          ## Gemini Automated PR Review\n\n          $RISK_BADGE\n\n          ### Summary\n          $SUMMARY\n\n          $ISSUES_MD\n\n          ### Positive Aspects\n          $POSITIVES\n\n          ---\n          **Recommendation:** \\`$APPROVAL\\`\n          EOF\n          )\n\n          # Post comment to PR\n          gh pr comment ${{ github.event.pull_request.number }} --body \"$COMMENT\"\n```\n\n## Step 3: Direct API Approach with Structured JSON Output (Recommended)\n\nIf the Gemini CLI does not reliably produce JSON, use the Gemini REST API directly with `responseMimeType` set to `application/json`. This forces the model to return valid JSON. Here is a standalone workflow that uses `curl` and `jq`:\n\n```yaml\nname: Gemini PR Review (API Direct)\n\non:\n  pull_request:\n    types: [opened, synchronize, reopened]\n\npermissions:\n  contents: read\n  pull-requests: write\n\njobs:\n  review:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0\n\n      - name: Generate review via Gemini API\n        id: review\n        env:\n          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}\n        run: |\n          git fetch origin ${{ github.base_ref }}\n          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD | head -c 30000)\n          FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)\n\n          # Build the prompt\n          PROMPT=\"You are a code reviewer. Review this PR diff. Changed files: ${FILES}. Diff: ${DIFF}. Return a JSON object with keys: summary (string), risk_level (low/medium/high), issues (array of {file, line, severity, message, suggestion}), positive_aspects (array of strings), approval_recommendation (approve/request_changes/comment).\"\n\n          # Call Gemini API with structured output\n          RESPONSE=$(curl -sf \\\n            \"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_API_KEY}\" \\\n            -H 'Content-Type: application/json' \\\n            -d \"$(jq -n --arg prompt \"$PROMPT\" '{\n              contents: [{parts: [{text: $prompt}]}],\n              generationConfig: {\n                responseMimeType: \"application/json\",\n                responseSchema: {\n                  type: \"object\",\n                  properties: {\n                    summary: {type: \"string\"},\n                    risk_level: {type: \"string\", enum: [\"low\", \"medium\", \"high\"]},\n                    issues: {\n                      type: \"array\",\n                      items: {\n                        type: \"object\",\n                        properties: {\n                          file: {type: \"string\"},\n                          line: {type: \"integer\"},\n                          severity: {type: \"string\", enum: [\"error\", \"warning\", \"suggestion\"]},\n                          message: {type: \"string\"},\n                          suggestion: {type: \"string\"}\n                        },\n                        required: [\"file\", \"severity\", \"message\"]\n                      }\n                    },\n                    positive_aspects: {type: \"array\", items: {type: \"string\"}},\n                    approval_recommendation: {type: \"string\", enum: [\"approve\", \"request_changes\", \"comment\"]}\n                  },\n                  required: [\"summary\", \"risk_level\", \"issues\", \"approval_recommendation\"]\n                },\n                temperature: 0.2\n              }\n            }')\")\n\n          # Extract the text content from the API response\n          REVIEW=$(echo \"$RESPONSE\" | jq -r '.candidates[0].content.parts[0].text')\n          echo \"$REVIEW\" > /tmp/review.json\n\n          # Validate\n          jq empty /tmp/review.json\n\n      - name: Post review comment\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          REVIEW_FILE=\"/tmp/review.json\"\n\n          SUMMARY=$(jq -r '.summary' \"$REVIEW_FILE\")\n          RISK=$(jq -r '.risk_level' \"$REVIEW_FILE\")\n          APPROVAL=$(jq -r '.approval_recommendation' \"$REVIEW_FILE\")\n\n          # Format issues as markdown\n          ISSUES=$(jq -r '\n            if (.issues | length) > 0 then\n              \"| Severity | File | Line | Issue | Fix |\\n|---|---|---|---|---|\\n\" +\n              (.issues[] | \"| \\(.severity) | `\\(.file)` | \\(.line // \"N/A\") | \\(.message) | \\(.suggestion // \"N/A\") |\")\n            else\n              \"_No issues found._\"\n            end\n          ' \"$REVIEW_FILE\")\n\n          POSITIVES=$(jq -r '(.positive_aspects // []) | map(\"- \" + .) | join(\"\\n\")' \"$REVIEW_FILE\")\n\n          BODY=$(cat <<EOF\n          ## Automated PR Review (Gemini)\n\n          **Risk Level:** \\`${RISK}\\` | **Recommendation:** \\`${APPROVAL}\\`\n\n          ### Summary\n          ${SUMMARY}\n\n          ### Issues\n          ${ISSUES}\n\n          ### What looks good\n          ${POSITIVES}\n          EOF\n          )\n\n          gh pr comment \"${{ github.event.pull_request.number }}\" --body \"$BODY\"\n```\n\n## Step 4: Using Gemini CLI with `--json` Flag (If Supported)\n\nSome versions of the Gemini CLI support a `--json` output flag. If your version does, the invocation simplifies to:\n\n```bash\ngemini --model gemini-2.0-flash \\\n  --api-key \"$GEMINI_API_KEY\" \\\n  --json \\\n  --prompt \"Review this code diff and return structured JSON: $DIFF\"\n```\n\nCheck your installed version's help output (`gemini --help`) to confirm available flags.\n\n## Step 5: Parsing the JSON Output in Downstream Steps\n\nOnce you have valid JSON in a file, you can use it in subsequent pipeline steps:\n\n```yaml\n      - name: Fail on high-risk reviews\n        run: |\n          RISK=$(jq -r '.risk_level' /tmp/review.json)\n          ERRORS=$(jq '[.issues[] | select(.severity == \"error\")] | length' /tmp/review.json)\n\n          echo \"Risk level: $RISK\"\n          echo \"Error count: $ERRORS\"\n\n          if [ \"$RISK\" = \"high\" ] || [ \"$ERRORS\" -gt 0 ]; then\n            echo \"::error::PR review found high risk or errors. Please address the issues.\"\n            exit 1\n          fi\n```\n\nYou can also use it in a matrix strategy or pass data between jobs:\n\n```yaml\n      - name: Export review data\n        id: export\n        run: |\n          echo \"risk=$(jq -r '.risk_level' /tmp/review.json)\" >> $GITHUB_OUTPUT\n          echo \"approval=$(jq -r '.approval_recommendation' /tmp/review.json)\" >> $GITHUB_OUTPUT\n          echo \"issue_count=$(jq '.issues | length' /tmp/review.json)\" >> $GITHUB_OUTPUT\n```\n\nThen reference in later steps or jobs:\n\n```yaml\n      - name: Auto-approve if clean\n        if: steps.export.outputs.approval == 'approve' && steps.export.outputs.issue_count == '0'\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          gh pr review \"${{ github.event.pull_request.number }}\" --approve --body \"Automated review passed with no issues.\"\n```\n\n## Key Points\n\n1. **`responseMimeType: \"application/json\"`** in the Gemini API `generationConfig` forces the model to return valid JSON. This is the most reliable way to get structured output.\n\n2. **`responseSchema`** lets you define the exact JSON schema you expect. The model will conform to it. This eliminates the need for post-processing or regex extraction.\n\n3. **Diff size limits**: Git diffs can be very large. Truncate them (e.g., `head -c 30000`) to stay within token limits. For large PRs, consider reviewing changed files individually.\n\n4. **Error handling**: Always validate the JSON output with `jq empty` before trying to parse it. Have a fallback path that posts raw output as a comment.\n\n5. **Rate limits**: The Gemini API has rate limits. For high-traffic repos, add retry logic or use the paid tier.\n\n6. **Security**: The `GEMINI_API_KEY` secret is never exposed in logs because GitHub Actions automatically masks secrets in output. The `GITHUB_TOKEN` is provided automatically by Actions with the permissions declared in the workflow.\n\n## Troubleshooting\n\n- **\"jq: parse error\"**: The model returned non-JSON text. Use `responseMimeType: \"application/json\"` to fix this.\n- **Empty response**: Check that `GEMINI_API_KEY` is set correctly in secrets and that the model name is valid.\n- **Diff too large**: Truncate the diff or split the review across multiple API calls, one per file.\n- **Permission denied on PR comment**: Ensure `permissions: pull-requests: write` is set in the workflow.\n"}], "grading": {"eval_id": 1, "eval_name": "cicd-setup", "configuration": "without_skill", "expectations": [{"text": "Uses -p or --prompt flag for headless/non-interactive mode", "passed": false, "evidence": "Pipes prompt via stdin (`echo \"$PROMPT\" | gemini`) instead of using -p flag. Does not demonstrate the correct headless invocation."}, {"text": "Includes --output-format json flag for structured output", "passed": false, "evidence": "Mentions `--json` in Step 4 but says 'if supported' and 'check your version'. Does not use the correct `--output-format json` flag."}, {"text": "Shows how to set GEMINI_API_KEY from GitHub secrets", "passed": true, "evidence": "Shows `GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}` in env block"}, {"text": "Provides a GitHub Actions workflow YAML snippet", "passed": true, "evidence": "Provides multiple complete workflow YAML files"}, {"text": "Shows how to parse JSON response (e.g., with jq .response)", "passed": false, "evidence": "Parses the REST API response structure (candidates[0].content.parts[0].text) rather than Gemini CLI's JSON envelope (.response). Falls back to curl-based API calls."}, {"text": "Does not invent non-existent CLI flags or options", "passed": false, "evidence": "Uses `--api-key` flag (doesn't exist), `--json` flag (doesn't exist), and references wrong npm package `@anthropic-ai/gemini-cli`"}]}}, {"id": "eval-2-mcp-server-config-with_skill", "prompt": "I need to connect a custom MCP server I built in Python to gemini cli. The server needs an API key from my environment. Also I want to restrict which tools it can expose. How do I configure this?", "eval_id": 2, "outputs": [{"name": "response.md", "type": "text", "content": "# Connecting a Custom Python MCP Server to Gemini CLI\n\nYou need to configure your server in `settings.json`, pass the API key via the `env` field, and use `includeTools` to restrict which tools are exposed. Here is the full walkthrough.\n\n---\n\n## 1. Configure the MCP Server in settings.json\n\nEdit (or create) `~/.gemini/settings.json` for a global configuration, or `<your-project>/.gemini/settings.json` for project-scoped configuration. Add your server under the `mcpServers` key:\n\n```json\n{\n  \"mcpServers\": {\n    \"myPythonServer\": {\n      \"command\": \"python\",\n      \"args\": [\"-m\", \"my_mcp_server\"],\n      \"cwd\": \"./mcp_tools\",\n      \"env\": {\n        \"MY_API_KEY\": \"$MY_API_KEY\"\n      },\n      \"timeout\": 15000,\n      \"trust\": false,\n      \"includeTools\": [\"search_records\", \"get_status\"]\n    }\n  }\n}\n```\n\n### What each field does\n\n| Field | Type | Description |\n|---|---|---|\n| `command` | string | The executable to run. Use `\"python\"` (or `\"python3\"` depending on your system). |\n| `args` | string[] | Arguments passed to the command. `[\"-m\", \"my_mcp_server\"]` runs your package as a module. You can also use a script path like `[\"./server.py\"]`. |\n| `cwd` | string | Working directory for the server process. Optional -- set this if your server needs to run from a specific directory. |\n| `env` | object | Environment variables passed to the server process. Use the `\"$VAR_NAME\"` syntax to reference variables from your shell environment. |\n| `timeout` | number | Connection timeout in milliseconds. Default is 10000 (10 seconds). Increase if your server takes longer to start. |\n| `trust` | boolean | When `true`, skips all tool confirmation dialogs. Keep this `false` unless you fully control and trust the server. |\n| `includeTools` | string[] | **Whitelist** -- only these tools will be exposed to Gemini. If set, all other tools from the server are hidden. |\n| `excludeTools` | string[] | **Blacklist** -- these specific tools will be hidden. All others remain available. |\n\n**Important:** `includeTools` and `excludeTools` are mutually exclusive approaches. Use one or the other:\n- Use `includeTools` when your server exposes many tools but you only want a few available.\n- Use `excludeTools` when you want most tools but need to block specific dangerous ones.\n\n---\n\n## 2. Pass the API Key Securely\n\nNever hardcode secrets directly in `settings.json`. Instead, use the `\"$VAR_NAME\"` pattern to reference environment variables from your shell:\n\n```json\n\"env\": {\n  \"MY_API_KEY\": \"$MY_API_KEY\"\n}\n```\n\nThen set the variable in your shell before running Gemini:\n\n```bash\nexport MY_API_KEY=\"sk-your-actual-api-key-here\"\ngemini\n```\n\nAlternatively, you can store the variable in a `.env` file that Gemini loads automatically:\n\n- `~/.gemini/.env` -- global env file\n- `./.gemini/.env` -- project-level env file\n\nExample `.gemini/.env`:\n\n```\nMY_API_KEY=sk-your-actual-api-key-here\n```\n\n**Security note:** Gemini CLI automatically redacts sensitive env vars (names containing `TOKEN`, `SECRET`, `KEY`, etc.) from MCP processes unless you explicitly list them in the `env` field. So you must include the variable in `env` for it to reach your server.\n\n---\n\n## 3. Full Working Example\n\nSuppose you have a Python MCP server at `~/mcp-servers/analytics_server.py` that exposes five tools (`query_data`, `generate_report`, `delete_records`, `export_csv`, `run_migration`), and you only want `query_data` and `generate_report` available. The server needs a `ANALYTICS_API_KEY`.\n\n### settings.json\n\n```json\n{\n  \"mcpServers\": {\n    \"analytics\": {\n      \"command\": \"python3\",\n      \"args\": [\"analytics_server.py\"],\n      \"cwd\": \"/home/user/mcp-servers\",\n      \"env\": {\n        \"ANALYTICS_API_KEY\": \"$ANALYTICS_API_KEY\"\n      },\n      \"timeout\": 15000,\n      \"trust\": false,\n      \"includeTools\": [\"query_data\", \"generate_report\"]\n    }\n  }\n}\n```\n\n### Shell setup\n\n```bash\nexport ANALYTICS_API_KEY=\"your-key-here\"\ngemini\n```\n\n### Using the server in a Gemini session\n\nOnce inside the interactive session, you can verify the server connected:\n\n```\n/mcp list\n/mcp status\n```\n\nThen invoke the tools by addressing the server:\n\n```\n> @analytics Query monthly active users for Q4 2025\n> @analytics Generate a report on user retention trends\n```\n\n---\n\n## 4. Alternative: Use excludeTools Instead\n\nIf you want all tools available except specific dangerous ones:\n\n```json\n{\n  \"mcpServers\": {\n    \"analytics\": {\n      \"command\": \"python3\",\n      \"args\": [\"analytics_server.py\"],\n      \"cwd\": \"/home/user/mcp-servers\",\n      \"env\": {\n        \"ANALYTICS_API_KEY\": \"$ANALYTICS_API_KEY\"\n      },\n      \"timeout\": 15000,\n      \"trust\": false,\n      \"excludeTools\": [\"delete_records\", \"run_migration\"]\n    }\n  }\n}\n```\n\nThis exposes everything except `delete_records` and `run_migration`.\n\n---\n\n## 5. Adding the Server via CLI (Alternative to Manual Editing)\n\nYou can also add MCP servers from the command line:\n\n```bash\ngemini mcp add analytics --command \"python3 analytics_server.py\"\n```\n\nHowever, the CLI `add` command provides less control over fields like `env`, `includeTools`, and `cwd`. For full configuration, editing `settings.json` directly is recommended.\n\nTo manage servers later:\n\n```bash\ngemini mcp list           # list all configured servers\ngemini mcp remove analytics  # remove a server\n```\n\nOr from within an interactive session:\n\n```\n/mcp list\n/mcp enable analytics\n/mcp disable analytics\n```\n\n---\n\n## 6. Minimal Python MCP Server Skeleton\n\nFor reference, here is the basic structure of a Python MCP server that your configuration would launch:\n\n```python\n# analytics_server.py\nimport os\nimport json\nimport sys\n\n# Read the API key passed through the env config\napi_key = os.environ.get(\"ANALYTICS_API_KEY\")\nif not api_key:\n    print(\"Error: ANALYTICS_API_KEY not set\", file=sys.stderr)\n    sys.exit(1)\n\n# Your MCP server implementation here\n# The server communicates over stdio using the MCP protocol\n# Tools like query_data, generate_report, etc. are registered\n# and handled according to the MCP specification.\n```\n\n---\n\n## Summary\n\n1. Add your server to `mcpServers` in `~/.gemini/settings.json` (or project-level).\n2. Pass the API key using `\"env\": { \"MY_API_KEY\": \"$MY_API_KEY\" }` -- never hardcode it.\n3. Restrict tools with `\"includeTools\": [\"tool1\", \"tool2\"]` (whitelist) or `\"excludeTools\": [\"bad_tool\"]` (blacklist).\n4. Set the actual environment variable in your shell or in a `.gemini/.env` file.\n5. Verify the connection with `/mcp list` and `/mcp status` inside an interactive session.\n"}], "grading": {"eval_id": 2, "eval_name": "mcp-server-config", "configuration": "with_skill", "expectations": [{"text": "Points to ~/.gemini/settings.json or project-level settings.json", "passed": true, "evidence": "Mentions both `~/.gemini/settings.json` (global) and `<project>/.gemini/settings.json` (project-scoped)"}, {"text": "Shows mcpServers config with command, args fields", "passed": true, "evidence": "Complete mcpServers config with command, args, cwd, env, timeout, trust, includeTools fields"}, {"text": "Shows $VAR pattern for environment variable references in env field", "passed": true, "evidence": "Shows `\"MY_API_KEY\": \"$MY_API_KEY\"` and explains the $VAR_NAME pattern explicitly"}, {"text": "Explains includeTools and/or excludeTools for restricting exposed tools", "passed": true, "evidence": "Explains both includeTools (whitelist) and excludeTools (blacklist) with examples and guidance on when to use each"}, {"text": "Mentions security considerations (trust field, not hardcoding keys, etc.)", "passed": true, "evidence": "Warns to never hardcode secrets, explains trust field, mentions auto-redaction of sensitive env vars"}, {"text": "Uses correct Python command (python/python3) in the command field", "passed": true, "evidence": "Uses both `python` and `python3` with note about system differences"}]}}, {"id": "eval-2-mcp-server-config-without_skill", "prompt": "I need to connect a custom MCP server I built in Python to gemini cli. The server needs an API key from my environment. Also I want to restrict which tools it can expose. How do I configure this?", "eval_id": 2, "outputs": [{"name": "response.md", "type": "text", "content": "# Connecting a Custom Python MCP Server to Gemini CLI\n\n## Overview\n\nGemini CLI supports MCP (Model Context Protocol) servers through its settings configuration. You can connect a custom Python MCP server, pass environment variables (like API keys), and restrict which tools are exposed.\n\n## 1. Build Your Python MCP Server\n\nFirst, make sure your Python MCP server uses the official MCP SDK. Here is a minimal example:\n\n```python\n# my_mcp_server.py\nimport os\nfrom mcp.server.fastmcp import FastMCP\n\n# Read the API key from environment\nAPI_KEY = os.environ.get(\"MY_API_KEY\")\nif not API_KEY:\n    raise ValueError(\"MY_API_KEY environment variable is required\")\n\nmcp = FastMCP(\"my-custom-server\")\n\n@mcp.tool()\ndef fetch_data(query: str) -> str:\n    \"\"\"Fetch data from the external API using the configured API key.\"\"\"\n    # Use API_KEY here for your requests\n    return f\"Results for '{query}' using key ending in ...{API_KEY[-4:]}\"\n\n@mcp.tool()\ndef analyze_data(data: str) -> str:\n    \"\"\"Analyze the provided data.\"\"\"\n    return f\"Analysis of: {data}\"\n\n@mcp.tool()\ndef internal_debug(info: str) -> str:\n    \"\"\"Internal debug tool - you may want to hide this from Gemini.\"\"\"\n    return f\"Debug: {info}\"\n\nif __name__ == \"__main__\":\n    mcp.run()\n```\n\nInstall the dependency:\n\n```bash\npip install mcp\n```\n\n## 2. Configure Gemini CLI to Use Your MCP Server\n\nGemini CLI reads MCP server configuration from a `settings.json` file. There are two levels:\n\n- **Global (user-level):** `~/.gemini/settings.json`\n- **Project-level:** `.gemini/settings.json` in your project root\n\n### Basic Configuration\n\nCreate or edit `~/.gemini/settings.json` (or `.gemini/settings.json` in your project):\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"cwd\": \"/absolute/path/to/your/server/directory\",\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-value-here\"\n      }\n    }\n  }\n}\n```\n\n### Configuration Fields Explained\n\n| Field | Description |\n|-------|-------------|\n| `command` | The executable to run (e.g., `python`, `python3`, `node`, `npx`) |\n| `args` | Array of arguments passed to the command |\n| `cwd` | Working directory for the server process (optional but recommended) |\n| `env` | Environment variables passed to the server process |\n\n### Using a Virtual Environment\n\nIf your MCP server has its own virtual environment:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"/path/to/venv/bin/python\",\n      \"args\": [\"/absolute/path/to/my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-value-here\"\n      }\n    }\n  }\n}\n```\n\nOn Windows, use the Windows path to the venv Python:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"C:\\\\path\\\\to\\\\venv\\\\Scripts\\\\python.exe\",\n      \"args\": [\"C:\\\\path\\\\to\\\\my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-value-here\"\n      }\n    }\n  }\n}\n```\n\n## 3. Passing the API Key Securely\n\nThere are several approaches to handle the API key:\n\n### Option A: Inline in settings.json (simplest, but key is in plaintext)\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"sk-abc123your-actual-key\"\n      }\n    }\n  }\n}\n```\n\n### Option B: Reference a shell variable at launch time\n\nSet the environment variable in your shell profile (`.bashrc`, `.zshrc`, etc.):\n\n```bash\nexport MY_API_KEY=\"sk-abc123your-actual-key\"\n```\n\nThen in `settings.json`, you can omit the `env` block entirely if the variable is already in your shell environment. The child process inherits the parent environment. However, to be explicit:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"${MY_API_KEY}\"\n      }\n    }\n  }\n}\n```\n\n> **Note:** Whether `${MY_API_KEY}` variable interpolation is supported depends on the Gemini CLI version. If it is not supported, you may need to use a wrapper script approach (see Option C).\n\n### Option C: Wrapper script (most flexible)\n\nCreate a wrapper script `run_server.sh`:\n\n```bash\n#!/bin/bash\n# run_server.sh\n# Load API key from a secure location\nsource ~/.secrets/my_api_keys.env\nexport MY_API_KEY\nexec python /path/to/my_mcp_server.py \"$@\"\n```\n\nThen configure:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"bash\",\n      \"args\": [\"/path/to/run_server.sh\"]\n    }\n  }\n}\n```\n\n## 4. Restricting Which Tools Are Exposed\n\nThere are two approaches to restricting tools: server-side and client-side.\n\n### Approach A: Server-Side Restriction (Recommended)\n\nOnly define the tools you want exposed in your MCP server. Simply remove or comment out any tools you do not want Gemini to access:\n\n```python\n# my_mcp_server.py - only expose the tools you want\nfrom mcp.server.fastmcp import FastMCP\nimport os\n\nAPI_KEY = os.environ.get(\"MY_API_KEY\")\nmcp = FastMCP(\"my-custom-server\")\n\n@mcp.tool()\ndef fetch_data(query: str) -> str:\n    \"\"\"Fetch data from the external API.\"\"\"\n    return f\"Results for '{query}'\"\n\n@mcp.tool()\ndef analyze_data(data: str) -> str:\n    \"\"\"Analyze the provided data.\"\"\"\n    return f\"Analysis of: {data}\"\n\n# internal_debug is NOT registered - Gemini will never see it\n\nif __name__ == \"__main__\":\n    mcp.run()\n```\n\n### Approach B: Conditional Tool Registration\n\nUse an environment variable or config file to control which tools are registered:\n\n```python\n# my_mcp_server.py - conditionally register tools\nfrom mcp.server.fastmcp import FastMCP\nimport os\nimport json\n\nAPI_KEY = os.environ.get(\"MY_API_KEY\")\nALLOWED_TOOLS = os.environ.get(\"ALLOWED_TOOLS\", \"\").split(\",\")\n\nmcp = FastMCP(\"my-custom-server\")\n\ndef register_if_allowed(tool_name):\n    \"\"\"Decorator factory that only registers a tool if it's in the allowed list.\"\"\"\n    def decorator(func):\n        if not ALLOWED_TOOLS or not ALLOWED_TOOLS[0]:\n            # No restriction set, register all tools\n            return mcp.tool()(func)\n        elif tool_name in ALLOWED_TOOLS:\n            return mcp.tool()(func)\n        return func\n    return decorator\n\n@register_if_allowed(\"fetch_data\")\ndef fetch_data(query: str) -> str:\n    \"\"\"Fetch data from the external API.\"\"\"\n    return f\"Results for '{query}'\"\n\n@register_if_allowed(\"analyze_data\")\ndef analyze_data(data: str) -> str:\n    \"\"\"Analyze the provided data.\"\"\"\n    return f\"Analysis of: {data}\"\n\n@register_if_allowed(\"internal_debug\")\ndef internal_debug(info: str) -> str:\n    \"\"\"Internal debug tool.\"\"\"\n    return f\"Debug: {info}\"\n\nif __name__ == \"__main__\":\n    mcp.run()\n```\n\nThen in your `settings.json`, pass the allowed tools list:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-here\",\n        \"ALLOWED_TOOLS\": \"fetch_data,analyze_data\"\n      }\n    }\n  }\n}\n```\n\nThis way, `internal_debug` is never registered and Gemini cannot call it.\n\n### Approach C: Gemini CLI Tool Filtering (if supported)\n\nSome MCP clients support an `allowedTools` or `toolFilter` field in the server configuration. If Gemini CLI supports this, the configuration would look like:\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"env\": {\n        \"MY_API_KEY\": \"your-api-key-here\"\n      },\n      \"toolFilter\": {\n        \"allow\": [\"fetch_data\", \"analyze_data\"]\n      }\n    }\n  }\n}\n```\n\n> **Note:** The exact field name and syntax for client-side tool filtering varies by MCP client implementation. Check the Gemini CLI documentation for the exact syntax. If Gemini CLI does not support this field, use the server-side approaches (A or B) described above.\n\n## 5. Complete Working Example\n\n### Directory Structure\n\n```\nmy-mcp-project/\n  my_mcp_server.py\n  requirements.txt\n  .gemini/\n    settings.json\n```\n\n### requirements.txt\n\n```\nmcp>=1.0.0\n```\n\n### my_mcp_server.py\n\n```python\nimport os\nfrom mcp.server.fastmcp import FastMCP\n\nAPI_KEY = os.environ.get(\"MY_API_KEY\")\nif not API_KEY:\n    raise ValueError(\"MY_API_KEY environment variable must be set\")\n\nALLOWED_TOOLS = [t.strip() for t in os.environ.get(\"ALLOWED_TOOLS\", \"\").split(\",\") if t.strip()]\n\nmcp = FastMCP(\"my-custom-server\")\n\n\ndef expose(tool_name):\n    \"\"\"Only register the tool if it passes the allowlist filter.\"\"\"\n    def decorator(func):\n        if not ALLOWED_TOOLS or tool_name in ALLOWED_TOOLS:\n            return mcp.tool()(func)\n        return func\n    return decorator\n\n\n@expose(\"fetch_data\")\ndef fetch_data(query: str) -> str:\n    \"\"\"Fetch data from the external API using the configured API key.\"\"\"\n    # Your real implementation here\n    return f\"Fetched results for '{query}'\"\n\n\n@expose(\"analyze_data\")\ndef analyze_data(data: str) -> str:\n    \"\"\"Analyze the provided data and return insights.\"\"\"\n    return f\"Analysis complete for: {data}\"\n\n\n@expose(\"dangerous_reset\")\ndef dangerous_reset() -> str:\n    \"\"\"Reset all data - restricted tool.\"\"\"\n    return \"Data reset complete\"\n\n\nif __name__ == \"__main__\":\n    mcp.run()\n```\n\n### .gemini/settings.json\n\n```json\n{\n  \"mcpServers\": {\n    \"my-custom-server\": {\n      \"command\": \"python\",\n      \"args\": [\"my_mcp_server.py\"],\n      \"cwd\": \"/absolute/path/to/my-mcp-project\",\n      \"env\": {\n        \"MY_API_KEY\": \"sk-your-api-key-here\",\n        \"ALLOWED_TOOLS\": \"fetch_data,analyze_data\"\n      }\n    }\n  }\n}\n```\n\nWith this configuration:\n- Gemini CLI launches your Python MCP server as a subprocess\n- The `MY_API_KEY` environment variable is passed securely to the server process\n- Only `fetch_data` and `analyze_data` are registered as tools; `dangerous_reset` is excluded\n- Gemini can discover and invoke the allowed tools during conversation\n\n## 6. Testing Your Server\n\nBefore connecting to Gemini CLI, test your server standalone:\n\n```bash\n# Test that the server starts correctly\nMY_API_KEY=\"test-key\" python my_mcp_server.py\n\n# Or with tool restrictions\nMY_API_KEY=\"test-key\" ALLOWED_TOOLS=\"fetch_data\" python my_mcp_server.py\n```\n\nYou can also use the MCP Inspector tool to verify your server works:\n\n```bash\nnpx @modelcontextprotocol/inspector python my_mcp_server.py\n```\n\nThen launch Gemini CLI in your project directory. It will automatically pick up the `.gemini/settings.json` and start your MCP server.\n\n## 7. Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Server not found | Use absolute paths in `command` and `args` |\n| API key not available | Verify the `env` block in settings.json; check with a print/log statement |\n| Tools not showing up | Confirm the tool functions are decorated with `@mcp.tool()` and not filtered out |\n| Permission errors | Ensure the Python executable and script have correct permissions |\n| Server crashes on start | Run the server manually from the terminal first to see error output |\n| Wrong Python version | Use the full path to the correct Python interpreter in `command` |\n"}], "grading": {"eval_id": 2, "eval_name": "mcp-server-config", "configuration": "without_skill", "expectations": [{"text": "Points to ~/.gemini/settings.json or project-level settings.json", "passed": true, "evidence": "Mentions both `~/.gemini/settings.json` and `.gemini/settings.json` in project root"}, {"text": "Shows mcpServers config with command, args fields", "passed": true, "evidence": "Shows mcpServers config with command, args, cwd, env fields"}, {"text": "Shows $VAR pattern for environment variable references in env field", "passed": false, "evidence": "Hardcodes API key directly in env (`\"MY_API_KEY\": \"sk-abc123your-actual-key\"`). Uses `${MY_API_KEY}` syntax but adds caveat that it might not be supported. Does not show the correct `$VAR` pattern."}, {"text": "Explains includeTools and/or excludeTools for restricting exposed tools", "passed": false, "evidence": "Does not mention includeTools or excludeTools. Instead suggests server-side tool filtering and uses non-existent `toolFilter` field with `allow` sub-field."}, {"text": "Mentions security considerations (trust field, not hardcoding keys, etc.)", "passed": false, "evidence": "Does not mention the trust field. Actually recommends hardcoding as Option A. Does not mention env var redaction behavior."}, {"text": "Uses correct Python command (python/python3) in the command field", "passed": true, "evidence": "Uses `python` in the command field correctly"}]}}, {"id": "eval-3-custom-slash-command-with_skill", "prompt": "How do I create a custom slash command in gemini cli that runs git diff --cached and asks the AI to write a commit message? I want it available in all my projects.", "eval_id": 3, "outputs": [{"name": "response.md", "type": "text", "content": "# Creating a Global Custom Slash Command for AI Commit Messages in Gemini CLI\n\nYou can create a custom slash command by writing a `.toml` file in the global commands directory. Since you want it available in all your projects, place it under `~/.gemini/commands/`.\n\n## Step 1: Create the Commands Directory\n\n```bash\nmkdir -p ~/.gemini/commands\n```\n\n## Step 2: Create the Slash Command TOML File\n\nCreate the file `~/.gemini/commands/commit.toml`:\n\n```toml\ndescription = \"Generate a commit message from staged changes\"\nprompt = \"\"\"\nWrite a concise conventional commit message for the following staged changes.\n\nUse the format: <type>(<scope>): <description>\n\nWhere type is one of: feat, fix, docs, style, refactor, test, chore\n\nOnly output the commit message itself, nothing else.\n\nStaged diff:\n!{git diff --cached}\n\"\"\"\n```\n\n### How This Works\n\n- The `prompt` field defines the instruction sent to Gemini when the command is invoked.\n- `!{git diff --cached}` is a **shell execution block** -- Gemini CLI runs the shell command inside `!{...}` and inlines its output into the prompt before sending it to the model.\n- `description` is optional but shows up when you run `/help` in an interactive session.\n\n## Step 3: Use the Command\n\nStart Gemini CLI in any project that has staged git changes:\n\n```bash\ncd your-project\ngit add -A   # or stage specific files\ngemini\n```\n\nThen in the interactive session, type:\n\n```\n/commit\n```\n\nGemini will execute `git diff --cached`, include the diff in the prompt, and return a conventional commit message.\n\n## Namespaced Alternative\n\nIf you prefer a namespaced command like `/git:commit`, use a subdirectory:\n\n```bash\nmkdir -p ~/.gemini/commands/git\n```\n\nCreate `~/.gemini/commands/git/commit.toml`:\n\n```toml\ndescription = \"Generate a commit message from staged changes\"\nprompt = \"\"\"\nWrite a concise conventional commit message for the following staged changes.\n\nUse the format: <type>(<scope>): <description>\n\nWhere type is one of: feat, fix, docs, style, refactor, test, chore\n\nOnly output the commit message itself, nothing else.\n\nStaged diff:\n!{git diff --cached}\n\"\"\"\n```\n\nThen invoke it with:\n\n```\n/git:commit\n```\n\n## Using It with Arguments\n\nYou can also make the command accept arguments with `{{args}}` to allow the user to pass additional context:\n\n```toml\ndescription = \"Generate a commit message from staged changes with optional context\"\nprompt = \"\"\"\nWrite a concise conventional commit message for the following staged changes.\n\nUse the format: <type>(<scope>): <description>\n\nWhere type is one of: feat, fix, docs, style, refactor, test, chore\n\nAdditional context from the user: {{args}}\n\nOnly output the commit message itself, nothing else.\n\nStaged diff:\n!{git diff --cached}\n\"\"\"\n```\n\nThis lets you provide extra hints:\n\n```\n/commit focus on the auth refactoring\n```\n\n## Fully Automated Shell Script (Bonus)\n\nIf you want to go beyond interactive mode and automate the entire commit flow from your shell, you can create a shell script. Note that custom slash commands currently do **not** work in headless mode (`-p` flag), so for automation you need to embed the prompt directly:\n\n```bash\n#!/bin/bash\n# Save as ~/bin/ai-commit or anywhere on your PATH\n\nDIFF=$(git diff --cached)\n\nif [ -z \"$DIFF\" ]; then\n  echo \"Error: Nothing staged. Run 'git add' first.\"\n  exit 1\nfi\n\nMSG=$(echo \"$DIFF\" | gemini -p \"Write a concise conventional commit message (type(scope): description format, max 72 chars). Only output the message, nothing else.\" --output-format json | jq -r '.response')\n\nif [ -z \"$MSG\" ]; then\n  echo \"Error: Failed to generate commit message.\"\n  exit 1\nfi\n\necho \"Proposed commit message:\"\necho \"  $MSG\"\necho \"\"\nread -p \"Commit with this message? [y/N] \" confirm\n\nif [[ \"$confirm\" =~ ^[Yy]$ ]]; then\n  git commit -m \"$MSG\"\n  echo \"Committed.\"\nelse\n  echo \"Aborted.\"\nfi\n```\n\nMake it executable:\n\n```bash\nchmod +x ~/bin/ai-commit\n```\n\nThen use it from any project:\n\n```bash\ngit add -A\nai-commit\n```\n\n## Summary of File Locations\n\n| Path | Scope | Invocation |\n|---|---|---|\n| `~/.gemini/commands/commit.toml` | Global (all projects) | `/commit` |\n| `~/.gemini/commands/git/commit.toml` | Global, namespaced | `/git:commit` |\n| `<project>/.gemini/commands/commit.toml` | Project only | `/commit` |\n\nThe global location (`~/.gemini/commands/`) is what makes the command available across all your projects without any per-project setup.\n"}], "grading": {"eval_id": 3, "eval_name": "custom-slash-command", "configuration": "with_skill", "expectations": [{"text": "Points to ~/.gemini/commands/ for global user-scoped commands", "passed": true, "evidence": "Correctly points to `~/.gemini/commands/commit.toml` for global scope"}, {"text": "Shows .toml file format with prompt field", "passed": true, "evidence": "Full .toml example with description and prompt fields"}, {"text": "Uses !{git diff --cached} shell execution syntax in the prompt", "passed": true, "evidence": "Correctly uses `!{git diff --cached}` in the prompt field"}, {"text": "Warns that custom slash commands don't work in headless/non-interactive mode", "passed": true, "evidence": "Explicitly warns: 'custom slash commands currently do not work in headless mode (`-p` flag)' and provides shell script alternative"}, {"text": "Provides a complete, copy-pasteable .toml file example", "passed": true, "evidence": "Multiple complete .toml examples including basic, namespaced, and with-arguments variants"}]}}, {"id": "eval-3-custom-slash-command-without_skill", "prompt": "How do I create a custom slash command in gemini cli that runs git diff --cached and asks the AI to write a commit message? I want it available in all my projects.", "eval_id": 3, "outputs": [{"name": "response.md", "type": "text", "content": "# Creating a Custom Slash Command in Gemini CLI for Commit Messages\n\n## Overview\n\nGemini CLI supports custom slash commands through its extensions system. To make a command available across all your projects, you place it in the global configuration directory at `~/.gemini/extensions/`. Extensions are defined using `gemini_extension.json` manifest files that describe the command and its behavior.\n\n## Step-by-Step Setup\n\n### 1. Create the Global Extensions Directory\n\n```bash\nmkdir -p ~/.gemini/extensions/commit-message\n```\n\n### 2. Create the Shell Script\n\nCreate `~/.gemini/extensions/commit-message/generate-commit-msg.sh`:\n\n```bash\n#!/bin/bash\n# generate-commit-msg.sh\n# Gets the staged diff and outputs it for the AI to process\n\nDIFF=$(git diff --cached)\n\nif [ -z \"$DIFF\" ]; then\n  echo \"ERROR: No staged changes found. Stage your changes with 'git add' first.\"\n  exit 1\nfi\n\necho \"Here is the staged git diff:\"\necho \"\"\necho '```diff'\necho \"$DIFF\"\necho '```'\necho \"\"\necho \"Please write a concise, conventional commit message for these changes.\"\necho \"Follow the Conventional Commits format: type(scope): description\"\necho \"\"\necho \"Types: feat, fix, docs, style, refactor, perf, test, build, ci, chore\"\necho \"\"\necho \"Provide:\"\necho \"1. A single-line subject (max 72 chars)\"\necho \"2. A blank line\"\necho \"3. An optional body explaining the 'why' behind the changes\"\n```\n\nMake it executable:\n\n```bash\nchmod +x ~/.gemini/extensions/commit-message/generate-commit-msg.sh\n```\n\n### 3. Create the Extension Manifest\n\nCreate `~/.gemini/extensions/commit-message/gemini_extension.json`:\n\n```json\n{\n  \"name\": \"commit-message\",\n  \"description\": \"Generate a commit message from staged changes using git diff --cached\",\n  \"version\": \"1.0.0\",\n  \"commands\": [\n    {\n      \"name\": \"commit-msg\",\n      \"description\": \"Analyze staged git changes and generate a conventional commit message\",\n      \"action\": {\n        \"type\": \"shell\",\n        \"command\": \"~/.gemini/extensions/commit-message/generate-commit-msg.sh\"\n      }\n    }\n  ]\n}\n```\n\n### 4. Usage\n\nOnce configured, open Gemini CLI in any project and type:\n\n```\n/commit-msg\n```\n\nThe command will:\n1. Run `git diff --cached` to capture your staged changes\n2. Pass the diff output into the conversation context\n3. Ask Gemini to generate a conventional commit message based on the diff\n\n## Alternative Approach: Using GEMINI.md with a Prompt-Based Workflow\n\nIf extensions are not behaving as expected in your version of Gemini CLI, you can achieve a similar result using a global `GEMINI.md` instruction file combined with a simple shell alias.\n\n### 1. Create a Global GEMINI.md\n\nCreate or edit `~/.gemini/GEMINI.md`:\n\n```markdown\n# Custom Commands\n\n## Commit Message Generation\n\nWhen I say \"generate commit message\" or provide a git diff, follow these rules:\n\n1. Analyze the diff carefully\n2. Write a commit message in Conventional Commits format\n3. Use the format: type(scope): short description\n4. Include a body if the changes are complex\n5. Keep the subject line under 72 characters\n6. Use imperative mood (\"add feature\" not \"added feature\")\n```\n\n### 2. Create a Shell Helper Script\n\nAdd to your `~/.bashrc` or `~/.zshrc`:\n\n```bash\n# Gemini CLI commit message helper\ngcm() {\n  local diff\n  diff=$(git diff --cached)\n\n  if [ -z \"$diff\" ]; then\n    echo \"No staged changes. Use 'git add' to stage files first.\"\n    return 1\n  fi\n\n  echo \"Staged diff:\"\n  echo '```diff'\n  echo \"$diff\"\n  echo '```'\n  echo \"\"\n  echo \"Generate a conventional commit message for the above changes.\"\n}\n```\n\nThen in Gemini CLI, you can run:\n\n```\n@run gcm\n```\n\nOr pipe the output directly:\n\n```bash\n# From your regular terminal, pipe into gemini\ngcm | gemini\n```\n\n## Alternative Approach: Inline Tool Use\n\nGemini CLI allows you to invoke shell commands inline. You can simply type within a Gemini CLI session:\n\n```\nPlease run `git diff --cached` and write me a commit message based on the output.\n```\n\nSince Gemini CLI has the ability to execute shell commands (with your approval), it will run the diff, read the output, and generate a commit message. This requires no configuration at all, but it is a manual process each time rather than a single slash command.\n\n## Applying the Generated Commit Message\n\nOnce Gemini generates a commit message you like, you can apply it directly:\n\n```bash\ngit commit -m \"feat(auth): add OAuth2 token refresh logic\n\nImplement automatic token refresh when access tokens expire.\nUses refresh token rotation for improved security.\"\n```\n\nOr ask Gemini CLI to run the commit for you:\n\n```\nPlease commit with that message.\n```\n\n## Directory Structure Summary\n\n```\n~/.gemini/\n  GEMINI.md                          # Global instructions (optional)\n  extensions/\n    commit-message/\n      gemini_extension.json          # Extension manifest\n      generate-commit-msg.sh         # Shell script for the command\n```\n\n## Tips\n\n- **Scope control**: Placing extensions in `~/.gemini/extensions/` makes them global. For project-specific commands, place them in `.gemini/extensions/` within the project root.\n- **Customizing the prompt**: Edit the echo statements at the end of `generate-commit-msg.sh` to adjust the instructions given to the AI (e.g., require a specific commit style, add ticket number references, etc.).\n- **Large diffs**: If the staged diff is very large, consider truncating it or summarizing file-level changes to stay within context limits. You can modify the script to include `--stat` output alongside a truncated diff.\n- **Security**: The shell script only reads data (via `git diff --cached`). It does not modify your repository or make commits on its own.\n\n## Troubleshooting\n\n- **\"Command not found\"**: Make sure the shell script is executable (`chmod +x`) and the path in `gemini_extension.json` is correct.\n- **\"No staged changes\"**: Run `git add <files>` before invoking the command.\n- **Extension not loading**: Verify the JSON manifest is valid (`cat ~/.gemini/extensions/commit-message/gemini_extension.json | python3 -m json.tool`) and restart Gemini CLI.\n- **Path issues on Windows**: Use forward slashes or the `%USERPROFILE%` equivalent in paths. The extensions directory would be at `%USERPROFILE%\\.gemini\\extensions\\`.\n"}], "grading": {"eval_id": 3, "eval_name": "custom-slash-command", "configuration": "without_skill", "expectations": [{"text": "Points to ~/.gemini/commands/ for global user-scoped commands", "passed": false, "evidence": "Points to `~/.gemini/extensions/commit-message/` instead. Does not know about the commands/ directory or .toml approach."}, {"text": "Shows .toml file format with prompt field", "passed": false, "evidence": "Uses gemini_extension.json manifest with commands array instead of .toml format"}, {"text": "Uses !{git diff --cached} shell execution syntax in the prompt", "passed": false, "evidence": "Creates a separate shell script to execute git diff. Does not know about the !{...} inline shell execution syntax."}, {"text": "Warns that custom slash commands don't work in headless/non-interactive mode", "passed": false, "evidence": "No mention of headless mode limitation for slash commands"}, {"text": "Provides a complete, copy-pasteable .toml file example", "passed": false, "evidence": "Provides a complete extension example, but it uses the wrong mechanism entirely (extensions vs custom commands)"}]}}], "previous_feedback": {}, "previous_outputs": {}, "benchmark": {"skill_name": "gemini-cli", "iteration": 1, "configurations": [{"name": "with_skill", "evals": [{"eval_name": "cicd-setup", "pass_rate": 1.0, "passed": 6, "total": 6, "tokens": 73417, "duration_seconds": 92.4}, {"eval_name": "mcp-server-config", "pass_rate": 1.0, "passed": 6, "total": 6, "tokens": 67196, "duration_seconds": 64.1}, {"eval_name": "custom-slash-command", "pass_rate": 1.0, "passed": 5, "total": 5, "tokens": 70530, "duration_seconds": 49.5}], "aggregate": {"mean_pass_rate": 1.0, "mean_tokens": 70381, "mean_duration_seconds": 68.7}}, {"name": "without_skill", "evals": [{"eval_name": "cicd-setup", "pass_rate": 0.33, "passed": 2, "total": 6, "tokens": 60035, "duration_seconds": 77.9}, {"eval_name": "mcp-server-config", "pass_rate": 0.33, "passed": 2, "total": 6, "tokens": 59258, "duration_seconds": 65.4}, {"eval_name": "custom-slash-command", "pass_rate": 0.0, "passed": 0, "total": 5, "tokens": 57648, "duration_seconds": 56.7}], "aggregate": {"mean_pass_rate": 0.22, "mean_tokens": 58980, "mean_duration_seconds": 66.7}}], "delta": {"pass_rate": "+0.78", "tokens": "+11401 (19% more)", "duration": "+2.0s (3% more)"}, "analyst_observations": ["The skill provides a massive accuracy improvement (100% vs 22% pass rate) with only marginal cost increase (~19% more tokens, ~3% more time).", "Eval 3 (custom slash command) shows the largest gap: the baseline gets the entire mechanism wrong (uses extensions instead of .toml commands), scoring 0/5.", "Eval 1 (CI/CD) baseline hallucinates non-existent flags (--api-key, --json) and even the wrong npm package name (@anthropic-ai/gemini-cli).", "Eval 2 (MCP config) baseline doesn't know about includeTools/excludeTools and recommends hardcoding API keys, which is a security anti-pattern.", "All assertions are discriminating - none pass for both configurations, confirming the skill provides genuine value over baseline knowledge."]}};
-    // ---- State ----
-    let feedbackMap = {};  // run_id -> feedback text
-    let currentIndex = 0;
-    let visitedRuns = new Set();
-    // ---- Init ----
-    async function init() {
-      // Load saved feedback from server — but only if this isn't a fresh
-      // iteration (indicated by previous_feedback being present). When
-      // previous feedback exists, the feedback.json on disk is stale from
-      // the prior iteration and should not pre-fill the textareas.
-      const hasPrevious = Object.keys(EMBEDDED_DATA.previous_feedback || {}).length > 0
-        || Object.keys(EMBEDDED_DATA.previous_outputs || {}).length > 0;
-      if (!hasPrevious) {
-        try {
-          const resp = await fetch("/api/feedback");
-          const data = await resp.json();
-          if (data.reviews) {
-            for (const r of data.reviews) feedbackMap[r.run_id] = r.feedback;
-          }
-        } catch { /* first run, no feedback yet */ }
-      }
-      document.getElementById("skill-name").textContent = EMBEDDED_DATA.skill_name;
-      showRun(0);
-      // Wire up feedback auto-save
-      const textarea = document.getElementById("feedback");
-      let saveTimeout = null;
-      textarea.addEventListener("input", () => {
-        clearTimeout(saveTimeout);
-        document.getElementById("feedback-status").textContent = "";
-        saveTimeout = setTimeout(() => saveCurrentFeedback(), 800);
-      });
-    }
-    // ---- Navigation ----
-    function navigate(delta) {
-      const newIndex = currentIndex + delta;
-      if (newIndex >= 0 && newIndex < EMBEDDED_DATA.runs.length) {
-        saveCurrentFeedback();
-        showRun(newIndex);
-      }
-    }
-    function updateNavButtons() {
-      document.getElementById("prev-btn").disabled = currentIndex === 0;
-      document.getElementById("next-btn").disabled =
-        currentIndex === EMBEDDED_DATA.runs.length - 1;
-    }
-    // ---- Show a run ----
-    function showRun(index) {
-      currentIndex = index;
-      const run = EMBEDDED_DATA.runs[index];
-      // Progress
-      document.getElementById("progress").textContent =
-        `${index + 1} of ${EMBEDDED_DATA.runs.length}`;
-      // Prompt
-      document.getElementById("prompt-text").textContent = run.prompt;
-      // Config badge
-      const badge = document.getElementById("config-badge");
-      const configMatch = run.id.match(/(with_skill|without_skill|new_skill|old_skill)/);
-      if (configMatch) {
-        const config = configMatch[1];
-        const isBaseline = config === "without_skill" || config === "old_skill";
-        badge.textContent = config.replace(/_/g, " ");
-        badge.className = "config-badge " + (isBaseline ? "config-baseline" : "config-primary");
-        badge.style.display = "inline-block";
-      } else {
-        badge.style.display = "none";
-      }
-      // Outputs
-      renderOutputs(run);
-      // Previous outputs
-      renderPrevOutputs(run);
-      // Grades
-      renderGrades(run);
-      // Previous feedback
-      const prevFb = (EMBEDDED_DATA.previous_feedback || {})[run.id];
-      const prevEl = document.getElementById("prev-feedback");
-      if (prevFb) {
-        document.getElementById("prev-feedback-text").textContent = prevFb;
-        prevEl.style.display = "block";
-      } else {
-        prevEl.style.display = "none";
-      }
-      // Feedback
-      document.getElementById("feedback").value = feedbackMap[run.id] || "";
-      document.getElementById("feedback-status").textContent = "";
-      updateNavButtons();
-      // Track visited runs and promote done button when all visited
-      visitedRuns.add(index);
-      const doneBtn = document.getElementById("done-btn");
-      if (visitedRuns.size >= EMBEDDED_DATA.runs.length) {
-        doneBtn.classList.add("ready");
-      }
-      // Scroll main content to top
-      document.querySelector(".main").scrollTop = 0;
-    }
-    // ---- Render outputs ----
-    function renderOutputs(run) {
-      const container = document.getElementById("outputs-body");
-      container.innerHTML = "";
-      const outputs = run.outputs || [];
-      if (outputs.length === 0) {
-        container.innerHTML = '<div class="empty-state">No output files</div>';
-        return;
-      }
-      for (const file of outputs) {
-        const fileDiv = document.createElement("div");
-        fileDiv.className = "output-file";
-        // Always show file header with download link
-        const header = document.createElement("div");
-        header.className = "output-file-header";
-        const nameSpan = document.createElement("span");
-        nameSpan.textContent = file.name;
-        header.appendChild(nameSpan);
-        const dlBtn = document.createElement("a");
-        dlBtn.className = "dl-btn";
-        dlBtn.textContent = "Download";
-        dlBtn.download = file.name;
-        dlBtn.href = getDownloadUri(file);
-        header.appendChild(dlBtn);
-        fileDiv.appendChild(header);
-        const content = document.createElement("div");
-        content.className = "output-file-content";
-        if (file.type === "text") {
-          const pre = document.createElement("pre");
-          pre.textContent = file.content;
-          content.appendChild(pre);
-        } else if (file.type === "image") {
-          const img = document.createElement("img");
-          img.src = file.data_uri;
-          img.alt = file.name;
-          content.appendChild(img);
-        } else if (file.type === "pdf") {
-          const iframe = document.createElement("iframe");
-          iframe.src = file.data_uri;
-          content.appendChild(iframe);
-        } else if (file.type === "xlsx") {
-          renderXlsx(content, file.data_b64);
-        } else if (file.type === "binary") {
-          const a = document.createElement("a");
-          a.className = "download-link";
-          a.href = file.data_uri;
-          a.download = file.name;
-          a.textContent = "Download " + file.name;
-          content.appendChild(a);
-        } else if (file.type === "error") {
-          const pre = document.createElement("pre");
-          pre.textContent = file.content;
-          pre.style.color = "var(--red)";
-          content.appendChild(pre);
-        }
-        fileDiv.appendChild(content);
-        container.appendChild(fileDiv);
-      }
-    }
-    // ---- XLSX rendering via SheetJS ----
-    function renderXlsx(container, b64Data) {
-      try {
-        const raw = Uint8Array.from(atob(b64Data), c => c.charCodeAt(0));
-        const wb = XLSX.read(raw, { type: "array" });
-        for (let i = 0; i < wb.SheetNames.length; i++) {
-          const sheetName = wb.SheetNames[i];
-          const ws = wb.Sheets[sheetName];
-          if (wb.SheetNames.length > 1) {
-            const sheetLabel = document.createElement("div");
-            sheetLabel.style.cssText =
-              "font-weight:600; font-size:0.8rem; color:#b0aea5; margin-top:0.5rem; margin-bottom:0.25rem;";
-            sheetLabel.textContent = "Sheet: " + sheetName;
-            container.appendChild(sheetLabel);
-          }
-          const htmlStr = XLSX.utils.sheet_to_html(ws, { editable: false });
-          const wrapper = document.createElement("div");
-          wrapper.innerHTML = htmlStr;
-          container.appendChild(wrapper);
-        }
-      } catch (err) {
-        container.textContent = "Error rendering spreadsheet: " + err.message;
-      }
-    }
-    // ---- Grades ----
-    function renderGrades(run) {
-      const section = document.getElementById("grades-section");
-      const content = document.getElementById("grades-content");
-      if (!run.grading) {
-        section.style.display = "none";
-        return;
-      }
-      const grading = run.grading;
-      section.style.display = "block";
-      // Reset to collapsed
-      content.classList.remove("open");
-      document.getElementById("grades-arrow").classList.remove("open");
-      const summary = grading.summary || {};
-      const expectations = grading.expectations || [];
-      let html = '<div style="padding: 1rem;">';
-      // Summary line
-      const passRate = summary.pass_rate != null
-        ? Math.round(summary.pass_rate * 100) + "%"
-        : "?";
-      const badgeClass = summary.pass_rate >= 0.8 ? "grade-pass" : summary.pass_rate >= 0.5 ? "" : "grade-fail";
-      html += '<div class="grades-summary">';
-      html += '<span class="grade-badge ' + badgeClass + '">' + passRate + '</span>';
-      html += '<span>' + (summary.passed || 0) + ' passed, ' + (summary.failed || 0) + ' failed of ' + (summary.total || 0) + '</span>';
-      html += '</div>';
-      // Assertions list
-      html += '<ul class="assertion-list">';
-      for (const exp of expectations) {
-        const statusClass = exp.passed ? "pass" : "fail";
-        const statusIcon = exp.passed ? "\u2713" : "\u2717";
-        html += '<li class="assertion-item">';
-        html += '<span class="assertion-status ' + statusClass + '">' + statusIcon + '</span>';
-        html += '<span>' + escapeHtml(exp.text) + '</span>';
-        if (exp.evidence) {
-          html += '<div class="assertion-evidence">' + escapeHtml(exp.evidence) + '</div>';
-        }
-        html += '</li>';
-      }
-      html += '</ul>';
-      html += '</div>';
-      content.innerHTML = html;
-    }
-    function toggleGrades() {
-      const content = document.getElementById("grades-content");
-      const arrow = document.getElementById("grades-arrow");
-      content.classList.toggle("open");
-      arrow.classList.toggle("open");
-    }
-    // ---- Previous outputs (collapsible) ----
-    function renderPrevOutputs(run) {
-      const section = document.getElementById("prev-outputs-section");
-      const content = document.getElementById("prev-outputs-content");
-      const prevOutputs = (EMBEDDED_DATA.previous_outputs || {})[run.id];
-      if (!prevOutputs || prevOutputs.length === 0) {
-        section.style.display = "none";
-        return;
-      }
-      section.style.display = "block";
-      // Reset to collapsed
-      content.classList.remove("open");
-      document.getElementById("prev-outputs-arrow").classList.remove("open");
-      // Render the files into the content area
-      content.innerHTML = "";
-      const wrapper = document.createElement("div");
-      wrapper.style.padding = "1rem";
-      for (const file of prevOutputs) {
-        const fileDiv = document.createElement("div");
-        fileDiv.className = "output-file";
-        const header = document.createElement("div");
-        header.className = "output-file-header";
-        const nameSpan = document.createElement("span");
-        nameSpan.textContent = file.name;
-        header.appendChild(nameSpan);
-        const dlBtn = document.createElement("a");
-        dlBtn.className = "dl-btn";
-        dlBtn.textContent = "Download";
-        dlBtn.download = file.name;
-        dlBtn.href = getDownloadUri(file);
-        header.appendChild(dlBtn);
-        fileDiv.appendChild(header);
-        const fc = document.createElement("div");
-        fc.className = "output-file-content";
-        if (file.type === "text") {
-          const pre = document.createElement("pre");
-          pre.textContent = file.content;
-          fc.appendChild(pre);
-        } else if (file.type === "image") {
-          const img = document.createElement("img");
-          img.src = file.data_uri;
-          img.alt = file.name;
-          fc.appendChild(img);
-        } else if (file.type === "pdf") {
-          const iframe = document.createElement("iframe");
-          iframe.src = file.data_uri;
-          fc.appendChild(iframe);
-        } else if (file.type === "xlsx") {
-          renderXlsx(fc, file.data_b64);
-        } else if (file.type === "binary") {
-          const a = document.createElement("a");
-          a.className = "download-link";
-          a.href = file.data_uri;
-          a.download = file.name;
-          a.textContent = "Download " + file.name;
-          fc.appendChild(a);
-        }
-        fileDiv.appendChild(fc);
-        wrapper.appendChild(fileDiv);
-      }
-      content.appendChild(wrapper);
-    }
-    function togglePrevOutputs() {
-      const content = document.getElementById("prev-outputs-content");
-      const arrow = document.getElementById("prev-outputs-arrow");
-      content.classList.toggle("open");
-      arrow.classList.toggle("open");
-    }
-    // ---- Feedback (saved to server -> feedback.json) ----
-    function saveCurrentFeedback() {
-      const run = EMBEDDED_DATA.runs[currentIndex];
-      const text = document.getElementById("feedback").value;
-      if (text.trim() === "") {
-        delete feedbackMap[run.id];
-      } else {
-        feedbackMap[run.id] = text;
-      }
-      // Build reviews array from map
-      const reviews = [];
-      for (const [run_id, feedback] of Object.entries(feedbackMap)) {
-        if (feedback.trim()) {
-          reviews.push({ run_id, feedback, timestamp: new Date().toISOString() });
-        }
-      }
-      fetch("/api/feedback", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ reviews, status: "in_progress" }),
-      }).then(() => {
-        document.getElementById("feedback-status").textContent = "Saved";
-      }).catch(() => {
-        // Static mode or server unavailable — no-op on auto-save,
-        // feedback will be downloaded on final submit
-        document.getElementById("feedback-status").textContent = "Will download on submit";
-      });
-    }
-    // ---- Done ----
-    function showDoneDialog() {
-      // Save current textarea to feedbackMap (but don't POST yet)
-      const run = EMBEDDED_DATA.runs[currentIndex];
-      const text = document.getElementById("feedback").value;
-      if (text.trim() === "") {
-        delete feedbackMap[run.id];
-      } else {
-        feedbackMap[run.id] = text;
-      }
-      // POST once with status: complete — include ALL runs so the model
-      // can distinguish "no feedback" (looks good) from "not reviewed"
-      const reviews = [];
-      const ts = new Date().toISOString();
-      for (const r of EMBEDDED_DATA.runs) {
-        reviews.push({ run_id: r.id, feedback: feedbackMap[r.id] || "", timestamp: ts });
-      }
-      const payload = JSON.stringify({ reviews, status: "complete" }, null, 2);
-      fetch("/api/feedback", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: payload,
-      }).then(() => {
-        document.getElementById("done-overlay").classList.add("visible");
-      }).catch(() => {
-        // Server not available (static mode) — download as file
-        const blob = new Blob([payload], { type: "application/json" });
-        const url = URL.createObjectURL(blob);
-        const a = document.createElement("a");
-        a.href = url;
-        a.download = "feedback.json";
-        a.click();
-        URL.revokeObjectURL(url);
-        document.getElementById("done-overlay").classList.add("visible");
-      });
-    }
-    function closeDoneDialog() {
-      // Reset status back to in_progress
-      saveCurrentFeedback();
-      document.getElementById("done-overlay").classList.remove("visible");
-    }
-    // ---- Toast ----
-    function showToast(message) {
-      const toast = document.getElementById("toast");
-      toast.textContent = message;
-      toast.classList.add("visible");
-      setTimeout(() => toast.classList.remove("visible"), 2000);
-    }
-    // ---- Keyboard nav ----
-    document.addEventListener("keydown", (e) => {
-      // Don't capture when typing in textarea
-      if (e.target.tagName === "TEXTAREA") return;
-      if (e.key === "ArrowLeft" || e.key === "ArrowUp") {
-        e.preventDefault();
-        navigate(-1);
-      } else if (e.key === "ArrowRight" || e.key === "ArrowDown") {
-        e.preventDefault();
-        navigate(1);
-      }
-    });
-    // ---- Util ----
-    function getDownloadUri(file) {
-      if (file.data_uri) return file.data_uri;
-      if (file.data_b64) return "data:application/octet-stream;base64," + file.data_b64;
-      if (file.type === "text") return "data:text/plain;charset=utf-8," + encodeURIComponent(file.content);
-      return "#";
-    }
-    function escapeHtml(text) {
-      const div = document.createElement("div");
-      div.textContent = text;
-      return div.innerHTML;
-    }
-    // ---- View switching ----
-    function switchView(view) {
-      document.querySelectorAll(".view-tab").forEach(t => t.classList.remove("active"));
-      document.querySelectorAll(".view-panel").forEach(p => p.classList.remove("active"));
-      document.querySelector(`[onclick="switchView('${view}')"]`).classList.add("active");
-      document.getElementById("panel-" + view).classList.add("active");
-    }
-    // ---- Benchmark rendering ----
-    function renderBenchmark() {
-      const data = EMBEDDED_DATA.benchmark;
-      if (!data) return;
-      // Show the tabs
-      document.getElementById("view-tabs").style.display = "flex";
-      const container = document.getElementById("benchmark-content");
-      const summary = data.run_summary || {};
-      const metadata = data.metadata || {};
-      const notes = data.notes || [];
-      let html = "";
-      // Header
-      html += "<h2 style='font-family: Poppins, sans-serif; margin-bottom: 0.5rem;'>Benchmark Results</h2>";
-      html += "<p style='color: var(--text-muted); font-size: 0.875rem; margin-bottom: 1.25rem;'>";
-      if (metadata.skill_name) html += "<strong>" + escapeHtml(metadata.skill_name) + "</strong> &mdash; ";
-      if (metadata.timestamp) html += metadata.timestamp + " &mdash; ";
-      if (metadata.evals_run) html += "Evals: " + metadata.evals_run.join(", ") + " &mdash; ";
-      html += (metadata.runs_per_configuration || "?") + " runs per configuration";
-      html += "</p>";
-      // Summary table
-      html += '<table class="benchmark-table">';
-      function fmtStat(stat, pct) {
-        if (!stat) return "—";
-        const suffix = pct ? "%" : "";
-        const m = pct ? (stat.mean * 100).toFixed(0) : stat.mean.toFixed(1);
-        const s = pct ? (stat.stddev * 100).toFixed(0) : stat.stddev.toFixed(1);
-        return m + suffix + " ± " + s + suffix;
-      }
-      function deltaClass(val) {
-        if (!val) return "";
-        const n = parseFloat(val);
-        if (n > 0) return "benchmark-delta-positive";
-        if (n < 0) return "benchmark-delta-negative";
-        return "";
-      }
-      // Discover config names dynamically (everything except "delta")
-      const configs = Object.keys(summary).filter(k => k !== "delta");
-      const configA = configs[0] || "config_a";
-      const configB = configs[1] || "config_b";
-      const labelA = configA.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase());
-      const labelB = configB.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase());
-      const a = summary[configA] || {};
-      const b = summary[configB] || {};
-      const delta = summary.delta || {};
-      html += "<thead><tr><th>Metric</th><th>" + escapeHtml(labelA) + "</th><th>" + escapeHtml(labelB) + "</th><th>Delta</th></tr></thead>";
-      html += "<tbody>";
-      html += "<tr><td><strong>Pass Rate</strong></td>";
-      html += "<td>" + fmtStat(a.pass_rate, true) + "</td>";
-      html += "<td>" + fmtStat(b.pass_rate, true) + "</td>";
-      html += '<td class="' + deltaClass(delta.pass_rate) + '">' + (delta.pass_rate || "—") + "</td></tr>";
-      // Time (only show row if data exists)
-      if (a.time_seconds || b.time_seconds) {
-        html += "<tr><td><strong>Time (s)</strong></td>";
-        html += "<td>" + fmtStat(a.time_seconds, false) + "</td>";
-        html += "<td>" + fmtStat(b.time_seconds, false) + "</td>";
-        html += '<td class="' + deltaClass(delta.time_seconds) + '">' + (delta.time_seconds ? delta.time_seconds + "s" : "—") + "</td></tr>";
-      }
-      // Tokens (only show row if data exists)
-      if (a.tokens || b.tokens) {
-        html += "<tr><td><strong>Tokens</strong></td>";
-        html += "<td>" + fmtStat(a.tokens, false) + "</td>";
-        html += "<td>" + fmtStat(b.tokens, false) + "</td>";
-        html += '<td class="' + deltaClass(delta.tokens) + '">' + (delta.tokens || "—") + "</td></tr>";
-      }
-      html += "</tbody></table>";
-      // Per-eval breakdown (if runs data available)
-      const runs = data.runs || [];
-      if (runs.length > 0) {
-        const evalIds = [...new Set(runs.map(r => r.eval_id))].sort((a, b) => a - b);
-        html += "<h3 style='font-family: Poppins, sans-serif; margin-bottom: 0.75rem;'>Per-Eval Breakdown</h3>";
-        const hasTime = runs.some(r => r.result && r.result.time_seconds != null);
-        const hasErrors = runs.some(r => r.result && r.result.errors > 0);
-        for (const evalId of evalIds) {
-          const evalRuns = runs.filter(r => r.eval_id === evalId);
-          const evalName = evalRuns[0] && evalRuns[0].eval_name ? evalRuns[0].eval_name : "Eval " + evalId;
-          html += "<h4 style='font-family: Poppins, sans-serif; margin: 1rem 0 0.5rem; color: var(--text);'>" + escapeHtml(evalName) + "</h4>";
-          html += '<table class="benchmark-table">';
-          html += "<thead><tr><th>Config</th><th>Run</th><th>Pass Rate</th>";
-          if (hasTime) html += "<th>Time (s)</th>";
-          if (hasErrors) html += "<th>Crashes During Execution</th>";
-          html += "</tr></thead>";
-          html += "<tbody>";
-          // Group by config and render with average rows
-          const configGroups = [...new Set(evalRuns.map(r => r.configuration))];
-          for (let ci = 0; ci < configGroups.length; ci++) {
-            const config = configGroups[ci];
-            const configRuns = evalRuns.filter(r => r.configuration === config);
-            if (configRuns.length === 0) continue;
-            const rowClass = ci === 0 ? "benchmark-row-with" : "benchmark-row-without";
-            const configLabel = config.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase());
-            for (const run of configRuns) {
-              const r = run.result || {};
-              const prClass = r.pass_rate >= 0.8 ? "benchmark-delta-positive" : r.pass_rate < 0.5 ? "benchmark-delta-negative" : "";
-              html += '<tr class="' + rowClass + '">';
-              html += "<td>" + configLabel + "</td>";
-              html += "<td>" + run.run_number + "</td>";
-              html += '<td class="' + prClass + '">' + ((r.pass_rate || 0) * 100).toFixed(0) + "% (" + (r.passed || 0) + "/" + (r.total || 0) + ")</td>";
-              if (hasTime) html += "<td>" + (r.time_seconds != null ? r.time_seconds.toFixed(1) : "—") + "</td>";
-              if (hasErrors) html += "<td>" + (r.errors || 0) + "</td>";
-              html += "</tr>";
-            }
-            // Average row
-            const rates = configRuns.map(r => (r.result || {}).pass_rate || 0);
-            const avgRate = rates.reduce((a, b) => a + b, 0) / rates.length;
-            const avgPrClass = avgRate >= 0.8 ? "benchmark-delta-positive" : avgRate < 0.5 ? "benchmark-delta-negative" : "";
-            html += '<tr class="benchmark-row-avg ' + rowClass + '">';
-            html += "<td>" + configLabel + "</td>";
-            html += "<td>Avg</td>";
-            html += '<td class="' + avgPrClass + '">' + (avgRate * 100).toFixed(0) + "%</td>";
-            if (hasTime) {
-              const times = configRuns.map(r => (r.result || {}).time_seconds).filter(t => t != null);
-              html += "<td>" + (times.length ? (times.reduce((a, b) => a + b, 0) / times.length).toFixed(1) : "—") + "</td>";
-            }
-            if (hasErrors) html += "<td></td>";
-            html += "</tr>";
-          }
-          html += "</tbody></table>";
-          // Per-assertion detail for this eval
-          const runsWithExpectations = {};
-          for (const config of configGroups) {
-            runsWithExpectations[config] = evalRuns.filter(r => r.configuration === config && r.expectations && r.expectations.length > 0);
-          }
-          const hasAnyExpectations = Object.values(runsWithExpectations).some(runs => runs.length > 0);
-          if (hasAnyExpectations) {
-            // Collect all unique assertion texts across all configs
-            const allAssertions = [];
-            const seen = new Set();
-            for (const config of configGroups) {
-              for (const run of runsWithExpectations[config]) {
-                for (const exp of (run.expectations || [])) {
-                  if (!seen.has(exp.text)) {
-                    seen.add(exp.text);
-                    allAssertions.push(exp.text);
-                  }
-                }
-              }
-            }
-            html += '<table class="benchmark-table" style="margin-top: 0.5rem;">';
-            html += "<thead><tr><th>Assertion</th>";
-            for (const config of configGroups) {
-              const label = config.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase());
-              html += "<th>" + escapeHtml(label) + "</th>";
-            }
-            html += "</tr></thead><tbody>";
-            for (const assertionText of allAssertions) {
-              html += "<tr><td>" + escapeHtml(assertionText) + "</td>";
-              for (const config of configGroups) {
-                html += "<td>";
-                for (const run of runsWithExpectations[config]) {
-                  const exp = (run.expectations || []).find(e => e.text === assertionText);
-                  if (exp) {
-                    const cls = exp.passed ? "benchmark-delta-positive" : "benchmark-delta-negative";
-                    const icon = exp.passed ? "\u2713" : "\u2717";
-                    html += '<span class="' + cls + '" title="Run ' + run.run_number + ': ' + escapeHtml(exp.evidence || "") + '">' + icon + "</span> ";
-                  } else {
-                    html += "— ";
-                  }
-                }
-                html += "</td>";
-              }
-              html += "</tr>";
-            }
-            html += "</tbody></table>";
-          }
-        }
-      }
-      // Notes
-      if (notes.length > 0) {
-        html += '<div class="benchmark-notes">';
-        html += "<h3>Analysis Notes</h3>";
-        html += "<ul>";
-        for (const note of notes) {
-          html += "<li>" + escapeHtml(note) + "</li>";
-        }
-        html += "</ul></div>";
-      }
-      container.innerHTML = html;
-    }
-    // ---- Start ----
-    init();
-    renderBenchmark();
-  </script>
-</body>
-</html>