@mrciphersmith/keryx 0.3.1 → 0.3.3

package/src/gdskills/bundled/stacks/ci-github-gitlab/governance/eval.json

@@ -0,0 +1,1295 @@
+{
+  "schemaVersion": "1.0.0",
+  "reports": [
+    {
+      "schemaVersion": "1.0.0",
+      "skillId": "ci-github-gitlab/ci-pipeline-implementation",
+      "strictness": "high",
+      "trials": 10,
+      "triggerAccuracy": {
+        "truePositive": 3,
+        "falsePositive": 0,
+        "positives": 6,
+        "negatives": 6
+      },
+      "evidence": "authored",
+      "scenarios": [
+        {
+          "id": "trigger-positive-1",
+          "kind": "trigger-positive",
+          "prompt": "Set up a workflow so our test suite runs automatically on incoming pull requests",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-2",
+          "kind": "trigger-positive",
+          "prompt": "I need pipeline stages in .gitlab-ci.yml for building, testing, and shipping the app",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-3",
+          "kind": "trigger-positive",
+          "prompt": "This CI job reinstalls dependencies every run -- can we speed it up with caching?",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-4",
+          "kind": "trigger-positive",
+          "prompt": "Multiple repos need the same build steps -- can we share one workflow definition?",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-5",
+          "kind": "trigger-positive",
+          "prompt": "The GITHUB_TOKEN in this workflow currently has no explicit scope -- lock it down",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-6",
+          "kind": "trigger-positive",
+          "prompt": "How do I keep our production deploy credentials from leaking to feature-branch pipelines?",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-1",
+          "kind": "trigger-negative",
+          "prompt": "Review this Dockerfile for insecure base image usage",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-2",
+          "kind": "trigger-negative",
+          "prompt": "Add a Kubernetes deployment manifest with resource limits",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-3",
+          "kind": "trigger-negative",
+          "prompt": "Run a general OWASP security review across this codebase",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-4",
+          "kind": "trigger-negative",
+          "prompt": "Edit this YAML config file to add a new logging field",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-5",
+          "kind": "trigger-negative",
+          "prompt": "Write unit tests for this Python function",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-6",
+          "kind": "trigger-negative",
+          "prompt": "Review this TypeScript diff for missing null checks",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "least-privilege-permissions",
+          "kind": "behavior",
+          "prompt": "I'm adding a new job to this GitHub Actions workflow that needs to comment on the pull request. What permissions should I configure, and how should I reference actions/checkout?",
+          "strictness": "high",
+          "trials": 10,
+          "passes": 10,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "judge",
+          "status": "ran",
+          "trialRecords": [
+            {
+              "output": "Configure it like this:\n\n**Permissions** — set `permissions: {}` (or omit and rely on the default, but be explicit) at the workflow level, then grant only the scope the commenting job needs at the job level:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read        # needed if the job checks out code\n      pull-requests: write  # needed to post/update the PR comment\n    steps:\n      ...\n```\n\nDon't add `pull-requests: write` at the workflow level if only this one job comments — scope it to this job so every other job in the file isn't handed write access it doesn't use. `contents: read` is only needed if the job actually checks out the repo; drop it otherwise. Avoid `permissions: write-all`.\n\n**Referencing `actions/checkout`** — pin it to a full commit SHA, not a tag:\n\n```yaml\n      - uses: actions/checkout@<full-40-char-sha> # v4.2.2\n```\n\nThe tag in the comment is for readability only; the SHA is what actually resolves. A tag is mutable and can be silently repointed by a compromised maintainer account or a compromised upstream; a commit SHA cannot.\n\n**One more thing worth flagging while you're in here:** if this job runs on `pull_request_target` (common for \"comment on the PR\" jobs, since posting a comment needs a write token the fork's `pull_request` run doesn't get), make sure it does **not** check out and execute the pull request's own head SHA/ref. A job that both has base-repo write access and runs the PR's untrusted code is the classic privilege-escalation path — it hands an attacker-controlled PR your repo's secrets and write token. If you need the PR's code for anything in this job, keep that in a separate `pull_request` job with read-only permissions, and have this `pull_request_target` job only post the comment.\n\nAnd if the comment body includes the PR title/body/branch name, route it through an `env:` entry before it hits a `run:` step rather than interpolating `${{ github.event.pull_request.title }}` directly into the shell string — a title containing quotes or `$(...)` breaks out of the generated script otherwise.",
+              "outputSha256": "bce08bf8dad7eb0cccd0eebe209e06b47b2bcb8893d06dddab97840e5f8b60ee",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Job-level permissions with pull-requests: write and contents: read shown in YAML (criterion 1). Explicitly says 'Avoid permissions: write-all' and warns against workflow-level write (criterion 2). Shows actions/checkout@<full-40-char-sha> with note that tags are mutable (criterion 3). No fail criteria met."
+              },
+              "passed": true
+            },
+            {
+              "output": "For a job that comments on the pull request:\n\n**Permissions** — set the workflow-level default to `permissions: {}` (deny everything), then grant only what this job needs at the job level:\n\n```yaml\njobs:\n  comment:\n    permissions:\n      contents: read          # needed to read the repo/checkout\n      pull-requests: write    # needed to post/update the PR comment\n    runs-on: ubuntu-latest\n    steps:\n      # ...\n```\n\nIf the job is only posting a comment and doesn't need the repository contents at all, you can drop `contents: read` — but keep it if you're calling `actions/checkout`. Do not use `permissions: write-all`. If this job runs as part of a `pull_request` trigger (not `pull_request_target`), note that fork PRs get a read-only token regardless of this block, so commenting from a `pull_request`-triggered job will silently fail on fork contributions — that's a trigger/design decision, not a permissions one, and worth flagging before you implement.\n\n**actions/checkout reference** — pin to a full commit SHA, never a mutable tag like `@v4`:\n\n```yaml\n- uses: actions/checkout@<full-40-char-sha>  # v4.x.y\n```\n\nThe trailing comment is just for readability; the reference itself must be the SHA, since a tag is exactly the kind of reference that can be silently moved.\n\n**One more thing for this specific job** — if the comment body includes anything derived from the pull request (title, author, commit message, a file path from the diff), do not interpolate it directly into a `run:` shell string like `gh pr comment --body \"${{ github.event.pull_request.title }}\"`. Route it through an `env:` entry first:\n\n```yaml\n- name: Comment on PR\n  env:\n    PR_TITLE: ${{ github.event.pull_request.title }}\n  run: gh pr comment \"$PR_NUMBER\" --body \"$PR_TITLE\"\n```\n\nA title containing a quote, backtick, or `$(...)` breaks out of the generated shell script otherwise. Also give the job a `timeout-minutes` so a hung comment step doesn't hold a runner indefinitely.\n\nBefore you call this done: confirm the trigger is what actually fits (a `pull_request_target` job that checks out and executes the PR's own head SHA would hand secrets to attacker-controlled code — avoid that combination entirely), confirm the checkout reference is a full SHA, and confirm the permissions block is scoped at the job level rather than left at a workflow-wide default.",
+              "outputSha256": "0f4d4fcb79bea706793ab39b139bb5239e5188c5fa5ce8be543496db408cb2dd",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer grants permissions at job level with pull-requests: write and contents: read for checkout (criterion 1). It explicitly says 'Do not use permissions: write-all' and sets workflow default to permissions: {} (criterion 2). It states checkout must be pinned to a full commit SHA, never a mutable tag like @v4, showing actions/checkout@<full-40-char-sha> (criterion 3). No fail criteria hold."
+              },
+              "passed": true
+            },
+            {
+              "output": "For a job that comments on the pull request, scope permissions at the **job** level (with `permissions: {}` at the workflow level as the safe default), granting only:\n\n```yaml\njobs:\n  comment:\n    permissions:\n      contents: read          # only if checkout/branch info is needed\n      pull-requests: write    # required to post the comment\n```\n\nKey points:\n\n- **`pull-requests: write`** is the scope that lets a job post/update a PR comment. Nothing broader (`write-all`, `issues: write`) is needed for commenting alone.\n- **`contents: read`** only if the job checks out the repo. If the job truly only posts a comment and never checks out code, drop it to `contents: none`.\n- Put the grant on the `comment:` job, not the whole workflow — other jobs shouldn't inherit write access they don't use.\n- Don't reach for `pull_request_target` just to get write access. `pull_request` runs read-only in the fork's context, but a `pull-requests: write` grant on a `pull_request`-triggered job still works for posting comments on the base repo (the token is scoped to the base repo even for `pull_request` events), so you rarely need `pull_request_target` for commenting. Reach for it only if you genuinely need base-repo secrets, and even then never check out and execute the PR's own head SHA in that job.\n\nFor the checkout action, pin it to a full commit SHA, not a tag:\n\n```yaml\n- uses: actions/checkout@<full-commit-sha>   # v4.x.y\n```\n\nA mutable tag like `@v4` can be silently moved (compromised maintainer account, supply-chain attack); a commit SHA cannot. Use `@v4` only as a trailing comment for readability.\n\nAlso, since you're commenting on a PR, whatever PR-controlled value you put into the comment (title, body, branch name) is untrusted — don't build the shell command by interpolating `${{ github.event.pull_request.title }}` straight into a `run:` string. Route it through `env:` first and reference `\"$PR_TITLE\"` inside the script so it can't break out of the shell.",
+              "outputSha256": "c73d0460091a8f6e5298985a1a0a261cc616efae45a4db60968e8e09c454d9ca",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer grants permissions at job level with pull-requests: write (and contents: read only if checkout needed), explicitly rejects write-all/issues: write as unneeded, and pins actions/checkout to a full commit SHA rather than @v4. All three pass criteria concretely shown; no fail criteria met."
+              },
+              "passed": true
+            },
+            {
+              "output": "For a job that comments on the pull request, configure `permissions:` at the job level, not the workflow level, and grant only the scopes that job actually needs:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read            # needed if the job checks out the repo at all\n      pull-requests: write      # needed to post/update the PR comment\n```\n\nIf the workflow file already has a workflow-level `permissions: {}` (or some other scoped block), leave that as-is and add this job-level block on top of it — the job-level block replaces the workflow-level one for that job only, so other jobs keep their own narrower grant. Do not add `permissions: write-all` or an unscoped default just to make this job work; `pull-requests: write` is the specific scope that lets a job create, update, or delete PR comments, and `contents: read` is only needed if the job reads repository contents (e.g., it checks out the repo to compute a diff). If the job only posts a comment from data already available in the event payload, `pull-requests: write` alone is enough.\n\nFor `actions/checkout`, pin it to a full commit SHA rather than a mutable tag, and comment the tag it corresponds to for readability:\n\n```yaml\n- uses: actions/checkout@<full-40-char-commit-sha>  # v4.x.x\n```\n\nIs the job only posting the comment, or does it also need to read the diff/repo contents? That determines whether you need `contents: read` alongside `pull-requests: write`. And is the comment triggered by `pull_request` or `pull_request_target`? That changes whether the job can even reach the token scope needed to comment on a fork PR — worth confirming before you write the trigger, since a plain `pull_request` job from a fork gets a read-only token regardless of the `permissions:` block.",
+              "outputSha256": "699c923419635403088ea21358b17b4233e11af546472ccc21afb8e2c891228b",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer grants permissions at job level with pull-requests: write (and contents: read if checkout) in the YAML block. It explicitly says 'Do not add permissions: write-all or an unscoped default just to make this job work.' It states actions/checkout should be 'pin it to a full commit SHA rather than a mutable tag,' showing actions/checkout@<full-40-char-commit-sha>. All pass criteria met; no fail criterion."
+              },
+              "passed": true
+            },
+            {
+              "output": "To comment on a pull request, you need to grant that specific job `pull-requests: write` (and typically `contents: read` for checking out code, since the job still needs to read the repo). Do this at the **job level**, not the workflow level.\n\n**Permissions**\n\nSet a restrictive workflow-level default and widen only where needed:\n\n```yaml\npermissions: {}          # workflow level: nothing by default\n\njobs:\n  comment:\n    permissions:\n      contents: read          # needed if the job checks out code\n      pull-requests: write    # needed to post the comment\n```\n\nA few notes on this specific scope:\n\n- `pull-requests: write` is what allows creating/updating an issue comment on a PR (PRs are issues under the hood). If you're only reading PR data, `pull-requests: read` suffices.\n- Don't reach for `write-all` or leave an unscoped default — only this one job needs the write scope, so only this job should get it.\n- If the workflow is triggered by `pull_request` from a fork, be aware the token is read-only regardless of what you set here — GitHub downgrades it for fork PRs. If the job genuinely must comment on fork PRs, you're now in `pull_request_target` territory, and that comes with the hard rule below.\n\n**Referencing actions/checkout**\n\nPin to a full commit SHA, not a mutable tag:\n\n```yaml\n- uses: actions/checkout@<full-40-char-sha>  # v4.x.x\n```\n\nUse the commit SHA the tag currently points to; add the tag as a trailing comment for readability only. A tag is a reference an attacker can silently move (compromised maintainer account, supply-chain compromise); a commit SHA cannot be moved.\n\n**The trap to avoid**\n\nIf you switch the trigger to `pull_request_target` so the comment job can get a write token / base-repo secrets for fork PRs, then:\n\n- **Never check out and execute the PR's own head SHA/ref in that job.** That combination — `pull_request_target` + untrusted checkout — hands attacker-controlled PR code your base-repo write token and secrets. If the job only needs to comment (not run the PR's code), don't check out the head ref at all; checkout the base branch, or skip checkout entirely.\n\n**Also apply to this job regardless:**\n\n- Add `timeout-minutes:` to the job.\n- Route any `${{ github.event.* }}` value (e.g., PR title/number) through an `env:` entry before it reaches a `run:` shell string — never interpolate it directly into the script text, since a crafted title can break out of the generated shell.\n\nSummary: job-level `permissions: { contents: read, pull-requests: write }` (no workflow-level `write-all`), `actions/checkout` pinned to a full commit SHA, and keep the trigger `pull_request` unless base-repo secrets are truly required — in which case never execute the PR head ref.",
+              "outputSha256": "156c3040235646018977060c0c2cef5ff19c1eeab38dc3286a19147382b62273",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer grants job-level permissions with pull-requests: write and contents: read (YAML block under jobs.comment.permissions), explicitly rejects write-all/unscoped default ('Don't reach for write-all or leave an unscoped default'), and pins actions/checkout to a full commit SHA ('actions/checkout@<full-40-char-sha>', 'not a mutable tag'). All pass criteria met; no fail criterion."
+              },
+              "passed": true
+            },
+            {
+              "output": "For a job that comments on a pull request:\n\n**Permissions** — scope them at the job level, not the workflow level:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read        # checkout needs this\n      pull-requests: write  # required to post/update the PR comment\n    steps:\n      - uses: actions/checkout@<full-commit-sha>\n```\n\nIf the rest of the workflow doesn't need any write access, set `permissions: {}` at the workflow level and grant `pull-requests: write` only on this job, so the broader file keeps a narrow default. Do not reach for `permissions: write-all` or an unscoped default just because one job needs write access — that hands every other job the same elevated access.\n\nOne caveat on `pull-requests: write`: it is only meaningful on jobs running in the base-repo context. If you're triggered via `pull_request_target`, that token scope lets the job comment on PRs — but that trigger mode comes with a hard constraint (see below). If you're on a plain `pull_request` trigger from a fork, the token is read-only and the comment will fail, so this job needs `pull_request_target` (or a `workflow_run` handoff) to actually have the write scope.\n\n**actions/checkout reference** — pin it to a full commit SHA, never a mutable tag:\n\n```yaml\n- uses: actions/checkout@<40-char-sha>  # v4.x.y, tag noted for readability\n```\n\nA tag like `@v4` is a reference an attacker (or a compromised upstream account) can silently move; a commit SHA cannot be moved.\n\n**Critical constraint on the trigger** — if this job runs on `pull_request_target` (which it must, to get `pull-requests: write` on fork PRs), do **not** follow the checkout by executing anything from the pull request's own head ref/SHA. Commenting on a PR title, diff, or metadata is fine; checking out and running the PR's head code under that trigger hands an attacker-controlled PR your base-repo secrets and write token. That is the exact `pull_request_target` + untrusted-checkout anti-pattern to avoid.\n\nTwo smaller things worth adding while you're in there: pass any `${{ github.event.pull_request.title }}`-style value the comment body uses through an `env:` entry rather than interpolating it directly into a `run:` shell string, and give the job a `timeout-minutes` and a `concurrency` group keyed on the PR number so rapid pushes don't leave stale comments racing each other.\n\nAfter editing, confirm: `permissions` sits at the job level with only `contents: read` and `pull-requests: write`, the `uses:` line names a full SHA, and no `pull_request_target` job checks out the PR's own head.",
+              "outputSha256": "33f3ede91b94ef6dcd70864a7de14cea8f3011a397f6751317f3edd473166d60",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Job-level permissions with pull-requests: write and contents: read shown in YAML (criterion 1). Explicitly rejects write-all/unscoped default: 'Do not reach for permissions: write-all or an unscoped default' (criterion 2). Pins checkout to full SHA: 'actions/checkout@<40-char-sha>' and warns @v4 tag is movable (criterion 3). No fail criterion holds."
+              },
+              "passed": true
+            },
+            {
+              "output": "For a job that comments on a pull request, configure permissions at the **job level**, not the workflow level:\n\n```yaml\njobs:\n  pr-comment:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read          # if you need to read the repo at all\n      pull-requests: write    # required to comment on the PR\n    steps:\n      - uses: actions/checkout@<full-commit-sha>\n        # pin to a full commit SHA, optionally with a comment naming the tag\n```\n\nKey points:\n\n- **`pull-requests: write`** is the permission that lets the job post a comment via `gh pr comment` or the GitHub API. Without it the job runs but the comment call fails with a 403.\n- **Only grant what this job needs.** If the job only comments and never checks out the repo, you can drop `contents: read`. If the workflow-level `permissions:` is already scoped to `contents: read`, override it at the job level as above — do not reach for `write-all`.\n- **If the job also needs to check out code**, keep `contents: read`; you very likely do not need `contents: write`.\n\nOn `actions/checkout`:\n\n- **Pin to a full commit SHA, not a tag.** `uses: actions/checkout@<sha>` — a version tag like `@v4` is a mutable reference an attacker (or a compromised maintainer account / supply-chain compromise) can silently repoint. Include the tag as a comment for readability, e.g. `uses: actions/checkout@<sha> # v4.1.7`.\n- **Confirm the trigger before wiring this up.** Commenting on a PR is almost always fine under `pull_request` (runs with the contributor's read-only token, no base-repo secrets). Only use `pull_request_target` if the job genuinely needs base-repo secrets or write access — and if you do, **never** check out and execute the pull request's own head SHA/ref inside that job, since that combination hands attacker-controlled code your privileged token.\n- **Do not interpolate `${{ github.event.* }}` directly** into a `run:` step when building the comment body (e.g. PR title, branch name). Route it through an `env:` entry first and reference the environment variable in the script, so a title containing quotes or `$(...)` can't break out of the generated shell command.\n\nA typical shape:\n\n```yaml\non:\n  pull_request:\n\npermissions: {}   # workflow level: lock everything down by default\n\njobs:\n  pr-comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n    steps:\n      - uses: actions/checkout@<full-commit-sha> # v4.x.x\n      - env:\n          PR_TITLE: ${{ github.event.pull_request.title }}\n        run: gh pr comment \"$PR_NUMBER\" --body \"$PR_TITLE\"\n```\n\nVerify before finishing: every `uses:` names a full commit SHA, `pull-requests: write` is scoped to this job only, no untrusted event field is interpolated directly into a `run:` string, and the trigger is `pull_request` (not `pull_request_target`) unless base-repo write access is genuinely required.",
+              "outputSha256": "59cffaf471ff48cc57f1b07017f29d3b2e52e0a1dd957a819c3c4ad1c5d20022",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Job-level permissions with pull-requests: write shown in YAML (jobs.pr-comment.permissions). Explicitly rejects write-all ('do not reach for write-all') and shows workflow-level permissions: {}. States checkout must be pinned to full commit SHA, not tag @v4, with example actions/checkout@<full-commit-sha>."
+              },
+              "passed": true
+            },
+            {
+              "output": "For a job that comments on the pull request:\n\n**Permissions block on that job (not workflow-level):**\n\n```yaml\njobs:\n  comment:\n    permissions:\n      contents: read          # needed if the job checks out the repo at all\n      pull-requests: write    # needed to post the comment\n```\n\nScope the workflow-level `permissions:` down to `{}` (or omit entirely if your org default is restrictive), then grant only these scopes at this specific job. If *only* this job posts a comment, don't grant `pull-requests: write` to the whole file — a job that only reads or builds doesn't need it. Drop `contents: read` if the job genuinely never checks out code.\n\n**Referencing actions/checkout:**\n\nPin to a full commit SHA, not a tag:\n\n```yaml\n- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2\n```\n\nThe trailing comment with the tag is just for human readability — the actual reference is the SHA. A version tag like `@v4` is a mutable reference the maintainer (or someone who compromises their account) can silently repoint at different code; a commit SHA cannot be moved.\n\nA couple of related points worth deciding now, since they shape the rest of the job:\n\n- If this job runs on `pull_request`, it gets a read-only token by default and can't post a comment from a fork's PR at all. Posting from a fork requires `pull_request_target` (or a separate `workflow_run`-triggered workflow) — and if you go that route, **do not check out and execute the PR's head SHA in that job.** That combination is the exact pattern that leaks base-repo secrets to attacker-controlled PRs. Use `pull_request_target` only for lightweight metadata/comment operations, not for running the contributed code.\n- Any PR-derived text you put into the comment (title, body, branch name, etc.) should be passed through an `env:` entry and referenced as a shell variable, not interpolated directly as `${{ github.event.pull_request.title }}` inside a `run:` block — a title containing quotes or `$(...)` otherwise breaks out of the generated shell script.\n- Add a `timeout-minutes` to the job, and a `concurrency:` group if you don't want overlapping runs posting duplicate comments on rapid pushes.\n\nOne caveat: I don't have the rest of your workflow or repo in front of me, so confirm against what's already there — the existing trigger (`pull_request` vs `pull_request_target`), whether an action like `peter-evans/create-or-update-comment` is already in use (pin that to a SHA too), and whether the repo's default token permissions are already restricted at the org level.",
+              "outputSha256": "9b4774020091e7efb110ff3c4014ece4e3a84453ff4f36507e3310f748bb9841",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Job-level permissions with pull-requests: write and contents: read shown in YAML (criterion 1). Explicitly says scope workflow-level down to {} and not to grant pull-requests: write to whole file (criterion 2). Pins actions/checkout to full SHA 11bd71901bbe5b1630ceea73d27597364c9af683, rejecting @v4 as mutable (criterion 3). No write-all recommendation."
+              },
+              "passed": true
+            },
+            {
+              "output": "Keep the two decisions separate — the comment permission is job-scoped and narrow, and the checkout reference is a pinning question that applies to every job, not just this one.\n\n**Permissions for a PR-commenting job**\n\nSet `permissions: {}` at the workflow level, then grant the posting job only what it needs:\n\n```yaml\njobs:\n  comment:\n    permissions:\n      pull-requests: write   # the only scope posting a PR comment needs\n    runs-on: ubuntu-latest\n    timeout-minutes: 5\n    steps:\n      - uses: actions/checkout@<full-commit-sha>  # e.g. 11bd71901bbe5b1630ceea73d27597364c9af683 (v4.2.2)\n        with:\n          persist-credentials: false\n      - name: Post comment\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n          PR_NUMBER: ${{ github.event.pull_request.number }}\n        run: |\n          gh pr comment \"$PR_NUMBER\" --body \"Build finished.\"\n```\n\n`pull-requests: write` is the scope that covers creating/updating a PR comment. `contents: write` is not needed for commenting; do not add it \"just in case.\" If the job only reads the diff or repo files (checkout does), `contents: read` is the default and can be granted at job level if the workflow-level `permissions: {}` stripped it — checkout will fail without at least `contents: read`.\n\n**Referencing `actions/checkout`**\n\nPin to a full commit SHA, never a mutable tag:\n\n```yaml\n- uses: actions/checkout@<full-commit-sha>   # comment the tag for readability\n```\n\nThe tag `v4` (or `v4.2.2`) is a reference the maintainer's account — or anything upstream in their supply chain — can silently move to point at different code. A full commit SHA cannot be moved. This applies to this job and every other job in the file; fix the whole file while you're here.\n\n**Two traps specific to this kind of job**\n\n1. **Don't reach for `pull_request_target` just to get a token that can comment.** `pull_request_target` runs with base-repo secrets and write scope, which is real power. If all this job does is post a comment, it does not need base-repo secrets — use the normal `pull_request` trigger and `GITHUB_TOKEN`. If you *do* use `pull_request_target` (e.g., to comment on PRs from forks), you must **not** check out the PR's own head SHA and run code from it:\n\n```yaml\n# pull_request_target ONLY if you never execute the PR's head ref:\non: pull_request_target\njobs:\n  comment:\n    permissions:\n      pull-requests: write\n    steps:\n      - uses: actions/checkout@<sha>   # checks out BASE ref, not PR head\n        with:\n          ref: ${{ github.event.pull_request.base.sha }}\n          persist-credentials: false\n```\n\n2. **Never interpolate PR-controlled text straight into the script.** A PR title or body containing `\"`, a backtick, or `$(...)` breaks out of the generated shell script the moment it's dropped into `run:`. Route it through `env:` first, exactly as the `PR_NUMBER`/`GH_TOKEN` example above does — never `run: gh pr comment ${{ github.event.pull_request.title }}`.\n\n**Quick checklist before you're done**\n\n- Trigger is `pull_request` unless base-repo secrets/write access are genuinely required.\n- No `pull_request_target` job checks out and runs the PR's own head SHA.\n- Every `uses:` is a full commit SHA.\n- No `${{ github.event.* }}` interpolated directly into `run:`.\n- `permissions:` scoped per job; no workflow-level `write-all`.\n- `timeout-minutes` on the new job.",
+              "outputSha256": "fdafa5d5434e29f96d01d899c6d43225fdf4de5326d658cae8a97688c0e713e1",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Job-level permissions with pull-requests: write shown in YAML ('permissions: pull-requests: write # the only scope posting a PR comment needs'), contents: read noted for checkout. Explicitly rejects workflow-level write-all ('no workflow-level write-all', sets permissions: {}). Checkout pinned to full commit SHA ('actions/checkout@<full-commit-sha>', 'never a mutable tag'). No fail criteria triggered."
+              },
+              "passed": true
+            },
+            {
+              "output": "For a job that comments on a pull request, scope permissions at the job level rather than inheriting a broad default:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read          # required so actions/checkout can fetch the repo\n      pull-requests: write    # required to post/update a PR comment\n    steps:\n      - uses: actions/checkout@<full-commit-sha>   # e.g. @08eba0b... # v4.2.2\n```\n\nA few points, per the workflow in the skill:\n\n- **Don't** set `permissions: write-all` at the workflow level just to give this one job comment access. Set `permissions: {}` at the workflow level and grant `contents: read` + `pull-requests: write` only on this job. If this job runs on `pull_request` from a fork, `pull-requests: write` is downgraded to read-only on forked PRs — plan for that (e.g. don't fail the job if the comment call is denied), and don't reach for `pull_request_target` to \"fix\" it unless the job genuinely needs base-repo write access.\n- **`actions/checkout` must be pinned to a full commit SHA**, not a tag: `uses: actions/checkout@<40-char-sha>`, optionally with `# v4.2.2` as a trailing comment for readability. A tag is a movable reference; a SHA is not.\n- If the comment body includes anything from `${{ github.event.* }}` (like the PR title), pass it through `env:` first and reference `\"$PR_TITLE\"` in the shell — never interpolate it directly into the `run:` string.\n- If your job uses `pull_request_target` to get write access on fork PRs, make sure it does **not** check out and execute the PR's own head SHA/ref — that combination is the classic secret-exfiltration setup.",
+              "outputSha256": "464be82d0f6be7f24c31724b9f31d96003cbd6b17a80550f8d12639e672aac4f",
+              "promptSha256": "13e5d5343f11f3cbcfb52f46eb20e2f5083d91fde86a196f1bb8754deb73bd73",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Job-level permissions with pull-requests: write and contents: read shown in YAML; explicitly rejects workflow-level write-all ('Don't set permissions: write-all at the workflow level'); states actions/checkout must be pinned to a full commit SHA, not a tag ('A tag is a movable reference; a SHA is not'). All pass criteria met, no fail criteria."
+              },
+              "passed": true
+            }
+          ]
+        },
+        {
+          "id": "safe-pr-trigger-choice",
+          "kind": "behavior",
+          "prompt": "This job builds and runs the test suite for incoming pull requests from any contributor, including forks. Which trigger should it use, and why?",
+          "strictness": "high",
+          "trials": 10,
+          "passes": 10,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "judge",
+          "status": "ran",
+          "trialRecords": [
+            {
+              "output": "Use `pull_request`, not `pull_request_target`.\n\nA `pull_request` trigger runs the job with the contributing fork's own token, which is read-only and has no access to the base repository's secrets. That's exactly what a build-and-test job needs: it only reads the checked-out code and runs the test suite, nothing more. Since it can't reach secrets or write back to the base repo, an untrusted fork's code is contained by the token itself. For a GitLab equivalent, that's the ordinary merge-request pipeline (`rules:` on `$CI_PIPELINE_SOURCE == \"merge_request_event\"`), which similarly runs with the source project's limited context rather than the target project's protected variables.\n\n`pull_request_target` exists for the narrower case where a job genuinely needs base-repo secrets or write access (posting a comment, applying a label, etc.). It runs in the context of the base repository, so it gets the base repo's token and secrets. The trap is combining it with a checkout of the pull request's own head SHA or ref: at that point you are executing attacker-controlled code with base-repo secrets in the environment. So `pull_request_target` is the wrong tool here both because this job doesn't need secrets (it just builds and tests), and because if it were used it would have to avoid checking out and executing the PR head at all — which defeats the purpose of testing the incoming change.\n\nSo: `pull_request` for the test job. It gets the contributor's read-only token, no secrets, and safely runs the fork's code. Alongside it, still scope `permissions: {}` at the workflow level and grant only `contents: read` at the job level, pin `actions/checkout` and any other third-party action to a full commit SHA, and pass any untrusted `${{ github.event.* }}` value through `env:` rather than interpolating it directly into a `run:` string.",
+              "outputSha256": "1283bace7f100323b5544aa4f6119096a09e60b7b772a01c98ff240965511f99",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends pull_request over pull_request_target (\"Use `pull_request`, not `pull_request_target`\"). It states pull_request runs with the fork's own read-only token with no access to base-repo secrets, containing untrusted code. It explains pull_request_target runs in base-repo context getting base token/secrets and write access, wrong here since job needs no secrets."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use `pull_request`, not `pull_request_target`.\n\nWhy:\n\n- The job only builds and runs the test suite on the contribution. It does not need base-repo secrets or write access to the base repository, so it has no reason to run in the elevated context `pull_request_target` provides.\n- `pull_request` runs with the contributor's own token, which is read-only for fork PRs, and it cannot reach the base repository's secrets. That is exactly the right blast radius for \"build and test an untrusted contribution.\"\n- `pull_request_target` runs in the context of the base repository and *does* have access to base-repo secrets and a read/write token. It exists for cases that genuinely need those privileges (for example, labeling or commenting on a PR from a fork). Using it for a job whose whole purpose is to check out and execute the contributor's code is the classic escalation: if that job then checks out the PR's head ref/SHA and runs its test/build scripts, the contributor's code executes with the base repo's secrets and write token in scope. An attacker can put arbitrary code in a test script or postinstall hook and exfiltrate the token or secrets.\n\nA couple of things that go with the choice, so the trigger isn't the only defense:\n\n- Keep `permissions: {}` at the workflow level and grant only what this job needs at the job level (typically `contents: read`), so there's no standing write access to abuse.\n- Pass any untrusted input the job happens to use (PR title, branch name, etc.) through `env:` rather than interpolating `${{ github.event.* }}` directly into a `run:` script.\n- Pin third-party actions to a full commit SHA so a moved tag can't inject code into a workflow that runs on untrusted input.\n\nSo: `on: pull_request`. It gives the job everything it needs to build and test the contribution while keeping base-repo secrets and write access out of reach of the contributor's code.",
+              "outputSha256": "63c9f69623942b14f9e4b3435a5b35b27c948a7cc6159386ef3ac6a95d78ba6f",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends pull_request over pull_request_target (\"Use `pull_request`, not `pull_request_target`\"). It states pull_request_target runs in base repo context with base-repo secrets and read/write token, unneeded here. It states pull_request runs with contributor's own read-only token and cannot reach base-repo secrets. All criteria met."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use `pull_request` (or GitLab's default merge-request pipeline). Do not use `pull_request_target` for this job.\n\nWhy: `pull_request_target` runs in the context of the base repository and is given the base repo's token and access to its secrets — that is its entire purpose. Since this job's job is to build and run code from an incoming PR, and that PR can come from a fork (i.e., from a contributor you have no trust relationship with), pairing `pull_request_target` with actually checking out and running that PR's head commit is the classic dangerous combination: the attacker-controlled PR content executes with the base repo's secrets and write-capable token. A `pull_request` run, by contrast, gets a token scoped to the fork/read-only and does not receive base-repo secrets, so even if the PR's code is malicious, there is nothing sensitive for it to reach.\n\nThe job here only needs to build and test — it does not need base-repo secrets or write access to the base repository — so nothing about it requires `pull_request_target`. Keep it on `pull_request`, and separately:\n\n- Set `permissions: {}` at the workflow level and grant only what the job needs at the job level (typically `contents: read`), so the run cannot write to the repo even in the base case.\n- Pin any third-party `uses:` action to a full commit SHA, since a forked-PR-triggered job is exactly the context where a movable tag is a supply-chain risk.\n- Route any untrusted event value (PR title, branch name, commit message) through `env:` before it reaches a `run:` shell string, rather than interpolating `${{ github.event.* }}` directly.\n\nIf a later job genuinely needs base-repo secrets or write access (posting a comment, deploying, uploading a release), split it into a separate workflow triggered by `pull_request_target` or `workflow_run` that does not execute the PR's own head code — never fold that need back into this build/test job.",
+              "outputSha256": "fd070ef79a12f088682bbd0e63100d0a09902489b0311eee626b47fc2a083ac1",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends pull_request and warns against pull_request_target (\"Use `pull_request`... Do not use `pull_request_target`\"). It explains pull_request_target runs in base repo context with base token/secrets and write access, dangerous when running fork PR head code. It states pull_request gets a fork/read-only token and no base-repo secrets, so malicious code reaches nothing sensitive."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use `pull_request` (or GitLab's merge-request pipeline), not `pull_request_target`.\n\nThe reason is where the credentials come from. A `pull_request`-triggered run executes with the contributor's own `GITHUB_TOKEN`, which is read-only/scoped by default and has no access to the base repository's secrets. That matches exactly what a build-and-test job needs — it should only need to read the code and maybe write a cache or post a check status.\n\n`pull_request_target` is the trap here, and it's the one people reach for precisely because they think a fork PR is somehow blocked or underprivileged. It runs in the context of the *base* repository, so it gets the base repo's secrets and a potentially write-capable token. That's the opposite of what an untrusted-contributor build needs. The dangerous combination — and the one this whole skill exists to prevent — is `pull_request_target` combined with checking out and executing the PR's head SHA: that hands an attacker-controlled PR your base-repo secrets. Since a build-and-test job's whole purpose is to check out and execute the contribution's code, `pull_request_target` would be maximally wrong here.\n\nSo: `pull_request` for the build/test job, with scoped `permissions` (workflow-level `permissions: {}`, job-level `contents: read` and only the scopes it actually needs). If a *separate* job later needs base-repo secrets or write access — say, posting a status comment — that belongs in its own `pull_request_target` job that does not check out the PR head, split out from the untrusted build/test job rather than merged into it.\n\nTwo related details while you're there: pin every third-party `uses:` action to a full commit SHA (not a tag), and route any `${{ github.event.* }}` value through an intermediate `env:` entry before it reaches a `run:` shell string, since PR titles/branches are attacker-controlled text.",
+              "outputSha256": "afeb67f5ee97014320663e5dbf104cc880fa771184835f04ae099c935ec84e48",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends pull_request over pull_request_target (\"Use `pull_request` ... not `pull_request_target`\"). It states pull_request runs with the contributor's own read-only GITHUB_TOKEN with no access to base-repo secrets. It explains pull_request_target runs in the base repo context, getting base-repo secrets and a write-capable token, wrong for untrusted fork code."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use `pull_request` (GitLab: a merge-request pipeline). Do not use `pull_request_target` for this job.\n\nWhy: `pull_request` runs in the context of the *contributor's* fork by default. The token available to the job is effectively read-only with respect to the base repository, and — critically — base-repo secrets are **not** exposed to it. Since the whole purpose here is to build and run tests on arbitrary, untrusted fork contributions, that's exactly the isolation you want: a malicious PR can run whatever it likes inside the job, but it has no secrets to reach and no write access to the base repo.\n\n`pull_request_target`, by contrast, runs in the context of the *base* repository and therefore *does* receive base-repo secrets and a write-capable token. It exists only for the case where a job genuinely needs base-repo secrets or write access (e.g., labeling, commenting, posting a status). It is the wrong trigger for build-and-test, and it becomes an outright security hole the moment it is combined with checking out and executing the pull request's own head SHA/ref — that hands attacker-controlled code your base-repo secrets. So:\n\n- Trigger: `pull_request`.\n- Set `permissions: {}` at the workflow level and grant only what the job needs (typically `contents: read`) at the job level.\n- Pin third-party `uses:` actions to full commit SHAs, not tags.\n- Route any `${{ github.event.* }}` through an intermediate `env:` entry before it reaches a `run:` shell string — never interpolate it directly.\n- Add `timeout-minutes` to the job.\n\nIf later you find you also want to post a comment or label back onto the PR, put that in a *separate* job with minimal scoped `permissions` — and still never check out and execute the PR's own head ref inside a `pull_request_target` job.",
+              "outputSha256": "7330fe113e5485691627eee5e4426635d0553ec009b2df08a6ac8472b5d701de",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends pull_request and explicitly says not to use pull_request_target. It states pull_request runs in contributor's fork context with effectively read-only token and base-repo secrets not exposed. It explains pull_request_target runs in base repo context, receives base-repo secrets and write-capable token, wrong for build/test."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use `pull_request` (or GitLab CI's default merge-request pipeline), not `pull_request_target`.\n\nThe reason is what token and secret surface each trigger exposes to the job:\n\n- `pull_request` runs the job in the context of the pull request itself. The token it gets is scoped to the contributor's own fork/branch and is read-only by default. `secrets` from the base repository are not exposed to the job at all. So a malicious PR from a fork can at worst do something to its own checkout — it cannot read or exfiltrate the base repo's secrets, cannot push to the base repo, and cannot write to the base repo's issues/PRs. That is exactly the blast radius a build-and-test job needs.\n\n- `pull_request_target` runs the job in the context of the base repository. It gets a token with the base repo's permissions (potentially write access) and does have access to base-repo secrets. It exists for jobs that genuinely need those (e.g., labeling, posting a comment, or reading a deploy secret) — but for a build-and-test job that just compiles and runs tests, none of that is required, so adopting it is pure added risk with no upside.\n\nThere is a common trap worth naming: someone reaches for `pull_request_target` when they find that `pull_request` jobs can't access secrets, then combines it with `actions/checkout` set to the PR's head SHA. That combination is the classic vulnerability — `pull_request_target` supplies the base repo's secrets and write token, and the checkout of the untrusted head SHA hands the attacker's code the ability to execute in that privileged context, so it can simply print or exfiltrate the secrets. If a job truly needs `pull_request_target`, it must not check out and execute the PR's own head ref.\n\nSo for this job:\n\n```yaml\non: pull_request\n\npermissions:\n  contents: read   # scoped at the job level; no write scopes needed to build and test\n\njobs:\n  test:\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    steps:\n      - uses: actions/checkout@<full-commit-sha>   # pinned to a SHA, not a tag\n      - run: <install and run tests>\n```\n\nA few corollaries that follow from the same reasoning:\n\n- Set `permissions:` to the minimum — for build-and-test, `contents: read` — rather than leaving the default or using `write-all`, so that even if a step is compromised it has no write surface.\n- Never interpolate `${{ github.event.* }}` (PR title, branch name, etc.) directly into a `run:` shell string; route it through `env:` first, since a fork contributor controls that text.\n- Pin every third-party action to a full commit SHA, since a mutable tag on an action that now runs against untrusted PR code is another way in.\n\nIf part of the work genuinely requires base-repo secrets or write access (e.g., posting a test-result comment), put that in a separate job or workflow using `pull_request_target` (or `workflow_run`) that does not check out and execute the contributor's code, rather than upgrading the whole test job to `pull_request_target`.",
+              "outputSha256": "8d2c8ce0a659d38e58f6bc088568c9f783f6645b4a676d6f5922fb4b1f1c46e2",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends `pull_request` (not pull_request_target) for the build/test job. It states pull_request's token is scoped to the contributor's fork and read-only by default, with base-repo secrets not exposed, so a fork PR cannot read/exfiltrate secrets. It explains pull_request_target runs in base-repo context with base permissions (write) and base-repo secrets, unneeded here. All criteria met."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use `pull_request` (GitLab equivalent: a merge-request pipeline), not `pull_request_target`.\n\nWhy:\n\n- `pull_request` jobs triggered by a fork's PR run with the fork contributor's own token, which is read-only by default and has no access to the base repository's secrets, write token, or repository-scoped credentials. That is exactly the right privilege level for \"build and run tests\" — nothing in that job needs to write back to the base repo or read base-repo secrets.\n- `pull_request_target` runs in the context of the base repository, with the base repo's token and secret access. That elevation is only justified when a job genuinely needs base-repo secrets or write permissions (e.g., auto-labeling, commenting back on the PR). A build-and-test job needs neither, so reach for `pull_request_target` only with explicit justification, never as a default.\n\nAdditional guardrails regardless of trigger:\n\n- If a `pull_request_target` job is ever added alongside this one, it must never check out and execute the pull request's own head SHA/ref — that combination hands attacker-controlled code your base-repo secrets.\n- Set `permissions: {}` (or an explicit minimal scoped set) at the workflow level and grant only what this job needs at the job level (typically `contents: read` for checkout); avoid a default `write-all`.\n- Pin third-party actions (e.g., `actions/checkout`) to a full commit SHA rather than a mutable tag.\n- Any `${{ github.event.* }}` value used in the build/test job (PR title, branch name, etc.) must be routed through `env:` before reaching a `run:` shell string, never interpolated directly.\n- Add `timeout-minutes` to the job so a stuck test run doesn't hold a runner indefinitely; skip `concurrency`-based cancellation for this job only if you want every PR's test run to complete rather than supersede an earlier one.\n\nSo the trigger for a build-and-test-on-incoming-PR job from any contributor, including forks, is `pull_request`.",
+              "outputSha256": "f5a86bf0821f59a97c8eb6d359435260ec7e2c074ef7a22240d5f43eb0b338a8",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends `pull_request` not `pull_request_target` (opening line, closing line). It states pull_request runs with the fork contributor's own read-only token with no access to base-repo secrets/write token. It explains pull_request_target runs in base repo context with base repo's token and secret access, unnecessary for build/test. All criteria met."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use the `pull_request` trigger for this job — not `pull_request_target`.\n\nThe reason is the trust model each trigger uses. `pull_request` runs the workflow in the context of the merge commit, using a token scoped to the fork/contributor rather than the base repository. It gets read-only `GITHUB_TOKEN` permissions by default and cannot access base-repo secrets. That's exactly the right posture for \"build and run the test suite for arbitrary incoming contributions\" — you want the job to prove the code compiles and passes tests, and you want nothing in that job to be able to reach the base repo's secrets or write access.\n\n`pull_request_target` is the wrong choice here despite its name suggesting it's the \"fuller\" version. It runs in the context of the base repository, which means it *does* get base-repo secrets and a write-capable token. That is only needed when a job genuinely requires base-repo secrets or write access (for example, labeling or commenting on a fork PR, or posting a deploy preview). If you use `pull_request_target` and then check out and execute the pull request's own head SHA — which is what a \"build and test the contribution\" job naturally wants to do — you are handing an attacker-controlled PR your base-repo secrets and write token. That combination is the classic privilege-escalation footgun and must be avoided.\n\nSo: `pull_request` for the untrusted build/test job, with `permissions: {}` at the workflow level and only `contents: read` at the job level, `actions/checkout` pinned to a full commit SHA, and any PR-derived value (title, branch name, commit message) passed through `env:` before it reaches a `run:` shell string rather than interpolated directly. If a *separate* need arises later that genuinely requires base-repo secrets or write access on a fork PR, handle that with a distinct `pull_request_target` job that does not check out or execute the PR's head code — it should operate only on trusted base-repo data, not run the contributor's build.",
+              "outputSha256": "af21e290c2baed178d88d5846a3122b30a349a593313ea597a638347346f6400",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends pull_request, not pull_request_target (\"Use the `pull_request` trigger for this job — not `pull_request_target`\"). It states pull_request_target \"runs in the context of the base repository, which means it *does* get base-repo secrets and a write-capable token,\" and that pull_request \"gets read-only `GITHUB_TOKEN` permissions by default and cannot access base-repo secrets.\" All three pass criteria are concretely met."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use `pull_request`, not `pull_request_target`.\n\nWhy: a job that only builds and runs the test suite on an incoming contribution needs nothing from the base repository — no secrets, no write access. `pull_request` already provides exactly that:\n\n- It runs with the contributor's own token, which for a fork PR is read-only, so a malicious PR cannot use the job to push commits, comment with a bot identity, or otherwise mutate the base repo.\n- Repository secrets are not exposed to `pull_request` runs from forks at all, so even a PR that tries to exfiltrate `secrets.*` gets nothing.\n\n`pull_request_target` is the wrong fit here for two reasons. It runs in the context of the base repository and *does* get base-repo secrets and the base repo's (potentially write-scoped) token. That is only justified when a job genuinely needs to act on the base repo — e.g. labelling, posting a summary comment, or deploying a preview — not for a job whose whole purpose is building and testing the contributor's code. And it is specifically dangerous when combined with checking out the pull request's head: `pull_request_target` + `actions/checkout` with `ref: ${{ github.event.pull_request.head.sha }}` (or head repo) followed by any `run:`/build step executes attacker-controlled code with the base repo's secrets and token. That is the exact combination to avoid.\n\nSo the design is:\n\n- `on: pull_request` (optionally scoped with `branches:` / `paths:` to the relevant targets).\n- `permissions: {}` at the workflow level, with a job-level `contents: read` — enough to check out and run tests, nothing more.\n- Third-party `uses:` pinned to full commit SHAs.\n- Any `${{ github.event.* }}` value (PR title, branch name, etc.) routed through `env:` before it reaches a `run:` string, never interpolated directly into the shell script.\n- A `timeout-minutes` on the job and a `concurrency:` group keyed on the PR number so superseded runs cancel.\n\nIf a separate, later need arises to comment on the PR or otherwise touch the base repo, that belongs in its own job with its own minimal `permissions:` (e.g. `pull-requests: write`) — and even then it must not check out and build the PR's head code.",
+              "outputSha256": "0276910fced935ef06fdbe5b7574ce5ed45a8a8df608f487f7e6d5c39bffbdb4",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends pull_request over pull_request_target (\"Use `pull_request`, not `pull_request_target`\"). It states pull_request runs with the contributor's own token, read-only for forks, and that fork secrets aren't exposed. It explains pull_request_target runs in base-repo context with base-repo secrets and write-scoped token, and warns against it for this build/test job."
+              },
+              "passed": true
+            },
+            {
+              "output": "Use `pull_request` (not `pull_request_target`).\n\nFor a job that only builds and runs the test suite on an incoming PR, nothing about that job needs base-repo secrets or write access. `pull_request` runs in the context of the fork/head branch, using the contributor's own token, which is read-only by default and cannot read the base repository's secrets. That's exactly the trust boundary you want: an arbitrary contributor's code executes, but it has nothing sensitive to reach.\n\n`pull_request_target` is the wrong choice here because it runs in the context of the base repository, with access to base-repo secrets and a token that can have write permissions. That's only warranted when a job genuinely needs to comment back on the PR, push labels, or otherwise act with base-repo authority. The specific danger is that `pull_request_target` combined with checking out and executing the PR's own head ref/SHA hands an attacker-controlled pull request your base-repo secrets and write token — the code under test is the attacker's, but the credentials are yours. If someone later needs a privileged follow-up action (like posting a summary comment), the correct shape is to keep the build/test job on `pull_request` and have a separate, minimal job triggered by `workflow_run` do only that privileged step, never executing the untrusted code.\n\nAlongside the trigger choice: set `permissions: {}` at the workflow level and grant only `contents: read` at the job level, since a build-and-test job needs nothing more. Pin every third-party `uses:` action to a full commit SHA, not a tag, since tags are mutable. And route any `${{ github.event.* }}` value — the PR title, branch name, etc. — through an intermediate `env:` entry before it reaches a `run:` script, never interpolating it directly into the shell string.",
+              "outputSha256": "e4962d9b002db2817c5b73730a132ea82070b1a888b9c28a7f4ff0814b2ac552",
+              "promptSha256": "2578a3d9f820e02e9596443736044b61293e68733568fb957af1e4b1d61e4c02",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer recommends pull_request (not pull_request_target) for the build/test job. It states pull_request runs in the fork/head context with the contributor's own read-only token that cannot read base-repo secrets, and explains pull_request_target runs in base-repo context with base-repo secrets and a write-capable token, which this job doesn't need."
+              },
+              "passed": true
+            }
+          ]
+        }
+      ],
+      "verdict": "fail",
+      "scope": "bundled",
+      "skillDigest": "7d932f985018e327c3210f1ecd1287976b1d243c780c950244b1148eb5013460",
+      "catalogDigest": "8600b35461e2a92efe928c3011b674fd4afa25360ad47066b44d9253cddb0d7c",
+      "judgePromptVersion": "2026-09-25.1",
+      "runner": "deepseek",
+      "model": "deepseek-chat",
+      "runnerPromptVersion": "2026-09-25.1",
+      "recordedAt": "2026-09-25T17:59:46.418Z",
+      "judge": "deepseek",
+      "judgeModel": "deepseek-chat"
+    },
+    {
+      "schemaVersion": "1.0.0",
+      "skillId": "ci-github-gitlab/ci-pipeline-code-review",
+      "strictness": "high",
+      "trials": 10,
+      "triggerAccuracy": {
+        "truePositive": 4,
+        "falsePositive": 0,
+        "positives": 6,
+        "negatives": 6
+      },
+      "evidence": "authored",
+      "scenarios": [
+        {
+          "id": "trigger-positive-1",
+          "kind": "trigger-positive",
+          "prompt": "Look over this GitHub Actions diff and flag anything risky before we merge it",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-2",
+          "kind": "trigger-positive",
+          "prompt": "Check whether this new .gitlab-ci.yml job could leak our deploy token",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-3",
+          "kind": "trigger-positive",
+          "prompt": "Does this pull_request_target job put our secrets at risk?",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-4",
+          "kind": "trigger-positive",
+          "prompt": "Audit these workflow changes for unpinned third-party actions",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-5",
+          "kind": "trigger-positive",
+          "prompt": "Is the permissions block on this workflow scoped tightly enough?",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-6",
+          "kind": "trigger-positive",
+          "prompt": "Check this CI diff for a place where the PR title gets passed straight into a shell command",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-1",
+          "kind": "trigger-negative",
+          "prompt": "Fix the script injection bug you found in this workflow",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-2",
+          "kind": "trigger-negative",
+          "prompt": "Review this Terraform plan for a publicly exposed S3 bucket",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-3",
+          "kind": "trigger-negative",
+          "prompt": "Review this Node.js diff for a prototype pollution vulnerability",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-4",
+          "kind": "trigger-negative",
+          "prompt": "Give this Python service a general code style review",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-5",
+          "kind": "trigger-negative",
+          "prompt": "Review this Kubernetes Helm chart for missing resource limits",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-6",
+          "kind": "trigger-negative",
+          "prompt": "Run our standard OWASP Top 10 review on the whole repo",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "pwn-request-review",
+          "kind": "behavior",
+          "prompt": "Review this GitHub Actions diff: a job triggered by pull_request_target checks out github.event.pull_request.head.sha and then runs `npm run build`. What do you find?",
+          "strictness": "high",
+          "trials": 10,
+          "passes": 10,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "judge",
+          "status": "ran",
+          "trialRecords": [
+            {
+              "output": "Finding — pwn-request pattern in the `pull_request_target` job:\n\n**Risk category:** `pull_request_target` combined with untrusted checkout.\n\n**What the diff does:** The job is triggered by `pull_request_target`, which runs in the context of the base repository with access to repository secrets and a default `GITHUB_TOKEN` that typically has write scope. It then checks out `github.event.pull_request.head.sha` — the contributor's own fork commit — and runs `npm run build` against that code.\n\n**Concrete consequence:** `pull_request_target` is designed to give the workflow privileged access for operations like labeling or commenting, precisely *because* it does not run untrusted code. By checking out the PR head SHA and executing it, this job now executes attacker-controlled code (`npm run build` runs the contributor's `package.json` scripts and any lifecycle hooks such as `preinstall`/`postinstall`) inside a privileged context. That code can read the repository's secrets and the write-scoped `GITHUB_TOKEN`, exfiltrate them, push commits, or otherwise act as the repository — a \"pwn request.\" The fact that the executed step looks innocuous (`npm run build`) is irrelevant; the danger is the *combination* of an untrusted checkout and an execute step under a privileged trigger, and a malformed fork `package.json` or a build-time dependency is enough to exploit it.\n\n**Fix direction (not applied — read-only review):**\n- Preferred: switch the trigger to `pull_request`, which runs untrusted code in an unprivileged context without access to repository secrets. If the build genuinely needs no secrets, this is sufficient.\n- If privileged post-processing of the PR is required (e.g., commenting results), split the work: keep an unprivileged `pull_request` job that checks out the head SHA and runs `npm run build`, uploads only safe artifacts (build output, a result file), and then a separate `pull_request_target` job that consumes those artifacts without ever checking out or executing the PR's code.\n- Never combine `pull_request_target` with a checkout of the fork's head SHA followed by an execute step. If `pull_request_target` must remain, do not check out `github.event.pull_request.head.sha` at all.\n\nThis finding is reported as-is and no edit has been made to the workflow.\n\nNote: I assessed only the specific pattern you described. A complete review would also need to verify the third-party `uses:` pinning (tag vs. full commit SHA), the `permissions:` block for least privilege, whether `github.event.*` values are interpolated directly into `run:`/`script:` strings, and (for any deploy job) protected-variable handling — none of which were included in your description.",
+              "outputSha256": "4fd8f71fb27f2ce0b417e7722a2570335016ca0b1cb600086af360311fd5c8db",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names the pull_request_target job checking out github.event.pull_request.head.sha and running npm run build as the pwn-request pattern (\"What the diff does\"). Explains concrete risk: base-repo secrets and write-scoped GITHUB_TOKEN reachable by attacker-controlled fork code. Gives fix direction (switch to pull_request, or split unprivileged build + privileged artifact-consuming job) explicitly as \"not applied — read-only review\" and states no edit made. No edits claimed."
+              },
+              "passed": true
+            },
+            {
+              "output": "This is a **pwn request** finding — a high-severity one. Report it; do not edit the workflow.\n\n**What I find:**\n\n- **Job trigger + checkout + execute (pwn-request pattern).** The job is triggered by `pull_request_target`, checks out `github.event.pull_request.head.sha` (the contributor's own fork commit), and then runs `npm run build` against that checkout. This is the textbook critical pattern: `pull_request_target` runs in the base repository's context with access to the base repo's secrets and a write-scoped token, while the checked-out code is fully attacker-controlled. Any contributor can open a PR whose commit modifies, for example, a `preinstall`/`build` npm script or a config file that `npm run build` executes, and that code runs with the trusted job's privileges.\n\n  \"Just `npm run build`\" is not a mitigating detail. The risk is the *combination* of the trigger, the untrusted checkout, and executing the checked-out tree — not how innocuous the individual command looks. This is flagged every time it appears, regardless of how trivial the executed step seems.\n\n**Fix direction (not to be applied here):**\n\n- Switch the trigger to `pull_request` if the job only needs read-only access, or\n- Split into two jobs: an unprivileged job triggered by `pull_request` that checks out and builds the fork code with no secrets and read-only permissions, and a separate, minimal privileged job (triggered by `pull_request_target`/`workflow_run`) that consumes only safe build artifacts, never the fork's source tree or head SHA directly.\n- Do not check out `github.event.pull_request.head.sha` (or any fork head ref) in a `pull_request_target` job that also executes the checked-out code.\n\n**Other checks I would still complete before calling this review done** (this diff's snippet only shows the trigger/checkout/execute lines):\n\n- Whether the third-party actions' `uses:` lines are pinned to full commit SHAs vs. mutable tags.\n- Whether the workflow grants `permissions: write-all` or omits `permissions:` entirely despite jobs not all needing write access.\n- Whether any `run:`/`script:` step interpolates `${{ github.event.* }}` (PR title, branch name, commit message) directly into the shell string rather than through an intermediate `env:` entry.\n- Whether any deploy secret in the job is reachable given the trigger.\n\nNote: this skill is read-only. The above is the finding and fix direction only — no change has been made or claimed against the workflow file.",
+              "outputSha256": "3d89148de65d356d7214bce200d5d3971ff741bd8583a06f0273cf0371eb02a3",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names the job's pull_request_target trigger + checkout of github.event.pull_request.head.sha + npm run build as the pwn-request pattern (criterion 1). It explains base-repo secrets and write-scoped token reachable from attacker-controlled fork code (criterion 2). Fix direction given as finding only: switch to pull_request or split unprivileged build + privileged artifact-consuming job, explicitly 'not to be applied here' (criterion 3). No edit claimed; states read-only, no change made (fail criterion 1 not met)."
+              },
+              "passed": true
+            },
+            {
+              "output": "This is the classic \"pwn request\" pattern, and it is a high-severity finding.\n\n**What the diff does:** a job runs on `pull_request_target`, checks out the pull request's own head commit (`github.event.pull_request.head.sha`), and then executes `npm run build` against that checked-out code.\n\n**Why it is dangerous (concrete consequence):**\n\n- `pull_request_target` runs in the context of the *base* repository, so the job gets the base repo's secrets, a write-capable `GITHUB_TOKEN`, and access to any deployment credentials — unlike `pull_request`, which runs untrusted forks with a read-only token and no secrets.\n- Because the job explicitly checks out the fork's `head.sha`, the code being built is attacker-controlled. Any contributor can open a PR whose head commit makes `npm run build` do whatever they want.\n- `npm run build` executes arbitrary code from that checkout (build scripts, `postinstall`/`preinstall` hooks, `prepare` scripts in the fork's `package.json`). That code runs *before* the build produces anything, with the privileged token and secrets already in the environment.\n- Net effect: an outside contributor can exfiltrate the repo's secrets, push to the base repository using the write token, or pivot to any environment the job can reach (e.g. OIDC-federated cloud roles). No repo write access or approval is needed.\n\nNote that the danger is the *combination* — privileged trigger + untrusted checkout + execution — not any single line. Even a \"trivial\" `npm run build` is enough, because the build itself runs fork-controlled code.\n\n**Fix direction (to hand back to the author — do not apply it here):**\n\n- Split the pipeline. Run untrusted build/test on `pull_request` (no secrets, read-only token), and if a privileged follow-up is needed, pass only safe artifacts (build outputs, reports) to a separate `pull_request_target`/`workflow_run` job that never checks out the fork's code and never executes it.\n- If `pull_request_target` genuinely must touch the PR, never check out `github.event.pull_request.head.sha` / the fork ref and never run fork code against it.\n- Alternatively, use the `pull_request` trigger, which is unprivileged by default.\n\n**Also worth flagging in the same diff (per this skill's categories):**\n\n- **Action pinning:** check whether any `uses:` in this workflow (e.g. `actions/checkout@v4`) is pinned to a mutable tag rather than a full commit SHA.\n- **Permissions:** check for a workflow-level `permissions: write-all` or a missing `permissions:` block on a job that does not need write access.\n- **Script injection:** check whether any `run:` step interpolates `${{ github.event.* }}` (PR title, body, branch name) directly into the shell instead of routing it through an `env:` entry — a separate injection path from the checkout issue above.\n\nThe checkout-and-execute finding is the one to report first; the others depend on the rest of the file, which I have not been shown.",
+              "outputSha256": "73494367e5079f0efd0c645da0e353e39d2f82cd33ce9bfe549232442247711e",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names the pwn-request pattern: pull_request_target + checkout of github.event.pull_request.head.sha + npm run build execution. Explains concrete risk: base-repo secrets, write-capable GITHUB_TOKEN, exfiltration/push. Fix direction given as finding only ('do not apply it here'): switch to pull_request or split unprivileged build + privileged job consuming safe artifacts. No edits claimed."
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings in the changed workflow:\n\n- Job trigger + checkout: pwn-request pattern. The job is triggered by `pull_request_target`, which runs in the context of the base repository with access to the base repo's secrets and a write-scoped `GITHUB_TOKEN`. It then checks out `${{ github.event.pull_request.head.sha }}` — the contributor's own fork code — and executes `npm run build` against it. This means arbitrary code from an untrusted fork runs with the privileges `pull_request_target` grants. This is the classic \"pwn request\": an attacker opens a PR from their fork, the maintainer's workflow runs it, and the fork-controlled build script (e.g. a modified `package.json` `build` script or a malicious `postinstall`) can exfiltrate secrets or push to the repo. Flag regardless of how innocuous `npm run build` looks — the risk is the combination, not the individual line.\n  - Fix direction: either switch the trigger to `pull_request` (which runs with a read-only token and no base-repo secrets), or split into two jobs — one unprivileged job that checks out and runs untrusted code under `pull_request`, and a separate privileged job (no untrusted checkout) that consumes only safe artifacts. Do not run untrusted fork code in a `pull_request_target` context.\n\n- Because the job executes untrusted code under an elevated context, also verify and likely flag:\n  - `permissions:` scope: if this workflow is missing a `permissions:` block or uses `permissions: write-all`, the untrusted build step inherits that write scope. Even a read-only token is not safe here because `pull_request_target` still exposes secrets.\n  - Action pinning: any third-party `uses:` line in this job pinned to a mutable tag (e.g. `@v4`) instead of a full commit SHA is a separate finding, and it compounds the above since the actions also run in the privileged context.\n\n- Script injection: if any `run:`/`script:` step in this job interpolates `${{ github.event.* }}` (PR title, branch name, commit message) directly into the shell string rather than routing it through an intermediate `env:` entry, that is an additional script-injection finding — and it is especially dangerous here because the injection occurs in the privileged `pull_request_target` context.\n\nNote on the checkout itself: checking out the fork's head SHA is not inherently wrong in a `pull_request` context; it is the combination of `pull_request_target` (privileged, secret-bearing context) with checking out untrusted code and then executing it that constitutes the vulnerability.\n\nFix direction summary: change the trigger to `pull_request` or split into unprivileged build + privileged consumer jobs; pin all third-party actions to full commit SHAs; set least-privilege `permissions:`; and pass any event data through `env:` rather than direct interpolation.\n\nI have not modified the workflow — this is a read-only report; applying any fix belongs to the implementation step or the author.",
+              "outputSha256": "e2adc1b9c3fe9ba2b11a8b87f557851939827b6bd4d9de6f6cda9ed00920511b",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names the job's pull_request_target trigger plus checkout of github.event.pull_request.head.sha and npm run build as the pwn-request pattern (first bullet). Explains base-repo secrets and write-scoped GITHUB_TOKEN reachable from untrusted fork code. Gives fix direction (switch to pull_request, or split unprivileged build + privileged consumer) as a finding, and explicitly states it made no edits (read-only report)."
+              },
+              "passed": true
+            },
+            {
+              "output": "This is a finding, and it is the highest-severity category this review covers. Let me walk through it against the risk categories.\n\n**Job scoped:** the diff adds (or modifies) a job under a `pull_request_target` trigger.\n\n**Finding 1 — pwn request (pull_request_target + untrusted checkout + execute).**\n\n- The job is triggered by `pull_request_target`. That trigger runs with the base repository's context: a write-scoped `GITHUB_TOKEN`, access to repository secrets, and whatever `permissions:` the workflow declares — regardless of who opened the pull request.\n- The job checks out `github.event.pull_request.head.sha`. That ref is the contributor's own fork's head commit. `pull_request_target` does *not* restrict what that ref contains — it only changes which context the job runs in. So the checkout pulls in attacker-controlled file contents.\n- It then runs `npm run build`. Depending on how `npm run build` resolves, this executes the checked-out `package.json` `build` script and whatever it calls (`tsc`, a bundler, a `prebuild`/`postinstall` hook, a locally vendored binary — any of these resolve from the checked-out tree). This is arbitrary code execution against the base repo's privileged context.\n- Consequence: the contributor can exfiltrate secrets available to the job, push to the repository using the write-scoped token, or tamper with the base repo's artifacts/caches. The fact that the step reads as an innocuous \"build\" does not reduce this — the risk is the *combination* of the privileged trigger and executing untrusted code, not the shape of any single step.\n- This is flagged every time it appears, regardless of how trivial the executed step looks.\n\n**Fix direction (do not apply here):** split the pipeline. Run the untrusted build under a plain `pull_request` trigger (no secrets, read-only token), and have a separate privileged job — triggered by `workflow_run` on that first job, or by `pull_request_target` but consuming only produced artifacts — do anything that genuinely needs write scope or secrets. Never check out and execute a fork ref inside a `pull_request_target` job.\n\n**Finding 2 — check the trigger's `permissions:` (related, needs the diff).**\n\n`pull_request_target` jobs default to the workflow/repository token permission rather than the restricted pull_request default. If the diff has a workflow-level `permissions: write-all`, or omits a `permissions:` key entirely, that compounds Finding 1 — the executing code inherits write scope. Flag `write-all`/missing-`permissions:` here alongside the pwn request. Fix direction: declare explicit least-privilege `permissions:` (e.g. `contents: read`) at workflow or job level, granting `id-token: write` only to the job that actually federates via OIDC.\n\n**Finding 3 — action pinning (needs the diff to confirm).**\n\nI can't confirm from the description alone whether the `actions/checkout` and any other third-party `uses:` lines in this job are pinned to a full commit SHA or to a mutable tag like `@v4`. If they are tag-pinned, that is a separate finding, and it matters more than usual here: a movable tag is a second avenue into the same privileged job. Fix direction: pin third-party actions to full commit SHAs.\n\n**Finding 4 — script injection (needs the diff to confirm).**\n\n`head.sha` is a SHA and is comparatively safe to interpolate. But check the rest of the job for `${{ github.event.pull_request.title }}`, body, or branch-name interpolations into `run:` strings; those are attacker-controlled free text and should route through an intermediate `env:` entry rather than inline into the shell. If the diff only uses `head.sha`, this category is clear for the lines shown.\n\n**Verification status:** the trigger (`pull_request_target`) and the untrusted ref (`head.sha`) plus execution (`npm run build`) are confirmed from the description — Finding 1 stands. Findings 2–4 cannot be fully confirmed without the actual diff; I'm noting them as items to check rather than reading clean.\n\n**No fix was applied.** This is a read-only review; the work of changing the workflow belongs to the implementation skill or the author.",
+              "outputSha256": "9223a261db34e258b369b3edf831dd8d416f4f83f70f81e34a1aa24e186c55ac",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names pull_request_target + checkout of github.event.pull_request.head.sha + npm run build as pwn request, explains base-repo secrets/write GITHUB_TOKEN reachable from untrusted fork code, and gives fix direction (split into pull_request build job + separate privileged job consuming artifacts) explicitly as 'do not apply here'. States 'No fix was applied.' No edits claimed."
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings in the reviewed workflow:\n\n- **`pull_request_target` + untrusted checkout-and-execute (pwn-request pattern).** The job is triggered by `pull_request_target`, which runs with the base repository's write-scoped token and secrets. It then checks out `github.event.pull_request.head.sha` — the fork contributor's own head commit — and runs `npm run build` against it. This is the classic \"pwn request\" combination: any contributor can open a PR whose checked-out code executes with the privileged base-repo context. `npm run build` is not innocuous here — it runs `package.json` scripts from the untrusted checkout, which can be trivially modified to exfiltrate the token/secrets or push to the repo. Flag this regardless of how trivial the executed step looks; the risk is the combination, not the single line.\n\n  Fix direction: either switch the trigger to `pull_request` (unprivileged, no secrets/write token on fork PRs), or split into two jobs — an unprivileged build job under `pull_request` that produces artifacts, and a separate `pull_request_target` job that only consumes those already-produced, safe artifacts (never re-checking-out and executing the fork's head). Never run untrusted code under `pull_request_target` with secrets available.\n\nAdditional checks I would still run and note, since they are not settled by the description:\n\n- **Action pinning.** Confirm every third-party `uses:` line (including the `actions/checkout` used by this job) is pinned to a full commit SHA, not a mutable tag like `@v4`. If it is on a tag, that is a separate finding.\n- **`permissions:` scope.** Check whether the workflow sets `permissions:` at all, or sets `write-all`. A `pull_request_target` job that does not restrict `permissions:` inherits broad write access, compounding the pwn-request exposure.\n- **Script injection.** Even after any restructuring, check whether any `run:`/`script:` step interpolates `${{ github.event.* }}` (PR title, branch name, commit message) directly into the shell string rather than passing it through an intermediate `env:` entry.\n\nPrimary finding: the `pull_request_target` + head-SHA-checkout + `npm run build` combination is a confirmed pwn-request pattern and should be reported as such every time it appears. This review is read-only — I am reporting the finding and the fix direction, not applying any change.",
+              "outputSha256": "61c209ab45572a99cc17c317cf485115869544198edc5fc4d5c739cffc160f2c",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Names the job's pull_request_target trigger + checkout of github.event.pull_request.head.sha + npm run build as the pwn-request pattern. Explains base-repo write token/secrets reachable from untrusted fork code. Gives fix direction (switch to pull_request, or split unprivileged build + privileged artifact-consuming job) as a finding. Explicitly read-only: 'not applying any change.'"
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings:\n\n- **`pull_request_target` + untrusted checkout + execution — pwn-request pattern.** The job is triggered by `pull_request_target`, which runs in the context of the base repository with access to its secrets and a write-scoped `GITHUB_TOKEN`. It then checks out `github.event.pull_request.head.sha` (the contributor's own fork/commit) and runs `npm run build` against it. That checkout-and-execute combination is the classic \"pwn request\": an attacker opens a PR from a fork containing a malicious `package.json`/build script, and that code executes with the base repo's privileged token and any secrets. This is a finding regardless of how innocuous `npm run build` looks, because the risk is the combination, not the step itself. Fix direction: use `pull_request` instead of `pull_request_target`; if base-context privileges are genuinely needed, split into an unprivileged job that builds/validates on `pull_request` and a separate privileged job that only consumes safe artifacts — never check out and execute the fork head in the privileged context.\n\n- **Action pinning (conditional on the diff).** If the `actions/checkout` (or any third-party) `uses:` line is pinned to a tag (`@v4`) or branch rather than a full commit SHA, that is a separate finding — flag it. A tag is mutable and the pin can be moved out from under you; a full commit SHA is the only safe pin.\n\n- **`permissions:` scope (check the diff).** If the workflow/ljob grants `permissions: write-all`, or has no `permissions:` key at all, on a job that only builds, that is a finding — grant least privilege (read-only by default, widened only where needed).\n\n- **Script injection (check the diff).** If any `run:`/`script:` step interpolates `${{ github.event.* }}` (PR title, branch name, commit message) directly into the shell string rather than passing it through an intermediate `env:` entry first, that is a finding too.\n\nThe single actionable finding from what you described is the `pull_request_target` + head-SHA checkout + `npm run build` combination. Report it and give the fix direction; do not edit the workflow.",
+              "outputSha256": "4abea46a47ae3dc041809c9bb4846d412bd7ca1f8f83100aa7955e9165adce67",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names the job's pull_request_target trigger + checkout of github.event.pull_request.head.sha + npm run build as the pwn-request pattern (first bullet). Explains base-repo secrets and write-scoped GITHUB_TOKEN reachable from untrusted fork code. Gives fix direction (switch to pull_request, or split unprivileged build + privileged artifact-consuming job) as a finding, and explicitly says do not edit the workflow — no edits claimed."
+              },
+              "passed": true
+            },
+            {
+              "output": "This is a pwn-request pattern, and it should be flagged as a finding.\n\n**Finding — `pull_request_target` + untrusted checkout and execute**\n\n- **Risk category:** `pull_request_target` combined with untrusted checkout.\n- **What the diff does:** the job is triggered by `pull_request_target` (which runs in the base repository's context, with access to the base repo's secrets and a write-scoped `GITHUB_TOKEN`), and it checks out `github.event.pull_request.head.sha` — i.e. the contributor's own head commit — then runs `npm run build` against that code.\n- **Concrete consequence:** `npm run build` executes code from the contributor's branch. Because `pull_request_target` runs in the privileged base context, that code executes with the base repo's `GITHUB_TOKEN` (potentially write scope) and any secrets available to the workflow. A contributor can modify their fork's `package.json` `build` script (or any file it pulls in) to exfiltrate `GITHUB_TOKEN`, secrets, or push to the repo. The trivial-looking `npm run build` step is not the mitigating factor — the risk is the combination of the privileged trigger and the untrusted checkout, so it is a finding regardless of how innocuous the executed step appears.\n- **Fix direction:** do not check out and execute the fork's head commit under `pull_request_target`. Either switch the trigger to `pull_request` (unprivileged), or split into an unprivileged build/test job on `pull_request` that produces artifacts, and a separate `pull_request_target` job that only consumes those safe artifacts without re-running contributor code. Never interpolate/execute contributor-controlled ref content in the privileged context.\n\n**Additional checks I cannot conclude from the description given (need the rest of the diff):**\n\n- **Action pinning:** whether the `actions/checkout` (and any other third-party) `uses:` line is pinned to a full commit SHA or a mutable tag like `@v4` — a tag would be a second finding.\n- **`permissions:` scope:** whether the workflow declares a least-privilege `permissions:` block, or leaves it at the default/write-all — another finding if absent or overly broad.\n- **Script injection:** whether any `run:`/`script:` step interpolates `${{ github.event.* }}` (PR title, branch name, commit message) directly into the shell string rather than routing it through an intermediate `env:` entry.\n- **Secrets:** which secrets this job can reach, and whether they are exposed to a `pull_request_target` job at all.\n\n**Verification status:** the `pull_request_target` + untrusted-checkout finding is confirmed. The pinning, `permissions:`, and interpolation checks are not yet verified from the description alone and must be checked against the full diff before the review is called complete.\n\nThis is reported as a finding with a fix direction only — no edit has been or will be applied to the workflow file.",
+              "outputSha256": "a9775b06dea5083127ffbf1475968f2d480be2890b325139a22417e32690fc23",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names the job's pull_request_target trigger plus checkout of github.event.pull_request.head.sha then npm run build as the pwn-request pattern (\"What the diff does\"). It explains base-repo secrets and write-scoped GITHUB_TOKEN reachable from fork code (\"Concrete consequence\"). Fix direction given as finding: switch to pull_request or split unprivileged build + privileged artifact-consuming job. Explicitly read-only: \"no edit has been or will be applied.\""
+              },
+              "passed": true
+            },
+            {
+              "output": "Finding — pwn-request pattern (pull_request_target + untrusted checkout + execute):\n\n- Trigger/lines: the job is triggered by `pull_request_target`, which runs in the context of the base repository and therefore with the base repo's secrets and a write-capable `GITHUB_TOKEN`. It then checks out `github.event.pull_request.head.sha` — i.e. the contributor's own fork commit — and runs `npm run build` against that checkout.\n- Consequence: this is the classic \"pwn request.\" Any external contributor can open a PR, modify `package.json`/build scripts/`postinstall`/a `.js` file that the build executes, and have arbitrary code run with the base repo's secrets and token. Even if the checked-out SHA is a fork commit, `pull_request_target` grants the *base* repo's privileges to whatever that code does. There is no scenario where executing an untrusted PR head under `pull_request_target` is safe by default.\n\nFix direction (do not apply here — read-only):\n- Prefer switching the trigger to `pull_request` if the job only needs to build/test, so it runs with the fork's restricted token and no secrets.\n- If base-repo privileges are genuinely required, split into two jobs: an unprivileged job under `pull_request` that checks out and builds the fork code and uploads safe artifacts, and a separate privileged `pull_request_target` job that consumes only those artifacts and never executes the fork's code.\n- If the fork head must be referenced, do not run its code in the privileged job; only pass data (artifacts, metadata) across the boundary.\n\nSecondary check to note in the same job (flagging because it commonly co-occurs, but confirm against the diff):\n\n- Action pinning: verify every third-party `uses:` (e.g. `actions/checkout@v4`) is pinned to a full commit SHA rather than a mutable tag. A tag pin is a separate finding even though it is not the cause of the pwn-request risk.\n- `permissions:` scope: if this job has a workflow-level `permissions: write-all` or no `permissions:` at all, flag that too for least-privilege.\n- Script injection: if any `run:` string in this job interpolates `${{ github.event.pull_request.title }}`, branch name, commit message, etc. directly into the shell, that is a further finding; route such values through `env:` first.\n\nNet: the headline finding is the `pull_request_target` trigger combined with checkout of `github.event.pull_request.head.sha` and immediate `npm run build`. Report it and the fix direction; do not edit the workflow.",
+              "outputSha256": "89e6d963a7f3fb3d1f36335a150295504b29590e1e328a60e284fd761f408118",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names the job's pull_request_target trigger + checkout of github.event.pull_request.head.sha + npm run build as the pwn-request (\"classic 'pwn request'\"). Explains base-repo secrets and write-capable GITHUB_TOKEN reachable from untrusted fork code. Gives fix direction (switch to pull_request, or split unprivileged build + privileged artifact-consuming job) explicitly as \"do not apply here — read-only\"; no edits claimed."
+              },
+              "passed": true
+            },
+            {
+              "output": "**Findings — this is a pwn-request pattern; do not merge as-is.**\n\n**Finding 1: `pull_request_target` + untrusted checkout + execute (pwn request)**\n\n- **Category:** `pull_request_target` combined with untrusted checkout.\n- **What's wrong:** The job is triggered by `pull_request_target`, which runs in the context of the base repository with access to its secrets and a write-capable `GITHUB_TOKEN`. It then checks out `github.event.pull_request.head.sha` — the contributor's own fork commit — and executes `npm run build` against that checkout. The checked-out code is fully attacker-controlled, and `npm run build` executes it.\n- **Concrete consequence:** A contributor can open a PR from a fork containing a malicious `package.json` (e.g., a `build` or `prebuild`/`postinstall` script) and get arbitrary code execution inside a job that holds the base repo's secrets and write token. That is the classic exfiltration path: read `secrets.*`, push to the repo, or use the token to tamper with releases/artifacts. `pull_request_target` was designed precisely so untrusted fork code does *not* get executed with this privilege; this job defeats that.\n- **Fix direction:** Either (a) switch the trigger to `pull_request` so the job runs without base-repo secrets/write scope, or (b) split it: an unprivileged `pull_request` job builds/tests the fork code, and a separate `pull_request_target` (or `workflow_run`) job consumes only the produced artifacts — never the fork's own working tree — and performs any privileged step. Do not \"fix\" it by merely adding `permissions: read-all`; the checkout-and-execute of the head SHA is the finding.\n\n**Secondary checks on the same diff (flag if present):**\n\n- **Action pinning:** the `actions/checkout` line — if it's `actions/checkout@v4` (a mutable tag) rather than a full commit SHA, that's a separate pinning finding. Note that in this job the pinning concern is secondary to Finding 1; a SHA-pinned checkout of an *untrusted* ref is still the pwn-request pattern.\n- **`permissions:` scope:** if the workflow has `permissions: write-all`, or omits `permissions:` entirely while this job does not need write access, flag it — but again, tightening permissions does not remediate Finding 1, only narrows the blast radius.\n- **Script injection:** confirm the `run:` step does not also interpolate `${{ github.event.pull_request.title }}` / branch name / commit message directly into the shell string. If it does, that's an additional injection finding (route through `env:` first).\n\n**Bottom line:** Finding 1 is blocking on its own. The trigger, the untrusted ref, and the executed step must not coexist; resolve by changing the trigger or by splitting privileged vs. unprivileged work. Reported as fix direction only — no edits applied.",
+              "outputSha256": "f8d3a9ce7846b90d07a3c485e039a73290f97a8b8946fe11135a1fd7b26110e2",
+              "promptSha256": "466f21c826cf2b5fb1b06e933913f0f7a736c69c9a2dc644c13c07454ecf9e25",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names pull_request_target + checkout of github.event.pull_request.head.sha + npm run build as the pwn-request pattern in this job. It explains base-repo secrets and write-capable GITHUB_TOKEN reachable from untrusted fork code (exfiltration path). Fix direction given as finding only: switch to pull_request or split unprivileged build + privileged artifact-consuming job. Explicitly 'no edits applied' — read-only."
+              },
+              "passed": true
+            }
+          ]
+        },
+        {
+          "id": "unpinned-action-permissions",
+          "kind": "behavior",
+          "prompt": "Review this workflow diff: a new job adds `uses: some-org/some-action@v2` and the workflow has no permissions: block at all. What do you find?",
+          "strictness": "high",
+          "trials": 10,
+          "passes": 10,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "judge",
+          "status": "ran",
+          "trialRecords": [
+            {
+              "output": "Findings:\n\n- `.github/workflows/<file>.yml`, job `<name>`: `uses: some-org/some-action@v2` — third-party action pinned to a mutable tag. The risk is that `v2` is a movable pointer: whoever controls `some-org/some-action` can repoint it at any commit at any time, and the next run of this workflow picks that up with no change on your side. Fix direction: pin to the action's full commit SHA (a `# v2.x.y` comment next to the SHA is fine for readability; the SHA itself must be the pin).\n\n- `.github/workflows/<file>.yml`: no `permissions:` block at all — the workflow runs with the repository's default token scope, which can be read/write depending on repo or org settings. Every job in this workflow implicitly inherits it, including anything the newly added job does with that token. Fix direction: add a workflow-level `permissions:` block set to the minimum needed (e.g. `contents: read`), and widen per-job only where a job demonstrably requires more (e.g. `id-token: write` on just the job doing OIDC federation). Do not leave the scope implicit.\n\nTwo related checks worth confirming before closing this out, since the diff doesn't show them:\n\n- Trigger: I only know the new job's `uses:` line and the missing `permissions:` block. I have not seen which event triggers this job. If it runs under `pull_request_target` and any step checks out the contributor's fork head and executes it, that is a separate and more serious pwn-request finding — it needs its own look, but I cannot assert it exists from what's given here.\n- Interpolation: no `run:`/`script:` step was shown, so there is nothing yet to flag for direct `${{ github.event.* }}` interpolation. If the new job has any `run:` step referencing event fields, that needs checking too.\n\nFix direction for both confirmed findings, not applied: the pin belongs on the `uses:` line, and the `permissions:` block belongs at workflow level (or job level if scopes genuinely differ per job). I have not edited the file — these are findings and directions only.",
+              "outputSha256": "f0fe43e2dcb020ac3739b6efabd77f7a54e53c1f333f8795b889c87afbde537b",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names `some-org/some-action@v2` as a mutable tag and gives fix: pin to full commit SHA. Finding 2 flags no `permissions:` block, explains default token scope can be read/write and every job inherits it, fix: add workflow-level minimum `permissions:` (e.g. contents: read), widen per-job. Both fixes given as directions, not applied ('I have not edited the file'). No dismissal of the missing block."
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings:\n\n1. **Third-party action pinned to a mutable tag** — `some-org/some-action@v2` references a tag, not a full commit SHA. A tag can be re-pointed by the action's maintainers (or by anyone who compromises that org's account/release process), so the code actually executed by this job can change without any visible change to this workflow file. Fix direction: pin the action to its full 40-character commit SHA (a `# v2` comment alongside the SHA is fine for readability), per the pinning rule in `rules/security.mdc`.\n\n2. **No `permissions:` block at all** — with no workflow-level or job-level `permissions:` key, this job inherits the repository/org default token scope, which is frequently `write-all` and is not guaranteed to be least-privilege. That grants the job's `GITHUB_TOKEN` far more access than this job presumably needs. Fix direction: add an explicit top-level `permissions: contents: read` (or narrower) and only elevate specific scopes on the specific job that genuinely requires them.\n\nI can't fully rule out the `pull_request_target` + untrusted-checkout pattern, script injection via `${{ github.event.* }}` interpolation, or GitLab-side concerns from what's given here — the trigger, the rest of the job body, and whether a `pull_request_target`-style checkout-and-execute combination exists weren't provided. Those would need checking against the full diff before calling the review complete; what I can confirm from the two details given is the unpinned action and the missing least-privilege permissions block.\n\nThis is a read-only review — I've reported the findings and fix directions only and have not edited the workflow.",
+              "outputSha256": "5e01aee7d0ea20c4fd85e96ae1e446c84992dcb39af9468a4a265a4325266b75",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names `some-org/some-action@v2` as a mutable tag and gives concrete fix: pin to full 40-char commit SHA. Finding 2 flags no permissions: block, explains inherited default may be write-all/broader than needed, fix: add explicit `permissions: contents: read` (or narrower). Both fixes stated as directions, not applied; answer explicitly stays read-only. No fail criterion met."
+              },
+              "passed": true
+            },
+            {
+              "output": "Two findings from this diff:\n\n**1. Unpinned third-party action (mutable tag)**\n\n`some-org/some-action@v2` is pinned to a movable tag, not a full commit SHA. Tags can be re-pointed by anyone with write access to that action's repo — and a compromise of that org later can silently swap the code your job runs, including on already-merged workflows. Even if `some-org` is well-known today, the pinning risk is about the tag being mutable, not about current trust.\n\n- Fix direction: pin to the action's full 40-character commit SHA (a `# v2.x.y` comment next to the SHA is fine for readability). If the repo enforces Dependabot/Renovate for GitHub Actions, let it manage the SHA bumps.\n\n**2. Missing `permissions:` block (unscoped token)**\n\nWith no `permissions:` key at all, the job falls back to the repository/org default token scope, which on many repos is still read-write across `contents`, `issues`, `pull_requests`, `packages`, etc. A job that only does a checkout/build/lint almost certainly needs nothing more than `contents: read`. Leaving the default wide-open means anything that ends up executing in that job — including the unpinned action above — inherits write capability it has no business having.\n\n- Fix direction: add an explicit least-privilege block at the workflow level, e.g.:\n  ```yaml\n  permissions:\n    contents: read\n  ```\n  and add per-job `permissions:` only where a job genuinely needs more (e.g. `id-token: write` on the one job doing OIDC, `pull-requests: write` on the one job posting a comment).\n\n**Why these compound**\n\nThe two findings interact, which is worth stating rather than reporting them as independent checkboxes: an unpinned action that is later compromised runs with whatever the token scope allows, and an unscoped token means that compromise reaches write access — pushing to the repo, opening/modifying PRs, publishing packages — rather than being confined to reads. Fixing the pin bounds *what code runs*; fixing `permissions:` bounds *what that code can do* if the pin is ever bypassed. Both are needed.\n\n**Not assessed / out of scope for this excerpt**\n\nI can't rule on the other categories without seeing more of the diff: whether the job runs under `pull_request_target` and checks out an untrusted head SHA (pwn-request pattern), whether any `run:` step interpolates `${{ github.event.* }}` or an untrusted variable directly into a shell string (script injection), and — if this is a GitLab pipeline — whether any deploy job consumes a credential variable without protected-branch/masked-variable restrictions. If you paste the full job (trigger, `run:` steps, and any checkout ref), I can check those too.\n\nNo edits were made — these are findings and fix directions only.",
+              "outputSha256": "c28971c8ed2c6f9478d5fe626a62d46d3ba8dbea8d99f04309f1d93c7fecce99",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names some-org/some-action@v2 as a mutable tag and directs pinning to a full 40-char commit SHA. Finding 2 flags the absent permissions: block, explains the wide default scope, and gives a concrete least-privilege block (contents: read) plus per-job scoping. Both fixes are directions only; answer states no edits were made, staying read-only. No fail criterion holds."
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings:\n\n- Third-party action pinned to a mutable tag: the new job's `uses: some-org/some-action@v2` references a tag, not a full commit SHA. Risk category: action pinning. Consequence: the tag `v2` is movable — whoever controls `some-org/some-action` (or anyone who compromises that account) can repoint it at different code, and this workflow will silently execute that new code on the next run without any change to this repository. Fix direction: pin the `uses:` line to the action's full 40-character commit SHA (a `# v2.x.x` comment alongside the SHA is fine for readability).\n\n- Missing `permissions:` block: the workflow has no `permissions:` key at all. Risk category: least-privilege permissions. Consequence: with no explicit `permissions:` block, the workflow falls back to the repository/org default token scope, which is frequently read-write across all scopes (`write-all`). That means every job — including this new one — gets a `GITHUB_TOKEN` with far more authority than it needs, so a compromised or merely buggy action can push commits, alter releases, or write to packages/issues using the ambient token. This is especially relevant here because the action is unpinned — an unpinned action running with an over-scoped token is a materially worse combination than either alone. Fix direction: add an explicit workflow-level `permissions:` block defaulting to `contents: read` (or `{}`/`read-all` as appropriate), and grant any additional scope only at the specific job that needs it (e.g. `id-token: write` only on an OIDC-federating job).\n\nBoth findings are independent and both should be fixed: pinning the action limits *what code runs*, and scoping `permissions:` limits *what that code can do* if it turns out to be malicious or compromised. Neither fix substitutes for the other.\n\nNot checked here because the diff excerpt does not include it: the job's trigger (so the `pull_request_target` + untrusted-checkout pattern can't be assessed from what was provided), and any `run:`/`script:` steps for direct interpolation of `${{ github.event.* }}` or untrusted CI variables. Those still need review against the full workflow before this change is signed off.",
+              "outputSha256": "1ceefe19aefce6478d3d836d187ccd7a7436d42043e725338fad748d115b0bba",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names `some-org/some-action@v2` as a mutable tag, not a full commit SHA, with fix: pin to full 40-char SHA. Finding 2 flags no `permissions:` key, explains default token scope is often write-all granting more authority than jobs need, fix: add explicit workflow-level `permissions:` (contents: read) scoped per job. Both fixes given as directions, not applied; no dismissal of the missing block."
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings in the diff:\n\n**Finding 1 — Third-party action pinned to a mutable tag.**\n`uses: some-org/some-action@v2` references the `v2` tag, not a full commit SHA. A tag is movable: whoever controls `some-org/some-action` (or anyone who compromises that account/repo) can repoint `v2` at different code, and this workflow picks up that change on its next run with zero review. Since this is a third-party action (`some-org`, outside the reviewing repo's own `.github/actions/`), it must be pinned to a full 40-character commit SHA. Both the action-selection risk and the tag-vs-SHA pinning requirement are flagged in every instance per this skill's rules, regardless of how trusted `some-org` currently looks.\nFix direction: replace `@v2` with the full commit SHA of the intended release, keeping a trailing `# v2` comment for human readability.\n\n**Finding 2 — No `permissions:` block at all.**\nThe workflow declares no workflow-level `permissions:` and (per the diff description) the new job adds none either. When `permissions:` is absent, the workflow inherits the repository/org default, which is commonly read-write `GITHUB_TOKEN` — so every job, including this new one, can get far more token scope than it needs. This is exactly the \"no `permissions:` at all alongside jobs that do not all need write access\" case this skill flags.\nFix direction: add an explicit least-privilege `permissions:` block. Set the workflow default to `contents: read` (or `{}`), then grant the specific scopes only to the jobs that genuinely require them (e.g. `id-token: write` only on a job doing OIDC federation). If the new job only runs the third-party action and needs no write access, it inherits read-only, which is correct.\n\n**Not applicable to this diff (checked, no finding):**\n- No `pull_request_target` trigger is described, so the untrusted-checkout/pwn-request pattern does not apply to this change as written. Note: if the surrounding workflow *is* `pull_request_target`-triggered and checks out a fork head SHA before running this action, that becomes a separate finding — but it isn't in the diff provided.\n- No `run:`/`script:` step interpolating `${{ github.event.* }}` is described, so no script-injection finding.\n- No GitLab-specific content in this diff.\n\nSummary: two findings — an unpinned third-party action (`@v2`), and a missing/overly-broad `permissions:` scope from the absent block. Both are read-only findings with fix direction only; no edit has been made to the workflow.",
+              "outputSha256": "39b8263af5245b67adbbe81c7c7ec9100318c075f4fd6236d674e3321cd27636",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names some-org/some-action@v2 as a mutable tag and requires a full 40-char commit SHA fix. Finding 2 flags the absent permissions: block, explains inherited read-write default grants broader scope than jobs need, and gives a concrete least-privilege fix (contents: read, per-job scopes). Both are fix directions only; answer stays read-only."
+              },
+              "passed": true
+            },
+            {
+              "output": "Two findings in this diff, both falling under the risk categories this review covers:\n\n**Finding 1 — third-party action pinned to a mutable tag.**\n`some-org/some-action@v2` is pinned to a tag, not a full commit SHA. Tags are movable: whoever controls `some-org/some-action` can re-point `v2` at a different commit at any time, and the next run of this workflow picks up whatever that commit does — with whatever access this workflow's token has. That is the risk, independent of whether `some-org` looks reputable today.\nFix direction: pin to the full 40-character commit SHA of the intended release, with `# v2.x.y` as a trailing comment for readability. Never apply this yourself — report it.\n\n**Finding 2 — no `permissions:` block on the workflow.**\nWith no `permissions:` key at workflow or job level, the workflow inherits the repository/org default token scope, which may be read-write across `contents`, `issues`, `pull-requests`, and more. Combined with Finding 1, that is the concrete consequence: a compromised or re-pointed `some-org/some-action@v2` would run with that inherited broad scope. The absence of a `permissions:` block is a finding here regardless of what the new job does, because the job has not declared that it needs less.\nFix direction: add an explicit top-level `permissions:` block set to the minimum the jobs actually need (commonly `contents: read`), and grant any additional scope (e.g. `id-token: write`) only on the specific job that requires it.\n\nNot flagged, to be explicit about scope: I cannot see the trigger of the new job or any `run:` step in this diff, so I am not asserting a `pull_request_target` + untrusted-checkout issue or a script-injection issue. If the new job runs under `pull_request_target`, or has a `run:` step interpolating `${{ github.event.* }}` directly into the shell string, those would need checking separately — the two findings above stand on their own from what is actually in this diff.\n\nThe workflow file was not edited, patched, or altered in any way by this review.",
+              "outputSha256": "e8d1a0b34372a6ebf19b8659d94e662f68aadb9163af24187f34f12d09c5ce11",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names some-org/some-action@v2 as a mutable tag and gives fix: pin to full 40-char commit SHA. Finding 2 flags missing permissions: block, explains inherited broad default scope, fix: explicit top-level permissions set to minimum (contents: read). Both fixes reported, not applied ('Never apply this yourself'; 'not edited'). No dismissal of unscoped default."
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings:\n\n- **Action pinning (the `uses:` line).** `some-org/some-action@v2` is pinned to a mutable tag, not a full commit SHA. Tag refs can be moved by whoever controls the upstream repo (or by anyone who compromises that account), so the code that actually runs can change without any change in this repository. Fix direction: pin to the action's full 40-character commit SHA (a `# v2` comment alongside it is fine and conventional); update it deliberately when you intend to upgrade.\n\n- **Missing `permissions:` block (workflow-level).** With no `permissions:` key, the workflow falls back to the repository/organization default token scope, which is frequently read-write (`write-all`) unless the repo has been hardened. Every job in the workflow gets that scope, so any job that only needs to read the repo still hands out write access to `contents`, `packages`, `actions`, etc. Fix direction: add an explicit workflow-level `permissions:` set to the minimum (commonly `contents: read`), then grant narrower extra scopes (`id-token: write`, `packages: write`, ...) only at the individual job that needs them.\n\nWhat I checked and did not flag from this diff:\n\n- No trigger was shown, so I could not evaluate the `pull_request_target` + untrusted-checkout-and-execute pattern; that only applies if this job is reachable from a `pull_request_target`-triggered workflow that also checks out the fork head. Worth confirming before calling the review complete.\n- No `run:`/`script:` steps were shown, so there is no evidence of `${{ github.event.* }}` (or an untrusted CI variable) being interpolated directly into a shell string. If the job has any such step, that is a separate finding.\n\nThese two findings are independent and both apply regardless of what the rest of the diff looks like: the mutable-tag pin and the missing/scoped-out `permissions:` block each stand on their own. This is a read-only review — I have not modified the workflow; the pin and the `permissions:` block are yours (or the implementation skill's) to apply.",
+              "outputSha256": "282ad3615973c7d9bff9364716d1d4626dcd8aef820317c02bf72e1de093149f",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names some-org/some-action@v2 as a mutable tag, not a full commit SHA, with fix: pin to full 40-char SHA. Finding 2 flags missing permissions: block, explains default write-all scope grants broader access than jobs need, fix: explicit workflow-level permissions (contents: read) plus per-job scopes. Both fixes given as directions, not applied; answer states it is read-only and did not modify the workflow."
+              },
+              "passed": true
+            },
+            {
+              "output": "Two findings.\n\n**Finding 1 — third-party action pinned to a mutable tag.**\n`uses: some-org/some-action@v2` references the `v2` tag, not a full commit SHA. The risk is not about who controls `some-org` today; it is that a tag is a movable pointer. The tag can be re-pointed to different code at any time — whether by a maintainer, a compromised maintainer account, or a supply-chain compromise of the upstream repo — and your workflow will silently pull the new code on its next run with no diff visible in your repository. A `@v2` reference gives you reproducibility of *name* but not of *content*.\nFix direction: pin to the full commit SHA that `v2` currently resolves to, e.g. `uses: some-org/some-action@<40-char-sha> # v2`. This costs nothing and is the standard hardening. Flag it every time a third-party `uses:` (anything outside the repo's own `.github/actions/`) is a tag or branch.\n\n**Finding 2 — no `permissions:` block at all.**\nThe workflow has no `permissions:` key. Under GitHub Actions' defaults, the `GITHUB_TOKEN` for the workflow inherits the repository/org default, which for many repos is read/write across a broad set of scopes — so this new job silently receives whatever write access the repo default grants, regardless of what the job actually needs. The absence of a `permissions:` block is itself the finding, not a neutral state.\nFix direction: add an explicit least-privilege `permissions:` block. Start from `permissions: contents: read` at the workflow level (or `permissions: {}` to deny all), then grant the specific scope each job actually needs at the job level — e.g. `id-token: write` only on a job doing OIDC federation, `contents: write` only on a job that must push. Do not grant `write-all`.\n\n**Things I could not assess from what you gave me, and which still need checking before this is a complete review:**\n\n- **Trigger.** You did not say what triggers this new job. If it runs under `pull_request_target` and any step checks out the contributor's fork head and then executes a build/test/lint step against it, that is a pwn-request and is a finding on top of the two above. If it runs under plain `pull_request`, this concern does not apply.\n- **Script injection.** I could not see any `run:`/`script:` steps, so I cannot tell whether `${{ github.event.* }}` (PR title, issue body, branch name, commit message) or an untrusted CI variable is interpolated directly into a shell string. If this job has such a step, that is a finding.\n- **What the action actually does with the token.** A broadly-scoped token handed to a tag-pinned third-party action is worse than either issue alone — the two compound. Worth confirming what `some-org/some-action` reads or writes.\n\nNote: this is a read-only review. I have not edited the workflow and have not applied the fixes above, not even partially — the pinning and the `permissions:` block should be changed by the author or the implementation skill.",
+              "outputSha256": "02ea6a17cb600c50e5807fcfcfe87441649787b712f209cb3000a3bd1fb37846",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names `some-org/some-action@v2` as a mutable tag and gives fix: pin to full commit SHA (`@<40-char-sha> # v2`). Finding 2 flags no `permissions:` block, explains broad default token access, fix: explicit least-privilege block (`contents: read`, `permissions: {}`, job-level scopes). States read-only, no changes applied. No fail criterion."
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings:\n\n1. **Mutable tag pinning (some-org/some-action@v2).** The new `uses:` line references `@v2`, a tag, rather than a full commit SHA. Tags are movable by whoever controls the action's repository, so if that account is compromised or the tag is repointed, the job silently runs different code than what was reviewed — with whatever access the job's token has. Fix direction: pin to `some-org/some-action`'s full commit SHA (a `# v2` comment next to it is fine and preserves readability).\n\n2. **Missing `permissions:` block.** With no workflow-level `permissions:` key, the workflow falls back to the repository's default token permissions, which is frequently read/write on all scopes depending on the repo's settings. Every job — including this new one — inherits whatever that default is, whether or not it needs it. Fix direction: add an explicit, minimal `permissions:` block at the workflow level (e.g. `contents: read`), and grant additional scopes per-job only where a job actually needs them (e.g. `id-token: write` only on the job that does OIDC federation).\n\nTwo caveats on what I could and could not check:\n\n- **Trigger scope unknown.** I can't tell from the description alone which trigger this job runs under. If it is `pull_request_target` and the job checks out the contributor's fork head and executes anything against it, that's a separate pwn-request finding that needs flagging independently of items 1–2 — the combination is the risk, not the pinning or permissions alone.\n- **Script injection unchecked.** I have no visibility into the job's `run:`/`script:` steps here, so `${{ github.event.* }}` (PR title, branch name, commit message) interpolated directly into a shell string is neither confirmed nor ruled out. That needs the actual step bodies before the review can be called complete.\n\nBoth findings above are reported, not fixed — I have made no edits, not even partial or proof-of-concept, to the workflow file.",
+              "outputSha256": "6a2e810da67ca35df89d83676617508bed9bfdad894c6b5cdbcc4c08185e8db8",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names some-org/some-action@v2 as a movable tag vs full commit SHA and gives fix: pin to full commit SHA. Finding 2 flags no permissions: block, explains default token perms may be read/write on all scopes inherited by every job, fix: add explicit minimal permissions: block (contents: read) with per-job scopes. Both reported, not applied; no dismissal of the default."
+              },
+              "passed": true
+            },
+            {
+              "output": "Findings:\n\n- **Third-party action pinned to a mutable tag** — `uses: some-org/some-action@v2` pins to the tag `v2` rather than a full commit SHA. The tag is movable: whoever controls `some-org/some-action` can retag `v2` to point at different code at any time, and this workflow would silently run that new code with whatever access this job has (repo token, secrets, etc.). Fix direction: pin to `some-org/some-action`'s full 40-character commit SHA, with a `# v2` comment alongside for human readability.\n\n- **Missing `permissions:` block** — the workflow (and, as described, the new job) declares no `permissions:` at all. Absent an explicit block, the workflow gets the repository/org default token scope, which is frequently read-write on contents, packages, etc. rather than least-privilege. Any job that doesn't need write access should not have it, and the combination of \"unpinned third-party action\" + \"default-or-broader token scope\" is the worst case of this pairing: a compromised action inherits whatever the default grants. Fix direction: add an explicit workflow-level `permissions: contents: read` (or `permissions: {}` if nothing is needed), then elevate per-job only for the job that actually needs write access or `id-token: write`.\n\nCaveats on what I could not check from this description alone:\n\n- **Trigger + checkout pattern** — I don't know whether this job runs under `pull_request_target`; if it does, and it also checks out the contributor's own head SHA and then executes any step against that checkout, that is a separate pwn-request finding on top of the two above. Worth confirming.\n- **Script injection** — I have not seen the job's `run:`/`script:` steps, so I can't say whether `${{ github.event.* }}` or an untrusted CI variable is interpolated directly into a shell string rather than routed through `env:`.\n- **Deploy-secret access** — if this job consumes a deploy credential, its exposure should be gated to the protected branch/tag (GitLab) or scoped to a protected environment with required reviewers (GitHub).\n\nReported as findings and fix directions only — nothing has been edited.",
+              "outputSha256": "8a9934a78726ac0b0adebe9954da5da3d8e518f620865d0ecbe3267ac8912b75",
+              "promptSha256": "a4cf741e3580dc8a66d5ec93f91e22e34288785cfc05d3319a6f6fc8317ef162",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Finding 1 names `some-org/some-action@v2` as a mutable tag vs full commit SHA, with fix: pin to full 40-char SHA. Finding 2 flags no permissions: block, explains default scope is broader than least-privilege, fix: add `permissions: contents: read`/`{}` and elevate per-job. Both fixes given as directions, not applied; answer stays read-only."
+              },
+              "passed": true
+            }
+          ]
+        }
+      ],
+      "verdict": "fail",
+      "scope": "bundled",
+      "skillDigest": "03c39837d079d53a11e8cda4ec097d66b76b8407e3718c9f4fb3cc67904e2bd8",
+      "catalogDigest": "8600b35461e2a92efe928c3011b674fd4afa25360ad47066b44d9253cddb0d7c",
+      "judgePromptVersion": "2026-09-25.1",
+      "runner": "deepseek",
+      "model": "deepseek-chat",
+      "runnerPromptVersion": "2026-09-25.1",
+      "recordedAt": "2026-09-25T17:59:58.892Z",
+      "judge": "deepseek",
+      "judgeModel": "deepseek-chat"
+    },
+    {
+      "schemaVersion": "1.0.0",
+      "skillId": "ci-github-gitlab/ci-pipeline-build-fix",
+      "strictness": "high",
+      "trials": 10,
+      "triggerAccuracy": {
+        "truePositive": 5,
+        "falsePositive": 4,
+        "positives": 6,
+        "negatives": 6
+      },
+      "evidence": "authored",
+      "scenarios": [
+        {
+          "id": "trigger-positive-1",
+          "kind": "trigger-positive",
+          "prompt": "This workflow won't even parse -- GitHub says there's a YAML syntax error",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-2",
+          "kind": "trigger-positive",
+          "prompt": "Our deploy job can't read the API_KEY variable even though it's set in GitLab",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-3",
+          "kind": "trigger-positive",
+          "prompt": "The bot says Resource not accessible by integration when posting a PR comment",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-4",
+          "kind": "trigger-positive",
+          "prompt": "GitLab keeps saying this job's rules never match so it just gets skipped",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-5",
+          "kind": "trigger-positive",
+          "prompt": "actions/checkout is erroring out with an unrecognized input on this job",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-positive-6",
+          "kind": "trigger-positive",
+          "prompt": "This job references another job in needs: that GitHub says doesn't exist",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-1",
+          "kind": "trigger-negative",
+          "prompt": "The build fails because a Python import is missing, fix the code",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-2",
+          "kind": "trigger-negative",
+          "prompt": "Our TypeScript compile step is failing with a type error, fix it",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-3",
+          "kind": "trigger-negative",
+          "prompt": "The Go binary fails go vet, please fix the source",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-4",
+          "kind": "trigger-negative",
+          "prompt": "This Dockerfile fails to build because of a missing apt package",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-5",
+          "kind": "trigger-negative",
+          "prompt": "Write a new workflow that lints our code on every push",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 0,
+          "passRate": 0,
+          "passAtK": 0,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "trigger-negative-6",
+          "kind": "trigger-negative",
+          "prompt": "Review this workflow for security issues before we merge it",
+          "strictness": "high",
+          "trials": 1,
+          "passes": 1,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "trigger-rank-fork-family",
+          "status": "ran",
+          "deterministic": true
+        },
+        {
+          "id": "permission-scope-fix",
+          "kind": "behavior",
+          "prompt": "This job fails with 'Resource not accessible by integration' when it tries to comment on the pull request. The workflow currently has no permissions: block. How do I fix it?",
+          "strictness": "high",
+          "trials": 10,
+          "passes": 9,
+          "passRate": 0.9,
+          "passAtK": 1,
+          "grader": "judge",
+          "status": "ran",
+          "trialRecords": [
+            {
+              "output": "Add a job-level `permissions:` block granting exactly `pull-requests: write` to the job that posts the comment — not a workflow-level block, and not `write-all`.\n\nTwo things to confirm while doing it:\n\n1. If that job also checks out or reads repo contents (most do), keep `contents: read` explicitly, since adding any `permissions:` block resets the token to no permissions at all except what you list. Without it, the checkout step can start failing for a new reason.\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read\n      pull-requests: write\n    steps:\n      # ...\n```\n\n2. Verify it's the PR-comment API call that's failing and not something else with the same error text — `Resource not accessible by integration` is the generic 403 GitHub returns for any under-scoped token, so confirm the failing step is the comment step before attributing it to the permission. If the workflow is triggered by `pull_request` (not `pull_request_target`) from a fork, the token is read-only regardless of what you list, and no `permissions:` change will fix it — that needs a different trigger model, not a wider scope.\n\nAfter editing, run `actionlint` on the file and re-run the same workflow to confirm the step now succeeds. Do not grant `pull-requests: write` at the workflow level or fall back to `write-all`; one job, one scope, is the fix.",
+              "outputSha256": "4aeceda68f77ae692f0a291f09978c97959e2bc507fd1a8900120dea63f2da7a",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names missing pull-requests: write as the fix for the 403 (\"granting exactly pull-requests: write\"), places it at job level (\"job-level permissions: block ... not a workflow-level block\"), and confirms no widening (\"not write-all\", \"one job, one scope\"). YAML shows job-level permissions with contents: read + pull-requests: write."
+              },
+              "passed": true
+            },
+            {
+              "output": "The error \"Resource not accessible by integration\" means the job's `GITHUB_TOKEN` lacks the scope needed to write the PR comment. With no `permissions:` block, the token falls back to the repository's default (often read-only for `pull-requests` depending on repo/org settings), so the comment POST is denied.\n\nFix: add a job-level `permissions:` block granting only the `pull-requests: write` scope the commenting step needs. Don't add it at the workflow level and don't use `write-all`.\n\nExample, assuming the job is named `comment`:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read          # keep read unless a step needs more\n      pull-requests: write    # the scope the failing step actually needs\n    steps:\n      - uses: actions/checkout@v4\n      - name: Comment on PR\n        uses: peter-evans/create-or-update-comment@v4\n        with:\n          issue-number: ${{ github.event.pull_request.number }}\n          body: \"Build finished.\"\n```\n\nNotes:\n\n- Put `permissions:` on the specific job, not at the top of the workflow — a workflow-level grant widens every other job too.\n- If the step also needs to read repo contents (e.g. `actions/checkout`), keep `contents: read` alongside it; set `contents: write` only if a step actually pushes.\n- If the commenting step is posting a review rather than a plain comment, the same `pull-requests: write` scope covers it.\n- If you still get the error after adding the scope, check the org/repo setting \"Workflow permissions\" — when it's set to read-only, a job-level `permissions:` block is still what overrides it, so the block above should suffice; if it doesn't, the org has locked it and the token genuinely can't be granted that scope from the workflow file.\n\nVerify with `actionlint`, then re-run the failing job: the comment step should now succeed, and no job in the file has gained privileges beyond `pull-requests: write`.",
+              "outputSha256": "031ed84f7affca152724eef35d6c91a3dfeeeeb0ee76b4e6f5e646f77cd1828a",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer identifies missing pull-requests: write as root cause ('token lacks the scope needed to write the PR comment'), adds it at job level ('jobs: comment: permissions: pull-requests: write'), and confirms no widening ('granting only the pull-requests: write scope', 'no job has gained privileges beyond pull-requests: write', warns against write-all)."
+              },
+              "passed": true
+            },
+            {
+              "output": "Add a job-level `permissions:` block granting only `pull-requests: write` (plus whatever else that job already relies on, if anything) to the job that posts the comment — not a workflow-level `write-all`.\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n    steps:\n      - uses: actions/github-script@v7\n        with:\n          script: |\n            await github.rest.issues.createComment({...})\n```\n\nWhy this and not `write-all`:\n\n- **\"Resource not accessible by integration\"** is GitHub telling you the `GITHUB_TOKEN` for that step lacks the scope the API call needs. Commenting on a PR goes through the Issues API, so the scope is `pull-requests: write` (and in some setups `issues: write`, since `createComment` targets an issue number). Trace the exact API call your step makes to its [documented required scope](https://docs.github.com/en/actions/reference/authentication-in-a-workflow) rather than guessing — if the script calls `issues.createComment`, grant `issues: write` too.\n\n- Scope it to **the job that fails**, not the whole workflow. A workflow-level `write-all` also silently escalates every other job in the file, which is a permissions widening, not a fix.\n\n- If the job also checks out code or reads other resources, keep the rest at their defaults (`contents: read`, etc.) and add only `pull-requests: write`. Any permission you don't list under a job-level block reverts to the default (`none` unless you set a top-level default), so if the job needs `contents: read` for `actions/checkout`, list it explicitly.\n\nAfter the edit, run `actionlint` (or push and let the workflow's own validator run) to confirm the file parses, then trigger the same failing run to confirm the comment step now succeeds rather than failing on a different scope.",
+              "outputSha256": "6ad9a32e380d5281bae0a8c21212ed8eae2d70bd93372564c827a3e2a31fbb6b",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer identifies missing pull-requests: write scope as root cause of 'Resource not accessible by integration' (\"GITHUB_TOKEN for that step lacks the scope\"), adds it at job level in the YAML (jobs.comment.permissions.pull-requests: write), and explicitly confirms no widening: \"not a workflow-level write-all\" and \"Scope it to the job that fails, not the whole workflow.\""
+              },
+              "passed": true
+            },
+            {
+              "output": "Add a job-level `permissions:` block granting only `pull-requests: write` to that commenting job — not a workflow-level `write-all`.\n\nThe error `Resource not accessible by integration` means the `GITHUB_TOKEN` the step is using doesn't have the scope needed to POST the PR comment. With no `permissions:` block, the token defaults to the repository/org default, which for most repos is read-only for `pull-requests` (or inherited from a restrictive org default), so the comment call is rejected.\n\nMinimal fix, scoped to just the one job:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n    steps:\n      - uses: actions/github-script@v7\n        with:\n          script: |\n            await github.rest.issues.createComment({...})\n```\n\nThings to check and avoid:\n\n- **Scope the grant to the job, not the workflow.** Putting `permissions: pull-requests: write` or `write-all` at the top level widens every other job in the file, including build/test jobs that only need read. Keep it on the commenting job.\n- **Grant exactly `pull-requests: write`, nothing else**, unless the step demonstrably also needs another scope (e.g. `issues: write` for `issues.createComment` on older API paths, or `contents: write` if it also pushes). Trace the failing call to the scope it needs rather than adding scopes speculatively.\n- **Also confirm the job isn't in a fork-triggered `pull_request` context** where the token is read-only regardless of `permissions:`. If the workflow runs on `pull_request` from a fork, `pull-requests: write` will still be denied by GitHub's fork-token restriction — in that case the real fix is moving the commenting step to a `pull_request_target` workflow (with the usual care about not checking out untrusted code), not changing the `permissions:` block.\n\nVerify: the step posts the comment successfully on the same PR that previously failed, and no other job's permissions changed.\n\n```\nFixed: .github/workflows/ci.yml\n  - comment job had no permissions block, so GITHUB_TOKEN was read-only\n    for pull-requests and the comment POST was rejected.\n  - Added job-level `permissions: { pull-requests: write }` only.\n  - Confirmed it's not a fork `pull_request` context (token write is\n    allowed here); re-run confirmed the comment step now succeeds.\n```",
+              "outputSha256": "89051eca515a866fb0e3dde65abf8d1ba893bfa96fc44e1496a753878b11d403",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer identifies missing pull-requests: write as root cause of 'Resource not accessible by integration' (token defaults read-only). Adds job-level permissions block with pull-requests: write in YAML. Explicitly warns against workflow-level write-all and says grant exactly pull-requests: write, nothing else, scoped to the commenting job. No fail criteria met."
+              },
+              "passed": true
+            },
+            {
+              "output": "A missing `permissions:` block means the workflow falls back to the repository's default token permissions, which is usually read-only for `pull-requests` — that's exactly why the comment step gets `Resource not accessible by integration`. The fix is to grant the specific scope the comment step needs:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n      contents: read   # only if the job also checks out the repo\n    steps:\n      # ... the step that posts the PR comment\n```\n\nKey points:\n\n- **Grant it at the job level, not the workflow level.** Putting `permissions:` at the top of the file would widen every job in the workflow, including ones that don't need to write anything. Scope it to the job that actually posts the comment.\n- **Only add `pull-requests: write`** (plus `contents: read` if that job uses `actions/checkout`). Don't reach for `write-all` or a list of scopes you haven't traced to a failing API call.\n- **`issues: write` is not a substitute.** PR comments are a pull-request resource, not an issue resource — if you only grant `issues: write`, the same error persists. Use `pull-requests: write`.\n- **Check the repo/org default too.** If an org-level or repo-level \"Workflow permissions\" setting is pinned to read-only, the job-level block still overrides it, but confirm nothing at the org level is blocking write tokens outright (e.g. a policy that strips write permissions).\n\nVerify with `actionlint` against the file, then re-run the same failing run to confirm the comment step succeeds.",
+              "outputSha256": "50ffd1747759c2c7441c301b6e0f8834e17d8e53b174c3d78e6292fa18c4e6b8",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states missing permissions block causes read-only pull-requests token, yielding 'Resource not accessible by integration' (criterion 1). It shows job-level `permissions: pull-requests: write` under jobs.comment (criterion 2). It explicitly says grant at job level not workflow level, and only add pull-requests: write, don't use write-all (criterion 3)."
+              },
+              "passed": true
+            },
+            {
+              "output": "The error `Resource not accessible by integration` means the job's `GITHUB_TOKEN` lacks a scope the commenting step needs. Since there's no `permissions:` block, the token falls back to either the repository default or, for pull requests from forks, a read-only token — either way, the write scope for PR comments isn't present.\n\nPosting a comment on a pull request requires `pull-requests: write`. Add a `permissions:` block **at the job level** (the narrowest scope), not at the workflow level:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n    steps:\n      - uses: actions/github-script@v7\n        with:\n          script: |\n            await github.rest.issues.createComment({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              issue_number: context.issue.number,\n              body: \"Working!\"\n            })\n```\n\nA few things to get right:\n\n1. **Scope choice.** Comments on a PR are issue comments under the hood, so `pull-requests: write` covers the `issues.createComment` / `pulls.createReviewComment` calls. If your step also needs to read repo code, keep `contents: read`. Don't reach for `write-all` — that widens every other job in the file that inherits the workflow-level default.\n\n2. **Job-level, not workflow-level.** Putting `permissions:` under the specific job means the other jobs keep their current (default) scope. If you put it at the top level it changes the token for the whole workflow.\n\n3. **Fork PRs.** If the workflow runs on `pull_request` from a fork, the token is **always** read-only regardless of what you put in `permissions:` — GitHub does this deliberately. If that's your case, you need `pull_request_target` instead (and handle the untrusted checkout carefully), not a bigger permissions grant. Check whether the run is from a fork before assuming the permissions block is the whole fix.\n\n4. **Verify.** Re-run `actionlint` on the file, then re-run the failing workflow on the same event and confirm the comment step succeeds. If it still fails on a fork PR, that's the fork-token restriction, not a missing scope.\n\nSo: add `permissions: { pull-requests: write }` to the commenting job, confirm it's not a fork PR, and re-run. That is the smallest root-cause fix — don't use `write-all` and don't move it to the workflow level.",
+              "outputSha256": "b976412dbb93db9d5620b2e768cd90dd2468c602315d30860f4a13aa9fdee129",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer identifies missing pull-requests: write as root cause of 'Resource not accessible by integration' (\"Posting a comment on a pull request requires pull-requests: write\"). Adds it at job level in YAML (permissions under jobs.comment). Confirms no widening: \"Don't reach for write-all\" and \"smallest root-cause fix — don't use write-all and don't move it to the workflow level.\""
+              },
+              "passed": true
+            },
+            {
+              "output": "Add a `permissions:` block that grants the one scope the commenting step needs — `pull-requests: write` — and keep it scoped to the job that posts the comment, not the whole workflow.\n\nWhy that specific scope: `Resource not accessible by integration` is GitHub's error when the workflow's token (`GITHUB_TOKEN`) lacks the permission for the API call being made. Posting or updating a comment on a PR is the pull-requests write operation, so the scope is `pull-requests: write`. Trace it against the docs for the exact endpoint the step calls rather than copying a scope from a similar workflow.\n\nMinimal fix:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n    steps:\n      # ... the step that posts the PR comment\n```\n\nNotes that matter here:\n\n- Put it at the job level, as above. Don't set `permissions: write-all` at the top of the file, and don't grant `pull-requests: write` workflow-wide — either one widens every other job's token for no reason. The job that needs to comment is the only one that should gain the write scope.\n- When you set any `permissions:` block, every scope you don't list defaults to `none`. If this same job also does a checkout and something else that needs a read scope, add only those explicitly (e.g. `contents: read`), and confirm by re-running rather than pre-emptively re-granting. If the jobs are separate, scoping it to just this job avoids that side effect entirely.\n- If the workflow triggers on `pull_request` from a fork, the token is read-only regardless of what you grant — `pull-requests: write` has no effect there. In that case the fix is to move the commenting step to a `pull_request_target`-triggered workflow (or a separate workflow run on the base repo), handled carefully so untrusted fork code isn't checked out. If your trigger is a same-repo `pull_request`, this doesn't apply and the scope above is sufficient.\n\nVerify by running the platform's linter against the changed file and re-running the same workflow to confirm the comment step now succeeds — don't stop at \"YAML looks right.\"",
+              "outputSha256": "53daaa332c49d7de2fd0686607a2d3cae568f41d055115c05ff355a683f10457",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names 'pull-requests: write' as the needed scope and ties 'Resource not accessible by integration' to the token lacking that permission. It places the grant at job level in the YAML ('jobs: comment: permissions: pull-requests: write'). It explicitly warns against write-all and workflow-wide grants, stating the job that comments is the only one that should gain the write scope."
+              },
+              "passed": true
+            },
+            {
+              "output": "Fix it by adding a job-level `permissions:` block that grants exactly the scope the comment step needs — not a workflow-wide grant.\n\n**Root cause:** GitHub Actions defaults the `GITHUB_TOKEN` to read-only for most scopes on pull requests. Posting a comment is a write to the pull-request/issue comments API, so it fails with `Resource not accessible by integration` until `pull-requests: write` (or `issues: write`, see below) is granted to that job.\n\n**Fix — add to the commenting job only:**\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n    steps:\n      - uses: actions/github-script@v7\n        with:\n          script: |\n            github.rest.issues.createComment({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              issue_number: context.issue.number,\n              body: \"Build result...\"\n            })\n```\n\nNotes on getting the scope right rather than guessing:\n\n- **PR comments go through the issues API** (`issues.createComment`), which historically requires `pull-requests: write` on pull-request events in current GitHub Actions; if your step instead hits `issues: write`-scoped endpoints directly, grant `issues: write`. Check which API call your step actually makes and grant that scope — that's the trace from failing call → required scope.\n- **Do not grant at the workflow level.** A top-level `permissions: write-all` (or putting `pull-requests: write` at the top of the file) silently widens every other job in the workflow. Put the block on the one job whose step needs it.\n- **If the comment step also needs to read repo contents** (e.g. `actions/checkout` earlier in the same job), add `contents: read` alongside it. Job-level `permissions:` is a complete replacement for the default set on that job, not additive, so list every scope that job needs.\n\n**Verify:**\n- Run `actionlint` against the file — it passes.\n- Trigger the same failing run once more and confirm the comment step now succeeds.\n- Confirm you added only the scope(s) the step needs, at job level, and changed nothing else in the file.\n\nThis is a genuine missing-permission fix, not a gate to work around — so granting the specific scope is correct here. Just keep it narrow and at the job, not the workflow, level.",
+              "outputSha256": "548f219d27836cbb3d3cb8d3d65f19d61cfec5ec72e089bff11c7882e64fec29",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer names missing pull-requests: write as root cause of 'Resource not accessible by integration' (\"until pull-requests: write ... is granted\"). Fix shown as job-level permissions block on the comment job, not workflow level (\"Do not grant at the workflow level\"). Confirms narrow scope: \"granting exactly the scope the comment step needs — not a workflow-wide grant\" and warns against write-all."
+              },
+              "passed": true
+            },
+            {
+              "output": "The error `Resource not accessible by integration` means the ephemeral `GITHUB_TOKEN` for that job does not have `pull-requests: write` (and depending on the call, possibly `issues: write`, since PR comments are technically issue comments). The default token permissions for a repo can be read-only or restricted, and with no `permissions:` block at all, the job just inherits whatever the repo/org default is.\n\nFix: add a `permissions:` block **at the job level** (not workflow level), granting only the scope the commenting step actually needs.\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n    steps:\n      - name: Comment on the PR\n        uses: actions/github-script@v7\n        with:\n          script: |\n            await github.rest.issues.createComment({\n              owner: context.repo.owner,\n              repo: context.repo.repo,\n              issue_number: context.issue.number,\n              body: \"Build finished.\"\n            })\n```\n\nWhy job-level and only this scope:\n\n- A workflow-level `permissions:` block applies to *every* job in the file, silently raising privileges for jobs that never needed them. Scoping it to the job that actually calls the comments API keeps everything else at its intended privilege.\n- The issue-comments endpoint requires `issues: write` in some token configurations and `pull-requests: write` in others — check which call your step makes. `github-script`'s `issues.createComment` typically needs `issues: write`; if you're using a step that updates the PR itself, use `pull-requests: write`. Grant the one the failing call requires, not both defensively.\n\nTwo things to verify if the scoped grant still fails:\n\n1. **Repo/org default permissions.** If the repo's default `GITHUB_TOKEN` permission is set to \"Read repository contents\" and \"Allow access via the API\" is restricted, a job-level block can only narrow the default — it cannot add a scope the repo has disabled at the org level. Check Settings → Actions → General → Workflow permissions.\n2. **Forked PRs.** If the job runs on `pull_request` from a fork, the token is read-only and no `permissions:` grant can override that. For that case you need `pull_request_target` with an explicit checkout of the PR head, and you must be deliberate about what untrusted code the job runs — do not treat this as a permissions tweak.\n\nDo not reach for `permissions: write-all`. It fixes this failure by giving every job in the file write access to everything, which is far more than the one commenting step needs.\n\nAfter the edit, run `actionlint` against the workflow, then re-run the same failing job to confirm the comment step succeeds and the error is gone.",
+              "outputSha256": "776d8c51004b0f290a2ed21893cfeb034314b937178ccd4f9d41ea79fba9ad91",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "fail",
+                "reason": "Criterion 1 fails: the answer hedges, saying the error means the token lacks pull-requests: write 'and depending on the call, possibly issues: write', then concludes github-script's issues.createComment 'typically needs issues: write' and to grant 'the one the failing call requires, not both'. It does not firmly identify pull-requests: write as the root cause. Criteria 2 and 3 are met (job-level block shown; warns against write-all)."
+              },
+              "passed": false
+            },
+            {
+              "output": "The error `Resource not accessible by integration` means the job's `GITHUB_TOKEN` lacks the API scope needed to write the PR comment. The default token permissions are read-only (or restricted), so a step that calls the REST API to create an issue/PR comment — e.g. `gh pr comment`, `actions/github-script`, or a direct `POST /repos/.../issues/{n}/comments` — gets denied.\n\nFix it by granting the specific scope the failing step needs, at the job level, not the workflow level:\n\n```yaml\njobs:\n  comment:\n    runs-on: ubuntu-latest\n    permissions:\n      pull-requests: write\n      contents: read        # keep whatever read access the job already relies on via the default\n    steps:\n      - uses: actions/checkout@v4\n      # ... the step that posts the PR comment\n```\n\nKey points:\n\n1. **Which scope.** Posting a comment on a PR is an *issue comment* under the hood, so `pull-requests: write` is the usual answer for PR-comment steps (it covers the PR's issue-comment endpoint). If the step also needs to read the repo contents (checkout), keep `contents: read`. Add only the scopes the failing step actually touches — check the exact API call it makes against GitHub's token-permissions reference rather than guessing.\n\n2. **Job level, not workflow level.** Put the `permissions:` block under the specific job that posts the comment. A workflow-level `permissions: write-all` would fix this failure but silently widen every other job in the file — don't do that.\n\n3. **If you must keep the default read-only token elsewhere**, you can set a restrictive default at the workflow level and then grant `pull-requests: write` only on the commenting job:\n\n   ```yaml\n   permissions:\n     contents: read        # workflow-wide default, unchanged\n   jobs:\n     comment:\n       permissions:\n         contents: read\n         pull-requests: write\n   ```\n\n4. **Other related scopes you may see in the same error**, depending on what the step does: `issues: write` (commenting on plain issues), `statuses: write`, `checks: write`, `deployments: write`. Grant the one the failing call needs — not all of them.\n\nAfter editing, run `actionlint` on the file to confirm it parses, then re-run the same workflow and confirm the comment step succeeds. If the error persists after adding `pull-requests: write`, the remaining cause is usually one of: the token being referenced is not `secrets.GITHUB_TOKEN`/`github.token` (e.g. a separate PAT that itself lacks scope), or the repo/org setting \"Workflow permissions\" is locked to read-only at the org level, in which case the workflow-level `permissions:` block cannot override it and the org setting must be changed.",
+              "outputSha256": "3975e06bab2d836bb846e8b90bc0ee84fea7b357c40998cd93e533dab2ebacc1",
+              "promptSha256": "180ed50c4cb104fed2a63f9d3d78ab09eb0e356b9cb579a27c98c6418c98ca9a",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer identifies missing pull-requests: write as root cause of 'Resource not accessible by integration' (para 1-2), adds it at job level under jobs.comment.permissions (YAML block), and explicitly warns workflow-level write-all would widen every other job — 'don't do that' — confirming scope is limited to the comment step's needs."
+              },
+              "passed": true
+            }
+          ]
+        },
+        {
+          "id": "protected-variable-fix",
+          "kind": "behavior",
+          "prompt": "The deploy job can't see DEPLOY_TOKEN even though it's defined in GitLab CI/CD variables. The pipeline runs on a feature branch. How do I fix this?",
+          "strictness": "high",
+          "trials": 10,
+          "passes": 10,
+          "passRate": 1,
+          "passAtK": 1,
+          "grader": "judge",
+          "status": "ran",
+          "trialRecords": [
+            {
+              "output": "The token not being visible is almost certainly GitLab's protected-variable scoping, not a typo or a missing variable. When a CI/CD variable is marked **Protected**, GitLab only injects it into pipelines running on protected branches or protected tags — a feature branch is neither, so the variable simply isn't in the job's environment. That's the root cause to confirm first, not a reason to loosen the protection.\n\n**Confirm it (in the GitLab UI, without changing anything yet):**\n\n1. Settings → CI/CD → Variables → find `DEPLOY_TOKEN`.\n2. Check whether it's flagged **Protected**. If it is, and the failing pipeline is on a feature branch, that's your answer — this is the expected behavior, not a bug.\n3. Check **Masked** and **Environment scope** too — a variable scoped to `production` also won't be present in a job whose `environment:` doesn't match.\n\n**The fix depends on what the deploy job is actually for:**\n\n- **If the deploy job should only ever run against the protected branch/tag** (the normal case for a deploy token), fix it by scoping the job there rather than by touching the variable. Add a `rules:` entry so the job only runs on the intended ref:\n\n  ```yaml\n  deploy:\n    rules:\n      - if: '$CI_COMMIT_BRANCH == \"main\"'\n    script:\n      - ./deploy.sh\n  ```\n\n  Then run it on `main` (a protected branch) — the protected variable will be injected there. The feature branch simply shouldn't be deploying.\n\n- **If you genuinely need a deploy-capable job to run from feature branches** (e.g. a review/staging environment per branch), do **not** unprotect `DEPLOY_TOKEN`. Instead create a separate, non-protected CI/CD variable scoped to a review/staging environment, with its own token that has only the access that environment needs. Keep the protected `DEPLOY_TOKEN` on the workflow that deploys to production from the protected branch.\n\n**What not to do:** don't uncheck **Protected** on `DEPLOY_TOKEN` to make the feature-branch pipeline go green. That exposes the credential to every branch's pipeline — including any fork pipeline if the project allows them — which is precisely what the protection exists to prevent.\n\nSo the concrete answer to \"how do I fix this\": either add a `rules:` condition so `deploy` runs on the protected branch and run it there, or introduce a separate non-protected variable/token for the branch-level use case. If you tell me the job name and whether it's meant to be a production deploy or a per-branch review deploy, I can write the exact `rules:` block.",
+              "outputSha256": "f99dc9762dc377cdc2a4444fbe19e72ed9a250c4c996607710a664779c7c7c05",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and only injected on protected branches/tags, so a feature branch gets nothing (criterion 1). It fixes via rules: '$CI_COMMIT_BRANCH == \"main\"' shown in YAML, or a separate non-protected variable for review/staging (criterion 2). It explicitly warns not to uncheck Protected (criterion 3), so no fail criterion holds."
+              },
+              "passed": true
+            },
+            {
+              "output": "Short answer: you almost certainly have `DEPLOY_TOKEN` marked **Protected** in the GitLab UI (or scoped to an environment), and a feature branch isn't a protected ref, so GitLab simply doesn't inject it into that job. That's a config-level mismatch, and the fix is to route the job correctly — **not** to unprotect the variable or let deploy run on a feature branch.\n\nFirst, confirm the cause before changing anything:\n\n1. **Settings → CI/CD → Variables** → find `DEPLOY_TOKEN` and look at its flags.\n   - *Protected* checked? → only exposed to pipelines on protected branches/tags (and only if the pipeline's ref is protected). A feature branch won't see it. This is your most likely culprit.\n   - *Environment scope* not `*`? → only exposed to jobs that declare that `environment:`. If the job has no `environment:` or a different one, it stays empty.\n2. Check the run log — GitLab usually warns that a variable is not available on this ref, or your job fails with something like `$DEPLOY_TOKEN: unbound variable` / an auth failure using an empty token.\n\nNow fix it, in order of preference:\n\n**Option A — this is the right fix in most cases: don't deploy from a feature branch.**\nDeploy jobs normally belong on a protected branch/tag (e.g. `main`, or a tag). Gate the job so it only runs there, where the protected variable *is* available:\n\n```yaml\ndeploy:\n  stage: deploy\n  rules:\n    - if: '$CI_COMMIT_BRANCH == \"main\"'      # or a protected tag pattern\n      when: on_success\n    # feature branches: job doesn't run at all\n  environment:\n    name: production\n  script:\n    - ./deploy.sh   # $DEPLOY_TOKEN is injected here\n```\n\nThen run the pipeline on `main` (or merge) and it sees the token. If you were relying on the feature-branch run to *test* the deploy, test it with a harmless dry-run/staging path that has its own non-protected variable instead.\n\n**Option B — you genuinely need a token on feature branches (e.g. a preview/Review App deploy, not production).**\nUse a **separate, non-protected** variable scoped to non-production, and keep the protected one for the production job:\n\n```yaml\npreview-deploy:\n  stage: deploy\n  rules:\n    - if: '$CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != \"main\"'\n  environment:\n    name: review/$CI_COMMIT_REF_SLUG\n  script:\n    - ./deploy.sh            # uses PREVIEW_DEPLOY_TOKEN (non-protected)\n```\n\nDefine `PREVIEW_DEPLOY_TOKEN` as a *non-protected* variable (ideally a lower-privilege credential that can only touch a preview environment). Leave `DEPLOY_TOKEN` protected for production.\n\n**Option C — the variable is scoped to an environment.**\nIf `DEPLOY_TOKEN`'s environment scope is, say, `production`, the job must declare `environment: { name: production }`. Add that; a job with no `environment:` won't receive it even on a protected branch.\n\nWhat **not** to do:\n\n- Don't uncheck *Protected* on `DEPLOY_TOKEN` — that exposes the production credential to every unprotected branch and, if fork pipelines are allowed, to fork merge requests.\n- Don't drop the job's `rules:` gate so it \"just runs\" on the feature branch — that's usually the very thing the gate exists to prevent.\n\nSo the concrete fix for your described situation: keep `DEPLOY_TOKEN` protected, add/adjust the deploy job's `rules:` so it runs on `main`/the protected tag, and deploy from there — or, if you really need a feature-branch deploy, give that job its own non-protected preview variable rather than the protected one.\n\n(Side note: if this were *GitHub* Actions, the analogous cause would be an `environment:`-scoped secret with no `environment:` declared on the job, or a missing `permissions:` scope — same principle: route/gate the job to where the credential is meant to be available, don't widen access to it.)",
+              "outputSha256": "8665d873844544ef04b6a56b1047e2f778c03d69934b2cb8e2d725db8656f48d",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and feature branch isn't a protected ref so GitLab doesn't inject it (cause #1). Fix: Option A gates deploy job with rules: to main/protected tag; Option B introduces separate non-protected PREVIEW_DEPLOY_TOKEN (criterion #2). Explicitly warns 'Don't uncheck Protected on DEPLOY_TOKEN' (criterion #3, no fail)."
+              },
+              "passed": true
+            },
+            {
+              "output": "The most likely cause: `DEPLOY_TOKEN` is marked **Protected** in GitLab CI/CD settings, and protected variables are only injected into pipelines running on protected branches/tags. Your feature branch isn't protected, so the variable is simply absent from the job's environment — that's why the job \"can't see\" it.\n\nThe fix depends on whether the deploy job *should* run on a feature branch at all:\n\n**Option A — the deploy job should only run on a protected branch (the correct fix for most cases)**\n\nRoute the job to the protected ref instead of touching the variable's protection. In the job's rules, restrict it so it only runs where the variable is actually available:\n\n```yaml\ndeploy:\n  stage: deploy\n  script:\n    - ./deploy.sh\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n      when: on_success\n    - when: never\n```\n\nIf the pipeline structure currently lets this job run on feature branches, that's the bug — a real deploy with a production credential generally shouldn't run from an unprotected branch anyway. Fixing the gate is the fix.\n\n**Option B — the job genuinely must run on a feature branch, but for a non-production purpose**\n\nDon't unprotect `DEPLOY_TOKEN`. Define a separate, non-protected variable (e.g. `DEPLOY_TOKEN_STAGING` scoped to a staging/preview environment) and have the feature-branch path use that, keeping the protected `DEPLOY_TOKEN` for the protected-branch job only:\n\n```yaml\ndeploy:staging:\n  stage: deploy\n  script:\n    - echo \"deploying to staging with $DEPLOY_TOKEN_STAGING\"\n  rules:\n    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH\n\ndeploy:prod:\n  stage: deploy\n  script:\n    - echo \"deploying to prod with $DEPLOY_TOKEN\"\n  rules:\n    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH\n```\n\n**What not to do**\n\n- Don't uncheck **Protected** on `DEPLOY_TOKEN` just to make the feature-branch job pass. That exposes the production credential to every unprotected branch's pipeline (and to forks, if the project allows them) — a much larger security hole than the one you're trying to close.\n- Don't delete the `rules:`/branch gate to make the job \"just run.\" If the gate is what's keeping the deploy off feature branches, removing it is the cause of the incident, not the fix.\n\n**To confirm the diagnosis before changing anything**\n\n- Check the variable's **Protected** flag (and any **Environment scope**) in Settings → CI/CD → Variables.\n- In the failing job log, confirm the variable is absent/empty rather than present-but-wrong.\n- Confirm the feature branch is genuinely unprotected (Settings → Repository → Protected branches).\n\nIf the variable turns out *not* to be protected, the next thing to check is environment scoping: a variable scoped to e.g. `production` won't be injected into a job with no matching `environment:` (or one scoped to a different environment). In that case the fix is to give the job the matching `environment:` block rather than adjusting the variable.\n\nI can't run `actionlint` or GitLab's CI Lint here (no tool access), so after editing, run the CI Lint page against the file and re-trigger the pipeline to confirm the variable now resolves and the specific error is gone.",
+              "outputSha256": "c99e76cbec1484a13de7e401ac6ece752df17f5672713a2d84d3a3706637ebaf",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and only injected on protected branches, so absent on the unprotected feature branch (criterion 1). Option A restricts deploy job via rules: to $CI_DEFAULT_BRANCH; Option B introduces non-protected DEPLOY_TOKEN_STAGING for feature branches (criterion 2). It explicitly warns against unchecking Protected (criterion 3, no fail criterion)."
+              },
+              "passed": true
+            },
+            {
+              "output": "The most likely cause: `DEPLOY_TOKEN` is defined as a **Protected** variable in GitLab CI/CD settings. Protected variables are only injected into pipelines running on protected branches/tags (typically `main`, release branches, or `*`-matching protected tags). Your feature branch is not protected, so the variable is simply absent at runtime — the job sees an empty value, not an error, which is why it looks like \"the variable isn't there\" rather than a permissions failure.\n\nThe fix depends on whether the deploy job *should* run on that feature branch:\n\n**If the deploy job should only ever run on the protected branch/tag (the correct default for a deploy):**\nChange the job's `rules:` so it does not run on feature branches at all, and runs only where the token is actually available:\n\n```yaml\ndeploy:\n  rules:\n    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'\n      when: on_success\n    - when: never\n```\n\nNow the job only runs where `DEPLOY_TOKEN` is protected and injected. If a feature branch needs to *validate* the deploy config without actually deploying, give it a separate job (e.g. a `deploy:dry-run`) that uses a distinct, non-protected variable or no secret at all.\n\n**If the job genuinely must run on feature branches and use a token:**\nDo not unprotect `DEPLOY_TOKEN` — that would expose the real deploy credential to every unprotected branch, including any fork/MR pipeline if allowed, which is exactly what Protected is there to prevent. Instead, define a separate variable, e.g. `DEPLOY_TOKEN_NONPROD`, that is *not* protected and scoped to a non-production environment, and reference that in the feature-branch job while the protected `DEPLOY_TOKEN` stays for the protected-branch job.\n\n**Also worth checking, since it produces the same symptom:**\n- The variable is scoped to an **Environment** in the GitLab UI and the job doesn't declare a matching `environment:` key — then it won't be injected regardless of protection.\n- The variable is masked and your value contains characters that break masking, which can cause it to be dropped — check the value against GitLab's masking rules.\n- A typo or case mismatch in the variable name in the YAML vs. the GitLab settings (`$DEPLOY_TOKEN` vs `$Deploy_Token`).\n\nQuick way to confirm the root cause before changing anything: add a temporary step printing `env | grep -i deploy` (without printing the value) — if the variable is simply absent rather than empty, it's the Protected-branch scoping; if it's present but empty, check Environment scoping and masking.\n\nThe minimal correct fix is almost always the first one: gate the deploy job to the protected branch rather than reaching for the protected variable's settings.",
+              "outputSha256": "119cdd0362b91a00e0522d84c8065efab6b1defa6d4b6cbf4f3e250d5ca5d59a",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is a Protected variable absent because the feature branch is unprotected (criterion 1). It fixes via rules: gating deploy to $CI_DEFAULT_BRANCH, or a separate non-protected DEPLOY_TOKEN_NONPROD (criterion 2). It explicitly says 'Do not unprotect DEPLOY_TOKEN' (criterion 3, no fail criterion)."
+              },
+              "passed": true
+            },
+            {
+              "output": "The key detail here is that it runs on a **feature branch**. The most likely cause: `DEPLOY_TOKEN` is marked as **Protected** in GitLab. Protected variables are only exposed to pipelines running on protected branches (or protected tags) — not feature branches. So the job genuinely can't see it, by design, and GitLab gives you no error saying so; the variable just comes through empty.\n\nBefore fixing anything, confirm that's actually the cause:\n\n- Settings → CI/CD → Variables → find `DEPLOY_TOKEN` and check whether \"Protected\" is ticked.\n- Check Settings → Repository → Protected branches to see which refs are actually protected. If your feature branch isn't in that list, that's the match.\n- Check the job log for the actual symptom — a 401/403 or an empty `$DEPLOY_TOKEN` — to confirm it's the variable and not a typo in the variable *name* (GitLab is case-sensitive) or the wrong environment scope.\n\nIf it is Protected, the fix depends on intent, and you should pick deliberately rather than just unchecking the box:\n\n**If deploy should only ever run from a protected branch (the common case):**\nDon't make the feature branch job work — gate the job so it only runs where the variable is legitimately available:\n\n```yaml\ndeploy:\n  stage: deploy\n  script:\n    - ./deploy.sh\n  rules:\n    - if: '$CI_COMMIT_BRANCH == \"main\"'\n```\n\nThen test the deploy path the way it's meant to run: merge to `main` (or push a protected tag) and let the pipeline run there. If you need to exercise it from a branch first, temporarily add that branch to Protected branches **and** set it to \"No one can push\" / restrict who can — but understand that anything with push access to that branch can now reach the token.\n\n**If the job genuinely must run on feature branches too:**\nKeep `DEPLOY_TOKEN` protected and create a *separate* variable (e.g. `DEPLOY_TOKEN_DEV`) with whatever reduced-scope credential is appropriate for non-protected refs, scoped via environment. Reference the protected one only in the protected-branch job and the dev one in the feature-branch job. This keeps the real deploy credential off unprotected refs while still letting the branch pipeline do something useful.\n\nWhat **not** to do: uncheck Protected on `DEPLOY_TOKEN` just to unblock this run. That exposes a production credential to every pipeline on every branch — including merge-request pipelines from forks if your project allows them — which is almost certainly worse than the deploy job being skipped.\n\nSo, to answer directly: decide whether deploy is supposed to run on feature branches at all. If no, add a `rules:` gate so it only runs on the protected ref and test there. If yes, give it a separate, appropriately-scoped non-protected variable rather than unprotected the production token. In neither case should the fix be to make `DEPLOY_TOKEN` visible on the feature branch.",
+              "outputSha256": "985d2b26853fe16f46f415901539694d448476fc4952ab2c96b55a3c13fc10b7",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and feature branches aren't protected, so it's empty by design (criterion 1). Fix: rules: gate to main/protected ref, or separate non-protected DEPLOY_TOKEN_DEV for feature branches (criterion 2). Explicitly warns against unchecking Protected (criterion 3, no fail criterion)."
+              },
+              "passed": true
+            },
+            {
+              "output": "The deploy job can't see `DEPLOY_TOKEN` because GitLab CI/CD variables marked **Protected** are only exposed to pipelines running on protected branches or protected tags. Your pipeline is on a feature branch, which is not protected, so the variable is deliberately withheld.\n\nThe fix depends on *why* the token is protected — pick the one that matches your situation, and do not just unprotect the variable:\n\n**Option A — the deploy job should only run on a protected ref anyway (most common).**\nA real deploy token almost always belongs on `main`/`production`/a release tag, not a feature branch. Scope the job so it doesn't even attempt to run on feature branches:\n\n```yaml\ndeploy:\n  stage: deploy\n  rules:\n    - if: '$CI_COMMIT_BRANCH == \"main\"'\n      when: on_success\n    - when: never\n  script:\n    - deploy --token \"$DEPLOY_TOKEN\"\n```\n\nNow the job runs only where the protected variable is available, and the feature-branch pipeline no longer tries to deploy at all.\n\n**Option B — you genuinely need a working pipeline on the feature branch for validation, but not the real token.**\nKeep `DEPLOY_TOKEN` protected, and give the feature-branch path its own non-protected, lower-privilege variable (e.g. a staging/preview token, or a dry-run flag) so the branch can exercise the job without the production credential:\n\n```yaml\ndeploy:\n  script:\n    - |\n      if [ -n \"$DEPLOY_TOKEN_STAGING\" ]; then\n        deploy --token \"$DEPLOY_TOKEN_STAGING\" --dry-run\n      fi\n  rules:\n    - if: '$CI_COMMIT_BRANCH == \"main\"'\n      variables:\n        DEPLOY_TOKEN: \"$DEPLOY_TOKEN\"        # protected, main only\n    - when: on_success                        # feature branches use staging token\n```\n\n**Do not** do this: uncheck \"Protected\" on `DEPLOY_TOKEN` just to make the feature-branch pipeline see it. That exposes the production credential to every unprotected branch's pipeline (including any fork/MR pipelines your config allows), which is almost certainly the reason it was marked protected in the first place.\n\n**Quick way to confirm the diagnosis:** in the job log on the feature branch, if the variable is simply absent from the environment (not empty, not an error), it's the protected-branch scoping — that matches your symptom exactly. An unprotected-branch job can never read a protected variable, no matter how the job is written.\n\nSo: route the deploy job to the protected ref it belongs on (Option A), or give the feature branch its own non-protected variable (Option B) — never unprotect the token.",
+              "outputSha256": "61aa69abbb1e19911758b302407388ad79eb464689420c76103577cb5c5e689f",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and withheld on unprotected feature branch (criterion 1). Option A restricts deploy job via rules: to main (criterion 2); Option B introduces non-protected DEPLOY_TOKEN_STAGING for feature branch (criterion 2). Explicitly warns against unchecking Protected (criterion 3, no fail criterion)."
+              },
+              "passed": true
+            },
+            {
+              "output": "The fix depends on *why* it's scoped that way — and the answer is not \"unprotect it.\"\n\n**Diagnose first.** The most common cause: `DEPLOY_TOKEN` is marked **Protected** in *Settings → CI/CD → Variables*. Protected variables are only exposed to pipelines running on protected branches or tags. A feature branch is neither, so the job sees an empty/unset variable (GitLab silently omits it rather than erroring, which is why it looks \"defined but invisible\").\n\nCheck both flags on the variable:\n- **Protected** — restricts to protected refs (this is your likely culprit).\n- **Masked** — unrelated to availability; only hides it in logs.\n\n**Fix it the right way — pick based on intent:**\n\n1. **The deploy is genuinely meant to run on feature branches (e.g. a review app / staging deploy).** Do **not** unprotect the token. Instead give that job its own non-protected variable scoped to the feature-branch context — e.g. a separate `DEPLOY_TOKEN_STAGING` that is *not* Protected, protected only to the environments it legitimately needs. Keep the protected `DEPLOY_TOKEN` for the production-branch job. Route the feature-branch deploy job to that variable via `rules:`/`environment:`.\n\n2. **The deploy job should only run on the protected branch/tag this token is scoped to.** Then the bug is the job ran on the wrong ref. Gate it:\n\n   ```yaml\n   deploy:\n     rules:\n       - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'   # or a protected tag pattern\n     environment: production\n     script:\n       - ./deploy.sh   # DEPLOY_TOKEN now available\n   ```\n\n   and let the feature branch use a separate job/variable for whatever it actually needs there.\n\n3. **You truly need this exact secret on a feature branch** (rare, and a security decision — not a quick fix): the correct move is to make the *branch* protected under *Settings → Repository → Protected branches*, then confirm it's in the token's allowed scope. Protecting a branch has real consequences (who can push/merge), so treat it as a deliberate change, not a workaround.\n\n**Do not** just uncheck **Protected** on `DEPLOY_TOKEN`. That exposes the production credential to every pipeline on every unprotected branch — including forks, if the project allows them. That turns a broken job into a leaked secret.\n\n**Verify:** re-run the feature-branch pipeline (or use the CI Lint / a `test` job that echoes `[ -n \"$DEPLOY_TOKEN\" ]`) to confirm the job now sees a value — and confirm you only reached that state by scoping the variable/job correctly, not by widening the token's exposure.",
+              "outputSha256": "a62ac567989e045424bc71b55507974c54839ff61f7c9bf70f6a490b1c6718e6",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and feature branch is unprotected, so GitLab omits it (\"Protected variables are only exposed to pipelines running on protected branches or tags\"). Fix options: rules: gating to $CI_DEFAULT_BRANCH/protected tag (yaml shown) or separate non-protected DEPLOY_TOKEN_STAGING. Explicitly warns against unchecking Protected. All criteria met."
+              },
+              "passed": true
+            },
+            {
+              "output": "The most likely cause is that `DEPLOY_TOKEN` is marked **Protected** in GitLab's CI/CD settings, and this project only considers a protected branch (or tag) able to see protected variables. A feature branch is not protected, so the variable is simply not injected into that job's environment at all — it's not that the value is empty, the variable is absent.\n\nSupporting evidence to confirm before changing anything: in the project's **Settings → CI/CD → Variables**, check whether `DEPLOY_TOKEN` has the \"Protected\" flag set. If it does, that's the whole explanation; the job log showing the variable as unset (rather than wrong-valued) is consistent with this.\n\n**The fix depends on why the job needs the token on a feature branch, and the two correct fixes are:**\n\n1. **The deploy genuinely should only happen on the protected branch/ref.** Add a `rules:` gate so the deploy job only runs on that ref, e.g.:\n\n   ```yaml\n   rules:\n     - if: '$CI_COMMIT_BRANCH == \"main\"'\n   ```\n\n   Then the job runs where the protected variable is available, and never runs on the feature branch at all. This is usually the right answer — a deploy token is typically scoped to production-like refs.\n\n2. **You genuinely need a token on feature branches too** (e.g. deploying to a per-branch review environment). In that case, keep `DEPLOY_TOKEN` protected and add a *separate*, non-protected variable — say `DEPLOY_TOKEN_SANDBOX` — scoped to the non-protected case, and have the job use that on feature branches. A sandbox/review-environment credential can be non-protected because it isn't the production credential.\n\n**What not to do:** unchecking \"Protected\" on `DEPLOY_TOKEN` would expose the real deploy credential to every unprotected branch's pipeline, including forks if the project allows them — that's a privilege widening, not a fix. Likewise, don't delete the branch/ref gate to make the job \"just run\"; the gate is what keeps the credential off branches that shouldn't have it.\n\nIf after checking you find `DEPLOY_TOKEN` is **not** protected, then the cause is something else — most commonly the job's `rules:` mean it isn't running on this ref at all (so nothing gets injected because the job never starts), or the job runs in a context (like a scheduled/manual pipeline on a different ref) where the variable's environment scope doesn't match. In that case, verify the job actually starts on the feature branch before assuming it's a variable-visibility problem.",
+              "outputSha256": "322177f09f72bf25e3fef0ff94821557658c2b96cc0749decb5f21a204f8b7c8",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and feature branch is unprotected so variable is absent (criterion 1). Fix 1 gives rules: gate to protected ref (if CI_COMMIT_BRANCH == main); Fix 2 adds separate non-protected DEPLOY_TOKEN_SANDBOX (criterion 2). Explicitly warns against unchecking Protected (criterion 3, fail criterion avoided)."
+              },
+              "passed": true
+            },
+            {
+              "output": "The most likely cause: `DEPLOY_TOKEN` is marked **Protected** in GitLab's CI/CD variable settings, and protected variables are only exposed to pipelines running on protected branches or tags. A feature branch is not protected, so the variable is simply absent from that job's environment — the job isn't missing a definition, it's being deliberately withheld from that ref.\n\nThe fix depends on why the job runs on a feature branch:\n\n**If the deploy job shouldn't run on a feature branch at all** (the usual case) — the real bug is the job's scoping, not the variable. Add or tighten a `rules:` gate so it only runs on the protected branch/tag the token is meant for:\n\n```yaml\ndeploy:\n  stage: deploy\n  script:\n    - ./deploy.sh\n  rules:\n    - if: '$CI_COMMIT_BRANCH == \"main\"'   # or your protected branch\n```\n\nThen the token is available exactly where it's supposed to be, and the feature-branch pipeline no longer has a deploy job trying to run without it.\n\n**If the job genuinely must run on feature branches** (e.g. a preview/staging deploy) — do not unprotect `DEPLOY_TOKEN`. Create a separate, non-protected variable scoped to that use case:\n\n```yaml\ndeploy:preview:\n  stage: deploy\n  script:\n    - ./deploy.sh --env preview\n  variables:\n    DEPLOY_TOKEN: $PREVIEW_DEPLOY_TOKEN   # separate, non-protected variable\n  rules:\n    - if: '$CI_COMMIT_BRANCH != \"main\"'\n```\n\nKeep the real `DEPLOY_TOKEN` protected for the production job only, and give the preview job its own credential with narrower privileges.\n\n**What not to do:** unchecking \"Protected\" on `DEPLOY_TOKEN` so the feature-branch job can see it. That exposes the production deploy credential to every unprotected branch's pipeline — including any branch a contributor can push, and any fork pipeline if the project allows them. That turns a broken job into a credential leak.\n\nTo confirm the diagnosis before changing anything, check Settings → CI/CD → Variables: the `DEPLOY_TOKEN` row should show a \"Protected\" badge. If it does, the above applies. If it doesn't, the cause is elsewhere (wrong environment scope, a typo in the variable name, or the variable scoped to a GitLab environment this job doesn't declare) — worth checking the environment scope column next.",
+              "outputSha256": "848f3b0d5ffc9ab60f503a6872015a6d3f4d6d9c275bfffe0acf6c0acf1c4c31",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and withheld from unprotected feature branch (\"protected variables are only exposed to pipelines running on protected branches or tags\"). Fix 1: rules: gate to protected branch (main). Fix 2: separate non-protected PREVIEW_DEPLOY_TOKEN. Explicitly warns against unchecking Protected, so no fail criterion holds."
+              },
+              "passed": true
+            },
+            {
+              "output": "**Don't unprotect the token or move the job's gating — first confirm the scoping, then fix the job's placement.**\n\n`DEPLOY_TOKEN` not being visible on a feature-branch pipeline almost always means one thing: the variable is marked **Protected** in *Settings → CI/CD → Variables*, and GitLab only exposes protected variables to pipelines running on **protected branches or protected tags** — not feature branches. That's the protection working as designed, not a bug.\n\nSo the fix depends on what the job is actually for:\n\n**If the deploy job should only ever run from a protected branch** (the normal case — you usually don't want feature branches deploying):\n- Don't try to make the token visible on the feature branch. Instead, add a `rules:` condition so the job only runs on the protected ref it's meant for, e.g. only on `main`/`production`:\n  ```yaml\n  deploy:\n    rules:\n      - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'\n  ```\n  The job then runs where the protected variable is visible, and never runs (and never needs the token) on feature branches. This is almost certainly the correct answer if \"deploy\" means deploy.\n\n**If you genuinely need this job to run on feature branches too** (e.g. a preview/ephemeral environment deploy):\n- Do **not** unprotect `DEPLOY_TOKEN`. That would expose the real production credential to every feature-branch pipeline, including fork pipelines if they're allowed — a real credential leak.\n- Instead, create a **separate, non-protected variable** (e.g. `DEPLOY_TOKEN_PREVIEW`) scoped to a limited credential for the preview case, and have the feature-branch job use that name. Keep the protected `DEPLOY_TOKEN` for the protected-branch job only. You can also use GitLab's **environment scoping** to bind a variable to a specific environment without unprotecting it globally.\n\n**Check the other two scoping axes while you're there**, since \"can't see it\" has three common causes and people often only check one:\n- **Environment scope**: if the variable is scoped to a GitLab environment (e.g. `production`), the job must declare that `environment:` to receive it.\n- **Protected status**: the one above — the most likely cause given \"feature branch.\"\n\nIf you can tell me whether the deploy job *should* run on feature branches, I'll give you the exact `rules:` block rather than the either/or.",
+              "outputSha256": "872fda0b6d2b228f7941d8be749c8c54674c937378f2954e2cfdd1eaefd1b0e5",
+              "promptSha256": "d10b2c486e5dd1170f80d9bdafce33ebde3dba302fced9ec0c3cca56e0507e71",
+              "deterministic": [],
+              "judge": {
+                "verdict": "pass",
+                "reason": "Answer states DEPLOY_TOKEN is Protected and GitLab only exposes protected variables to protected branches/tags, not feature branches (criterion 1). It gives concrete rules: `if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'` to restrict the deploy job, and for feature branches recommends a separate non-protected variable DEPLOY_TOKEN_PREVIEW (criterion 2). It explicitly says do not unprotect DEPLOY_TOKEN (criterion 3), so no fail criterion holds."
+              },
+              "passed": true
+            }
+          ]
+        }
+      ],
+      "verdict": "fail",
+      "scope": "bundled",
+      "skillDigest": "21f83195f99265f66927d1dac0850c0a2850aea79d34ac8e610bc77c46ca3c84",
+      "catalogDigest": "8600b35461e2a92efe928c3011b674fd4afa25360ad47066b44d9253cddb0d7c",
+      "judgePromptVersion": "2026-09-25.1",
+      "runner": "deepseek",
+      "model": "deepseek-chat",
+      "runnerPromptVersion": "2026-09-25.1",
+      "recordedAt": "2026-09-25T18:00:02.193Z",
+      "judge": "deepseek",
+      "judgeModel": "deepseek-chat"
+    }
+  ]
+}