@cursor/july 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channels/slack/post-update-delivery.d.ts +85 -0
- package/dist/channels/slack/post-update-delivery.d.ts.map +1 -0
- package/dist/docs/404.html +2 -2
- package/dist/docs/ab.html +3 -3
- package/dist/docs/assets/{app.CKFFLL4Z.js → app.DjHC-Ggf.js} +1 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.DZtu0ZIc.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.BXk19JBN.js → VPLocalSearchBox.DwviwmIq.js} +1 -1
- package/dist/docs/assets/chunks/{theme.BXSPBMyn.js → theme.BIpeYSon.js} +2 -2
- package/dist/docs/assets/{example-agents_index.md.BPXjUtRB.js → example-agents_index.md.D2PEVSXl.js} +2 -2
- package/dist/docs/assets/{example-agents_index.md.BPXjUtRB.lean.js → example-agents_index.md.D2PEVSXl.lean.js} +1 -1
- package/dist/docs/assets/example-agents_security-reviewer.md.BZSNycMf.js +19 -0
- package/dist/docs/assets/{example-agents_security-reviewer.md.CE9v-aKm.lean.js → example-agents_security-reviewer.md.BZSNycMf.lean.js} +1 -1
- package/dist/docs/building-with-agents.html +3 -3
- package/dist/docs/concepts.html +3 -3
- package/dist/docs/deployment.html +3 -3
- package/dist/docs/evals.html +3 -3
- package/dist/docs/example-agents/approval-buddy.html +3 -3
- package/dist/docs/example-agents/benny.html +3 -3
- package/dist/docs/example-agents/bugbot.html +3 -3
- package/dist/docs/example-agents/codebase-wiki.html +3 -3
- package/dist/docs/example-agents/codeowners-review.html +3 -3
- package/dist/docs/example-agents/concierge.html +3 -3
- package/dist/docs/example-agents/fsd.html +3 -3
- package/dist/docs/example-agents/index.html +5 -5
- package/dist/docs/example-agents/knowledge-base.html +3 -3
- package/dist/docs/example-agents/oncall.html +3 -3
- package/dist/docs/example-agents/security-reviewer.html +14 -14
- package/dist/docs/example-agents/slack-agent.html +3 -3
- package/dist/docs/example-agents/weather-agent.html +3 -3
- package/dist/docs/guides/agent-to-agent.html +3 -3
- package/dist/docs/guides/cloud-runtime.html +3 -3
- package/dist/docs/guides/github.html +3 -3
- package/dist/docs/guides/human-in-the-loop.html +3 -3
- package/dist/docs/guides/mcp-oauth.html +3 -3
- package/dist/docs/guides/slack.html +3 -3
- package/dist/docs/guides/webhooks.html +3 -3
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +3 -3
- package/dist/docs/index.html +3 -3
- package/dist/docs/quickstart.html +3 -3
- package/dist/docs/reference/agent-config.html +3 -3
- package/dist/docs/reference/channels.html +3 -3
- package/dist/docs/reference/cli.html +3 -3
- package/dist/docs/reference/connections.html +3 -3
- package/dist/docs/reference/hooks.html +3 -3
- package/dist/docs/reference/http-api.html +3 -3
- package/dist/docs/reference/instructions.html +3 -3
- package/dist/docs/reference/playground.html +3 -3
- package/dist/docs/reference/project-layout.html +3 -3
- package/dist/docs/reference/prompt.html +3 -3
- package/dist/docs/reference/schedules.html +3 -3
- package/dist/docs/reference/sessions.html +3 -3
- package/dist/docs/reference/skills.html +3 -3
- package/dist/docs/reference/subagents.html +3 -3
- package/dist/docs/reference/tools.html +3 -3
- package/dist/docs/scaffolding-agents.html +3 -3
- package/dist/docs/storage.html +3 -3
- package/dist/docs/troubleshooting.html +3 -3
- package/dist/evals.d.ts +26 -0
- package/dist/evals.d.ts.map +1 -1
- package/dist/evals.js +30 -0
- package/dist/internal/cli-ax.d.ts.map +1 -1
- package/dist/internal/cli-ax.js +5 -2
- package/dist/internal/eval-run-store.d.ts.map +1 -1
- package/dist/internal/eval-run-store.js +7 -0
- package/dist/internal/eval-runner.d.ts +2 -0
- package/dist/internal/eval-runner.d.ts.map +1 -1
- package/dist/internal/eval-runner.js +17 -2
- package/dist/internal/evals-client.d.ts.map +1 -1
- package/dist/internal/evals-client.js +4 -3
- package/dist/internal/persistence-coordinator.d.ts +127 -0
- package/dist/internal/persistence-coordinator.d.ts.map +1 -0
- package/dist/persistence.d.ts +184 -0
- package/dist/persistence.d.ts.map +1 -0
- package/dist/playground/assets/index-Bwl4Onx4.js +85 -0
- package/dist/playground/assets/index-ByV4nfch.css +1 -0
- package/dist/playground/index.html +2 -2
- package/docs/example-agents/index.md +1 -1
- package/docs/example-agents/security-reviewer.md +23 -21
- package/package.json +1 -1
- package/skills/evals/SKILL.md +10 -0
- package/src/evals.ts +52 -0
- package/src/internal/cli-ax.ts +3 -0
- package/src/internal/eval-run-store.ts +8 -0
- package/src/internal/eval-runner.ts +22 -1
- package/src/internal/evals-client.ts +10 -2
- package/dist/channels/github/instrument.d.ts +0 -20
- package/dist/channels/github/instrument.d.ts.map +0 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.DGWkvXJc.js +0 -1
- package/dist/docs/assets/example-agents_security-reviewer.md.CE9v-aKm.js +0 -19
- package/dist/internal/json-dir-store.js +0 -100
- package/dist/playground/assets/index-CuXb4j4V.css +0 -1
- package/dist/playground/assets/index-DeyYpOit.js +0 -86
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import{_ as s,c as t,o as a,ag as i}from"./chunks/framework.CAZyNGu9.js";const u=JSON.parse('{"title":"Run staged security reviews from GitHub events","description":"Orchestrate parallel reviewer agents, streamed triage, deterministic stages, live playground progress, durable sessions, and severity A/B metrics.","frontmatter":{"title":"Run staged security reviews from GitHub events","description":"Orchestrate parallel reviewer agents, streamed triage, deterministic stages, live playground progress, durable sessions, and severity A/B metrics."},"headers":[],"relativePath":"example-agents/security-reviewer.md","filePath":"example-agents/security-reviewer.md"}'),r={name:"example-agents/security-reviewer.md"};function n(l,e,o,d,h,p){return a(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="run-staged-security-reviews-from-github-events" tabindex="-1">Run staged security reviews from GitHub events <a class="header-anchor" href="#run-staged-security-reviews-from-github-events" aria-label="Permalink to "Run staged security reviews from GitHub events""></a></h1><p>Security Reviewer turns a pull request into a staged host-side review. One tool prepares the diff and selects modules. A second fans out specialized reviewers and triages candidates as they arrive. A third deduplicates the confirmed findings, writes artifacts, and may publish a GitHub review.</p><p>Use this example when the workflow needs several model workers, but the host must own orchestration, progress, artifacts, and the final write.</p><p><a href="./../../examples/security-reviewer/">Browse the Security Reviewer source.</a></p><h2 id="run-a-three-stage-host-pipeline" tabindex="-1">Run a three-stage host pipeline <a class="header-anchor" href="#run-a-three-stage-host-pipeline" aria-label="Permalink to "Run a three-stage host pipeline""></a></h2><p>Security Reviewer is a pipeline, not one long agent turn:</p><table tabindex="0"><thead><tr><th>Stage</th><th>Tool</th><th>Result</th></tr></thead><tbody><tr><td>Prepare</td><td><code>prepare_review</code></td><td>Fetch metadata and diff, create a <code>runId</code>, and select security modules.</td></tr><tr><td>Review and triage</td><td><code>run_reviewers</code></td><td>Run module reviewers in parallel and start triage as each candidate arrives.</td></tr><tr><td>Finalize</td><td><code>finalize_review</code></td><td>Apply thresholds, deduplicate findings, write artifacts, and optionally post a review.</td></tr></tbody></table><p><code>run_triage</code> remains available as a compatibility stage. In the normal flow, triage has already completed inside <code>run_reviewers</code>, so it reports existing results. If candidates exist without triage output, it starts triage workers and writes their state.</p><p>The configured root agent chooses and sequences tools in chat. The review workers use a model selected by the host pipeline. They are created programmatically with the agent SDK, not discovered from <code>agent/subagents/</code>.</p><h2 id="follow-a-github-wake" tabindex="-1">Follow a GitHub wake <a class="header-anchor" href="#follow-a-github-wake" aria-label="Permalink to "Follow a GitHub wake""></a></h2><ol><li>A non-draft <code>pull_request.opened</code> or <code>pull_request.synchronize</code> event arrives for an allowlisted repository.</li><li>The GitHub channel returns a host <code>{ task }</code>, so the webhook gets a 202 response before the long review starts.</li><li>Host code tries to post a pending commit status and creates a playground session.</li><li>The root model sends one acknowledgement. It doesn't run review tools on this path.</li><li>The task calls <code>prepare_review</code>, <code>run_reviewers</code>, and <code>finalize_review</code> deterministically inside that session.</li><li>Reviewer candidates stream into duplicate gating and triage.</li><li>Finalization writes artifacts and tries to post the GitHub review.</li><li>The host tries to set a success commit status when no findings remain, a failure status when findings remain, or an error status when the pipeline throws.</li><li>The final response is appended to the session as an assistant message.</li></ol><p>The session records each stage as a normal tool event, even though host code selected the tools.</p><p>Review and status posting are best-effort. The channel chooses its final status from the finding count even when the review posting result says <code>posted: false</code>.</p><h2 id="map-the-framework-features" tabindex="-1">Map the framework features <a class="header-anchor" href="#map-the-framework-features" aria-label="Permalink to "Map the framework features""></a></h2><table tabindex="0"><thead><tr><th>Capability</th><th>Source</th><th>Role</th></tr></thead><tbody><tr><td>Root agent</td><td><a href="../../examples/security-reviewer/agent/agent.ts"><code>agent/agent.ts</code></a>, <a href="./../../examples/security-reviewer/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Configure local chat and explain the three-stage contract.</td></tr><tr><td>Server tools</td><td><a href="./../../examples/security-reviewer/agent/tools/"><code>agent/tools/</code></a></td><td>Expose each review stage to chat and host orchestration.</td></tr><tr><td>GitHub channel</td><td><a href="../../examples/security-reviewer/agent/channels/github.ts"><code>agent/channels/github.ts</code></a></td><td>Filter wakes, run background tasks, and publish status.</td></tr><tr><td>Progress channel</td><td><a href="../../examples/security-reviewer/agent/channels/asr-progress.ts"><code>agent/channels/asr-progress.ts</code></a></td><td>Serve live reviewer and triage state by <code>runId</code>.</td></tr><tr><td>Playground renderer</td><td><a href="./../../examples/security-reviewer/agent/playground/tools/run_reviewers.tsx.html"><code>agent/playground/tools/run_reviewers.tsx</code></a></td><td>Replace the generic tool chip with live module rows.</td></tr><tr><td>SDK review pipeline</td><td><a href="./../../examples/security-reviewer/agent/lib/asr/"><code>agent/lib/asr/</code></a>, <a href="../../examples/security-reviewer/agent/lib/review-stages.ts"><code>review-stages.ts</code></a></td><td>Select modules, call model workers, triage, deduplicate, and write artifacts.</td></tr><tr><td>Storage</td><td><a href="../../examples/security-reviewer/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persist framework sessions with <code>cursorHostedStorage</code> (lazy restore).</td></tr><tr><td>A/B</td><td><a href="../../examples/security-reviewer/agent/ab.ts"><code>agent/ab.ts</code></a>, <a href="../../examples/security-reviewer/agent/ab.config.ts"><code>agent/ab.config.ts</code></a></td><td>Compare all-severity versus high-only GitHub comments.</td></tr><tr><td>Eval</td><td><a href="./../../examples/security-reviewer/evals/"><code>evals/</code></a></td><td>Check stage-tool presence against a pinned sample.</td></tr></tbody></table><p>There is no Slack channel, authored skill, discovered subagent, MCP connection, schedule, reminder, hook, tool approval, or cloud runtime.</p><h2 id="prepare-the-host" tabindex="-1">Prepare the host <a class="header-anchor" href="#prepare-the-host" aria-label="Permalink to "Prepare the host""></a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential for the root turn and review workers.</li><li>GitHub read access for preparation.</li><li>GitHub write access for webhook-driven reviews and commit statuses.</li></ul><p>The pipeline exposes settings for:</p><ul><li>the worker model,</li><li>reviewer and triage parallelism,</li><li>reviewer, triage, duplicate-gate, and final-dedupe timeouts, and</li><li>prior-comment loading.</li></ul><p>The active names live beside the orchestration in <a href="../../examples/security-reviewer/agent/lib/review-stages.ts"><code>review-stages.ts</code></a>.</p><h2 id="validate-the-discovered-agent" tabindex="-1">Validate the discovered agent <a class="header-anchor" href="#validate-the-discovered-agent" aria-label="Permalink to "Validate the discovered agent""></a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span></span>
|
|
2
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span>
|
|
3
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span></code></pre></div><p>The manifest should show four server tools, two authored channels, one storage definition, and one A/B experiment. The eval listing should show one case.</p><h2 id="know-the-chat-path-s-write-boundary" tabindex="-1">Know the chat path's write boundary <a class="header-anchor" href="#know-the-chat-path-s-write-boundary" aria-label="Permalink to "Know the chat path's write boundary""></a></h2><p>In chat, the root instructions ask the model to use this order:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>prepare_review -> run_reviewers -> finalize_review</span></span></code></pre></div><p>They also ask the model to set <code>postComment: true</code> only on request. This is prompt policy, not a deterministic safety gate. The model chooses tool arguments, and <code>finalize_review</code> has no human approval. Use the direct stage calls below when a no-post proof must be enforced.</p><h2 id="call-stages-directly-without-publishing" tabindex="-1">Call stages directly without publishing <a class="header-anchor" href="#call-stages-directly-without-publishing" aria-label="Permalink to "Call stages directly without publishing""></a></h2><p>Call each stage and pass <code>postComment: false</code> yourself:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> prepare_review</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
4
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
5
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> '{"prUrl":"https://github.com/owner/repo/pull/123"}'</span></span>
|
|
6
|
-
<span class="line"></span>
|
|
7
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run_reviewers</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
8
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
9
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> '{"runId":"<run-id>"}'</span></span>
|
|
10
|
-
<span class="line"></span>
|
|
11
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> finalize_review</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
12
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
13
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> '{"runId":"<run-id>","postComment":false}'</span></span></code></pre></div><p>Review state lives under the project's run-artifact directory, so later stages can open the prepared <code>runId</code>.</p><div class="caution custom-block github-alert"><p class="custom-block-title">CAUTION</p><p><code>finalize_review</code> with <code>postComment: true</code> writes to GitHub. The webhook path always requests that write. Chat instructions alone don't prevent it.</p></div><h2 id="watch-parallel-work-in-the-playground" tabindex="-1">Watch parallel work in the playground <a class="header-anchor" href="#watch-parallel-work-in-the-playground" aria-label="Permalink to "Watch parallel work in the playground""></a></h2><p>Run the dev server:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span></span></code></pre></div><p>Open the printed playground and start a review. The custom <code>run_reviewers</code> renderer polls the progress channel's <code>GET /:runId</code> route.</p><p>It refreshes every 500 ms while the stage runs. Each row shows a reviewer module's state, candidates, reviewed areas, and failure. A second section shows triage jobs and confirmed or rejected counts.</p><p>This is an authored playground extension. Agentkit discovers it by the tool name, so the generic <code>run_reviewers</code> chip becomes a domain-specific view without changing the framework playground.</p><h2 id="fan-out-reviewers-while-triage-starts" tabindex="-1">Fan out reviewers while triage starts <a class="header-anchor" href="#fan-out-reviewers-while-triage-starts" aria-label="Permalink to "Fan out reviewers while triage starts""></a></h2><p>Module selection uses repository and path rules. The current module set covers:</p><ul><li>agent tooling trust boundaries,</li><li>privileged service RPCs,</li><li>product-specific security risks,</li><li>dependency and supply-chain changes,</li><li>deployment and infrastructure code,</li><li>filesystem and workspace boundaries,</li><li>privacy, and</li><li>general security review.</li></ul><p>Selected modules may run more than once. Candidates pass through a duplicate gate, then bounded triage. Reviewer or triage failures can produce partial results. A final dedupe failure stops finalization.</p><p>The pipeline writes JSONL journals as work completes. Final artifacts include the review bundle, patch, reviewer outputs, candidates, triage decisions, findings, accounting, and audit events.</p><h2 id="separate-session-storage-from-review-artifacts" tabindex="-1">Separate session storage from review artifacts <a class="header-anchor" href="#separate-session-storage-from-review-artifacts" aria-label="Permalink to "Separate session storage from review artifacts""></a></h2><p><code>defineStorage</code> + <code>cursorHostedStorage</code> sends agentkit session and event records to Cursor-hosted Bugbot storage through the control-plane proxy. Security Reviewer sets <code>restore: "off"</code> so startup doesn't load old review sessions in bulk. A continuation lookup can still fetch a needed session.</p><p>The staged review files are separate from session storage. Session-store durability doesn't preserve those files. All stages for one <code>runId</code> must see the same filesystem.</p><p>This split is useful when conversation history needs shared durability but large review artifacts belong on attached storage or an object store.</p><h2 id="compare-live-comment-variants" tabindex="-1">Compare live comment variants <a class="header-anchor" href="#compare-live-comment-variants" aria-label="Permalink to "Compare live comment variants""></a></h2><p>The comment-severity experiment uses sticky session assignment with a 5% holdout:</p><ul><li><code>control</code> posts every finding.</li><li><code>treatment</code> posts only high and critical findings.</li></ul><p>Finalization enforces the comment filter. The treatment also adds an instruction overlay asking chat and playground summaries to lead with high and critical findings. Full artifacts, <code>finalResponse</code>, finding counts, and status still include every finding. Stage-tool counters appear in the playground A/B view. Local sample and snapshot files persist under <code>.agent-serve/</code>.</p><p>When a treatment session has only low or medium findings, the filtered review body currently says no vulnerabilities were found even though artifacts and status retain findings. Account for that mismatch before using this experiment as a publishing policy.</p><p>Eval sessions skip A/B enrollment.</p><h2 id="test-the-github-channel-carefully" tabindex="-1">Test the GitHub channel carefully <a class="header-anchor" href="#test-the-github-channel-carefully" aria-label="Permalink to "Test the GitHub channel carefully""></a></h2><p>The channel currently accepts two configured repositories. It wakes on <code>opened</code> and <code>synchronize</code>, skips drafts, and requests <code>pr-write</code> access.</p><p>Inspect its event surface:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> github</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> events</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
14
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
15
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>Replay reaches the full publishing path:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">TEST_PR_URL</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">https://github.com/your-org/allowlisted-test-repo/pull/123</span></span>
|
|
16
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> github</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> replay</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
17
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$TEST_PR_URL</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
18
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
|
|
19
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --action</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> opened</span></span></code></pre></div><p>Set <code>TEST_PR_URL</code> to a PR in the channel's configured repository allowlist. Run the command only against a PR intended for test reviews. It posts a commit status and may post findings.</p><h2 id="inspect-the-eval-before-running-it" tabindex="-1">Inspect the eval before running it <a class="header-anchor" href="#inspect-the-eval-before-running-it" aria-label="Permalink to "Inspect the eval before running it""></a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/security-reviewer</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span></code></pre></div><p>The case reads pinned metadata from the committed fixture and checks for all three tool names. It doesn't assert their order. The current <code>prepare_review</code> still fetches the live PR, so the case needs GitHub access and isn't fully offline. It also doesn't assert finding location, severity, agreement with <code>gold.json</code>, or <code>postComment: false</code>.</p><p>Don't use this committed case as a no-post proof with write-capable GitHub credentials. The prompt asks for no comment, but the model can still pass <code>postComment: true</code>.</p><p>When you adapt the pipeline, add assertions for confirmed findings and make the prepare stage accept a materialized fixture if repeatable offline evals matter.</p><h2 id="build-another-staged-pipeline" tabindex="-1">Build another staged pipeline <a class="header-anchor" href="#build-another-staged-pipeline" aria-label="Permalink to "Build another staged pipeline""></a></h2><p>Use staged host orchestration when:</p><ul><li>each phase needs its own timeout and artifact,</li><li>model workers should run in bounded parallel,</li><li>later work can start as soon as partial results arrive,</li><li>a webhook must acknowledge before the work finishes, or</li><li>operators need live progress beyond one tool spinner.</li></ul><p>Keep external writes in finalization. Pass a <code>runId</code> between stages, journal progress before publishing, and make partial-worker failures visible in the result.</p><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to "Where to go next""></a></h2><ul><li><a href="./../guides/github.html">GitHub</a></li><li><a href="./../reference/tools.html">Tools</a></li><li><a href="./../reference/channels.html">Channels</a></li><li><a href="./../reference/playground.html">Playground</a></li><li><a href="./../storage.html">Storage</a></li><li><a href="./../ab.html">Live A/B metrics</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,73)])])}const g=s(r,[["render",n]]);export{u as __pageData,g as default};
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Small filesystem helpers for one-JSON-file-per-id persistence under a
|
|
3
|
-
* project-relative directory (eval run snapshots, A/B aggregate snapshots).
|
|
4
|
-
*/
|
|
5
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
6
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
7
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
8
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
9
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
10
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
11
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
12
|
-
});
|
|
13
|
-
};
|
|
14
|
-
import { randomBytes } from "node:crypto";
|
|
15
|
-
import { mkdir, readdir, readFile, rename, rm, writeFile, } from "node:fs/promises";
|
|
16
|
-
import { isAbsolute, join } from "node:path";
|
|
17
|
-
export function assertNonEmptyDir(dir, fnName) {
|
|
18
|
-
if (typeof dir !== "string" || dir.trim() === "") {
|
|
19
|
-
throw new Error(`${fnName}: dir must be a non-empty string`);
|
|
20
|
-
}
|
|
21
|
-
return dir;
|
|
22
|
-
}
|
|
23
|
-
/** Resolve `dir` against `projectRoot` when relative. */
|
|
24
|
-
export function resolveProjectRelativeDir(args) {
|
|
25
|
-
return isAbsolute(args.dir) ? args.dir : join(args.projectRoot, args.dir);
|
|
26
|
-
}
|
|
27
|
-
function isEnoent(error) {
|
|
28
|
-
return (error !== null &&
|
|
29
|
-
typeof error === "object" &&
|
|
30
|
-
"code" in error &&
|
|
31
|
-
error.code === "ENOENT");
|
|
32
|
-
}
|
|
33
|
-
/** List `*.json` basenames in `root` (any order). Missing dir → []. */
|
|
34
|
-
export function listJsonBasenames(root) {
|
|
35
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
36
|
-
let names;
|
|
37
|
-
try {
|
|
38
|
-
names = yield readdir(root);
|
|
39
|
-
}
|
|
40
|
-
catch (error) {
|
|
41
|
-
if (isEnoent(error)) {
|
|
42
|
-
return [];
|
|
43
|
-
}
|
|
44
|
-
throw error;
|
|
45
|
-
}
|
|
46
|
-
return names.filter((name) => name.endsWith(".json"));
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Read and JSON.parse every `*.json` file under `root`. Missing dir → [].
|
|
51
|
-
* Unreadable or corrupt files are skipped so one bad snapshot cannot wipe
|
|
52
|
-
* the rest of the history.
|
|
53
|
-
*/
|
|
54
|
-
export function readJsonDir(root, options) {
|
|
55
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
56
|
-
let names = yield listJsonBasenames(root);
|
|
57
|
-
if ((options === null || options === void 0 ? void 0 : options.sortBasenames) === "asc") {
|
|
58
|
-
names = [...names].sort();
|
|
59
|
-
}
|
|
60
|
-
else if ((options === null || options === void 0 ? void 0 : options.sortBasenames) === "desc") {
|
|
61
|
-
names = [...names].sort().reverse();
|
|
62
|
-
}
|
|
63
|
-
const out = [];
|
|
64
|
-
for (const name of names) {
|
|
65
|
-
try {
|
|
66
|
-
const raw = yield readFile(join(root, name), "utf8");
|
|
67
|
-
out.push(JSON.parse(raw));
|
|
68
|
-
}
|
|
69
|
-
catch (_a) {
|
|
70
|
-
// skip corrupt / unreadable files
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
return out;
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
/** Atomically write JSON under `root`/`fileName` (temp file + rename). */
|
|
77
|
-
export function writeJsonFile(args) {
|
|
78
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
79
|
-
const { root, fileName, value } = args;
|
|
80
|
-
// Serialize before any await so callers get a stable snapshot even if they
|
|
81
|
-
// mutate `value` afterward.
|
|
82
|
-
const body = `${JSON.stringify(value, null, 2)}\n`;
|
|
83
|
-
yield mkdir(root, { recursive: true });
|
|
84
|
-
const finalPath = join(root, fileName);
|
|
85
|
-
const tmpPath = `${finalPath}.${randomBytes(4).toString("hex")}.tmp`;
|
|
86
|
-
yield writeFile(tmpPath, body, "utf8");
|
|
87
|
-
yield rename(tmpPath, finalPath);
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
/** Best-effort delete; ignores missing files. */
|
|
91
|
-
export function deleteFileInDir(args) {
|
|
92
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
93
|
-
try {
|
|
94
|
-
yield rm(join(args.root, args.fileName), { force: true });
|
|
95
|
-
}
|
|
96
|
-
catch (_a) {
|
|
97
|
-
// ignore
|
|
98
|
-
}
|
|
99
|
-
});
|
|
100
|
-
}
|