@sreetej510/pi-shipd-checks 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 omega
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,104 @@
+# @sreetej510/pi-shipd-checks
+
+A [pi](https://github.com/earendil-works/pi) coding agent extension that runs a strict,
+multi-agent review of a benchmark task's `agent_prompt.md`, `test.patch`, and `solution.patch`
+against a fairness rubric, and finds behavioral test-coverage gaps that could let an incorrect
+agent solution slip past the hidden tests.
+
+## What it does
+
+For the flags you pass, `/checks`:
+
+1. Snapshots the current git `HEAD` into a throwaway temp directory (via
+   `git archive HEAD | tar -x`) — it never touches your working directory, staged, or
+   uncommitted changes.
+2. Copies `agent_prompt.md`, `solution.patch`, and `test.patch` from your project root into
+   that temp dir.
+3. Spawns one read-only reviewer agent per focus area you selected — **description**, **tests**,
+   **solution** — each restricted to `read`/`grep`/`find`/`ls` tools plus a single
+   `submit_review_report` tool it must call with a structured `PASS`/`FAIL` verdict, reasons,
+   and notes.
+4. Optionally runs a 2-stage behavioral test-gap analysis: an exhaustive researcher agent
+   proposes candidate gaps (required-but-untested edge cases), then a strict, independent
+   filter agent re-verifies each one against the fairness rules. This never turns a `PASS` into
+   a `FAIL` — it's purely informational.
+5. Posts a one-line chat summary and merges the results into `shipd_report.json` in your
+   project root. Running flags separately, in any order, builds up one combined report instead
+   of overwriting it — `overall` only becomes a confident `PASS`/`FAIL` once all 3 focus
+   reviewers have run at least once.
+
+## Commands
+
+All flags except `--config` are additive/combinable, e.g. `/checks --tests --gap-finder` runs
+just the tests reviewer plus the gap-finder/filter stages. `--config` must be used alone.
+
+| Command | Effect |
+|---|---|
+| `/checks` | List available options (runs nothing) |
+| `/checks --all` | Run all 3 focus reviewers + test-gap analysis |
+| `/checks --review` | Run only the 3 focus reviewer agents |
+| `/checks --description` | Run only the problem-description reviewer |
+| `/checks --tests` | Run only the tests reviewer |
+| `/checks --solution` | Run only the solution reviewer |
+| `/checks --gap-finder` | Run only the test-gap finder + filter agents |
+| `/checks --config` | Set the reviewer model and thinking level |
+
+**Shortcut:** `Ctrl+Shift+X` cancels an in-progress `/checks` run.
+
+## Configuration
+
+`/checks --config` lets you pick a model from your `enabledModels` list in `settings.json`,
+then (if the model supports more than one) a thinking level. The currently configured
+model/level is highlighted `[current]` in the picker. Settings are saved globally to
+`~/.pi/agent/checks-config.json` and shared by all reviewer/gap-finder/validator agents.
+
+## Install
+
+```bash
+npm install -g @sreetej510/pi-shipd-checks
+```
+
+Then add it to your pi `settings.json`:
+
+```json
+{
+  "packages": ["npm:@sreetej510/pi-shipd-checks"]
+}
+```
+
+Or, for local development, point at the entry point directly:
+
+```json
+{
+  "extensions": ["/absolute/path/to/pi-extensions/extensions/pi-shipd-checks/src/index.ts"]
+}
+```
+
+## File layout
+
+| File | Responsibility |
+|---|---|
+| `src/index.ts` | Extension entry point: message renderer, cancel shortcut, command registration |
+| `src/command.ts` | The `/checks` command: argument parsing, `--config` flow, run orchestration |
+| `src/agents.ts` | Spawns and races the reviewer + gap-finder/validator agent sessions |
+| `src/prompts.ts` | All prompt text sent to those agents |
+| `src/tools.ts` | Custom tools the agents call to submit their structured results |
+| `src/rubric.ts` | Embedded guidelines/fairness rubric text + per-role section loaders |
+| `src/roles.ts` | The 3 reviewer roles (description/tests/solution) metadata |
+| `src/report.ts` | `shipd_report.json` load/merge/summary logic |
+| `src/config.ts` | `~/.pi/agent/checks-config.json` + `settings.json` helpers (models, thinking levels, shell path) |
+| `src/git.ts` | Clean, non-mutating git `HEAD` snapshot into a scratch directory |
+| `src/progress.ts` | Progress-bar widget rendering |
+| `src/state.ts` | Shared "run in progress" / cancel state between the command and the shortcut |
+| `src/types.ts` | Shared TypeScript types |
+
+To change reviewer strictness or wording, edit `prompts.ts`. To change the rubric/fairness text
+itself, edit `rubric.ts`. To add a new tool, add it in `tools.ts` and wire it up in `agents.ts`.
+
+## Development
+
+```bash
+npm install
+npm run --workspace @sreetej510/pi-shipd-checks check     # biome + typecheck
+npm run --workspace @sreetej510/pi-shipd-checks format
+```

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@sreetej510/pi-shipd-checks",
+  "version": "0.1.1",
+  "description": "Pi extension that runs a strict, multi-agent fairness review of a benchmark task's agent_prompt.md, test.patch, and solution.patch, plus behavioral test-gap analysis, via /checks.",
+  "type": "module",
+  "license": "MIT",
+  "private": false,
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "pi",
+    "review",
+    "checks",
+    "shipd"
+  ],
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "pi": {
+    "extensions": [
+      "./src/index.ts"
+    ]
+  },
+  "scripts": {
+    "check": "biome check . && npm run typecheck",
+    "format": "biome check --write .",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "typebox": "^1.3.3"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.5.3",
+    "@earendil-works/pi-coding-agent": "^0.80.3",
+    "@earendil-works/pi-tui": "^0.80.3",
+    "typescript": "^5.7.3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Sreetej510/pi-extensions",
+    "directory": "extensions/pi-shipd-checks"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/agents.ts

@@ -0,0 +1,198 @@
+/**
+ * Spawns the throwaway reviewer / gap-finder / gap-validator agent sessions,
+ * races each against a timeout + external cancel signal, and pulls the
+ * structured result back out of the tool-call capture object.
+ */
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { createAgentSession, SessionManager } from "@earendil-works/pi-coding-agent";
+import { buildGapFinderPrompt, buildGapValidatorPrompt, buildReviewerPrompt } from "./prompts.js";
+import {
+  createGapFinderTool,
+  createGapValidatorTool,
+  createReportTool,
+  GAP_FINDER_TOOL_NAME,
+  GAP_VALIDATOR_TOOL_NAME,
+  REPORT_TOOL_NAME,
+} from "./tools.js";
+import type {
+  GapStageResult,
+  ReviewerRole,
+  ReviewReport,
+  TestGapCandidate,
+  TestGapFinal,
+  ThinkingLevel,
+} from "./types.js";
+
+export const REVIEWER_TIMEOUT_MS = 15 * 60 * 1000;
+export const REVIEWER_TOOLS = ["read", "grep", "find", "ls"] as const;
+
+/** Outcome of racing an agent turn against a timeout and an external cancel signal. */
+type AgentTurnOutcome = "done" | "timedOut" | "cancelled";
+
+async function raceAgentTurn(work: () => Promise<void>, cancelSignal: AbortSignal): Promise<AgentTurnOutcome> {
+  let outcome: AgentTurnOutcome = cancelSignal.aborted ? "cancelled" : "done";
+  await Promise.race([
+    work().then(() => {
+      if (!cancelSignal.aborted) outcome = "done";
+    }),
+    new Promise<void>((resolve) => {
+      setTimeout(() => {
+        outcome = "timedOut";
+        resolve();
+      }, REVIEWER_TIMEOUT_MS);
+    }),
+    new Promise<void>((resolve) => {
+      if (cancelSignal.aborted) return resolve();
+      cancelSignal.addEventListener(
+        "abort",
+        () => {
+          outcome = "cancelled";
+          resolve();
+        },
+        { once: true },
+      );
+    }),
+  ]);
+  return outcome;
+}
+
+export async function runReviewer(opts: {
+  pi: ExtensionAPI;
+  role: ReviewerRole;
+  tempDir: string;
+  model: unknown;
+  thinkingLevel: ThinkingLevel;
+  rubric: string;
+  fairnessRules: string;
+  cancelSignal: AbortSignal;
+}): Promise<ReviewReport> {
+  const capture: { report?: ReviewReport } = {};
+  // biome-ignore lint: model typed loosely to avoid depending on internal Model<Api> generics
+  const model = opts.model as any;
+  const { session } = await createAgentSession({
+    cwd: opts.tempDir,
+    model,
+    thinkingLevel: opts.thinkingLevel === "off" ? undefined : (opts.thinkingLevel as any),
+    tools: [...REVIEWER_TOOLS, REPORT_TOOL_NAME],
+    customTools: [createReportTool(capture)],
+    sessionManager: SessionManager.inMemory(),
+  });
+
+  try {
+    const outcome = await raceAgentTurn(async () => {
+      await session.prompt(buildReviewerPrompt(opts.role, opts.rubric, opts.fairnessRules));
+      await session.waitForIdle();
+    }, opts.cancelSignal);
+
+    if (outcome === "cancelled") {
+      await session.abort();
+      return {
+        verdict: "FAIL",
+        summary: `${opts.role.label} reviewer cancelled`,
+        reasons: ["Cancelled by user."],
+        notes: [],
+      };
+    }
+
+    if (outcome === "timedOut") {
+      await session.abort();
+      return {
+        verdict: "FAIL",
+        summary: `${opts.role.label} reviewer timed out`,
+        reasons: [`Reviewer did not finish within ${REVIEWER_TIMEOUT_MS / 1000}s.`],
+        notes: [],
+      };
+    }
+  } catch (err) {
+    return {
+      verdict: "FAIL",
+      summary: `${opts.role.label} reviewer errored`,
+      reasons: [`Reviewer agent failed: ${err instanceof Error ? err.message : String(err)}`],
+      notes: [],
+    };
+  }
+
+  if (!capture.report) {
+    return {
+      verdict: "FAIL",
+      summary: `${opts.role.label} reviewer did not submit a report`,
+      reasons: [`The reviewer agent finished without calling ${REPORT_TOOL_NAME}.`],
+      notes: [],
+    };
+  }
+  return capture.report;
+}
+
+export async function runGapFinder(opts: {
+  tempDir: string;
+  model: unknown;
+  thinkingLevel: ThinkingLevel;
+  testRubric: string;
+  fairnessRules: string;
+  cancelSignal: AbortSignal;
+}): Promise<GapStageResult<TestGapCandidate>> {
+  const capture: { gaps?: TestGapCandidate[] } = {};
+  // biome-ignore lint: model typed loosely to avoid depending on internal Model<Api> generics
+  const model = opts.model as any;
+  const { session } = await createAgentSession({
+    cwd: opts.tempDir,
+    model,
+    thinkingLevel: opts.thinkingLevel === "off" ? undefined : (opts.thinkingLevel as any),
+    tools: [...REVIEWER_TOOLS, GAP_FINDER_TOOL_NAME],
+    customTools: [createGapFinderTool(capture)],
+    sessionManager: SessionManager.inMemory(),
+  });
+
+  try {
+    const outcome = await raceAgentTurn(async () => {
+      await session.prompt(buildGapFinderPrompt(opts.testRubric, opts.fairnessRules));
+      await session.waitForIdle();
+    }, opts.cancelSignal);
+    if (outcome !== "done") {
+      await session.abort();
+      return { status: outcome, gaps: [] };
+    }
+  } catch {
+    return { status: "error", gaps: [] };
+  }
+  if (!capture.gaps) return { status: "noSubmission", gaps: [] };
+  return { status: "ok", gaps: capture.gaps };
+}
+
+export async function runGapValidator(opts: {
+  tempDir: string;
+  model: unknown;
+  thinkingLevel: ThinkingLevel;
+  testRubric: string;
+  fairnessRules: string;
+  candidates: TestGapCandidate[];
+  cancelSignal: AbortSignal;
+}): Promise<GapStageResult<TestGapFinal>> {
+  const capture: { gaps?: TestGapFinal[] } = {};
+  // biome-ignore lint: model typed loosely to avoid depending on internal Model<Api> generics
+  const model = opts.model as any;
+  const { session } = await createAgentSession({
+    cwd: opts.tempDir,
+    model,
+    thinkingLevel: opts.thinkingLevel === "off" ? undefined : (opts.thinkingLevel as any),
+    tools: [...REVIEWER_TOOLS, GAP_VALIDATOR_TOOL_NAME],
+    customTools: [createGapValidatorTool(capture)],
+    sessionManager: SessionManager.inMemory(),
+  });
+
+  try {
+    const outcome = await raceAgentTurn(async () => {
+      await session.prompt(buildGapValidatorPrompt(opts.candidates, opts.testRubric, opts.fairnessRules));
+      await session.waitForIdle();
+    }, opts.cancelSignal);
+    if (outcome !== "done") {
+      await session.abort();
+      return { status: outcome, gaps: [] };
+    }
+  } catch {
+    return { status: "error", gaps: [] };
+  }
+  if (!capture.gaps) return { status: "noSubmission", gaps: [] };
+  return { status: "ok", gaps: capture.gaps };
+}