@sreetej510/pi-shipd-checks 0.1.1

package/src/prompts.ts

@@ -0,0 +1,202 @@
+/**
+ * All prompt text sent to the reviewer / gap-finder / gap-validator agents.
+ * Keeping this in one file means future prompt tweaks never require touching
+ * agents.ts or command.ts.
+ */
+
+import { GAP_FINDER_TOOL_NAME, GAP_VALIDATOR_TOOL_NAME, REPORT_TOOL_NAME } from "./tools.js";
+import type { ReviewerRole, ReviewerRoleKey, TestGapCandidate } from "./types.js";
+
+/** Per-role instructions on what to look at and, for `tests`/`solution`, mandatory extra checks. */
+const ROLE_FOCUS: Record<ReviewerRoleKey, string> = {
+  description:
+    "Focus area: the task description in `agent_prompt.md`. Judge it strictly against rubric items P1-P5 below. " +
+    "You do not need to judge the tests or solution — other reviewers cover those.",
+  tests:
+    "Focus area: the tests added in `test.patch` (a unified diff). Judge them strictly against rubric items T1-T6 below. " +
+    "You cannot execute code or apply the patch, so read the diff carefully and reason about determinism, coverage, and " +
+    "strictness directly from the added code. Read `agent_prompt.md` for context on what behavior is in scope, and skim " +
+    "`solution.patch` only to understand what the tests are checking. Do not judge the description or the solution's code quality.\n" +
+    "Mandatory symbol-fairness check: for every non-trivial method/function/property/export name that a test calls, " +
+    "mutates, mocks, or asserts on, use grep/read on the repository (the pre-existing code, not solution.patch) to " +
+    "confirm it actually exists there, OR confirm it is explicitly named in `agent_prompt.md`. Pay special attention to " +
+    "any such name that is new/invented and that duplicates, shadows, or conflicts with an existing, differently-named " +
+    "public API doing the same thing (e.g. a test calling `setFoo(...)` when the repo's real, visible API is `setBar(...)`) " +
+    "— that is a textbook unfair/undiscoverable test per the fairness methodology, and is blocking on its own even if " +
+    "every other test in the patch is fine.",
+  solution:
+    "Focus area: the golden solution in `solution.patch` (a unified diff). Judge it strictly against rubric items S1-S4 below. " +
+    "Read `agent_prompt.md` for the requirements and `test.patch` to see what must pass, and use read/grep/ls/find on the rest " +
+    "of the repository to check for regressions, inconsistent style, and irrelevant/unexplained changes. Do not judge the " +
+    "description's wording or the tests' coverage.\n" +
+    "Mandatory dead-code check: scan every added/changed line in `solution.patch` for unused code — variables, " +
+    "parameters, imports, functions/methods, or fields that are declared/assigned but never read or called anywhere " +
+    "(including by `test.patch`) — and for dead/unreachable code (branches, conditions, or statements that can never " +
+    "execute, or code left behind after a return/throw/break that makes it unreachable). Use grep to confirm a symbol " +
+    "truly has no other usages in the repo before flagging it. This is a rubric S4 violation and is blocking on its " +
+    "own, even if the rest of the solution is otherwise excellent.",
+};
+
+export function buildReviewerPrompt(role: ReviewerRole, rubric: string, fairnessRules: string): string {
+  const parts = [
+    "You are a careful, calibrated reviewer for a coding-agent benchmark task.",
+    "You are working inside a throwaway, read-only copy of a git repository (this is your current directory). " +
+      "You have access to read/grep/find/ls tools only — you cannot execute code, apply patches, or edit files.",
+    "The repo root may contain: `agent_prompt.md` (the task description), `test.patch` (unified diff adding tests), " +
+      "and `solution.patch` (unified diff of the golden solution). Read whichever are relevant to your focus, and read " +
+      "the rest of the repository as needed via grep/read/ls/find to judge things against real repo context.",
+    "",
+    ROLE_FOCUS[role.key],
+    "",
+    "Checklist for your focus area:",
+    rubric,
+  ];
+
+  if (fairnessRules) {
+    parts.push(
+      "",
+      "Fairness methodology (use this to judge whether an issue is actually blocking, and to distinguish agent-fault " +
+        "from prompt-ambiguity from test-flaw problems):",
+      fairnessRules,
+    );
+  }
+
+  parts.push(
+    "",
+    "How to set the verdict — calibrate, don't nitpick:",
+    "FAIL only for a genuine BLOCKING issue: a checklist item is clearly violated, a requirement stated in " +
+      "`agent_prompt.md` is untested or contradicted, a test asserts something unfair/undiscoverable per the " +
+      "fairness methodology above (private internals, exact class names not in the prompt, exact call order, " +
+      "reference-solution-only structure, etc.), a test is genuinely non-deterministic in a way that risks real " +
+      "CI flakiness (real network calls, unseeded randomness, race-prone ordering), or the solution has a real " +
+      "regression, missing requirement, or unrelated/unexplained change.",
+    "Do NOT fail for: optional coverage suggestions, 'would also be nice to test X', dead/unused code that doesn't " +
+      "affect correctness, minor style inconsistencies, or defensible implementation choices the prompt didn't " +
+      "forbid. These are exactly the kind of thing a real reviewer leaves as a 'Minor/optional' note without " +
+      "failing the task — put them in `notes` and still return PASS.",
+    "When genuinely torn between PASS and FAIL, default to PASS with the concern captured in `notes`, unless the " +
+      "issue would let a materially incorrect agent solution pass the hidden tests, or would unfairly fail a " +
+      "correct one — that is always blocking.",
+    "A single blocking issue is enough to FAIL, even if it affects only one test or one line out of many, and even " +
+      "if the rest of the suite is excellent. Do NOT average it away or let a large, otherwise-strong test suite " +
+      "talk you into a PASS — one unfair or undiscoverable test (e.g. requiring a private/invented API name that " +
+      "doesn't exist in the repo and isn't named in the prompt, especially one that conflicts with an existing, " +
+      "differently-named public API) is exactly as blocking as many.",
+    "",
+    `When you are done analyzing, call the \`${REPORT_TOOL_NAME}\` tool exactly once with your structured verdict. ` +
+      "That tool call is your only way to report a result.",
+  );
+
+  return parts.join("\n");
+}
+
+export function buildGapFinderPrompt(testRubric: string, fairnessRules: string): string {
+  const parts = [
+    "You are an exhaustive, research-minded test-coverage analyst for a coding-agent benchmark task. Your mandate " +
+      "is to dig as deep as possible and surface EVERY genuine behavioral test gap you can find — not just the " +
+      "first one or two obvious ones. Treat a short list as a signal that you stopped too early, not as success.",
+    "You are working inside a throwaway, read-only copy of a git repository (this is your current directory). " +
+      "You have access to read/grep/find/ls tools only — you cannot execute code, apply patches, or edit files.",
+    "The repo root contains `agent_prompt.md` (the task description given to a coding agent), `test.patch` (a " +
+      "unified diff adding the hidden tests that will grade that agent's solution), and `solution.patch` (a " +
+      "unified diff of one golden/reference solution).",
+    "",
+    "Your job: find real BEHAVIORAL TEST GAPS. A gap is required or clearly-implied behavior from " +
+      "`agent_prompt.md` (and, where relevant, obvious existing repo conventions) that `test.patch` does NOT " +
+      "actually verify — such that a plausible alternative implementation could satisfy `agent_prompt.md` on its " +
+      "face, differ from `solution.patch`, get that behavior wrong or skip it entirely, and STILL pass every test " +
+      "in `test.patch` as written.",
+    "",
+    "Be systematic and exhaustive — do not stop after finding one or two gaps. Work through ALL of the following " +
+      "passes before you consider yourself done:",
+    "1. Go through `agent_prompt.md` sentence by sentence. For every distinct requirement, constraint, or implied " +
+      "rule, explicitly check which test(s) in `test.patch` exercise it, and how thoroughly.",
+    "2. Go through `solution.patch` branch by branch — every conditional, loop, early return, error path, and " +
+      "state transition. For each one, ask whether `test.patch` actually forces that branch to be taken and its " +
+      "outcome checked, or whether an implementation that got that branch wrong would still pass.",
+    "3. Systematically consider standard edge-case categories against the required behavior: boundary/limit " +
+      "values, empty/missing/null/zero inputs, duplicate or repeated inputs, ordering and interleaving, " +
+      "concurrent or repeated invocations, error/failure/rollback paths, interaction between two or more " +
+      "required behaviors at once (not just each in isolation), and state left behind after an operation.",
+    "4. Cross-check overlapping/interacting requirements — behaviors that are each tested alone but never tested " +
+      "together — since that's exactly where a plausible-looking but incomplete implementation slips through.",
+    "Do not filter yourself or self-censor for volume. List every gap that survives your own check against the " +
+      "ground rules below — a long, thorough list is expected and desired. A separate, independent agent will " +
+      "strictly filter this list afterward, so your job here is coverage and recall, not brevity.",
+    "",
+    "Ground rules — do not overreach:",
+    "- Every gap must trace back to a specific requirement or sentence in `agent_prompt.md`, or to behavior that " +
+      "is unambiguous from the existing, visible repo. Do not invent requirements the prompt doesn't support.",
+    "- Do not propose gaps for things `agent_prompt.md` leaves intentionally open, or for implementation " +
+      "details/style the prompt doesn't mandate.",
+    "- Read `test.patch` carefully before deciding something is untested — do not propose a gap that an existing " +
+      "test already covers (even indirectly).",
+    "",
+    "For reference, here is the checklist for the tests focus area (use it to calibrate what good coverage looks " +
+      "like, not as a list of gaps to report verbatim):",
+    testRubric,
+  ];
+
+  if (fairnessRules) {
+    parts.push("", "Fairness methodology (context on what a fair, in-scope requirement looks like):", fairnessRules);
+  }
+
+  parts.push(
+    "",
+    "For each gap, describe: (1) the specific untested behavior/edge case, in plain terms, and (2) concretely why " +
+      "a plausible-but-incorrect implementation would still pass every given test despite missing or " +
+      "misimplementing it.",
+    `When you are done — after completing ALL the passes above — call the \`${GAP_FINDER_TOOL_NAME}\` tool exactly ` +
+      "once with your full candidate list (empty only if, after genuinely exhaustive analysis, none exist). " +
+      "That tool call is your only way to report a result.",
+  );
+
+  return parts.join("\n");
+}
+
+export function buildGapValidatorPrompt(
+  candidates: TestGapCandidate[],
+  testRubric: string,
+  fairnessRules: string,
+): string {
+  const parts = [
+    "You are a strict, skeptical fairness auditor for a coding-agent benchmark task.",
+    "You are working inside a throwaway, read-only copy of a git repository (this is your current directory). " +
+      "You have access to read/grep/find/ls tools only — you cannot execute code, apply patches, or edit files.",
+    "Another research agent reviewed this same task (`agent_prompt.md`, `test.patch`, `solution.patch`, and the " +
+      "repo) and proposed the following CANDIDATE test gaps — behaviors it believes are required but untested:",
+    "",
+    JSON.stringify(candidates, null, 2),
+    "",
+    "Your job is to independently re-verify the files yourself and FILTER this list down to only candidates that " +
+      "are ALL of the following:",
+    "1. Genuinely grounded — actually required by a specific statement in `agent_prompt.md`, or unambiguous from " +
+      "clearly visible, existing repo behavior. Drop anything speculative, nice-to-have, or invented beyond what " +
+      "the prompt actually asks for.",
+    "2. Fair to test — verifying it would not require undiscoverable private internals, an invented/unnamed API, " +
+      "or exact incidental structure that only `solution.patch` happens to use. Judge this precisely against the " +
+      "fairness methodology below.",
+    "3. A real, distinct coverage hole — re-check `test.patch` yourself; drop any candidate an existing test " +
+      "already covers, and drop near-duplicate candidates (keep only the clearest phrasing of each distinct gap).",
+    "",
+    "Be strict: when genuinely unsure whether a candidate holds up, drop it rather than keep it. It is fine — " +
+      "expected, even — to return an empty list if none of the candidates survive scrutiny.",
+    "",
+    "Checklist for the tests focus area, for calibration:",
+    testRubric,
+  ];
+
+  if (fairnessRules) {
+    parts.push("", "Fairness methodology:", fairnessRules);
+  }
+
+  parts.push(
+    "",
+    "For every gap you keep, give a short justification citing where in `agent_prompt.md` or the repo it is " +
+      "grounded.",
+    `When you are done, call the \`${GAP_VALIDATOR_TOOL_NAME}\` tool exactly once with your final filtered list ` +
+      "(which may be empty). That tool call is your only way to report a result.",
+  );
+
+  return parts.join("\n");
+}

package/src/report.ts

@@ -0,0 +1,142 @@
+/**
+ * shipd_report.json read/merge/summarize logic. Running --review, --description,
+ * --tests, --solution, and --gap-finder separately (in any order) should build
+ * up one combined report rather than each overwriting the others' results.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { ROLES } from "./roles.js";
+import type { ChecksConfig, ReviewerRole, ReviewReport, TestGapFinal, Verdict } from "./types.js";
+
+export const REQUIRED_FILES = ["agent_prompt.md", "solution.patch", "test.patch"] as const;
+
+/** Load a prior shipd_report.json (if any) so a later run can merge into it instead of clobbering it. */
+export function loadExistingReport(reportPath: string): Record<string, unknown> {
+  try {
+    if (!existsSync(reportPath)) return {};
+    const parsed = JSON.parse(readFileSync(reportPath, "utf-8"));
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+export interface MergeReportInput {
+  existingReport: Record<string, unknown>;
+  config: ChecksConfig;
+  runReviewers: boolean;
+  byRole: Record<string, ReviewReport>;
+  runGapStages: boolean;
+  testGaps: TestGapFinal[];
+  gapAnalysisIncomplete: boolean;
+  gapFinderStatus: string;
+  gapFilterStatus: string;
+}
+
+/** Merges this run's results into the prior report, keeping any roles/gaps not touched this time. */
+export function mergeReport(input: MergeReportInput): Record<string, unknown> {
+  const {
+    existingReport,
+    config,
+    runReviewers,
+    byRole,
+    runGapStages,
+    testGaps,
+    gapAnalysisIncomplete,
+    gapFinderStatus,
+    gapFilterStatus,
+  } = input;
+
+  const merged: Record<string, unknown> = {
+    ...existingReport,
+    timestamp: new Date().toISOString(),
+    model: `${config.provider}/${config.modelId}`,
+    thinkingLevel: config.thinkingLevel,
+  };
+
+  if (runReviewers) {
+    const existingReports =
+      existingReport.reports && typeof existingReport.reports === "object"
+        ? (existingReport.reports as Record<string, ReviewReport>)
+        : {};
+    const mergedReports = { ...existingReports, ...byRole };
+    merged.reports = mergedReports;
+    // `overall` only reflects a confident PASS/FAIL once all 3 focus
+    // reviewers have actually run (possibly across separate invocations
+    // of --description/--tests/--solution) — otherwise it's incomplete.
+    if (ROLES.every((role) => mergedReports[role.key])) {
+      merged.overall = ROLES.every((role) => mergedReports[role.key].verdict === "PASS") ? "PASS" : "FAIL";
+    } else {
+      delete merged.overall;
+    }
+  }
+
+  if (runGapStages) {
+    merged.testGaps = testGaps;
+    if (gapAnalysisIncomplete) {
+      merged.testGapAnalysisNote = `Gap analysis did not fully complete (finder: ${gapFinderStatus}, filter: ${gapFilterStatus}); testGaps may be incomplete.`;
+    } else {
+      delete merged.testGapAnalysisNote;
+    }
+  }
+
+  return merged;
+}
+
+export interface RunSummary {
+  content: string;
+  details: {
+    overall?: Verdict;
+    hasTestGaps: boolean;
+    gapsCount: number;
+    roleVerdicts?: Record<string, Verdict>;
+    showGaps: boolean;
+  };
+}
+
+/** Builds the one-line chat summary + renderer details for this run's results. */
+export function buildRunSummary(opts: {
+  merged: Record<string, unknown>;
+  runReviewers: boolean;
+  activeRoles: ReviewerRole[];
+  byRole: Record<string, ReviewReport>;
+  runGapStages: boolean;
+}): RunSummary {
+  const { merged, runReviewers, activeRoles, byRole, runGapStages } = opts;
+
+  const overall = typeof merged.overall === "string" ? (merged.overall as Verdict) : undefined;
+  const gapsCount = Array.isArray(merged.testGaps) ? merged.testGaps.length : 0;
+
+  // Always surface whichever reviewer(s) actually just ran, even when
+  // `overall` is still incomplete (e.g. --tests alone, before --description
+  // and --solution have run) — a partial run must never look like it
+  // produced no PASS/FAIL information at all.
+  const roleVerdicts: Record<string, Verdict> | undefined = runReviewers
+    ? Object.fromEntries(
+        activeRoles.filter((role) => byRole[role.key]).map((role) => [role.key, byRole[role.key].verdict]),
+      )
+    : undefined;
+
+  const summaryParts: string[] = [];
+  if (roleVerdicts) {
+    summaryParts.push(activeRoles.map((role) => `${role.label}: ${roleVerdicts[role.key] ?? "?"}`).join(", "));
+  }
+  if (overall) {
+    const suffix = overall === "PASS" && gapsCount > 0 ? " (with test gaps)" : "";
+    summaryParts.push(`Overall: ${overall}${suffix}`);
+  }
+  if (runGapStages) {
+    summaryParts.push(gapsCount > 0 ? `${gapsCount} test gap(s) found` : "no test gaps found");
+  }
+
+  return {
+    content: `Checks: ${summaryParts.join("  |  ")}`,
+    details: {
+      overall,
+      hasTestGaps: gapsCount > 0,
+      gapsCount,
+      roleVerdicts,
+      showGaps: runGapStages,
+    },
+  };
+}

package/src/roles.ts

@@ -0,0 +1,20 @@
+import type { ReviewerRole } from "./types.js";
+
+/** The 3 focus reviewers. Their prompt text (`focus`) lives in prompts.ts, not here. */
+export const ROLES: ReviewerRole[] = [
+  {
+    key: "description",
+    label: "Description",
+    rubricHeading: /^## The problem description/i,
+  },
+  {
+    key: "tests",
+    label: "Tests",
+    rubricHeading: /^## The tests/i,
+  },
+  {
+    key: "solution",
+    label: "Solution",
+    rubricHeading: /^## The solution/i,
+  },
+];

package/src/rubric.ts

@@ -0,0 +1,32 @@
+/**
+ * Static rubric text embedded directly in the extension for consistency
+ * across machines: the per-focus P1-P5/T1-T6/S1-S4 checklist (was
+ * ~/.pi/agent/guidelines.txt) and the fairness methodology — agent-fault vs
+ * prompt-ambiguity vs test-flaw, fair/unfair test examples (was
+ * ~/.pi/agent/rules.md).
+ */
+
+import { ROLES } from "./roles.js";
+import type { ReviewerRoleKey } from "./types.js";
+
+const GUIDELINES_TEXT =
+  "## The problem description (aka the task)\r\nP1: Aligns with the repo's philosophy\r\nP2: Self-contained — solvable from the repo and description alone\r\nP3: Clear, concise, and unambiguous — describes what to build or fix and don't leave points for guessing\r\nP4: Verifiable — success is objectively testable\r\nP5: Not prescriptive — don't leak the solution, we want to challenge the agents, remember!!\r\n\r\nA note on how it should read: write it the way a maintainer writes an issue —\r\nnatural prose, full sentences. Open with the ask itself (“Add X to Y”, “Fix Z when …”) and let the first line stand on its own without the title. Skip the motivation and the “what the repo currently lacks” preamble.\r\nKeep it out of spec-sheet territory — no bulleted requirement lists, no headings, no code snippets doing the describing. And don't spell out what a developer in the repo would find on their own (internal class names, helpers, field names, the file layout);\r\ndescribe the behavior, not the implementation. That said, use your judgment\r\n— if a detail is genuinely part of the contract and the task can't be pinned down without it, state it. A task nobody can implement is worse than one that names a field.\r\n\r\n## The tests\r\nYour tests are the backbone; the agent only gets the repo and the problem description and is expected to create a solution to pass your tests (they'll be hidden 🤫). They should follow these requirements:\r\nT1: They highlight the missing or incorrect behavior (depending on the task's category — feature request, bug fix, etc). They should 100% fail at the base commit and 100% pass after adding your solution.\r\nT2: The tests should be deterministic (no timing, randomness, or ordering; nothing that could change across multiple runs or across different machines).\r\nT3: Strong tests 💪. The tests shouldn't be permissive enough to let inaccurate agent solutions pass.\r\nT4: Extensive coverage. The tests should cover the requested behavior and all the obvious edge cases.\r\nT5: DO NOT check for unspecified or undiscoverable behavior — this will be unfair to the agents to expect them to implement something not in the description or discoverable from the repo 😞.\r\nT6: Don't over-pin the output — don't assert on exact output (error text, messages, wording, formatting) unless the description says so or it's obvious from the repo's existing patterns. Otherwise it's unfair to an agent that gets the behavior right but words it differently; checking the behavior holds is enough.\r\n\r\n## The solution\r\nThat's the golden solution — the proof that your task is solvable and that we can pass the tests. The solution should follow these requirements:\r\nS1: The solution should meet all the requirements. (If it's missing a requirement and it manages to pass your tests, you're in a bad position 😠.)\r\nS2: No regressions and follow existing code patterns. Don't break existing working code by mistake (we'll still run the repo's existing tests).\r\nS3: No irrelevant changes to the code — if something is unrelated to your task, keep it as it is.\r\nS4: No AI slop (weird comments, unexplained defensive code, new coding patterns, etc).";
+
+/** Full fairness methodology (embedded — was ~/.pi/agent/rules.md): agent-fault vs prompt-ambiguity vs test-flaw, fair/unfair examples. */
+const RULES_TEXT =
+  "# Agent Evaluation Guidelines\r\n\r\nThis file describes how to align a task prompt, reference solution, and tests for fair coding-agent evaluation. It is generic and can be reused across repositories and tasks.\r\n\r\n## Core Principle\r\n\r\nThe prompt, solution, and tests should form a consistent triangle:\r\n\r\n- **Prompt**: defines required behavior and public contracts.\r\n- **Solution**: demonstrates one valid implementation.\r\n- **Tests**: verify the prompt requirements, not incidental choices from the solution.\r\n\r\nA test is fair only when it checks behavior required by the prompt, strongly implied by existing code, or exposed through a public API named in the prompt.\r\n\r\nA useful mental model is: **test what the user, caller, or public contract can observe; avoid testing how the implementation got there unless that mechanism is itself part of the contract.**\r\n\r\n---\r\n\r\n## Prompt Rules\r\n\r\nThe task prompt should be written in paragraphs, not bullet points or numbered lists, should be valid UTF-8. Keep it concise with a soft maximum of about 300 words, unless the task truly requires more detail to be fair.\r\n\r\nDo not restate the same requirement in multiple ways. Do not include long explanations for why a requirement exists. State the required behavior, public API, data shape, selector, or side effect clearly and move on.\r\n\r\nInclude implementation details only when tests or compatibility require them. If a selector, translation key, exported function name, file path, data attribute, timing guarantee, option ID, or confirmation message is required by tests, name it explicitly in the prompt. If multiple implementations should be valid, describe the required outcome rather than one mechanism.\r\n\r\n# 1. Early Stage: No Agent Evaluations Yet\r\n\r\nUse this section while creating or refining the problem before agent runs exist.\r\n\r\n## 1.1 Prompt-to-Test Mapping\r\n\r\nFor every test, identify the prompt sentence that justifies it.\r\n\r\nIf no prompt sentence exists, either:\r\n\r\n- add the requirement to the prompt, or\r\n- remove/relax the test.\r\n\r\nTests may also rely on existing public repo behavior, but avoid relying on private internals unless the task is explicitly about those internals.\r\n\r\n## 1.2 Solution Is Not the Spec\r\n\r\nThe reference solution is only one valid implementation. Do not copy its incidental choices into tests unless the prompt requires them.\r\n\r\nAvoid testing:\r\n\r\n- exact local helper names\r\n- exact component file paths\r\n- exact DOM nesting\r\n- exact CSS classes\r\n- exact call order\r\n- exact framework implementation style\r\n- exact module boundaries when multiple placements are valid\r\n- exact translation key names when only visible text matters\r\n- exact callback return shape when only the eventual effect matters\r\n- exact registry keys, menu IDs, or config entry names when only the visible action matters\r\n- one specific state source when multiple repo-valid sources expose the same data\r\n- one specific internal call path when another repo-valid path produces the same public effect\r\n\r\nPrefer testing:\r\n\r\n- public exports\r\n- rendered behavior\r\n- visible labels/text\r\n- required data attributes\r\n- state changes\r\n- side effects\r\n- user-triggered actions\r\n\r\n## 1.3 Public Interface Rules\r\n\r\nA test may assert an API name if:\r\n\r\n- the prompt names it, or\r\n- it already exists as a public API and the task says to use it.\r\n\r\nA test should not assert private helper names introduced only by the reference solution.\r\n\r\nIf a new exported API is required, name it in the prompt.\r\n\r\n## 1.4 Selector and UI Rules\r\n\r\nIf tests need a selector or DOM hook, mention it in the prompt.\r\n\r\nFair required hooks only user visible text include:\r\n\r\n- exact visible labels\r\n- exact `title`\r\n- repo covention common class\r\n\r\nUnfair unless specified:\r\n\r\n- requiring test or data attributes\r\n- requiring `<button>` instead of clickable elements\r\n- requiring `.btn`, `.ctrl`, `.popup-btn`, etc, unless repo convention.\r\n- requiring a row class like `.item`, `.tab`, `.entry`\r\n- requiring specific DOM nesting\r\n\r\nWhen testing controls, find them by required user-facing contract, not style-only class names. If broad discovery is necessary, keep the final assertion strict: exact label/key plus correct behavior.\r\n\r\n## 1.5 Async and Refresh Rules\r\n\r\nIf the prompt says something refreshes or updates after an operation, allow reasonable async timing unless synchronous behavior is explicitly required.\r\n\r\nUse polling helpers or fake timers as appropriate. Do not fail valid debounced or event-driven implementations unless the prompt requires immediate state.\r\n\r\nAvoid requiring refresh to happen inside a specific wrapper if it could fairly happen through a delegated operation, event handler, or shared service.\r\n\r\nDo not assume that an event handler or callback must return a promise unless the prompt or existing public API requires that contract. If the required behavior is eventual, trigger the callback and then flush or poll for the resulting effect.\r\n\r\n## 1.6 Fixture and Mock Rules\r\n\r\nMocks should preserve real code invariants.\r\n\r\nIf real code uses both a list and an ID map, populate both. If real code uses both native and reactive fields, set both when relevant.\r\n\r\nPrefer partial mocks over full replacement when the implementation may reasonably use additional existing exports from the same module. Full mocks are appropriate only when the hidden test truly intends to forbid every other interaction with that module.\r\n\r\nWhen a mocked module exports mutable state, prefer mutating existing arrays and objects in place rather than reassigning them. Reassignment can break valid implementations that hold live references to the original exported values.\r\n\r\nMocks should not remove valid implementation paths. For example:\r\n\r\n- do not mock away a delegated helper and then require the delegated side effect\r\n- do not stub the child component if testing behavior rendered by the child\r\n- include framework probe fields in module mocks when templates need them\r\n- do not replace a service module so narrowly that a valid implementation fails only because it uses another existing export from that module\r\n- do not observe only one internal call path when the same user-visible effect can happen through another repo-valid path\r\n- do not make one mocked function call the only accepted proof that an effect happened if the same effect can be observed through another repo-valid mechanism\r\n\r\nWhen testing text, prefer real dictionaries or neutral translation stubs that resolve the visible strings. Do not make tests pass or fail based on arbitrary translation key naming unless the prompt explicitly names the key.\r\n\r\n## 1.7 Early-Stage Checklist\r\n\r\nBefore running agents, verify:\r\n\r\n- every test maps to prompt text or existing public behavior\r\n- every required selector/key/attribute is named in the prompt\r\n- tests accept valid alternative implementations\r\n- tests avoid private internals\r\n- async behavior is waited for fairly\r\n- fixtures match real runtime invariants\r\n- solution-specific structure is not treated as mandatory\r\n- Keep the prompt behavioral and public-facing; avoid internal API coupling.\r\n- Fairness matters: tests should avoid implementation details and brittle DOM assumptions.\r\n- Prefer existing repo patterns/components (TextInput, ToggleInput, SetupPage.registerEl, debounced saves).\r\n- Preserve baseline behavior; new tests must fail before the feature and pass after.\r\n- Sandbox should be limited/approved, not broad live-app access.\r\n- If tests assert specific visible text, that text must be named in the prompt.\r\n- Avoid DOM class selectors or custom attributes unless explicitly stated in the prompt.\r\n\r\n---\r\n\r\n# 2. After Agent Evaluations Exist\r\n\r\nUse this section when agent runs, solution patches, or failure reports are available.\r\n\r\n## 2.1 Analyze Before Changing Tests\r\n\r\nFor each failed agent:\r\n\r\n1. List failed tests.\r\n2. Inspect the agent's solution patch.\r\n3. Compare the implementation to the prompt, not just the reference solution.\r\n4. Decide whether the failure is agent fault, prompt ambiguity, or test flaw.\r\n\r\nDo not loosen tests merely to make agents pass. Relax tests only when the failed implementation is genuinely valid under the prompt.\r\n\r\n## 2.2 Agent Fault\r\n\r\nKeep the test unchanged when the agent missed a clear requirement.\r\n\r\nAgent fault examples:\r\n\r\n- prompt names a required public API and it is missing\r\n- prompt requires loading data and the implementation does not load it\r\n- prompt requires confirmation and implementation skips it\r\n- prompt requires refresh after an operation and implementation never refreshes\r\n- prompt requires a visible/data attribute and it is absent\r\n- prompt requires a user action to work and it does not work\r\n\r\nRepeated failures are not automatically test flaws. If several agents miss the same explicit requirement, the task may simply be challenging.\r\n\r\n## 2.3 Prompt Ambiguity\r\n\r\nChange the prompt or tests when multiple reasonable interpretations exist.\r\n\r\nAmbiguity examples:\r\n\r\n- tests require logic to live in a specific layer, but prompt only requires outcome\r\n- tests require synchronous behavior, but prompt allows debounced/async behavior\r\n- tests require direct platform calls, but a public helper is equally valid\r\n- tests require a specific control class, but prompt only requires a visible action\r\n- tests assert exact call shape, but prompt only requires the effect\r\n- tests assume a callback is awaitable, but the visible contract only requires eventual behavior\r\n- tests assume one service or module boundary, but the repo allows multiple equivalent integration points\r\n- tests assume one translation key pattern, but the prompt only requires visible text\r\n- tests assume one exact exported mutable object identity, but a valid implementation reads the same public state through a preserved reference\r\n- tests treat one mocked call as the only accepted proof of behavior, but the same visible effect can be observed another way\r\n\r\nFix ambiguity by either:\r\n\r\n- clarifying the prompt, if the detail is truly required, or\r\n- relaxing the test, if multiple implementations should be accepted.\r\n\r\n## 2.4 Test Flaw\r\n\r\nFix the test when it enforces an undocumented implementation detail.\r\n\r\nTest flaw examples:\r\n\r\n- exact CSS class names not in the prompt\r\n- exact DOM hierarchy not in the prompt\r\n- importing private files not named in the task\r\n- mocks inconsistent with real app state\r\n- mocked helpers hide valid delegated behavior\r\n- tests pass only because the reference solution used a specific structure\r\n- translation stubs that only recognize reference-solution key names\r\n- full module mocks that omit repo-valid exports a reasonable solution may use\r\n- reassigning mocked exported state in a way that breaks valid consumers of the original exported references\r\n- asserting one mocked internal function was called when the same visible effect could be produced through another public or repo-valid path\r\n- awaiting a callback and treating resolution timing as a requirement when the contract only promises the eventual side effect\r\n- requiring one exact config entry name, option ID, or registry key when the prompt only requires the visible capability\r\n\r\n## 2.6 Evaluation Checklist\r\n\r\nFor every failure cluster, answer:\r\n\r\n- Is this behavior named in the prompt?\r\n- Is the tested interface public?\r\n- Is the test checking outcome or implementation?\r\n- Could another reasonable implementation satisfy the prompt but fail this test?\r\n- Did mocks/fixtures remove a valid path?\r\n- Is the reference solution being treated as the only solution?\r\n\r\n---\r\n\r\n# 3. Fair vs Unfair Examples\r\n\r\n## Fair\r\n\r\n- Required exported function exists.\r\n- Required state includes specified fields.\r\n- Required UI appears only under specified conditions.\r\n- Required data attribute is present.\r\n- Required control label/key appears and triggers the required action.\r\n- Required confirmation appears before destructive action.\r\n- Required state refresh eventually occurs after operation.\r\n- A callback or user action eventually produces the required side effect, regardless of whether the implementation returns a promise.\r\n- A translated visible string appears, regardless of which internal translation key produced it.\r\n- A feature works through any repo-valid integration point that satisfies the prompt.\r\n- A mock preserves enough of the real module shape that reasonable uses of existing exports still work.\r\n\r\n## Unfair\r\n\r\n- Requires a private helper name.\r\n- Requires a particular component file path not specified.\r\n- Requires direct browser/platform API call when helper delegation is valid.\r\n- Requires exact class names used by the reference solution only.\r\n- Requires immediate update when debounced update is valid.\r\n- Requires exact function call arity when only message/effect matters.\r\n- Uses incomplete fixtures that break valid code paths.\r\n- Requires exact translation key names when the prompt only names the visible text.\r\n- Requires one mocked service export to be called when another repo-valid path produces the same effect.\r\n- Requires a callback to be awaitable or fully settled on return when the visible contract does not say so.\r\n- Requires one exact option ID or registry key when the prompt only requires the user-visible action.\r\n- Reassigns mocked exported state and then blames the implementation for not following the new object identity.\r\n- Treats one internal observation point as mandatory even though the same required outcome can be observed through another public or repo-valid path.\r\n\r\n---\r\n\r\n# 4. Patch Hygiene\r\n\r\nKeep changes separated when possible:\r\n\r\n- **Test patch**: tests, test setup, test runner/config.\r\n- **Solution patch**: implementation, styles, locales, app config.\r\n- Avoid unrelated dependency or lockfile changes.\r\n- Regenerate patches after every prompt/test/solution change.\r\n\r\n---";
+
+export function loadGuidelinesSections(): Record<ReviewerRoleKey, string> {
+  const parts = GUIDELINES_TEXT.split(/\r?\n(?=## )/g);
+  const sections: Partial<Record<ReviewerRoleKey, string>> = {};
+  for (const role of ROLES) {
+    const match = parts.find((p) => role.rubricHeading.test(p.trim()));
+    sections[role.key] = (match ?? GUIDELINES_TEXT).trim();
+  }
+  return sections as Record<ReviewerRoleKey, string>;
+}
+
+/** Full fairness methodology (agent-fault vs prompt-ambiguity vs test-flaw, fair/unfair examples). */
+export function loadFairnessRules(): string {
+  return RULES_TEXT.trim();
+}

package/src/state.ts

@@ -0,0 +1,31 @@
+/**
+ * Module-level run state shared between the /checks command handler and the
+ * cancel shortcut — encapsulated behind functions so both sides stay in sync
+ * without either needing to reassign another module's bindings directly.
+ */
+
+let reviewInProgress = false;
+let currentReviewAbort: AbortController | undefined;
+
+export function isReviewInProgress(): boolean {
+  return reviewInProgress;
+}
+
+/** Marks a run as started and returns the AbortController to pass down to the agents. */
+export function startReview(): AbortController {
+  reviewInProgress = true;
+  currentReviewAbort = new AbortController();
+  return currentReviewAbort;
+}
+
+export function endReview(): void {
+  reviewInProgress = false;
+  currentReviewAbort = undefined;
+}
+
+/** Returns true if a cancel was actually requested (i.e. a run was in progress and not already cancelled). */
+export function cancelReview(): boolean {
+  if (!reviewInProgress || !currentReviewAbort || currentReviewAbort.signal.aborted) return false;
+  currentReviewAbort.abort();
+  return true;
+}

package/src/tools.ts

@@ -0,0 +1,140 @@
+/**
+ * Custom tools the review/gap agents call to submit their structured results.
+ * Each tool just writes into a `capture` object passed in by the caller —
+ * that's how agents.ts pulls the final result back out of the agent session.
+ */
+
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import type { ReviewReport, TestGapCandidate, TestGapFinal } from "./types.js";
+
+export const REPORT_TOOL_NAME = "submit_review_report";
+export const GAP_FINDER_TOOL_NAME = "submit_test_gap_candidates";
+export const GAP_VALIDATOR_TOOL_NAME = "submit_filtered_test_gaps";
+
+// ── Reviewer report tool ─────────────────────────────────────────────
+
+const reportToolParams = Type.Object({
+  verdict: Type.Union([Type.Literal("PASS"), Type.Literal("FAIL")], {
+    description:
+      "FAIL only for a genuine blocking issue (clear rubric violation, agent-fault-worthy gap, unfair/undiscoverable " +
+      "test requirement, or a real determinism/regression risk). Optional, minor, or stylistic points are NOT grounds " +
+      "for FAIL — put those in `notes` instead and use PASS.",
+  }),
+  summary: Type.String({ description: "One short sentence summarizing the verdict." }),
+  reasons: Type.Array(Type.String(), {
+    description:
+      "Specific BLOCKING justifications only, citing rubric item IDs and concrete evidence from the files you read. " +
+      "Required (non-empty) when verdict is FAIL. Use an empty array when verdict is PASS.",
+  }),
+  notes: Type.Array(Type.String(), {
+    description:
+      "Non-blocking, optional/minor observations or suggested improvements — the kind of feedback a real reviewer " +
+      "leaves as 'Minor/optional' without failing the task. Include these regardless of verdict; use an empty array " +
+      "if you truly have none.",
+  }),
+});
+
+export function createReportTool(capture: { report?: ReviewReport }): ToolDefinition<typeof reportToolParams> {
+  return {
+    name: REPORT_TOOL_NAME,
+    label: "Submit Review Report",
+    description:
+      "Submit your final verdict for this review. This is the ONLY way to report your result — " +
+      "call it exactly once, as your last action, after you have finished reading and analyzing the relevant files. " +
+      "Do not write a plain-text final answer instead of calling this tool.",
+    parameters: reportToolParams,
+    async execute(_toolCallId, params) {
+      capture.report = {
+        verdict: params.verdict,
+        summary: params.summary,
+        reasons: params.reasons ?? [],
+        notes: params.notes ?? [],
+      };
+      return {
+        content: [{ type: "text", text: `Report recorded: ${params.verdict}` }],
+        details: undefined,
+      };
+    },
+  };
+}
+
+// ── Test-gap finder / filter tools ────────────────────────────────────
+
+const gapFinderToolParams = Type.Object({
+  gaps: Type.Array(
+    Type.Object({
+      description: Type.String({ description: "The specific untested behavior or edge case, in plain terms." }),
+      risk: Type.String({
+        description:
+          "Concretely why a plausible-but-incorrect implementation could still pass every test in test.patch " +
+          "despite missing or misimplementing this behavior.",
+      }),
+    }),
+    {
+      description:
+        "Candidate behavioral test gaps grounded in agent_prompt.md or clear existing repo behavior. Use an empty " +
+        "array if you found none — do not manufacture gaps just to report something.",
+    },
+  ),
+});
+
+export function createGapFinderTool(capture: {
+  gaps?: TestGapCandidate[];
+}): ToolDefinition<typeof gapFinderToolParams> {
+  return {
+    name: GAP_FINDER_TOOL_NAME,
+    label: "Submit Candidate Test Gaps",
+    description:
+      "Submit your candidate list of behavioral test gaps. This is the ONLY way to report your result — call it " +
+      "exactly once, as your last action, after you have finished reading and analyzing the relevant files.",
+    parameters: gapFinderToolParams,
+    async execute(_toolCallId, params) {
+      const gaps = params.gaps ?? [];
+      capture.gaps = gaps;
+      return {
+        content: [{ type: "text", text: `Recorded ${gaps.length} candidate gap(s)` }],
+        details: undefined,
+      };
+    },
+  };
+}
+
+const gapValidatorToolParams = Type.Object({
+  gaps: Type.Array(
+    Type.Object({
+      description: Type.String({ description: "The confirmed, real, fair test gap (may be reworded for clarity)." }),
+      justification: Type.String({
+        description:
+          "Why this is genuinely grounded in agent_prompt.md or the repo, fair to test per the fairness " +
+          "methodology, and a real (non-duplicate) coverage hole in test.patch.",
+      }),
+    }),
+    {
+      description:
+        "The filtered, final list of confirmed test gaps. Use an empty array if none of the candidates survive " +
+        "strict scrutiny.",
+    },
+  ),
+});
+
+export function createGapValidatorTool(capture: {
+  gaps?: TestGapFinal[];
+}): ToolDefinition<typeof gapValidatorToolParams> {
+  return {
+    name: GAP_VALIDATOR_TOOL_NAME,
+    label: "Submit Filtered Test Gaps",
+    description:
+      "Submit your final, strictly filtered list of confirmed test gaps. This is the ONLY way to report your " +
+      "result — call it exactly once, as your last action, after you have independently verified each candidate.",
+    parameters: gapValidatorToolParams,
+    async execute(_toolCallId, params) {
+      const gaps = params.gaps ?? [];
+      capture.gaps = gaps;
+      return {
+        content: [{ type: "text", text: `Confirmed ${gaps.length} gap(s) after filtering` }],
+        details: undefined,
+      };
+    },
+  };
+}

package/src/types.ts

@@ -0,0 +1,56 @@
+/**
+ * Shared types for the shipd-checks extension. Kept dependency-free (no
+ * imports from pi/typebox) so every other module can import from here
+ * without pulling in extra runtime surface.
+ */
+
+export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "off";
+
+export type Verdict = "PASS" | "FAIL";
+
+export interface ChecksConfig {
+  provider: string;
+  modelId: string;
+  thinkingLevel: ThinkingLevel;
+}
+
+export interface ReviewReport {
+  verdict: Verdict;
+  summary: string;
+  /** Blocking justifications. Required (non-empty) when verdict is FAIL, empty when PASS. */
+  reasons: string[];
+  /** Non-blocking, optional/minor suggestions — present regardless of verdict. */
+  notes: string[];
+}
+
+/** A candidate behavioral test gap proposed by the (unfiltered) gap-finder researcher agent. */
+export interface TestGapCandidate {
+  description: string;
+  risk: string;
+}
+
+/** A gap that survived the strict fairness-filter agent — goes into the final report. */
+export interface TestGapFinal {
+  description: string;
+  justification: string;
+}
+
+export type ReviewerRoleKey = "description" | "tests" | "solution";
+
+export interface ReviewerRole {
+  key: ReviewerRoleKey;
+  label: string;
+  rubricHeading: RegExp;
+}
+
+/** `ok` means the agent actually finished and submitted (possibly an empty list on purpose). */
+export interface GapStageResult<T> {
+  status: "ok" | "timedOut" | "cancelled" | "error" | "noSubmission";
+  gaps: T[];
+}
+
+export interface CommandOption {
+  value: string;
+  label: string;
+  description: string;
+}