@jwilger/pi-development-system 0.54.0 → 0.55.0

package/extensions/development-system.ts

@@ -20,6 +20,7 @@ import { createRequestApprovalTool } from "../src/gates/request-approval-tool.ts
 import { registerTestGuard } from "../src/gates/test-guard.ts";
 import { createJevHolder } from "../src/jev/holder.ts";
 import { createIntakeTool } from "../src/planning/intake-tool.ts";
+import { createTaskCheckTool } from "../src/planning/task-check-tool.ts";
 import { createReviewRecordTool, createReviewStartTool } from "../src/review/review-tools.ts";
 import { registerCiCommand } from "../src/state/ci-command.ts";
 import { loadConfig } from "../src/state/config.ts";
@@ -140,6 +141,7 @@ export function createDevelopmentSystem(pi: ExtensionAPI) {
   pi.registerTool(createModelsTool());
   pi.registerTool(createRouteTaskTool({ jev: (ctx) => jevHolder.forContext(ctx) }));
   pi.registerTool(createIntakeTool({ state, jev: (ctx) => jevHolder.forContext(ctx) }));
+  pi.registerTool(createTaskCheckTool({ jev: (ctx) => jevHolder.forContext(ctx) }));
   const reviewDeps = {
     state,
     jev: (ctx: ExtensionContext) => jevHolder.forContext(ctx),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jwilger/pi-development-system",
-  "version": "0.54.0",
+  "version": "0.55.0",
   "description": "A pi extension package representing a seasoned approach to software development using a full AI SDLC.",
   "keywords": [
     "pi-package"

package/src/jev/questions/readiness.ts

@@ -0,0 +1,82 @@
+import type { ClassifierBoolQuestion } from "@earendil-works/pi-ai";
+import { redactSecrets } from "../../core/redact.ts";
+import { err, ok, type Result } from "../../core/result.ts";
+import type { TaskRecord } from "../../planning/task-record.ts";
+import type { Jev, JevError } from "../client.ts";
+
+const vague = (instructions: string): ClassifierBoolQuestion => ({
+  type: "bool",
+  instructions,
+  criteria: {
+    true: "A weaker model implementing this alone would have to guess",
+    false: "It is specific enough to act on without guessing",
+  },
+});
+
+/** One narrow judgement per aspect; `tooBig` is separate because the fix is to split, not to add detail. */
+export const READINESS_QUESTIONS = {
+  goal: vague(
+    "`task.goal` should be one observable sentence: something a person or test could see change. Is it vague, a restatement of the title, or about activity rather than outcome?",
+  ),
+  interfaces: vague(
+    "`task.interfaces` should give exact signatures or contracts for what is created or changed. Is it missing signatures, hand-wavy ('a helper', 'some function') or inconsistent with `task.files`?",
+  ),
+  firstFailingTest: vague(
+    "`task.firstFailingTest` should name a test file and test and say what it asserts. Does it fail to say what would be asserted, or assert something unrelated to `task.goal`?",
+  ),
+  steps: vague(
+    "`task.steps` should be small steps a reviewer could reject independently. Are any steps large, combined ('implement and wire everything'), or out of order with the test-first step?",
+  ),
+  tooBig: {
+    type: "bool",
+    instructions:
+      "Is this more than one task: does it touch unrelated areas, or need more than one failing test to describe, so that it should be split before anyone implements it?",
+    criteria: {
+      true: "It should be split into two or more task records",
+      false: "It is one coherent task a single implementer can finish",
+    },
+  },
+} as const satisfies Record<string, ClassifierBoolQuestion>;
+
+type Readiness = "ready" | "needs-detail" | "too-big";
+export type ReadinessJudgement = { readiness: Readiness; missing: string[] };
+
+/** Probability at or above which an aspect counts as vague. */
+const VAGUE_AT = 0.6;
+
+const clip = (text: string, max: number): string =>
+  redactSecrets(text.slice(0, max * 2)).slice(0, max);
+
+export async function judgeTaskReadiness(
+  jev: Jev,
+  record: TaskRecord,
+): Promise<Result<ReadinessJudgement, JevError>> {
+  const asked = await jev.ask(
+    {
+      task: {
+        title: clip(record.title, 200),
+        goal: clip(record.goal, 600),
+        files: record.files.slice(0, 30).map((f) => clip(f, 200)),
+        interfaces: clip(record.interfaces, 1500),
+        firstFailingTest: clip(record.firstFailingTest, 600),
+        steps: record.steps.slice(0, 10).map((s) => clip(s, 300)),
+      },
+    },
+    READINESS_QUESTIONS,
+  );
+  if (!asked.ok) return asked;
+  const probability = new Map<string, number>();
+  for (const key of Object.keys(READINESS_QUESTIONS)) {
+    const answer = asked.value[key];
+    if (answer?.type !== "bool" || !Number.isFinite(answer.probability)) {
+      return err({ kind: "provider", message: `missing ${key} answer` });
+    }
+    probability.set(key, answer.probability);
+  }
+  if ((probability.get("tooBig") ?? 0) >= VAGUE_AT)
+    return ok({ readiness: "too-big", missing: [] });
+  const missing = Object.keys(READINESS_QUESTIONS).filter(
+    (k) => k !== "tooBig" && (probability.get(k) ?? 0) >= VAGUE_AT,
+  );
+  return ok({ readiness: missing.length > 0 ? "needs-detail" : "ready", missing });
+}

package/src/planning/task-check-tool.ts

@@ -0,0 +1,73 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute, relative, resolve } from "node:path";
+import type { ExtensionContext, ToolDefinition } from "@earendil-works/pi-coding-agent";
+import { type Static, Type } from "typebox";
+import { isParseError } from "../core/types.ts";
+import type { Jev } from "../jev/client.ts";
+import { judgeTaskReadiness } from "../jev/questions/readiness.ts";
+import { parseTaskRecord } from "./task-record.ts";
+
+const Parameters = Type.Object({
+  path: Type.String({
+    description: "Path to the task record markdown file, inside the repository.",
+  }),
+});
+
+const reply = (text: string, isError = false) => ({
+  content: [{ type: "text" as const, text }],
+  details: undefined,
+  isError,
+});
+
+const insideRepo = (cwd: string, target: string): boolean => {
+  const rel = relative(cwd, target);
+  return rel !== "" && !rel.startsWith("..") && !isAbsolute(rel);
+};
+
+/** `devsys_task_check`: structural readiness is deterministic; Jev adds "specific enough" and "one task". */
+export function createTaskCheckTool(deps: {
+  jev: (ctx: ExtensionContext) => Jev;
+}): ToolDefinition<typeof Parameters> {
+  return {
+    name: "devsys_task_check",
+    label: "Check task record",
+    description:
+      "Check a task record file against the task-record format and judge whether a weaker implementer could act on it. " +
+      "Run before handing a task record to an implementer subagent.",
+    promptSnippet: "Check a task record is ready for an implementer",
+    parameters: Parameters,
+    exposure: "direct",
+    async execute(_id, params: Static<typeof Parameters>, _signal, _onUpdate, ctx) {
+      const target = resolve(ctx.cwd, params.path);
+      if (!insideRepo(ctx.cwd, target))
+        return reply(`${params.path} is outside the repository`, true);
+      let markdown: string;
+      try {
+        markdown = await readFile(target, "utf8");
+      } catch (cause) {
+        return reply(
+          `cannot read ${params.path}: ${cause instanceof Error ? cause.message : String(cause)}`,
+          true,
+        );
+      }
+      const record = parseTaskRecord(markdown);
+      if (isParseError(record))
+        return reply(`${params.path} is not ready: ${record.message}`, true);
+      const judged = await judgeTaskReadiness(deps.jev(ctx), record);
+      if (!judged.ok) {
+        return reply(
+          `${record.id}: structure ok. Jev unavailable (${judged.error.kind}), so specificity and size were not judged; review the record yourself.`,
+        );
+      }
+      const { readiness, missing } = judged.value;
+      if (readiness === "ready")
+        return reply(`${record.id}: ready. Structure ok and specific enough for an implementer.`);
+      if (readiness === "too-big") {
+        return reply(
+          `${record.id}: too-big. Split it into task records that each need one failing test, then check each.`,
+        );
+      }
+      return reply(`${record.id}: needs-detail. Make these more specific: ${missing.join(", ")}.`);
+    },
+  };
+}

package/src/planning/task-record.ts

@@ -0,0 +1,121 @@
+import { type ParseError, parseError } from "../core/types.ts";
+
+/** Appendix C of the plan: the one-task-per-implementer format a weaker model can follow. */
+export type TaskRecord = {
+  readonly id: string;
+  readonly title: string;
+  readonly goal: string;
+  readonly files: readonly string[];
+  readonly interfaces: string;
+  readonly firstFailingTest: string;
+  readonly steps: readonly string[];
+  readonly run: string;
+  readonly expected: string;
+  readonly outOfScope: string;
+};
+
+const SECTIONS = [
+  "Goal",
+  "Files",
+  "Interfaces",
+  "First failing test",
+  "Steps",
+  "Run",
+  "Expected",
+  "Out of scope",
+] as const;
+type Section = (typeof SECTIONS)[number];
+
+const HEADER = /^## (\S+) [—-] (.+)$/;
+const SECTION_LINE = /^\*\*([^:*]+):\*\*\s*(.*)$/;
+const STEP = /^\s*(?:\d+[.)]|[-*])\s+(.*)$/;
+const STEPS_MIN = 3;
+const STEPS_MAX = 7;
+/** Words that stand in for a command or an observable result. */
+const PLACEHOLDER = /^(?:n\/?a|none|todo|tbc|-|…|\.\.\.|works?|it works|passes|ok)\.?$/i;
+
+const isSection = (name: string): name is Section => SECTIONS.some((s) => s === name);
+
+/** Splits the body into section texts keyed by name; text may continue over several lines. */
+function splitSections(lines: readonly string[]): Map<Section, string> {
+  const found = new Map<Section, string[]>();
+  let current: string[] | undefined;
+  for (const line of lines) {
+    const match = SECTION_LINE.exec(line);
+    const name = match?.[1]?.trim();
+    if (match !== null && name !== undefined && isSection(name)) {
+      current = match[2] === "" ? [] : [match[2] ?? ""];
+      found.set(name, current);
+    } else current?.push(line);
+  }
+  return new Map([...found].map(([name, text]) => [name, text.join("\n").trim()]));
+}
+
+const stripTicks = (text: string): string => text.replace(/^`+|`+$/g, "").trim();
+
+const stepsOf = (text: string): string[] =>
+  text.split("\n").flatMap((line) => {
+    const item = STEP.exec(line)?.[1]?.trim();
+    return item === undefined || item === "" ? [] : [item];
+  });
+
+function concrete(name: "Run" | "Expected", text: string): string | undefined {
+  const bare = stripTicks(text);
+  if (PLACEHOLDER.test(bare))
+    return `${name} must be a concrete ${name === "Run" ? "command" : "observable result"}, not "${bare}"`;
+  if (name === "Expected" && bare.split(/\s+/).length < 2) {
+    return "Expected must say what is observed (at least two words, e.g. a count, output or exit status)";
+  }
+  return undefined;
+}
+
+const get = (sections: ReadonlyMap<Section, string>, s: Section): string => sections.get(s) ?? "";
+
+/** Every readiness problem in the sections, so the author fixes them in one pass. */
+function problemsOf(markdown: string, sections: ReadonlyMap<Section, string>): string[] {
+  const problems: string[] = [];
+  const missing = SECTIONS.filter((s) => get(sections, s) === "");
+  if (missing.length > 0) problems.push(`missing section: ${missing.join(", ")}`);
+  if (/\bTBD\b/.test(markdown)) problems.push("contains TBD; decide it or split the task");
+  const steps = stepsOf(get(sections, "Steps"));
+  if (!missing.includes("Steps") && (steps.length < STEPS_MIN || steps.length > STEPS_MAX)) {
+    problems.push(`Steps must be ${STEPS_MIN}-${STEPS_MAX} steps, found ${steps.length}`);
+  }
+  for (const name of ["Run", "Expected"] as const) {
+    const why = missing.includes(name) ? undefined : concrete(name, get(sections, name));
+    if (why !== undefined) problems.push(why);
+  }
+  return problems;
+}
+
+const filesOf = (text: string): string[] =>
+  text
+    .split(/[,\n]/)
+    .map((f) => stripTicks(f.trim()))
+    .filter((f) => f !== "");
+
+/** Parses one task record, reporting every problem together. */
+export function parseTaskRecord(markdown: string): TaskRecord | ParseError {
+  const lines = markdown.split("\n");
+  const headerAt = lines.findIndex((l) => l.startsWith("## "));
+  const header = headerAt < 0 ? null : HEADER.exec(lines[headerAt] ?? "");
+  if (header === null) {
+    return parseError("header must be `## <id> — <title>` (an id, an em dash, then the title)");
+  }
+  const sections = splitSections(lines.slice(headerAt + 1));
+  const problems = problemsOf(markdown, sections);
+  if (problems.length > 0) return parseError(problems.join("; "));
+  const text = (s: Section): string => get(sections, s);
+  return {
+    id: header[1] ?? "",
+    title: header[2]?.trim() ?? "",
+    goal: text("Goal"),
+    files: filesOf(text("Files")),
+    interfaces: text("Interfaces"),
+    firstFailingTest: text("First failing test"),
+    steps: stepsOf(text("Steps")),
+    run: stripTicks(text("Run")),
+    expected: text("Expected"),
+    outOfScope: text("Out of scope"),
+  };
+}