@jwilger/pi-development-system 0.1.2 → 0.3.0

package/src/jev/client.ts

@@ -0,0 +1,127 @@
+import { createHash } from "node:crypto";
+import type {
+  ClassifierAnswer,
+  ClassifierApi,
+  ClassifierContext,
+  ClassifierModel,
+  ClassifierQuestion,
+  ClassifierResult,
+  JsonObject,
+} from "@earendil-works/pi-ai";
+import { err, ok, type Result } from "../core/result.ts";
+import { splitModelRef } from "./models.ts";
+
+export type JevAvailability = "online" | "offline" | "unknown";
+export type JevError =
+  | { kind: "no-model" }
+  | { kind: "timeout" }
+  | { kind: "provider"; message: string }
+  | { kind: "aborted" };
+
+type Classifier = ClassifierModel<ClassifierApi>;
+
+/** The slice of pi's `ctx.modelRegistry` that Jev needs; tests pass a fake. */
+export interface ClassifierRegistry {
+  findOfType(type: "classifier", provider: string, modelId: string): Classifier | undefined;
+  // biome-ignore lint/suspicious/noExplicitAny: structural bridge to the host's Model<Api> parameter
+  hasConfiguredAuth(model: any): boolean;
+  classify(
+    model: Classifier,
+    context: ClassifierContext,
+    options?: { signal?: AbortSignal },
+  ): Promise<ClassifierResult>;
+}
+
+export type JevOptions = {
+  registry: ClassifierRegistry;
+  candidates: readonly string[];
+  timeoutMs: number;
+  cache: Map<string, ClassifierResult>;
+  now(): number;
+};
+
+export interface Jev {
+  ask(
+    state: JsonObject,
+    questions: Record<string, ClassifierQuestion>,
+  ): Promise<Result<Record<string, ClassifierAnswer>, JevError>>;
+  availability(): JevAvailability;
+  model(): string | undefined;
+}
+
+/** First candidate `provider/id` that exists in the registry AND has configured auth. */
+export function resolveJevModel(
+  registry: ClassifierRegistry,
+  candidates: readonly string[],
+): Classifier | undefined {
+  for (const ref of candidates) {
+    const parts = splitModelRef(ref);
+    if (parts === undefined) continue;
+    const model = registry.findOfType("classifier", parts.provider, parts.id);
+    if (model !== undefined && registry.hasConfiguredAuth(model)) return model;
+  }
+  return undefined;
+}
+
+const hashOf = (state: JsonObject, questions: Record<string, ClassifierQuestion>): string =>
+  createHash("sha256").update(JSON.stringify({ state, questions })).digest("hex");
+
+/** Wraps the host classifier: model resolution, timeout, caching and typed errors. Never rejects. */
+export function createJev(opts: JevOptions): Jev {
+  let availability: JevAvailability = "unknown";
+  let current: string | undefined;
+
+  const run = async (
+    model: Classifier,
+    context: ClassifierContext,
+  ): Promise<Result<ClassifierResult, JevError>> => {
+    const controller = new AbortController();
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const timeout = new Promise<"timeout">((resolve) => {
+      timer = setTimeout(() => {
+        controller.abort();
+        resolve("timeout");
+      }, opts.timeoutMs);
+    });
+    try {
+      const outcome = await Promise.race([
+        opts.registry.classify(model, context, { signal: controller.signal }),
+        timeout,
+      ]);
+      if (outcome === "timeout") return err({ kind: "timeout" });
+      if (outcome.stopReason === "aborted") return err({ kind: "aborted" });
+      if (outcome.stopReason === "error")
+        return err({ kind: "provider", message: outcome.errorMessage ?? "classifier error" });
+      return ok(outcome);
+    } catch (e) {
+      return err({ kind: "provider", message: e instanceof Error ? e.message : String(e) });
+    } finally {
+      if (timer !== undefined) clearTimeout(timer);
+    }
+  };
+
+  return {
+    async ask(state, questions) {
+      const model = resolveJevModel(opts.registry, opts.candidates);
+      if (model === undefined) {
+        availability = "offline";
+        current = undefined;
+        return err({ kind: "no-model" });
+      }
+      current = `${model.provider}/${model.id}`;
+      const key = hashOf(state, questions);
+      const cached = opts.cache.get(key);
+      if (cached !== undefined) return ok(cached.answers);
+      const result = await run(model, { state, questions });
+      if (!result.ok) {
+        availability = result.error.kind === "aborted" ? availability : "offline";
+        return result;
+      }
+      availability = "online";
+      opts.cache.set(key, result.value);
+      return ok(result.value.answers);
+    },
+    availability: () => availability,
+    model: () => current,
+  };
+}

package/src/jev/holder.ts

@@ -0,0 +1,54 @@
+import type { ClassifierResult } from "@earendil-works/pi-ai";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import {
+  type ClassifierRegistry,
+  createJev,
+  type Jev,
+  type JevAvailability,
+  resolveJevModel,
+} from "./client.ts";
+import { DEFAULT_JEV_CANDIDATES } from "./models.ts";
+
+export type JevHolder = {
+  /** The Jev bound to this context's model registry (one per registry, shared cache). */
+  forContext(ctx: Pick<ExtensionContext, "modelRegistry">): Jev;
+  /** Cheap, offline probe of whether a Jev credential is configured. */
+  probe(ctx: Pick<ExtensionContext, "modelRegistry">): {
+    availability: JevAvailability;
+    model: string | undefined;
+  };
+};
+
+export function createJevHolder(opts?: {
+  candidates?: readonly string[];
+  timeoutMs?: number;
+  now?: () => number;
+}): JevHolder {
+  const candidates = opts?.candidates ?? DEFAULT_JEV_CANDIDATES;
+  const cache = new Map<string, ClassifierResult>();
+  const bound = new WeakMap<object, Jev>();
+  const registryOf = (ctx: Pick<ExtensionContext, "modelRegistry">): ClassifierRegistry =>
+    ctx.modelRegistry;
+  return {
+    forContext(ctx) {
+      const registry = registryOf(ctx);
+      const existing = bound.get(registry);
+      if (existing !== undefined) return existing;
+      const jev = createJev({
+        registry,
+        candidates,
+        timeoutMs: opts?.timeoutMs ?? 15_000,
+        cache,
+        now: opts?.now ?? Date.now,
+      });
+      bound.set(registry, jev);
+      return jev;
+    },
+    probe(ctx) {
+      const model = resolveJevModel(registryOf(ctx), candidates);
+      return model === undefined
+        ? { availability: "offline", model: undefined }
+        : { availability: "online", model: `${model.provider}/${model.id}` };
+    },
+  };
+}

package/src/jev/models.ts

@@ -0,0 +1,15 @@
+/** Default Jev classifier candidates, tried in order; overridable by `[models] jev`. */
+export const DEFAULT_JEV_CANDIDATES: readonly string[] = [
+  "typesafe/jev-latest",
+  "openrouter/typesafe/jev-latest",
+  "opencode/jev-1.13",
+  "cloudflare-workers-ai/typesafe/jev",
+  "vercel-ai-gateway/typesafe-ai/jev",
+];
+
+/** Splits `provider/id` on the first slash (ids may themselves contain slashes). */
+export function splitModelRef(ref: string): { provider: string; id: string } | undefined {
+  const at = ref.indexOf("/");
+  if (at <= 0 || at === ref.length - 1) return undefined;
+  return { provider: ref.slice(0, at), id: ref.slice(at + 1) };
+}

package/src/jev/questions/shell-intent.ts

@@ -0,0 +1,55 @@
+import type { ClassifierChoiceQuestion } from "@earendil-works/pi-ai";
+import type { GitIntent } from "../../core/git-intent.ts";
+import { err, ok, type Result } from "../../core/result.ts";
+import type { Jev, JevError } from "../client.ts";
+
+export const SHELL_INTENT_THRESHOLD = 0.6;
+
+export const SHELL_INTENT_QUESTION: ClassifierChoiceQuestion = {
+  type: "choice",
+  instructions:
+    "A coding agent is about to run the shell command in `command`. Which irreversible git operation, if any, would running it perform? Consider wrappers, subshells, variables, pipes and scripts. Choose `ordinary` when it performs none of them (including commands that only mention git text).",
+  criteria: {
+    "force-push":
+      "Pushes with --force, --force-with-lease, --mirror, or a +refspec, overwriting remote history",
+    "branch-delete-remote": "Deletes a branch or tag on a remote (push --delete, push :ref)",
+    "history-rewrite": "Rewrites local history: commit --amend, rebase, filter-branch, filter-repo",
+    "destructive-reset":
+      "Discards work irrecoverably: reset --hard, clean -fd, checkout -- . over uncommitted changes",
+    "no-verify":
+      "Skips commit/push hooks: --no-verify, -n on commit, core.hooksPath override, LEFTHOOK=0 / HUSKY=0",
+    ordinary: "None of the above; read-only or reversible work",
+  },
+};
+
+const INTENTS: readonly GitIntent[] = [
+  "force-push",
+  "branch-delete-remote",
+  "history-rewrite",
+  "destructive-reset",
+  "no-verify",
+  "ordinary",
+];
+
+const isIntent = (value: string): value is GitIntent => INTENTS.some((i) => i === value);
+
+/** Pure policy: a confident Jev choice maps to its intent; low confidence or an unknown label → `unknown`. */
+export function intentFromChoice(choice: string, confidence: number): GitIntent {
+  if (confidence < SHELL_INTENT_THRESHOLD) return "unknown";
+  return isIntent(choice) ? choice : "unknown";
+}
+
+export async function judgeShellIntent(
+  jev: Jev,
+  command: string,
+): Promise<Result<{ intent: GitIntent; confidence: number }, JevError>> {
+  const asked = await jev.ask({ command }, { intent: SHELL_INTENT_QUESTION });
+  if (!asked.ok) return asked;
+  const answer = asked.value.intent;
+  if (answer?.type !== "choice")
+    return err({ kind: "provider", message: "missing shell-intent answer" });
+  return ok({
+    intent: intentFromChoice(answer.choice, answer.confidence),
+    confidence: answer.confidence,
+  });
+}

package/src/jev/questions/test-change.ts

@@ -0,0 +1,76 @@
+import type { ClassifierBoolQuestion, ClassifierChoiceQuestion } from "@earendil-works/pi-ai";
+import { err, ok, type Result } from "../../core/result.ts";
+import type { Jev, JevError } from "../client.ts";
+
+export const WEAKENS_THRESHOLD = 0.7;
+export const MOTIVE_THRESHOLD = 0.6;
+
+export type TestChangeMotive = "requirement-change" | "gate-gaming" | "refactor" | "unclear";
+export type TestChangeInput = {
+  path: string;
+  before?: string;
+  after?: string;
+  recentFailure?: string;
+};
+
+export const WEAKENS_QUESTION: ClassifierBoolQuestion = {
+  type: "bool",
+  instructions:
+    "Does the change to the test file in `path` (from `before` to `after`; `after` absent means deleted) weaken what the tests verify — removing or loosening assertions, skipping or deleting tests, widening expected values, or catching and ignoring failures?",
+  criteria: {
+    true: "The tests verify less behaviour afterwards (assertions removed or loosened, tests skipped, disabled or deleted)",
+    false:
+      "The tests verify the same or more behaviour (added tests, stronger assertions, renames, pure refactors)",
+  },
+};
+
+export const MOTIVE_QUESTION: ClassifierChoiceQuestion = {
+  type: "choice",
+  instructions:
+    "Why is the test file in `path` being changed from `before` to `after`? `recentFailure` is the most recent failing output, if any. Choose `gate-gaming` only when the change makes a failing check pass by weakening the check instead of fixing the code.",
+  criteria: {
+    "requirement-change":
+      "The behaviour under test was deliberately changed, so the test must change with it",
+    "gate-gaming": "A failing test is being weakened, skipped or deleted to get a green result",
+    refactor: "Structure of the test changes with no change in what is verified",
+    unclear: "Cannot tell from the evidence",
+  },
+};
+
+const MOTIVES: readonly TestChangeMotive[] = [
+  "requirement-change",
+  "gate-gaming",
+  "refactor",
+  "unclear",
+];
+const isMotive = (value: string): value is TestChangeMotive => MOTIVES.some((m) => m === value);
+
+/** Pure policy: low confidence or an unexpected label → `unclear`. */
+export const motiveFromChoice = (choice: string, confidence: number): TestChangeMotive =>
+  confidence >= MOTIVE_THRESHOLD && isMotive(choice) ? choice : "unclear";
+
+export async function judgeTestChange(
+  jev: Jev,
+  input: TestChangeInput,
+): Promise<Result<{ weakens: number; motive: TestChangeMotive; confidence: number }, JevError>> {
+  const clip = (text: string | undefined): string => (text ?? "").slice(0, 6000);
+  const state = {
+    path: input.path,
+    before: clip(input.before),
+    after: input.after === undefined ? "" : clip(input.after),
+    deleted: input.after === undefined,
+    recentFailure: clip(input.recentFailure),
+  };
+  const asked = await jev.ask(state, { weakens: WEAKENS_QUESTION, motive: MOTIVE_QUESTION });
+  if (!asked.ok) return asked;
+  const weakens = asked.value.weakens;
+  const motive = asked.value.motive;
+  if (weakens?.type !== "bool" || motive?.type !== "choice") {
+    return err({ kind: "provider", message: "missing test-change answers" });
+  }
+  return ok({
+    weakens: weakens.probability,
+    motive: motiveFromChoice(motive.choice, motive.confidence),
+    confidence: motive.confidence,
+  });
+}

package/src/state/decision-log.ts

@@ -0,0 +1,28 @@
+import { appendFile, mkdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { type Departure, renderDepartureMarkdown } from "../core/departure.ts";
+
+const exists = async (path: string): Promise<boolean> =>
+  stat(path).then(
+    () => true,
+    () => false,
+  );
+
+/** Appends a departure to `docs/decisions/YYYY-MM.md` (UTC month of `now`), creating it with a header. */
+export async function appendDecision(
+  repoRoot: string,
+  d: Departure,
+  now: Date,
+): Promise<{ path: string }> {
+  const month = now.toISOString().slice(0, 7);
+  const dir = join(repoRoot, "docs", "decisions");
+  const path = join(dir, `${month}.md`);
+  await mkdir(dir, { recursive: true });
+  const entry = renderDepartureMarkdown(d);
+  if (await exists(path)) {
+    await appendFile(path, `\n${entry}`);
+  } else {
+    await appendFile(path, `# Decisions — ${month}\n\n${entry}`);
+  }
+  return { path };
+}

package/src/state/session-state.ts

@@ -1,4 +1,5 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { parseDeparture } from "../core/departure.ts";
 import {
   type Departure,
   type DevsysState,
@@ -28,36 +29,18 @@ const JEV: ReadonlyArray<JevStatus> = ["online", "offline", "unknown"];
 const isRecord = (v: unknown): v is Record<string, unknown> =>
   typeof v === "object" && v !== null && !Array.isArray(v);
 
-const isString = (v: unknown): v is string => typeof v === "string";
-
-/** Structural check for a persisted Departure; I1.1's parseDeparture should supersede it. */
-function isDeparture(v: unknown): v is Departure {
-  if (!isRecord(v)) return false;
-  const scope = v["scope"];
-  const scopeOk =
-    isRecord(scope) &&
-    (scope["kind"] === "session" ||
-      (scope["kind"] === "slice" && isString(scope["slice"])) ||
-      (scope["kind"] === "once" && isString(scope["toolCallId"])));
-  return (
-    scopeOk &&
-    ["id", "gate", "default", "chosen", "why", "costIfWrong", "recordedAt"].every((k) =>
-      isString(v[k]),
-    ) &&
-    (v["tier"] === "soft" || v["tier"] === "hard") &&
-    (v["approver"] === "agent" || v["approver"] === "user") &&
-    (v["revisitWhen"] === undefined || isString(v["revisitWhen"]))
-  );
-}
-
 /** Boundary parse for persisted state: the only place state is cast from `unknown`. */
 export function parseDevsysState(input: unknown): DevsysState | ParseError {
   if (!isRecord(input)) return parseError("devsys state must be an object");
   const { phase, sizing, activeSlice, openDepartures, jev, lastPushAt } = input;
   if (!PHASES.includes(phase as Phase)) return parseError(`unknown phase: ${String(phase)}`);
   if (!JEV.includes(jev as JevStatus)) return parseError(`unknown jev status: ${String(jev)}`);
-  if (!Array.isArray(openDepartures) || !openDepartures.every(isDeparture)) {
-    return parseError("openDepartures must be an array of valid departures");
+  if (!Array.isArray(openDepartures)) return parseError("openDepartures must be an array");
+  const departures: Departure[] = [];
+  for (const raw of openDepartures) {
+    const departure = parseDeparture(raw);
+    if (isParseError(departure)) return departure;
+    departures.push(departure);
   }
   if (sizing !== undefined && !SIZINGS.includes(sizing as Sizing)) {
     return parseError(`unknown sizing: ${String(sizing)}`);
@@ -71,7 +54,7 @@ export function parseDevsysState(input: unknown): DevsysState | ParseError {
   return {
     phase: phase as Phase,
     jev: jev as JevStatus,
-    openDepartures: openDepartures,
+    openDepartures: departures,
     ...(sizing !== undefined ? { sizing: sizing as Sizing } : {}),
     ...(activeSlice !== undefined ? { activeSlice: activeSlice as SliceRef } : {}),
     ...(lastPushAt !== undefined ? { lastPushAt } : {}),