@wyattjoh/demur 0.5.0 → 0.7.0

package/README.md

@@ -48,11 +48,13 @@ or sensitive names. Review TypeSafe's service terms and data-handling policy
 before enabling demur in a sensitive repository. Do not run secrets directly in
 shell arguments when the guard is active.
 
-Pi training capture stores the complete command, working directory, operating
-mode, full verdict and judgments, and resulting host action locally. These
-records can therefore contain secrets or sensitive names from shell arguments
-and paths. Training capture is off by default and is unavailable while the Pi
-integration is disabled.
+Pi training capture stores the complete command, exact TypeSafe state, local
+static-analysis result, model/question/policy versions, exact policy thresholds,
+operating mode, full verdict and judgments, and resulting host action locally.
+These records can therefore contain secrets or sensitive names from shell
+arguments, paths, Git
+context, and deterministic command analysis. Training capture is off by default
+and is unavailable while the Pi integration is disabled.
 
 ## Requirements
 
@@ -98,7 +100,7 @@ Pin a specific release when reproducibility matters:
 
 <!-- x-release-please-start-version -->
 ```sh
-pi install npm:@wyattjoh/demur@0.5.0
+pi install npm:@wyattjoh/demur@0.7.0
 ```
 <!-- x-release-please-end -->
 
@@ -111,7 +113,9 @@ pi
 The extension intercepts `bash` tool calls. Because Pi runs extensions under
 Node.js while demur uses `Bun.secrets`, the extension launches a package-local
 Bun worker for each judgment. The API key remains inside that worker; only the
-command request and resulting verdict cross its local stdio pipes. `ask` opens
+command request and resulting verdict cross its local stdio pipes. When training
+capture is enabled, the exact model state and local policy evidence return with
+the verdict for private local persistence. `ask` opens
 an interactive confirmation dialog; without an interactive UI, demur blocks the
 command.
 
@@ -166,8 +170,11 @@ Set `DEMUR_STATE_HOME` to place `usage.json`, `training.jsonl`, and
 demur-specific override takes precedence over the XDG and home-directory
 locations. Training records are retained until the user removes them. Human
 reviews are appended separately; accepted and corrected records are linked by a
-stable record ID, leaving the original evidence unchanged. A training-write
-failure is reported but never changes whether the command runs.
+stable record ID, leaving the original evidence unchanged. Corrected reviews
+also carry a machine-readable reason so recurring question, context, policy, and
+service failures can be measured without mining free-form notes. Version-one
+records and reviews remain readable. A training-write failure is reported but
+never changes whether the command runs.
 
 Pi packages execute with the user's full system permissions. Review this
 repository before installing it.
@@ -218,33 +225,58 @@ demur auth login
 demur auth status
 demur auth logout
 demur training review
-# Force the line-oriented interface for pipes or basic terminals:
-demur training review --plain
+demur training list --status=unreviewed --cwd=/workspace
+demur training list --status=deny --json
+demur training evaluate --json
+demur training review <record-id> --decision=deny --reason=recoverability --note="would destroy work" --json
 demur judge "git reset --hard HEAD~3"
 ```
 
 Bare `demur` opens the central OpenTUI interface when stdin and stdout are
 interactive. `demur training review` remains an explicit alias for the same
-interface. Its header shows the persisted global estimated cost, and each queue
-row shows that evaluation's estimated input cost. It starts in an `all` view;
-Tab and Shift-Tab rotate between `all`, `not reviewed`, `approved` (`allow`),
-`ask`, and `deny` views. The queue is focused initially: arrow keys navigate it,
-Up from its first result focuses a
-fuzzy working-directory filter, and another Up focuses the tab strip. Left and
-Right select adjacent focused tabs, while Down returns through the filter to the
-queue. Right from the queue focuses the scrollable detail pane.
+interface. `[` opens Reviews and `]` opens Settings. You can also navigate Up
+to the top-level section strip, use Left and Right to switch sections, and press
+Down or Enter to open one. Settings exposes every option from Pi's `/demur`
+menu—operating mode, training capture,
+and failure policy—and atomically saves each change to the same global
+configuration file. Use Up and Down to select a setting, Left and Right to
+change it in either direction, or Enter/Space to choose its next value.
+Disabling demur also turns training capture off, and training remains unavailable
+until an active mode is selected.
+
+The Reviews header shows the persisted global estimated cost, and each queue row
+shows the model decision in a muted semantic color. It starts in an `all` view;
+Tab and Shift-Tab rotate between `all`, `not reviewed`, `approved` (`allow`), `ask`,
+and `deny` views. The queue is focused initially: arrow keys navigate it, Up
+from its first result focuses a fuzzy working-directory filter, and another Up
+focuses the tab strip. Left and Right select adjacent focused tabs, while Down
+returns through the filter to the queue. Right from the queue focuses the
+scrollable detail pane.
 The detail pane supports arrows or `j`/`k`; Left returns to the queue. Page Up
 and Page Down page within the focused pane, and queue navigation stops at its
 first and last entries. Mouse clicks select tabs, records, the filter, or either
 pane; the wheel scrolls the queue and detail pane. Enter selects the original
 decision, `1`/`2`/`3` choose `allow`/`ask`/`deny`, `s`
-leaves a record for a later pass, and `q` or Escape stops. Previously reviewed
-records remain available, and changing an answer appends a review revision while
-preserving its visible history. The TUI remains open when a view is empty and
+leaves a record for a later pass, and `q` or Escape stops. A correction first
+asks for one structured reason, then accepts an optional note. Previously
+reviewed records remain available, and changing an answer appends a review
+revision while preserving its visible history. The TUI remains open when a view is empty and
 polls training state for newly captured or externally reviewed evaluations.
-Non-interactive invocations automatically use the line-oriented reviewer; pass
-`--plain` to select it explicitly. Reviews remain separate from the original
-evidence so they can later be curated into independently licensed eval fixtures.
+
+Flag-based training commands provide the same review operations without the
+TUI. `demur training list` accepts `--status` and fuzzy `--cwd` filters. Status
+is derived from the latest human review, so `allow`, `ask`, and `deny` select
+reviewed records while `unreviewed` selects records without a review. Record a
+new append-only review revision by passing a record ID and a required
+`--decision=<allow|ask|deny>`. Corrections also require `--reason` with one of
+`inert-or-read-only`, `sensitive-data`, `security-boundary`, `recoverability`,
+`shared-infrastructure`, `blast-radius`, `static-uncertainty`, `missing-context`,
+or `service-failure`; `--note` remains optional. Add `--json` to list, evaluate,
+or review to emit one versioned JSON document; JSON failures are written to
+stdout with a nonzero exit code. Interactive `demur training review` still opens
+the TUI, but without a terminal it requires an explicit list or record-ID review
+operation. Reviews remain separate from the original evidence so they can later
+be curated into independently licensed eval fixtures.
 
 From a development checkout, `bun run judge "<command>"` remains available.
 
@@ -305,11 +337,36 @@ and deterministic policy controls alongside demur.
 - `src/guard.internal.ts` — Effect-native orchestration and fail-closed recovery
 - `src/guard.ts` — managed runtime and Promise boundary
 - `src/training-review-model.ts` — historical review status and cwd filtering
+- `src/training-evaluation.ts` — offline correction metrics and threshold comparisons
 - `src/training-review-tui.tsx` — interactive OpenTUI training-review queue
 - `extensions/demur/` — Pi `tool_call` integration and training-state storage
 - `src/adapters/claude-code.ts` — Claude Code `PreToolUse` integration
 - `eval/` — safe synthetic contrast cases and the live evaluation runner
 
+## Training feedback evaluation
+
+`demur training evaluate` joins each record with its latest review and replays
+stored raw judgments through the shared `decide()` policy. By default it makes
+no TypeSafe requests. The report includes a decision matrix, an asymmetric weighted loss
+that penalizes unsafe false allows most heavily, correction counts grouped by
+structured reason, replay fidelity, and up to five single-threshold candidates
+that improve the observed records.
+
+Candidates are exploratory and are never applied automatically. The command
+scores them on the same private records used to discover them, so validate a
+candidate on an independent holdout and the synthetic corpus before changing
+`THRESHOLDS`. Newly captured version-two records include the exact model state
+and static-gate analysis required for complete replay; legacy records support
+policy-only replay.
+
+Pass `--replay` explicitly to send reviewed version-two records' previously
+captured model state to TypeSafe again using the current question set. This is a
+cost-bearing operation and re-discloses the stored command and context described
+above. It is capped at 20 requests by default; use `--limit=<1-100>` to choose a
+different bound. Commands remain data and are never executed. The replay report shows
+improvements, regressions, failures, token usage, and per-record raw judgments;
+use it when changing one question at a time.
+
 ## Synthetic evaluation
 
 The synthetic evaluation measures whether the two policy-qualification questions

package/extensions/demur/index.ts

@@ -7,7 +7,11 @@ import {
   type ToolCallEvent,
   type ToolCallEventResult,
 } from "@earendil-works/pi-coding-agent";
-import type { Verdict } from "../../src/types.ts";
+import type {
+  GuardEvaluation,
+  GuardEvidence,
+  Verdict,
+} from "../../src/types.ts";
 import {
   estimateInputCostUsd,
   formatUsd,
@@ -54,9 +58,15 @@ export async function handleToolCall(
   if (command.trim() === "") return undefined;
 
   const evaluationStartedAt = performance.now();
-  let verdict: Verdict;
+  let evaluation: GuardEvaluation;
   try {
-    verdict = await runGuardWorker(command, ctx.cwd, ctx.signal);
+    evaluation = await runGuardWorkerRequest(
+      command,
+      ctx.cwd,
+      ctx.signal,
+      process.env,
+      settings.training,
+    );
   } catch (error: unknown) {
     const evaluationMs = performance.now() - evaluationStartedAt;
     if (ctx.signal?.aborted) {
@@ -74,16 +84,20 @@ export async function handleToolCall(
       };
     }
 
-    verdict = {
-      decision: "deny",
-      reason: `demur: guard worker crashed — ${errorDetail(error)}`,
-      judgments: undefined,
-      failure: "unexpected",
-      latencyMs: evaluationMs,
-      usage: undefined,
+    evaluation = {
+      verdict: {
+        decision: "deny",
+        reason: `demur: guard worker crashed — ${errorDetail(error)}`,
+        judgments: undefined,
+        failure: "unexpected",
+        latencyMs: evaluationMs,
+        usage: undefined,
+      },
+      evidence: undefined,
     };
   }
 
+  const verdict = evaluation.verdict;
   const evaluationMs = performance.now() - evaluationStartedAt;
   const inputTokens = verdict.usage?.inputTokens;
   const accumulatedCostUsd = await recordAccumulatedCost(inputTokens);
@@ -109,6 +123,7 @@ export async function handleToolCall(
       ctx,
       settings.mode,
       verdict,
+      evaluation.evidence,
       result,
     );
   }
@@ -302,6 +317,18 @@ export function runGuardWorker(
   signal: AbortSignal | undefined,
   environment: NodeJS.ProcessEnv = process.env,
 ): Promise<Verdict> {
+  return runGuardWorkerRequest(command, cwd, signal, environment, false).then(
+    (evaluation) => evaluation.verdict,
+  );
+}
+
+function runGuardWorkerRequest(
+  command: string,
+  cwd: string,
+  signal: AbortSignal | undefined,
+  environment: NodeJS.ProcessEnv,
+  includeEvidence: boolean,
+): Promise<GuardEvaluation> {
   return new Promise((resolve, reject) => {
     if (signal?.aborted) {
       reject(new Error("guard request cancelled"));
@@ -333,7 +360,6 @@ export function runGuardWorker(
     signal?.addEventListener("abort", abort, { once: true });
     child.stdout.on("data", (chunk: Buffer) => {
       stdout = appendBounded(stdout, chunk);
-      if (stdout.length >= MAX_WORKER_OUTPUT_BYTES) child.kill();
     });
     child.stderr.on("data", (chunk: Buffer) => {
       stderr = appendBounded(stderr, chunk);
@@ -349,7 +375,7 @@ export function runGuardWorker(
         }
 
         try {
-          resolve(parseVerdict(stdout));
+          resolve(parseGuardWorkerResponse(stdout, includeEvidence));
         } catch (error: unknown) {
           reject(new Error(`invalid guard worker response: ${errorDetail(error)}`));
         }
@@ -357,7 +383,7 @@ export function runGuardWorker(
     });
 
     child.stdin.on("error", (error) => finish(() => reject(error)));
-    child.stdin.end(JSON.stringify({ command, cwd }));
+    child.stdin.end(JSON.stringify({ command, cwd, includeEvidence }));
   });
 }
 
@@ -496,14 +522,22 @@ async function recordTrainingResult(
   ctx: ExtensionContext,
   mode: "enforce" | "passive",
   verdict: Verdict,
+  evidence: GuardEvidence | undefined,
   result: ToolCallEventResult | undefined,
 ): Promise<void> {
   try {
+    if (verdict.judgments !== undefined && evidence === undefined) {
+      ctx.ui.notify(
+        "demur: recording training evaluation without replay evidence",
+        "warning",
+      );
+    }
     await recordTrainingEvaluation({
       command,
       cwd: ctx.cwd,
       mode,
       verdict,
+      evidence,
       hostAction: result?.block === true ? "block" : "allow",
     });
   } catch (error: unknown) {
@@ -630,8 +664,36 @@ function formatDecimal(value: number, fractionDigits: number): string {
     .replace(/\.0+$/, "");
 }
 
-function parseVerdict(output: string): Verdict {
-  const value: unknown = JSON.parse(output);
+function parseGuardWorkerResponse(
+  output: string,
+  includeEvidence: boolean,
+): GuardEvaluation {
+  if (!includeEvidence) {
+    return {
+      verdict: parseVerdictValue(JSON.parse(output)),
+      evidence: undefined,
+    };
+  }
+
+  const separator = output.indexOf("\n");
+  if (separator < 0) {
+    throw new Error("training guard response did not contain a verdict line");
+  }
+
+  const verdict = parseVerdictValue(JSON.parse(output.slice(0, separator)));
+  const encodedEvidence = output.slice(separator + 1);
+  try {
+    return {
+      verdict,
+      evidence: parseGuardEvidence(JSON.parse(encodedEvidence)),
+    };
+  } catch {
+    // Training evidence is best-effort and must never change the guard verdict.
+    return { verdict, evidence: undefined };
+  }
+}
+
+function parseVerdictValue(value: unknown): Verdict {
   if (value === null || typeof value !== "object") {
     throw new Error("verdict must be an object");
   }
@@ -657,6 +719,28 @@ function parseVerdict(output: string): Verdict {
   };
 }
 
+function parseGuardEvidence(value: unknown): GuardEvidence | undefined {
+  if (value === undefined || value === null) return undefined;
+  if (typeof value !== "object") {
+    throw new Error("guard evidence must be an object");
+  }
+
+  const evidence = value as Record<string, unknown>;
+  if (
+    evidence.modelState === null ||
+    typeof evidence.modelState !== "object" ||
+    typeof evidence.model !== "string" ||
+    !Number.isInteger(evidence.questionSetVersion) ||
+    !Number.isInteger(evidence.policyVersion) ||
+    evidence.policyThresholds === null ||
+    typeof evidence.policyThresholds !== "object"
+  ) {
+    throw new Error("guard evidence is incomplete");
+  }
+
+  return evidence as GuardEvidence;
+}
+
 function errorDetail(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }