@wyattjoh/demur 0.6.0 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wyattjoh/demur",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "type": "module",
   "description": "A proof-of-concept destructive-command guard for coding agents.",
   "license": "MIT",
@@ -39,6 +39,7 @@
     "src/questions.ts",
     "src/state.ts",
     "src/settings-model.ts",
+    "src/training-evaluation.ts",
     "src/training-review-model.ts",
     "src/training-review-tui.tsx",
     "src/types.ts"

package/src/adapters/pi-worker.ts

@@ -1,14 +1,22 @@
 #!/usr/bin/env bun
-import { guard } from "../guard.ts";
+import { guard, guardWithEvidence } from "../guard.ts";
 
 type GuardRequest = {
   command: string;
   cwd: string;
+  includeEvidence: boolean;
 };
 
 const request = parseRequest(await Bun.stdin.text());
-const verdict = await guard(request.command, request.cwd, "pi");
-process.stdout.write(JSON.stringify(verdict));
+if (request.includeEvidence) {
+  const evaluation = await guardWithEvidence(request.command, request.cwd, "pi");
+  process.stdout.write(`${JSON.stringify(evaluation.verdict)}\n`);
+  process.stdout.write(JSON.stringify(evaluation.evidence ?? null));
+} else {
+  process.stdout.write(
+    JSON.stringify(await guard(request.command, request.cwd, "pi")),
+  );
+}
 
 function parseRequest(input: string): GuardRequest {
   const value: unknown = JSON.parse(input);
@@ -16,10 +24,16 @@ function parseRequest(input: string): GuardRequest {
     throw new Error("guard request must be an object");
   }
 
-  const { command, cwd } = value as Record<string, unknown>;
-  if (typeof command !== "string" || typeof cwd !== "string") {
-    throw new Error("guard request must contain string command and cwd fields");
+  const { command, cwd, includeEvidence } = value as Record<string, unknown>;
+  if (
+    typeof command !== "string" ||
+    typeof cwd !== "string" ||
+    (includeEvidence !== undefined && typeof includeEvidence !== "boolean")
+  ) {
+    throw new Error(
+      "guard request must contain string command and cwd fields and an optional boolean includeEvidence field",
+    );
   }
 
-  return { command, cwd };
+  return { command, cwd, includeEvidence: includeEvidence === true };
 }

package/src/cli.ts

@@ -7,6 +7,7 @@ import {
   type DemurSettings,
 } from "../extensions/demur/settings.ts";
 import {
+  isTrainingCorrectionReason,
   loadTrainingRecords,
   loadTrainingReviews,
   recordTrainingReview,
@@ -15,12 +16,17 @@ import {
   type TrainingReviewInput,
 } from "../extensions/demur/training-store.ts";
 import { guard } from "./guard.ts";
+import { judgeRenderedState, type JudgeResult } from "./judge.ts";
 import {
   deleteApiKey,
   resolveApiKey,
   storeApiKey,
   type ResolvedApiKey,
 } from "./key.ts";
+import {
+  analyzeTrainingFeedback,
+  replayTrainingQuestions,
+} from "./training-evaluation.ts";
 import {
   buildTrainingReviewEntries,
   createTrainingReviewInput,
@@ -32,7 +38,7 @@ import {
   type TrainingReviewSnapshot,
 } from "./training-review-model.ts";
 import type { TrainingReviewTuiResult } from "./training-review-tui.tsx";
-import type { Verdict } from "./types.ts";
+import type { RenderedCommandState, Verdict } from "./types.ts";
 
 const USAGE = `Usage:
   demur
@@ -40,8 +46,9 @@ const USAGE = `Usage:
   demur auth status
   demur auth logout
   demur training list [--status=<all|unreviewed|allow|ask|deny>] [--cwd=<query>] [--json]
+  demur training evaluate [--replay] [--limit=<1-100>] [--json]
   demur training review
-  demur training review <record-id> --decision=<allow|ask|deny> [--note=<text>] [--json]
+  demur training review <record-id> --decision=<allow|ask|deny> [--reason=<correction-reason>] [--note=<text>] [--json]
   demur judge "<command>" [--cwd=<path>]`;
 
 /**
@@ -49,6 +56,7 @@ const USAGE = `Usage:
  */
 export type CliDependencies = {
   judge(command: string, cwd: string): Promise<Verdict>;
+  judgeTrainingState(state: RenderedCommandState): Promise<JudgeResult>;
   resolveApiKey(): Promise<ResolvedApiKey | undefined>;
   storeApiKey(value: string): Promise<void>;
   deleteApiKey(): Promise<boolean>;
@@ -74,6 +82,7 @@ export type CliDependencies = {
 
 const defaultDependencies: CliDependencies = {
   judge: (command, cwd) => guard(command, cwd, "cli"),
+  judgeTrainingState: judgeRenderedState,
   resolveApiKey,
   storeApiKey,
   deleteApiKey,
@@ -246,6 +255,10 @@ async function runTraining(
     return runTrainingList(args.slice(1), dependencies);
   }
 
+  if (command === "evaluate") {
+    return runTrainingEvaluate(args.slice(1), dependencies);
+  }
+
   if (command === "review") {
     if (args.length === 1) {
       return runInteractiveTrainingReview(dependencies);
@@ -256,7 +269,7 @@ async function runTraining(
   return writeTrainingUsageError(
     trainingOperation(["training", ...args]),
     args.includes("--json"),
-    "expected `training list` or `training review`",
+    "expected `training list`, `training evaluate`, or `training review`",
     dependencies,
   );
 }
@@ -370,6 +383,120 @@ async function runTrainingList(
   return 0;
 }
 
+async function runTrainingEvaluate(
+  args: ReadonlyArray<string>,
+  dependencies: CliDependencies,
+): Promise<number> {
+  const operation = "training.evaluate";
+  const parsedResult = parseArguments(
+    args,
+    new Set(["json", "replay"]),
+    new Set(["limit"]),
+  );
+  const json = args.includes("--json");
+  if (parsedResult.error !== undefined) {
+    return writeTrainingUsageError(
+      operation,
+      json,
+      parsedResult.error,
+      dependencies,
+    );
+  }
+
+  const parsed = parsedResult.parsed;
+  if (parsed.positionals.length !== 0) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`training evaluate` does not accept positional arguments",
+      dependencies,
+    );
+  }
+
+  const limitValue = getStringOption(parsed, "limit");
+  if (limitValue !== undefined && !hasOption(parsed, "replay")) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`--limit` requires `--replay`",
+      dependencies,
+    );
+  }
+  const replayLimit = limitValue === undefined ? 20 : Number(limitValue);
+  if (
+    !Number.isInteger(replayLimit) ||
+    replayLimit < 1 ||
+    replayLimit > 100
+  ) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`--limit` must be an integer from 1 to 100",
+      dependencies,
+    );
+  }
+
+  const entries = await loadTrainingEntries(dependencies);
+  const report = analyzeTrainingFeedback(entries);
+  const replay = hasOption(parsed, "replay")
+    ? await replayTrainingQuestions(
+      entries,
+      dependencies.judgeTrainingState,
+      replayLimit,
+    )
+    : undefined;
+  if (hasOption(parsed, "json")) {
+    writeTrainingJsonSuccess(
+      operation,
+      replay === undefined ? report : { offline: report, replay },
+      dependencies,
+    );
+    return 0;
+  }
+
+  dependencies.stdout(
+    `Reviewed ${report.reviewed}; evaluable ${report.evaluable}; unavailable ${report.unavailable}.`,
+  );
+  dependencies.stdout(
+    `Replay fidelity: ${report.completeReplayRecords} complete, ${report.policyOnlyRecords} policy-only legacy.`,
+  );
+  dependencies.stdout(
+    `Current policy: ${report.current.matches}/${report.current.evaluated} match; weighted loss ${report.current.weightedLoss}.`,
+  );
+
+  const reasons = Object.entries(report.correctionsByReason);
+  if (reasons.length > 0) {
+    dependencies.stdout(
+      `Correction reasons: ${reasons.map(([reason, count]) => `${reason}=${count}`).join(", ")}.`,
+    );
+  }
+
+  if (report.candidates.length === 0) {
+    dependencies.stdout("No single-threshold candidate improved observed weighted loss.");
+  } else {
+    dependencies.stdout("Exploratory single-threshold candidates:");
+    for (const candidate of report.candidates) {
+      dependencies.stdout(
+        `  ${candidate.field}=${candidate.value}: ${candidate.metrics.matches}/${candidate.metrics.evaluated} match; weighted loss ${candidate.metrics.weightedLoss}`,
+      );
+    }
+  }
+
+  if (report.warning !== undefined) dependencies.stdout(`Warning: ${report.warning}`);
+  dependencies.stdout(
+    "Candidates are not applied automatically; validate them on an independent holdout and the synthetic corpus.",
+  );
+  if (replay !== undefined) {
+    dependencies.stdout(
+      `Current questions replay: ${replay.metrics.matches}/${replay.metrics.evaluated} match; ${replay.improved} improved; ${replay.regressed} regressed; ${replay.unavailable} unavailable; ${replay.skipped} skipped by limit.`,
+    );
+    dependencies.stdout(
+      `Replay usage: ${replay.inputTokens} input / ${replay.outputTokens} output tokens. Stored commands remained data and were never executed.`,
+    );
+  }
+  return 0;
+}
+
 async function runTrainingReview(
   args: ReadonlyArray<string>,
   dependencies: CliDependencies,
@@ -378,7 +505,7 @@ async function runTrainingReview(
   const parsedResult = parseArguments(
     args,
     new Set(["json"]),
-    new Set(["decision", "note"]),
+    new Set(["decision", "reason", "note"]),
   );
   const json = args.includes("--json");
   if (parsedResult.error !== undefined) {
@@ -423,15 +550,37 @@ async function runTrainingReview(
     );
   }
 
+  const corrected = decisionValue !== entry.record.verdict.decision;
+  const reasonValue = getStringOption(parsed, "reason");
+  if (corrected && !isTrainingCorrectionReason(reasonValue)) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "corrected reviews require `--reason` with a supported correction reason",
+      dependencies,
+    );
+  }
+  if (!corrected && reasonValue !== undefined) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`--reason` is only valid when correcting the model decision",
+      dependencies,
+    );
+  }
+
+  const correctionReason = corrected && isTrainingCorrectionReason(reasonValue)
+    ? reasonValue
+    : undefined;
   const previousReview = getLatestTrainingReview(entry) ?? null;
   const review = await dependencies.recordTrainingReview(
     createTrainingReviewInput(
       entry.record,
       decisionValue,
+      correctionReason,
       getStringOption(parsed, "note"),
     ),
   );
-  const corrected = decisionValue !== entry.record.verdict.decision;
 
   if (hasOption(parsed, "json")) {
     writeTrainingJsonSuccess(

package/src/guard.internal.ts

@@ -6,14 +6,91 @@ import {
   Predicate,
   Result,
 } from "effect";
-import { judgeEffect, Judgment } from "./judge.ts";
+import { judgeEffect, Judgment, TYPESAFE_MODEL } from "./judge.ts";
 import { Environment } from "./key.ts";
-import { applyStaticGate, decide } from "./policy.ts";
-import { gatherStateEffect, GitCommand } from "./state.ts";
-import type { CommandState, Host, Verdict } from "./types.ts";
+import {
+  applyStaticGate,
+  decide,
+  POLICY_VERSION,
+  THRESHOLDS,
+} from "./policy.ts";
+import { QUESTION_SET_VERSION } from "./questions.ts";
+import { gatherStateEffect, GitCommand, renderState } from "./state.ts";
+import type {
+  CommandState,
+  GuardEvaluation,
+  Host,
+  Verdict,
+} from "./types.ts";
 
 const PREFIX = "demur:";
 
+/**
+ * Effect-native guard implementation with replayable evidence.
+ *
+ * @param command - The shell command the agent wants to run
+ * @param cwd - Absolute working directory for the command
+ * @param agent - Which coding agent is asking
+ * @returns A fail-closed verdict and the exact evidence behind it
+ */
+export const guardEvaluationEffect = Effect.fn("guardEvaluationEffect")(
+  function* (
+    command: string,
+    cwd: string,
+    agent: Host,
+  ): Effect.fn.Return<
+    GuardEvaluation,
+    never,
+    Environment | GitCommand | Judgment
+  > {
+    const started = yield* Clock.monotonicTimeNanos;
+    const core = Effect.gen(function* () {
+      const environment = yield* Environment;
+      const disabled = isDisabled(yield* environment.get("DEMUR_DISABLE"));
+
+      if (disabled) {
+        const verdict = yield* completeVerdict(started, {
+          ...emptyEvidence,
+          decision: "allow",
+          reason: `${PREFIX} disabled via DEMUR_DISABLE.`,
+        });
+        return { verdict, evidence: undefined } satisfies GuardEvaluation;
+      }
+
+      if (command.trim() === "") {
+        const verdict = yield* completeVerdict(started, {
+          ...emptyEvidence,
+          decision: "allow",
+          reason: `${PREFIX} empty command.`,
+        });
+        return { verdict, evidence: undefined } satisfies GuardEvaluation;
+      }
+
+      const state = yield* gatherStateEffect(command, cwd, agent);
+      const verdict = yield* judgeStateCore(state, started, 0);
+      return {
+        verdict,
+        evidence: {
+          modelState: renderState(state),
+          analysis: state.analysis,
+          model: TYPESAFE_MODEL,
+          questionSetVersion: QUESTION_SET_VERSION,
+          policyVersion: POLICY_VERSION,
+          policyThresholds: { ...THRESHOLDS },
+        },
+      } satisfies GuardEvaluation;
+    });
+
+    const exit = yield* Effect.exit(core);
+    if (Exit.isSuccess(exit)) return exit.value;
+
+    return {
+      verdict: yield* unexpectedVerdict(started, exit.cause, 0),
+      evidence: undefined,
+    };
+  },
+);
+
 /**
  * Effect-native guard implementation used by the Promise boundary.
  *
@@ -27,35 +104,8 @@ export const guardEffect = Effect.fn("guardEffect")(function* (
   cwd: string,
   agent: Host,
 ): Effect.fn.Return<Verdict, never, Environment | GitCommand | Judgment> {
-  const started = yield* Clock.monotonicTimeNanos;
-  const core = Effect.gen(function* () {
-    const environment = yield* Environment;
-    const disabled = isDisabled(yield* environment.get("DEMUR_DISABLE"));
-
-    if (disabled) {
-      return yield* completeVerdict(started, {
-        ...emptyEvidence,
-        decision: "allow",
-        reason: `${PREFIX} disabled via DEMUR_DISABLE.`,
-      });
-    }
-
-    if (command.trim() === "") {
-      return yield* completeVerdict(started, {
-        ...emptyEvidence,
-        decision: "allow",
-        reason: `${PREFIX} empty command.`,
-      });
-    }
-
-    const state = yield* gatherStateEffect(command, cwd, agent);
-    return yield* judgeStateCore(state, started, 0);
-  });
-
-  const exit = yield* Effect.exit(core);
-  if (Exit.isSuccess(exit)) return exit.value;
-
-  return yield* unexpectedVerdict(started, exit.cause, 0);
+  const evaluation = yield* guardEvaluationEffect(command, cwd, agent);
+  return evaluation.verdict;
 });
 
 /**

package/src/guard.ts

@@ -1,9 +1,18 @@
 import { Layer, ManagedRuntime, Predicate } from "effect";
-import { guardEffect, judgeStateEffect } from "./guard.internal.ts";
+import {
+  guardEffect,
+  guardEvaluationEffect,
+  judgeStateEffect,
+} from "./guard.internal.ts";
 import { Judgment } from "./judge.ts";
 import { Environment } from "./key.ts";
 import { GitCommand } from "./state.ts";
-import type { CommandState, Host, Verdict } from "./types.ts";
+import type {
+  CommandState,
+  GuardEvaluation,
+  Host,
+  Verdict,
+} from "./types.ts";
 
 const PREFIX = "demur:";
 
@@ -41,6 +50,37 @@ export function guard(
     .catch((error: unknown) => unexpectedVerdict(error, started));
 }
 
+/**
+ * Judge one command while retaining the exact state and policy versions.
+ *
+ * This boundary is used only when a host has enabled local training capture;
+ * ordinary guard callers continue to receive the smaller {@link Verdict}.
+ *
+ * @param command - The shell command the agent wants to run
+ * @param cwd - Absolute working directory for the command
+ * @param agent - Which coding agent is asking
+ * @param signal - Optional cancellation signal from the host
+ * @returns The verdict and replayable evidence, when state collection occurred
+ */
+export function guardWithEvidence(
+  command: string,
+  cwd: string,
+  agent: Host,
+  signal: AbortSignal | undefined = undefined,
+): Promise<GuardEvaluation> {
+  const started = performance.now();
+
+  return runtime
+    .runPromise(
+      guardEvaluationEffect(command, cwd, agent),
+      signal === undefined ? undefined : { signal },
+    )
+    .catch((error: unknown) => ({
+      verdict: unexpectedVerdict(error, started),
+      evidence: undefined,
+    }));
+}
+
 /**
  * Judge a command from state that has already been collected.
  *

package/src/judge.ts

@@ -19,7 +19,12 @@ import {
 } from "./key.ts";
 import { COMMAND_JUDGMENTS } from "./questions.ts";
 import { renderState } from "./state.ts";
-import type { CommandState, FailureKind, Judgments } from "./types.ts";
+import type {
+  CommandState,
+  FailureKind,
+  Judgments,
+  RenderedCommandState,
+} from "./types.ts";
 
 /**
  * Default per-attempt timeout for the judgment call.
@@ -29,7 +34,10 @@ import type { CommandState, FailureKind, Judgments } from "./types.ts";
  */
 const DEFAULT_TIMEOUT_MS = 4000;
 
-const TYPESAFE_MODEL = "jev-latest";
+/**
+ * TypeSafe model used for command judgments.
+ */
+export const TYPESAFE_MODEL = "jev-latest";
 
 /**
  * Build the TypeSafe-backed Effect decision model for one API key.
@@ -90,7 +98,9 @@ export class JudgmentError extends Schema.TaggedError<JudgmentError>()(
 export class Judgment extends Context.Service<
   Judgment,
   {
-    judge(state: CommandState): Effect.Effect<JudgeSuccess, JudgmentError>;
+    judge(
+      state: RenderedCommandState,
+    ): Effect.Effect<JudgeSuccess, JudgmentError>;
   }
 >()("demur/judge/Judgment") {
   static readonly layerNoDeps = Layer.effect(
@@ -134,11 +144,11 @@ export class Judgment extends Context.Service<
       );
 
       const judge = Effect.fn("Judgment.judge")(function* (
-        state: CommandState,
+        state: RenderedCommandState,
       ): Effect.fn.Return<JudgeSuccess, JudgmentError> {
         const activeDecisionLayer = yield* getDecisionLayer();
         const result = yield* DecisionModel.decide(COMMAND_JUDGMENTS, {
-          input: renderState(state),
+          input: state,
         }).pipe(
           Effect.provide(activeDecisionLayer),
           Effect.timeout(timeoutMs),
@@ -188,9 +198,24 @@ export const judgeEffect = Effect.fn("judgeEffect")(function* (
   state: CommandState,
 ): Effect.fn.Return<JudgeSuccess, JudgmentError, Judgment> {
   const judgment = yield* Judgment;
-  return yield* judgment.judge(state);
+  return yield* judgment.judge(renderState(state));
 });
 
+/**
+ * Ask the configured judgment service about already-rendered TypeSafe state.
+ *
+ * @param state - Exact JSON state captured from an earlier invocation
+ * @returns The current question set's judgments in the Effect error channel
+ */
+export const judgeRenderedStateEffect = Effect.fn("judgeRenderedStateEffect")(
+  function* (
+    state: RenderedCommandState,
+  ): Effect.fn.Return<JudgeSuccess, JudgmentError, Judgment> {
+    const judgment = yield* Judgment;
+    return yield* judgment.judge(state);
+  },
+);
+
 const runtime = ManagedRuntime.make(Judgment.layer);
 
 /**
@@ -206,9 +231,30 @@ const runtime = ManagedRuntime.make(Judgment.layer);
 export function judge(
   state: CommandState,
   signal: AbortSignal | undefined = undefined,
+): Promise<JudgeResult> {
+  return runJudgment(judgeEffect(state), signal);
+}
+
+/**
+ * Re-run the current question set against exact previously captured state.
+ *
+ * @param state - Exact model-visible state from a version-two training record
+ * @param signal - Optional cancellation signal
+ * @returns Current raw judgments or a typed failure
+ */
+export function judgeRenderedState(
+  state: RenderedCommandState,
+  signal: AbortSignal | undefined = undefined,
+): Promise<JudgeResult> {
+  return runJudgment(judgeRenderedStateEffect(state), signal);
+}
+
+function runJudgment(
+  effect: Effect.Effect<JudgeSuccess, JudgmentError, Judgment>,
+  signal: AbortSignal | undefined,
 ): Promise<JudgeResult> {
   const program = Effect.gen(function* () {
-    const result = yield* Effect.result(judgeEffect(state));
+    const result = yield* Effect.result(effect);
     if (Result.isFailure(result)) {
       return {
         ok: false,

package/src/policy.ts

@@ -1,6 +1,15 @@
 import type { CommandAnalysis } from "./analyze.ts";
 import type { Decision, Judgments } from "./types.ts";
 
+/**
+ * Version of the policy ordering and threshold semantics.
+ *
+ * Increment this whenever composition behavior changes. Threshold values are
+ * captured separately by offline evaluation, so candidate sweeps do not need a
+ * version bump until a candidate is promoted.
+ */
+export const POLICY_VERSION = 2;
+
 /**
  * Decision thresholds applied to raw judgments.
  *
@@ -81,7 +90,7 @@ export const THRESHOLDS: Thresholds = {
   denyUnrecoverable: 0.45,
   denySharedInfrastructure: 0.5,
   denyBlastRadius: 2.0,
-  minBlastRadiusConfidence: 0.5,
+  minBlastRadiusConfidence: 0.95,
   uncertaintyBand: 0.04,
   uncertaintyBandScore: 0.12,
   staticGateMinExecution: 0.8,

package/src/questions.ts

@@ -1,6 +1,14 @@
 import { Schema } from "effect";
 import { Decision } from "effect/unstable/ai";
 
+/**
+ * Version of the complete TypeSafe question set.
+ *
+ * Increment this whenever question instructions, criteria, or primitives change
+ * so captured training evidence remains attributable to its exact semantics.
+ */
+export const QUESTION_SET_VERSION = 2;
+
 /**
  * The judgment set demur asks about every command.
  *
@@ -35,7 +43,7 @@ export const QUESTIONS = {
     criteria: {
       true: "It reads a value from an environment variable, file, credential or secret store, database, or remote system and prints, logs, copies, or transmits it. Read-only output still counts.",
       false:
-        "It reveals only names, metadata, schema, attributes, or redacted values; rotates or revokes without revealing; or quotes or prints secret-looking names or commands as documentation, comments, or other inert payload.",
+        "It reveals only names, metadata, schema, attributes, or redacted values; looks up or describes a credential item without requesting its secret value; rotates or revokes without revealing; or quotes or prints secret-looking names or commands as documentation, comments, or other inert payload. For example, `security find-generic-password -s NAME` without `-w` prints item attributes rather than the password value.",
     },
   }),