@hank-warren/pi-auto-permissions 0.1.0 → 0.1.2

package/NOTICE.md

@@ -4,4 +4,4 @@ This package is a fork of [`@ogulcancelik/pi-auto-permissions`](https://github.c
 
 The original work is Copyright (c) 2026 Can Celik and is used under the MIT License included in [`LICENSE`](LICENSE).
 
-Fork-specific changes (the `reviewEvidence.userAnswerTools` dialog-answer evidence feature, proposed upstream in [ogulcancelik/pi-extensions#24](https://github.com/ogulcancelik/pi-extensions/issues/24) / [PR #25](https://github.com/ogulcancelik/pi-extensions/pull/25)) are Copyright (c) 2026 Hank Warren and are released under the same MIT License.
+Fork-specific changes are Copyright (c) 2026 Hank Warren and are released under the same MIT License. They include the `reviewEvidence.userAnswerTools` dialog-answer evidence feature, proposed upstream in [ogulcancelik/pi-extensions#24](https://github.com/ogulcancelik/pi-extensions/issues/24) / [PR #25](https://github.com/ogulcancelik/pi-extensions/pull/25), and the prompted-review evaluation log with explicit user feedback.

package/README.md

@@ -183,6 +183,33 @@ A successful result from an allowlisted tool qualifies when its `details` are `{
 
 The allowlist matches what you wrote: a bare name such as `ask_user_question` matches that tool in any namespace (`functions.ask_user_question` included), while a dotted name matches exactly. The default is an empty list.
 
+## Prompted-review evaluation log
+
+Auto Permissions can append a private JSONL regression record whenever the guardian asks for confirmation and the user gives explicit feedback:
+
+```json
+{
+  "evaluationLog": {
+    "enabled": true,
+    "path": "./review-evals.jsonl"
+  }
+}
+```
+
+The path defaults to `review-evals.jsonl` beside the Auto Permissions config and resolves relative to that config. The file is created with mode `0600`.
+
+When logging is enabled, prompted reviews offer three choices:
+
+- **Allow — asking was unnecessary** executes the command and records `userChoice: "allow_unnecessary"` with `expectedDecision: "approve"`.
+- **Block** remains the second choice, blocks the command, and records `userChoice: "block"` with `expectedDecision: "ask_user"`.
+- **Allow — asking was appropriate** executes the command and records `userChoice: "allow_appropriate"` with `expectedDecision: "ask_user"`.
+
+When logging is disabled, the prompt retains the normal **Allow** and **Block** choices. Prompts from Pi or other extensions are unchanged.
+
+Each version 2 record contains the collected user request, exact command, compact reviewer evidence, guardian reason, gate and session metadata, raw user choice, and both labels used for evaluation. The guardian's `actualDecision` is `ask_user`. Automatic-review failures are identified separately with `decisionSource: "review_failure"`. Existing version 1 records can remain in the same JSONL file.
+
+Logging is disabled by default. Records can contain sensitive conversation text and shell commands, so keep the file private and out of repositories. Cancelled or interrupted prompts are not labeled or logged.
+
 ## Review display
 
 The default UI shows guardian progress in a temporary widget below the editor. Configure it with:

package/config.ts

@@ -23,6 +23,10 @@ export interface AutoPermissionsConfig {
     projectInstructions: boolean;
     userAnswerTools: string[];
   };
+  evaluationLog: {
+    enabled: boolean;
+    path: string;
+  };
   rules: Gate[];
   ui: {
     enabled: boolean;
@@ -121,6 +125,28 @@ function resolveReviewEvidence(raw: Record<string, unknown>): AutoPermissionsCon
   };
 }
 
+function resolveEvaluationLog(
+  raw: Record<string, unknown>,
+  configFilePath: string,
+): AutoPermissionsConfig["evaluationLog"] {
+  const defaultPath = resolve(dirname(configFilePath), "review-evals.jsonl");
+  if (raw.evaluationLog === undefined) return { enabled: false, path: defaultPath };
+  if (!raw.evaluationLog || typeof raw.evaluationLog !== "object" || Array.isArray(raw.evaluationLog)) {
+    throw new Error("evaluationLog must be an object");
+  }
+  const evaluationLog = raw.evaluationLog as Record<string, unknown>;
+  if (evaluationLog.enabled !== undefined && typeof evaluationLog.enabled !== "boolean") {
+    throw new Error("evaluationLog.enabled must be boolean");
+  }
+  const configuredPath = optionalString(evaluationLog.path, "evaluationLog.path");
+  if (!configuredPath) return { enabled: evaluationLog.enabled === true, path: defaultPath };
+  const expanded = configuredPath.startsWith("~/") ? join(homedir(), configuredPath.slice(2)) : configuredPath;
+  return {
+    enabled: evaluationLog.enabled === true,
+    path: isAbsolute(expanded) ? expanded : resolve(dirname(configFilePath), expanded),
+  };
+}
+
 function resolveUi(raw: Record<string, unknown>): AutoPermissionsConfig["ui"] {
   if (raw.ui === undefined) return { enabled: true, resultDisplayMs: 2500, placement: "widget" };
   if (!raw.ui || typeof raw.ui !== "object" || Array.isArray(raw.ui)) {
@@ -172,6 +198,7 @@ export function loadAutoPermissionsConfig(path = configPath()): AutoPermissionsC
     reviewer: resolveReviewer(raw),
     systemPrompt: resolvePrompt(raw, path),
     reviewEvidence: resolveReviewEvidence(raw),
+    evaluationLog: resolveEvaluationLog(raw, path),
     rules: customRules,
     ui: resolveUi(raw),
   };

package/evaluation-log.ts

@@ -0,0 +1,72 @@
+import { chmodSync, mkdirSync, appendFileSync } from "node:fs";
+import { dirname } from "node:path";
+import type { PermissionDecision, ReviewEvidenceRecord } from "./review.js";
+
+export const PROMPT_FEEDBACK_OPTIONS = {
+  allowUnnecessary: "Allow — asking was unnecessary",
+  allowAppropriate: "Allow — asking was appropriate",
+  block: "Block",
+} as const;
+
+export type PromptEvaluationUserChoice = "allow_unnecessary" | "allow_appropriate" | "block";
+
+export interface PromptChoiceClassification {
+  allowsExecution: boolean;
+  userChoice?: PromptEvaluationUserChoice;
+}
+
+export function permissionPromptOptions(evaluationLoggingEnabled: boolean): string[] {
+  return evaluationLoggingEnabled
+    ? [
+      PROMPT_FEEDBACK_OPTIONS.allowUnnecessary,
+      PROMPT_FEEDBACK_OPTIONS.block,
+      PROMPT_FEEDBACK_OPTIONS.allowAppropriate,
+    ]
+    : ["Allow", "Block"];
+}
+
+export interface PromptEvaluationRecord {
+  version: 2;
+  timestamp: string;
+  sessionId: string;
+  cwd: string;
+  tool: string;
+  gate: {
+    label: string;
+    group: string;
+  };
+  userRequest: string;
+  command: string;
+  relevantContext: ReviewEvidenceRecord[];
+  actualDecision: "ask_user";
+  actualReason: string;
+  decisionSource: "guardian" | "review_failure";
+  userChoice: PromptEvaluationUserChoice;
+  expectedDecision: Extract<PermissionDecision, "approve" | "ask_user">;
+}
+
+export function classifyPromptChoice(choice: string | undefined): PromptChoiceClassification | undefined {
+  if (choice === "Allow") return { allowsExecution: true };
+  if (choice === PROMPT_FEEDBACK_OPTIONS.allowUnnecessary) {
+    return { allowsExecution: true, userChoice: "allow_unnecessary" };
+  }
+  if (choice === PROMPT_FEEDBACK_OPTIONS.allowAppropriate) {
+    return { allowsExecution: true, userChoice: "allow_appropriate" };
+  }
+  if (choice === PROMPT_FEEDBACK_OPTIONS.block) {
+    return { allowsExecution: false, userChoice: "block" };
+  }
+  return undefined;
+}
+
+export function expectedDecisionForChoice(
+  choice: PromptEvaluationRecord["userChoice"],
+): PromptEvaluationRecord["expectedDecision"] {
+  return choice === "allow_unnecessary" ? "approve" : "ask_user";
+}
+
+export function appendPromptEvaluation(path: string, record: PromptEvaluationRecord): void {
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  appendFileSync(path, `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 });
+  chmodSync(path, 0o600);
+}

package/index.ts

@@ -8,6 +8,13 @@ import { randomBytes, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { loadAutoPermissionsConfig, type AutoPermissionsConfig } from "./config.js";
+import {
+  appendPromptEvaluation,
+  classifyPromptChoice,
+  expectedDecisionForChoice,
+  permissionPromptOptions,
+  type PromptEvaluationUserChoice,
+} from "./evaluation-log.js";
 import { findGates, type Gate } from "./gates.js";
 import {
   buildReviewEnvelope,
@@ -191,6 +198,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
   const allowedConventionCommands = new Set<string>();
   let trustedGroups = new Set<string>();
   let lastConfigError: string | undefined;
+  let lastEvaluationLogError: string | undefined;
   let sessionActive = true;
   let bashRendererRegistered = false;
   let bashRendererSourcePath: string | undefined;
@@ -519,6 +527,50 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
     }
   }
 
+  function logPromptEvaluation(
+    gate: Gate,
+    command: string,
+    detail: string,
+    config: AutoPermissionsConfig,
+    ctx: ExtensionContext,
+    target: ReviewTarget,
+    relevantContext: ReviewEvidenceRecord[],
+    decisionSource: "guardian" | "review_failure",
+    userChoice: PromptEvaluationUserChoice,
+  ): void {
+    if (!config.evaluationLog.enabled) return;
+    try {
+      const userRequest = relevantContext
+        .filter((record) => record.source === "user")
+        .map((record) => record.text)
+        .join("\n");
+      appendPromptEvaluation(config.evaluationLog.path, {
+        version: 2,
+        timestamp: new Date().toISOString(),
+        sessionId: ctx.sessionManager.getSessionId(),
+        cwd: ctx.cwd,
+        tool: target.toolName,
+        gate: { label: gate.label, group: gate.group },
+        userRequest,
+        command,
+        relevantContext,
+        actualDecision: "ask_user",
+        actualReason: detail,
+        decisionSource,
+        userChoice,
+        expectedDecision: expectedDecisionForChoice(userChoice),
+      });
+      lastEvaluationLogError = undefined;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (message !== lastEvaluationLogError) {
+        lastEvaluationLogError = message;
+        console.error(`[pi-auto-permissions] could not append evaluation log: ${message}`);
+        ctx.ui.notify("Auto Permissions could not append its evaluation log.", "warning");
+      }
+    }
+  }
+
   async function askUser(
     gate: Gate,
     command: string,
@@ -528,6 +580,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
     ctx: ExtensionContext,
     target: ReviewTarget,
     lifecycleSignal: AbortSignal,
+    decisionSource: "guardian" | "review_failure",
   ): Promise<BlockResult | undefined> {
     const lifecycleStale = () => lifecycleSignal.aborted || reviewerLifecycleController.signal !== lifecycleSignal;
     const promptSignal = signal ? AbortSignal.any([signal, lifecycleSignal]) : lifecycleSignal;
@@ -542,13 +595,20 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
       return { block: true, reason: `${gate.label} requires user approval: ${detail}` };
     }
 
+    const evaluationContext = config.evaluationLog.enabled
+      ? collectReviewEvidence(
+        ctx.sessionManager.buildContextEntries(),
+        target.toolCallId,
+        config.reviewEvidence.userAnswerTools,
+      )
+      : [];
     setHerdrBlocked(true, gate.label);
     try {
       let choice: string | undefined;
       try {
         choice = await ctx.ui.select(
           `${gate.label} — Auto Permissions needs approval\n\n${detail}\n\n${command}`,
-          ["Allow", "Block"],
+          permissionPromptOptions(config.evaluationLog.enabled),
           { signal: promptSignal },
         );
       } catch (error) {
@@ -562,7 +622,21 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
         if (sessionActive) clearReviewDisplay(ctx, target);
         return cancelled();
       }
-      if (choice === "Allow") {
+      const classification = classifyPromptChoice(choice);
+      if (config.evaluationLog.enabled && classification?.userChoice) {
+        logPromptEvaluation(
+          gate,
+          command,
+          detail,
+          config,
+          ctx,
+          target,
+          evaluationContext,
+          decisionSource,
+          classification.userChoice,
+        );
+      }
+      if (classification?.allowsExecution) {
         showReviewDisplay(ctx, config, gate, command, target, "approved", "approved by user", true);
         return undefined;
       }
@@ -628,14 +702,24 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
           reason: `Auto Permissions requested revision: ${verdict.reason}\nRevise the command and try again.`,
         };
       }
-      return askUser(gate, command, verdict.reason, config, signal, ctx, target, lifecycleSignal);
+      return askUser(gate, command, verdict.reason, config, signal, ctx, target, lifecycleSignal, "guardian");
     } catch (error) {
       if (lifecycleStale() || reviewCancelled(signal)) {
         if (!lifecycleStale() && sessionActive) clearReviewDisplay(ctx, target);
         return { block: true, reason: "Auto Permissions review cancelled" };
       }
       const reason = error instanceof Error ? error.message : String(error);
-      return askUser(gate, command, `Automatic review failed: ${reason}`, config, signal, ctx, target, lifecycleSignal);
+      return askUser(
+        gate,
+        command,
+        `Automatic review failed: ${reason}`,
+        config,
+        signal,
+        ctx,
+        target,
+        lifecycleSignal,
+        "review_failure",
+      );
     }
   });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hank-warren/pi-auto-permissions",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Fork of @ogulcancelik/pi-auto-permissions with dialog-answer user evidence: context-aware Bash permissions for Pi with automated guardian review.",
   "type": "module",
   "keywords": [
@@ -40,6 +40,7 @@
     "NOTICE.md",
     "index.ts",
     "config.ts",
+    "evaluation-log.ts",
     "gates.ts",
     "review.ts",
     "README.md",