@hank-warren/pi-auto-permissions 0.1.0 → 0.1.1

package/README.md

@@ -183,6 +183,25 @@ A successful result from an allowlisted tool qualifies when its `details` are `{
 
 The allowlist matches what you wrote: a bare name such as `ask_user_question` matches that tool in any namespace (`functions.ask_user_question` included), while a dotted name matches exactly. The default is an empty list.
 
+## Prompted-review evaluation log
+
+Auto Permissions can append a private JSONL regression record whenever the guardian asks for confirmation and the user chooses Allow or Block:
+
+```json
+{
+  "evaluationLog": {
+    "enabled": true,
+    "path": "./review-evals.jsonl"
+  }
+}
+```
+
+The path defaults to `review-evals.jsonl` beside the Auto Permissions config and resolves relative to that config. The file is created with mode `0600`.
+
+Each record contains the collected user request, exact command, compact reviewer evidence, guardian reason, gate and session metadata, raw user choice, and both labels used for evaluation. The guardian's `actualDecision` is `ask_user`; Allow maps to `expectedDecision: "approve"`, while Block maps to `expectedDecision: "ask_user"`. Automatic-review failures are identified separately with `decisionSource: "review_failure"`.
+
+Logging is disabled by default. Records can contain sensitive conversation text and shell commands, so keep the file private and out of repositories. Cancelled or interrupted prompts are not labeled or logged.
+
 ## Review display
 
 The default UI shows guardian progress in a temporary widget below the editor. Configure it with:

package/config.ts

@@ -23,6 +23,10 @@ export interface AutoPermissionsConfig {
     projectInstructions: boolean;
     userAnswerTools: string[];
   };
+  evaluationLog: {
+    enabled: boolean;
+    path: string;
+  };
   rules: Gate[];
   ui: {
     enabled: boolean;
@@ -121,6 +125,28 @@ function resolveReviewEvidence(raw: Record<string, unknown>): AutoPermissionsCon
   };
 }
 
+function resolveEvaluationLog(
+  raw: Record<string, unknown>,
+  configFilePath: string,
+): AutoPermissionsConfig["evaluationLog"] {
+  const defaultPath = resolve(dirname(configFilePath), "review-evals.jsonl");
+  if (raw.evaluationLog === undefined) return { enabled: false, path: defaultPath };
+  if (!raw.evaluationLog || typeof raw.evaluationLog !== "object" || Array.isArray(raw.evaluationLog)) {
+    throw new Error("evaluationLog must be an object");
+  }
+  const evaluationLog = raw.evaluationLog as Record<string, unknown>;
+  if (evaluationLog.enabled !== undefined && typeof evaluationLog.enabled !== "boolean") {
+    throw new Error("evaluationLog.enabled must be boolean");
+  }
+  const configuredPath = optionalString(evaluationLog.path, "evaluationLog.path");
+  if (!configuredPath) return { enabled: evaluationLog.enabled === true, path: defaultPath };
+  const expanded = configuredPath.startsWith("~/") ? join(homedir(), configuredPath.slice(2)) : configuredPath;
+  return {
+    enabled: evaluationLog.enabled === true,
+    path: isAbsolute(expanded) ? expanded : resolve(dirname(configFilePath), expanded),
+  };
+}
+
 function resolveUi(raw: Record<string, unknown>): AutoPermissionsConfig["ui"] {
   if (raw.ui === undefined) return { enabled: true, resultDisplayMs: 2500, placement: "widget" };
   if (!raw.ui || typeof raw.ui !== "object" || Array.isArray(raw.ui)) {
@@ -172,6 +198,7 @@ export function loadAutoPermissionsConfig(path = configPath()): AutoPermissionsC
     reviewer: resolveReviewer(raw),
     systemPrompt: resolvePrompt(raw, path),
     reviewEvidence: resolveReviewEvidence(raw),
+    evaluationLog: resolveEvaluationLog(raw, path),
     rules: customRules,
     ui: resolveUi(raw),
   };

package/evaluation-log.ts

@@ -0,0 +1,35 @@
+import { chmodSync, mkdirSync, appendFileSync } from "node:fs";
+import { dirname } from "node:path";
+import type { PermissionDecision, ReviewEvidenceRecord } from "./review.js";
+
+export interface PromptEvaluationRecord {
+  version: 1;
+  timestamp: string;
+  sessionId: string;
+  cwd: string;
+  tool: string;
+  gate: {
+    label: string;
+    group: string;
+  };
+  userRequest: string;
+  command: string;
+  relevantContext: ReviewEvidenceRecord[];
+  actualDecision: "ask_user";
+  actualReason: string;
+  decisionSource: "guardian" | "review_failure";
+  userChoice: "allow" | "block";
+  expectedDecision: Extract<PermissionDecision, "approve" | "ask_user">;
+}
+
+export function expectedDecisionForChoice(
+  choice: PromptEvaluationRecord["userChoice"],
+): PromptEvaluationRecord["expectedDecision"] {
+  return choice === "allow" ? "approve" : "ask_user";
+}
+
+export function appendPromptEvaluation(path: string, record: PromptEvaluationRecord): void {
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  appendFileSync(path, `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 });
+  chmodSync(path, 0o600);
+}

package/index.ts

@@ -8,6 +8,7 @@ import { randomBytes, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { loadAutoPermissionsConfig, type AutoPermissionsConfig } from "./config.js";
+import { appendPromptEvaluation, expectedDecisionForChoice } from "./evaluation-log.js";
 import { findGates, type Gate } from "./gates.js";
 import {
   buildReviewEnvelope,
@@ -191,6 +192,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
   const allowedConventionCommands = new Set<string>();
   let trustedGroups = new Set<string>();
   let lastConfigError: string | undefined;
+  let lastEvaluationLogError: string | undefined;
   let sessionActive = true;
   let bashRendererRegistered = false;
   let bashRendererSourcePath: string | undefined;
@@ -519,6 +521,50 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
     }
   }
 
+  function logPromptEvaluation(
+    gate: Gate,
+    command: string,
+    detail: string,
+    config: AutoPermissionsConfig,
+    ctx: ExtensionContext,
+    target: ReviewTarget,
+    relevantContext: ReviewEvidenceRecord[],
+    decisionSource: "guardian" | "review_failure",
+    userChoice: "allow" | "block",
+  ): void {
+    if (!config.evaluationLog.enabled) return;
+    try {
+      const userRequest = relevantContext
+        .filter((record) => record.source === "user")
+        .map((record) => record.text)
+        .join("\n");
+      appendPromptEvaluation(config.evaluationLog.path, {
+        version: 1,
+        timestamp: new Date().toISOString(),
+        sessionId: ctx.sessionManager.getSessionId(),
+        cwd: ctx.cwd,
+        tool: target.toolName,
+        gate: { label: gate.label, group: gate.group },
+        userRequest,
+        command,
+        relevantContext,
+        actualDecision: "ask_user",
+        actualReason: detail,
+        decisionSource,
+        userChoice,
+        expectedDecision: expectedDecisionForChoice(userChoice),
+      });
+      lastEvaluationLogError = undefined;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (message !== lastEvaluationLogError) {
+        lastEvaluationLogError = message;
+        console.error(`[pi-auto-permissions] could not append evaluation log: ${message}`);
+        ctx.ui.notify("Auto Permissions could not append its evaluation log.", "warning");
+      }
+    }
+  }
+
   async function askUser(
     gate: Gate,
     command: string,
@@ -528,6 +574,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
     ctx: ExtensionContext,
     target: ReviewTarget,
     lifecycleSignal: AbortSignal,
+    decisionSource: "guardian" | "review_failure",
   ): Promise<BlockResult | undefined> {
     const lifecycleStale = () => lifecycleSignal.aborted || reviewerLifecycleController.signal !== lifecycleSignal;
     const promptSignal = signal ? AbortSignal.any([signal, lifecycleSignal]) : lifecycleSignal;
@@ -542,6 +589,13 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
       return { block: true, reason: `${gate.label} requires user approval: ${detail}` };
     }
 
+    const evaluationContext = config.evaluationLog.enabled
+      ? collectReviewEvidence(
+        ctx.sessionManager.buildContextEntries(),
+        target.toolCallId,
+        config.reviewEvidence.userAnswerTools,
+      )
+      : [];
     setHerdrBlocked(true, gate.label);
     try {
       let choice: string | undefined;
@@ -563,9 +617,13 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
         return cancelled();
       }
       if (choice === "Allow") {
+        logPromptEvaluation(gate, command, detail, config, ctx, target, evaluationContext, decisionSource, "allow");
         showReviewDisplay(ctx, config, gate, command, target, "approved", "approved by user", true);
         return undefined;
       }
+      if (choice === "Block") {
+        logPromptEvaluation(gate, command, detail, config, ctx, target, evaluationContext, decisionSource, "block");
+      }
       showReviewDisplay(ctx, config, gate, command, target, "blocked", "blocked by user", true);
       return { block: true, reason: "Blocked by user" };
     } finally {
@@ -628,14 +686,24 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
           reason: `Auto Permissions requested revision: ${verdict.reason}\nRevise the command and try again.`,
         };
       }
-      return askUser(gate, command, verdict.reason, config, signal, ctx, target, lifecycleSignal);
+      return askUser(gate, command, verdict.reason, config, signal, ctx, target, lifecycleSignal, "guardian");
     } catch (error) {
       if (lifecycleStale() || reviewCancelled(signal)) {
         if (!lifecycleStale() && sessionActive) clearReviewDisplay(ctx, target);
         return { block: true, reason: "Auto Permissions review cancelled" };
       }
       const reason = error instanceof Error ? error.message : String(error);
-      return askUser(gate, command, `Automatic review failed: ${reason}`, config, signal, ctx, target, lifecycleSignal);
+      return askUser(
+        gate,
+        command,
+        `Automatic review failed: ${reason}`,
+        config,
+        signal,
+        ctx,
+        target,
+        lifecycleSignal,
+        "review_failure",
+      );
     }
   });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hank-warren/pi-auto-permissions",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Fork of @ogulcancelik/pi-auto-permissions with dialog-answer user evidence: context-aware Bash permissions for Pi with automated guardian review.",
   "type": "module",
   "keywords": [
@@ -40,6 +40,7 @@
     "NOTICE.md",
     "index.ts",
     "config.ts",
+    "evaluation-log.ts",
     "gates.ts",
     "review.ts",
     "README.md",