@hank-warren/pi-auto-permissions 0.1.1 → 0.1.2

package/NOTICE.md

@@ -4,4 +4,4 @@ This package is a fork of [`@ogulcancelik/pi-auto-permissions`](https://github.c
 
 The original work is Copyright (c) 2026 Can Celik and is used under the MIT License included in [`LICENSE`](LICENSE).
 
-Fork-specific changes (the `reviewEvidence.userAnswerTools` dialog-answer evidence feature, proposed upstream in [ogulcancelik/pi-extensions#24](https://github.com/ogulcancelik/pi-extensions/issues/24) / [PR #25](https://github.com/ogulcancelik/pi-extensions/pull/25)) are Copyright (c) 2026 Hank Warren and are released under the same MIT License.
+Fork-specific changes are Copyright (c) 2026 Hank Warren and are released under the same MIT License. They include the `reviewEvidence.userAnswerTools` dialog-answer evidence feature, proposed upstream in [ogulcancelik/pi-extensions#24](https://github.com/ogulcancelik/pi-extensions/issues/24) / [PR #25](https://github.com/ogulcancelik/pi-extensions/pull/25), and the prompted-review evaluation log with explicit user feedback.

package/README.md

@@ -185,7 +185,7 @@ The allowlist matches what you wrote: a bare name such as `ask_user_question` ma
 
 ## Prompted-review evaluation log
 
-Auto Permissions can append a private JSONL regression record whenever the guardian asks for confirmation and the user chooses Allow or Block:
+Auto Permissions can append a private JSONL regression record whenever the guardian asks for confirmation and the user gives explicit feedback:
 
 ```json
 {
@@ -198,7 +198,15 @@ Auto Permissions can append a private JSONL regression record whenever the guard
 
 The path defaults to `review-evals.jsonl` beside the Auto Permissions config and resolves relative to that config. The file is created with mode `0600`.
 
-Each record contains the collected user request, exact command, compact reviewer evidence, guardian reason, gate and session metadata, raw user choice, and both labels used for evaluation. The guardian's `actualDecision` is `ask_user`; Allow maps to `expectedDecision: "approve"`, while Block maps to `expectedDecision: "ask_user"`. Automatic-review failures are identified separately with `decisionSource: "review_failure"`.
+When logging is enabled, prompted reviews offer three choices:
+
+- **Allow — asking was unnecessary** executes the command and records `userChoice: "allow_unnecessary"` with `expectedDecision: "approve"`.
+- **Block** remains the second choice, blocks the command, and records `userChoice: "block"` with `expectedDecision: "ask_user"`.
+- **Allow — asking was appropriate** executes the command and records `userChoice: "allow_appropriate"` with `expectedDecision: "ask_user"`.
+
+When logging is disabled, the prompt retains the normal **Allow** and **Block** choices. Prompts from Pi or other extensions are unchanged.
+
+Each version 2 record contains the collected user request, exact command, compact reviewer evidence, guardian reason, gate and session metadata, raw user choice, and both labels used for evaluation. The guardian's `actualDecision` is `ask_user`. Automatic-review failures are identified separately with `decisionSource: "review_failure"`. Existing version 1 records can remain in the same JSONL file.
 
 Logging is disabled by default. Records can contain sensitive conversation text and shell commands, so keep the file private and out of repositories. Cancelled or interrupted prompts are not labeled or logged.
 

package/evaluation-log.ts

@@ -2,8 +2,31 @@ import { chmodSync, mkdirSync, appendFileSync } from "node:fs";
 import { dirname } from "node:path";
 import type { PermissionDecision, ReviewEvidenceRecord } from "./review.js";
 
+export const PROMPT_FEEDBACK_OPTIONS = {
+  allowUnnecessary: "Allow — asking was unnecessary",
+  allowAppropriate: "Allow — asking was appropriate",
+  block: "Block",
+} as const;
+
+export type PromptEvaluationUserChoice = "allow_unnecessary" | "allow_appropriate" | "block";
+
+export interface PromptChoiceClassification {
+  allowsExecution: boolean;
+  userChoice?: PromptEvaluationUserChoice;
+}
+
+export function permissionPromptOptions(evaluationLoggingEnabled: boolean): string[] {
+  return evaluationLoggingEnabled
+    ? [
+      PROMPT_FEEDBACK_OPTIONS.allowUnnecessary,
+      PROMPT_FEEDBACK_OPTIONS.block,
+      PROMPT_FEEDBACK_OPTIONS.allowAppropriate,
+    ]
+    : ["Allow", "Block"];
+}
+
 export interface PromptEvaluationRecord {
-  version: 1;
+  version: 2;
   timestamp: string;
   sessionId: string;
   cwd: string;
@@ -18,14 +41,28 @@ export interface PromptEvaluationRecord {
   actualDecision: "ask_user";
   actualReason: string;
   decisionSource: "guardian" | "review_failure";
-  userChoice: "allow" | "block";
+  userChoice: PromptEvaluationUserChoice;
   expectedDecision: Extract<PermissionDecision, "approve" | "ask_user">;
 }
 
+export function classifyPromptChoice(choice: string | undefined): PromptChoiceClassification | undefined {
+  if (choice === "Allow") return { allowsExecution: true };
+  if (choice === PROMPT_FEEDBACK_OPTIONS.allowUnnecessary) {
+    return { allowsExecution: true, userChoice: "allow_unnecessary" };
+  }
+  if (choice === PROMPT_FEEDBACK_OPTIONS.allowAppropriate) {
+    return { allowsExecution: true, userChoice: "allow_appropriate" };
+  }
+  if (choice === PROMPT_FEEDBACK_OPTIONS.block) {
+    return { allowsExecution: false, userChoice: "block" };
+  }
+  return undefined;
+}
+
 export function expectedDecisionForChoice(
   choice: PromptEvaluationRecord["userChoice"],
 ): PromptEvaluationRecord["expectedDecision"] {
-  return choice === "allow" ? "approve" : "ask_user";
+  return choice === "allow_unnecessary" ? "approve" : "ask_user";
 }
 
 export function appendPromptEvaluation(path: string, record: PromptEvaluationRecord): void {

package/index.ts

@@ -8,7 +8,13 @@ import { randomBytes, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { loadAutoPermissionsConfig, type AutoPermissionsConfig } from "./config.js";
-import { appendPromptEvaluation, expectedDecisionForChoice } from "./evaluation-log.js";
+import {
+  appendPromptEvaluation,
+  classifyPromptChoice,
+  expectedDecisionForChoice,
+  permissionPromptOptions,
+  type PromptEvaluationUserChoice,
+} from "./evaluation-log.js";
 import { findGates, type Gate } from "./gates.js";
 import {
   buildReviewEnvelope,
@@ -530,7 +536,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
     target: ReviewTarget,
     relevantContext: ReviewEvidenceRecord[],
     decisionSource: "guardian" | "review_failure",
-    userChoice: "allow" | "block",
+    userChoice: PromptEvaluationUserChoice,
   ): void {
     if (!config.evaluationLog.enabled) return;
     try {
@@ -539,7 +545,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
         .map((record) => record.text)
         .join("\n");
       appendPromptEvaluation(config.evaluationLog.path, {
-        version: 1,
+        version: 2,
         timestamp: new Date().toISOString(),
         sessionId: ctx.sessionManager.getSessionId(),
         cwd: ctx.cwd,
@@ -602,7 +608,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
       try {
         choice = await ctx.ui.select(
           `${gate.label} — Auto Permissions needs approval\n\n${detail}\n\n${command}`,
-          ["Allow", "Block"],
+          permissionPromptOptions(config.evaluationLog.enabled),
           { signal: promptSignal },
         );
       } catch (error) {
@@ -616,14 +622,24 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
         if (sessionActive) clearReviewDisplay(ctx, target);
         return cancelled();
       }
-      if (choice === "Allow") {
-        logPromptEvaluation(gate, command, detail, config, ctx, target, evaluationContext, decisionSource, "allow");
+      const classification = classifyPromptChoice(choice);
+      if (config.evaluationLog.enabled && classification?.userChoice) {
+        logPromptEvaluation(
+          gate,
+          command,
+          detail,
+          config,
+          ctx,
+          target,
+          evaluationContext,
+          decisionSource,
+          classification.userChoice,
+        );
+      }
+      if (classification?.allowsExecution) {
         showReviewDisplay(ctx, config, gate, command, target, "approved", "approved by user", true);
         return undefined;
       }
-      if (choice === "Block") {
-        logPromptEvaluation(gate, command, detail, config, ctx, target, evaluationContext, decisionSource, "block");
-      }
       showReviewDisplay(ctx, config, gate, command, target, "blocked", "blocked by user", true);
       return { block: true, reason: "Blocked by user" };
     } finally {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hank-warren/pi-auto-permissions",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Fork of @ogulcancelik/pi-auto-permissions with dialog-answer user evidence: context-aware Bash permissions for Pi with automated guardian review.",
   "type": "module",
   "keywords": [