@hank-warren/pi-auto-permissions 0.1.1 → 0.2.0

package/NOTICE.md

@@ -4,4 +4,4 @@ This package is a fork of [`@ogulcancelik/pi-auto-permissions`](https://github.c
 
 The original work is Copyright (c) 2026 Can Celik and is used under the MIT License included in [`LICENSE`](LICENSE).
 
-Fork-specific changes (the `reviewEvidence.userAnswerTools` dialog-answer evidence feature, proposed upstream in [ogulcancelik/pi-extensions#24](https://github.com/ogulcancelik/pi-extensions/issues/24) / [PR #25](https://github.com/ogulcancelik/pi-extensions/pull/25)) are Copyright (c) 2026 Hank Warren and are released under the same MIT License.
+Fork-specific changes are Copyright (c) 2026 Hank Warren and are released under the same MIT License. They include the `reviewEvidence.userAnswerTools` dialog-answer evidence feature, proposed upstream in [ogulcancelik/pi-extensions#24](https://github.com/ogulcancelik/pi-extensions/issues/24) / [PR #25](https://github.com/ogulcancelik/pi-extensions/pull/25), and the prompted-review evaluation log with explicit user feedback.

package/README.md

@@ -185,7 +185,7 @@ The allowlist matches what you wrote: a bare name such as `ask_user_question` ma
 
 ## Prompted-review evaluation log
 
-Auto Permissions can append a private JSONL regression record whenever the guardian asks for confirmation and the user chooses Allow or Block:
+Auto Permissions can append a private JSONL regression record whenever the guardian asks for confirmation and the user gives explicit feedback:
 
 ```json
 {
@@ -198,10 +198,41 @@ Auto Permissions can append a private JSONL regression record whenever the guard
 
 The path defaults to `review-evals.jsonl` beside the Auto Permissions config and resolves relative to that config. The file is created with mode `0600`.
 
-Each record contains the collected user request, exact command, compact reviewer evidence, guardian reason, gate and session metadata, raw user choice, and both labels used for evaluation. The guardian's `actualDecision` is `ask_user`; Allow maps to `expectedDecision: "approve"`, while Block maps to `expectedDecision: "ask_user"`. Automatic-review failures are identified separately with `decisionSource: "review_failure"`.
+When logging is enabled, prompted reviews offer three choices:
+
+- **Allow — asking was unnecessary** executes the command and records `userChoice: "allow_unnecessary"` with `expectedDecision: "approve"`.
+- **Block** remains the second choice, blocks the command, and records `userChoice: "block"` with `expectedDecision: "ask_user"`.
+- **Allow — asking was appropriate** executes the command and records `userChoice: "allow_appropriate"` with `expectedDecision: "ask_user"`.
+
+When logging is disabled, the prompt retains the normal **Allow** and **Block** choices. Prompts from Pi or other extensions are unchanged.
+
+Each version 2 record contains the collected user request, exact command, compact reviewer evidence, guardian reason, gate and session metadata, raw user choice, and both labels used for evaluation. The guardian's `actualDecision` is `ask_user`. Automatic-review failures are identified separately with `decisionSource: "review_failure"`. Existing version 1 records can remain in the same JSONL file.
 
 Logging is disabled by default. Records can contain sensitive conversation text and shell commands, so keep the file private and out of repositories. Cancelled or interrupted prompts are not labeled or logged.
 
+## Reviewer usage sidecar
+
+Guardian reviews are model calls made outside the Pi agent loop, so they never appear in the session transcript and are invisible to tools that total usage from session files. Auto Permissions therefore appends one content-free record per completed review:
+
+```json
+{"v":1,"id":"3f2b…","ts":"2026-08-11T22:41:03.118Z","source":"auto-permissions","label":"guardian","provider":"anthropic","model":"claude-fable-5","usage":{"input":812,"output":96,"cacheRead":18442,"cacheWrite":0,"reasoning":48,"cost":0.0121}}
+```
+
+The record carries identity, timing, and counters only. It never contains prompts, commands, reviewer evidence, verdicts, or responses, which is what makes it safe to keep on by default:
+
+```json
+{
+  "usageLog": {
+    "enabled": true,
+    "path": "./usage.jsonl"
+  }
+}
+```
+
+The path defaults to `usage.jsonl` beside the Auto Permissions config and resolves relative to it. The file is created with mode `0600` and rotates to `usage.jsonl.1` once it passes 16 MB, keeping one previous generation. Writing is best effort: a failing sidecar never blocks or changes a permission decision.
+
+[`@hank-warren/pi-stats`](../pi-stats) reads `<agent dir>/<extension>/usage.jsonl` sidecars and shows this usage as its own `provider/model (guardian)` row. Set `"enabled": false` to stop recording.
+
 ## Review display
 
 The default UI shows guardian progress in a temporary widget below the editor. Configure it with:

package/config.ts

@@ -27,6 +27,10 @@ export interface AutoPermissionsConfig {
     enabled: boolean;
     path: string;
   };
+  usageLog: {
+    enabled: boolean;
+    path: string;
+  };
   rules: Gate[];
   ui: {
     enabled: boolean;
@@ -147,6 +151,26 @@ function resolveEvaluationLog(
   };
 }
 
+function resolveUsageLog(
+  raw: Record<string, unknown>,
+  configFilePath: string,
+): AutoPermissionsConfig["usageLog"] {
+  const defaultPath = resolve(dirname(configFilePath), "usage.jsonl");
+  if (raw.usageLog === undefined) return { enabled: true, path: defaultPath };
+  if (!raw.usageLog || typeof raw.usageLog !== "object" || Array.isArray(raw.usageLog)) {
+    throw new Error("usageLog must be an object");
+  }
+  const usageLog = raw.usageLog as Record<string, unknown>;
+  if (usageLog.enabled !== undefined && typeof usageLog.enabled !== "boolean") {
+    throw new Error("usageLog.enabled must be boolean");
+  }
+  const enabled = usageLog.enabled !== false;
+  const configuredPath = optionalString(usageLog.path, "usageLog.path");
+  if (!configuredPath) return { enabled, path: defaultPath };
+  const expanded = configuredPath.startsWith("~/") ? join(homedir(), configuredPath.slice(2)) : configuredPath;
+  return { enabled, path: isAbsolute(expanded) ? expanded : resolve(dirname(configFilePath), expanded) };
+}
+
 function resolveUi(raw: Record<string, unknown>): AutoPermissionsConfig["ui"] {
   if (raw.ui === undefined) return { enabled: true, resultDisplayMs: 2500, placement: "widget" };
   if (!raw.ui || typeof raw.ui !== "object" || Array.isArray(raw.ui)) {
@@ -199,6 +223,7 @@ export function loadAutoPermissionsConfig(path = configPath()): AutoPermissionsC
     systemPrompt: resolvePrompt(raw, path),
     reviewEvidence: resolveReviewEvidence(raw),
     evaluationLog: resolveEvaluationLog(raw, path),
+    usageLog: resolveUsageLog(raw, path),
     rules: customRules,
     ui: resolveUi(raw),
   };

package/evaluation-log.ts

@@ -2,8 +2,31 @@ import { chmodSync, mkdirSync, appendFileSync } from "node:fs";
 import { dirname } from "node:path";
 import type { PermissionDecision, ReviewEvidenceRecord } from "./review.js";
 
+export const PROMPT_FEEDBACK_OPTIONS = {
+  allowUnnecessary: "Allow — asking was unnecessary",
+  allowAppropriate: "Allow — asking was appropriate",
+  block: "Block",
+} as const;
+
+export type PromptEvaluationUserChoice = "allow_unnecessary" | "allow_appropriate" | "block";
+
+export interface PromptChoiceClassification {
+  allowsExecution: boolean;
+  userChoice?: PromptEvaluationUserChoice;
+}
+
+export function permissionPromptOptions(evaluationLoggingEnabled: boolean): string[] {
+  return evaluationLoggingEnabled
+    ? [
+      PROMPT_FEEDBACK_OPTIONS.allowUnnecessary,
+      PROMPT_FEEDBACK_OPTIONS.block,
+      PROMPT_FEEDBACK_OPTIONS.allowAppropriate,
+    ]
+    : ["Allow", "Block"];
+}
+
 export interface PromptEvaluationRecord {
-  version: 1;
+  version: 2;
   timestamp: string;
   sessionId: string;
   cwd: string;
@@ -18,14 +41,28 @@ export interface PromptEvaluationRecord {
   actualDecision: "ask_user";
   actualReason: string;
   decisionSource: "guardian" | "review_failure";
-  userChoice: "allow" | "block";
+  userChoice: PromptEvaluationUserChoice;
   expectedDecision: Extract<PermissionDecision, "approve" | "ask_user">;
 }
 
+export function classifyPromptChoice(choice: string | undefined): PromptChoiceClassification | undefined {
+  if (choice === "Allow") return { allowsExecution: true };
+  if (choice === PROMPT_FEEDBACK_OPTIONS.allowUnnecessary) {
+    return { allowsExecution: true, userChoice: "allow_unnecessary" };
+  }
+  if (choice === PROMPT_FEEDBACK_OPTIONS.allowAppropriate) {
+    return { allowsExecution: true, userChoice: "allow_appropriate" };
+  }
+  if (choice === PROMPT_FEEDBACK_OPTIONS.block) {
+    return { allowsExecution: false, userChoice: "block" };
+  }
+  return undefined;
+}
+
 export function expectedDecisionForChoice(
   choice: PromptEvaluationRecord["userChoice"],
 ): PromptEvaluationRecord["expectedDecision"] {
-  return choice === "allow" ? "approve" : "ask_user";
+  return choice === "allow_unnecessary" ? "approve" : "ask_user";
 }
 
 export function appendPromptEvaluation(path: string, record: PromptEvaluationRecord): void {

package/index.ts

@@ -8,8 +8,15 @@ import { randomBytes, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { loadAutoPermissionsConfig, type AutoPermissionsConfig } from "./config.js";
-import { appendPromptEvaluation, expectedDecisionForChoice } from "./evaluation-log.js";
+import {
+  appendPromptEvaluation,
+  classifyPromptChoice,
+  expectedDecisionForChoice,
+  permissionPromptOptions,
+  type PromptEvaluationUserChoice,
+} from "./evaluation-log.js";
 import { findGates, type Gate } from "./gates.js";
+import { appendUsageRecord, buildUsageLogRecord } from "./usage-log.js";
 import {
   buildReviewEnvelope,
   collectReviewEvidence,
@@ -351,6 +358,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
       if (response.stopReason === "error") {
         throw new Error(response.errorMessage ?? "reviewer request failed");
       }
+      recordReviewerUsage(config, model, response.usage);
       const verdict = parsePermissionVerdict(assistantText(response.content));
       if (!sessionActive || signal?.aborted || reviewerGeneration !== attemptGeneration) {
         throw new Error("review timed out or was cancelled");
@@ -373,6 +381,19 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
     }
   }
 
+  /**
+   * Reviewer calls never reach the session transcript, so their usage is invisible to
+   * tooling that reads session files. Record content-free counters in a sidecar instead.
+   */
+  function recordReviewerUsage(config: AutoPermissionsConfig, model: { provider: string; id: string }, usage: unknown): void {
+    if (!config.usageLog.enabled) return;
+    try {
+      appendUsageRecord(config.usageLog.path, buildUsageLogRecord(model.provider, model.id, usage));
+    } catch {
+      // Usage accounting is optional and must never block a permission decision.
+    }
+  }
+
   function untrustedMatches(command: string, config: AutoPermissionsConfig): Gate[] {
     return findGates(command, config.rules).filter((gate) => !trustedGroups.has(gate.group));
   }
@@ -530,7 +551,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
     target: ReviewTarget,
     relevantContext: ReviewEvidenceRecord[],
     decisionSource: "guardian" | "review_failure",
-    userChoice: "allow" | "block",
+    userChoice: PromptEvaluationUserChoice,
   ): void {
     if (!config.evaluationLog.enabled) return;
     try {
@@ -539,7 +560,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
         .map((record) => record.text)
         .join("\n");
       appendPromptEvaluation(config.evaluationLog.path, {
-        version: 1,
+        version: 2,
         timestamp: new Date().toISOString(),
         sessionId: ctx.sessionManager.getSessionId(),
         cwd: ctx.cwd,
@@ -602,7 +623,7 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
       try {
         choice = await ctx.ui.select(
           `${gate.label} — Auto Permissions needs approval\n\n${detail}\n\n${command}`,
-          ["Allow", "Block"],
+          permissionPromptOptions(config.evaluationLog.enabled),
           { signal: promptSignal },
         );
       } catch (error) {
@@ -616,14 +637,24 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
         if (sessionActive) clearReviewDisplay(ctx, target);
         return cancelled();
       }
-      if (choice === "Allow") {
-        logPromptEvaluation(gate, command, detail, config, ctx, target, evaluationContext, decisionSource, "allow");
+      const classification = classifyPromptChoice(choice);
+      if (config.evaluationLog.enabled && classification?.userChoice) {
+        logPromptEvaluation(
+          gate,
+          command,
+          detail,
+          config,
+          ctx,
+          target,
+          evaluationContext,
+          decisionSource,
+          classification.userChoice,
+        );
+      }
+      if (classification?.allowsExecution) {
         showReviewDisplay(ctx, config, gate, command, target, "approved", "approved by user", true);
         return undefined;
       }
-      if (choice === "Block") {
-        logPromptEvaluation(gate, command, detail, config, ctx, target, evaluationContext, decisionSource, "block");
-      }
       showReviewDisplay(ctx, config, gate, command, target, "blocked", "blocked by user", true);
       return { block: true, reason: "Blocked by user" };
     } finally {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hank-warren/pi-auto-permissions",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Fork of @ogulcancelik/pi-auto-permissions with dialog-answer user evidence: context-aware Bash permissions for Pi with automated guardian review.",
   "type": "module",
   "keywords": [
@@ -43,6 +43,7 @@
     "evaluation-log.ts",
     "gates.ts",
     "review.ts",
+    "usage-log.ts",
     "README.md",
     "LICENSE"
   ],

package/usage-log.ts

@@ -0,0 +1,84 @@
+import { appendFileSync, chmodSync, mkdirSync, renameSync, statSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { dirname } from "node:path";
+
+/** Rotate once the sidecar grows past this size; one previous generation is kept. */
+export const USAGE_LOG_ROTATE_BYTES = 16 * 1024 * 1024;
+
+export interface UsageLogTotals {
+  input: number;
+  output: number;
+  cacheRead: number;
+  cacheWrite: number;
+  reasoning: number;
+  cost: number;
+}
+
+/**
+ * One model call recorded outside the Pi session transcript. The record is deliberately
+ * content free: it carries identity, timing, and counters only, never prompts, commands,
+ * evidence, or responses.
+ */
+export interface UsageLogRecord {
+  v: 1;
+  id: string;
+  ts: string;
+  source: string;
+  label: string;
+  provider: string;
+  model: string;
+  usage: UsageLogTotals;
+}
+
+function count(value: unknown): number {
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
+}
+
+/** Normalize a pi-ai usage payload, whose cost is an object with a total. */
+export function usageLogTotals(usage: unknown): UsageLogTotals {
+  const record = (usage && typeof usage === "object" ? usage : {}) as Record<string, unknown>;
+  const cost = (record.cost && typeof record.cost === "object" ? record.cost : {}) as Record<string, unknown>;
+  return {
+    input: count(record.input),
+    output: count(record.output),
+    cacheRead: count(record.cacheRead),
+    cacheWrite: count(record.cacheWrite),
+    reasoning: count(record.reasoning),
+    cost: count(typeof record.cost === "number" ? record.cost : cost.total),
+  };
+}
+
+export function buildUsageLogRecord(
+  provider: string,
+  model: string,
+  usage: unknown,
+  label = "guardian",
+  source = "auto-permissions",
+): UsageLogRecord {
+  return {
+    v: 1,
+    id: randomUUID(),
+    ts: new Date().toISOString(),
+    source,
+    label,
+    provider,
+    model,
+    usage: usageLogTotals(usage),
+  };
+}
+
+function rotateIfLarge(path: string): void {
+  try {
+    if (statSync(path).size < USAGE_LOG_ROTATE_BYTES) return;
+    renameSync(path, `${path}.1`);
+  } catch {
+    // A missing or unrotatable sidecar simply keeps appending.
+  }
+}
+
+export function appendUsageRecord(path: string, record: UsageLogRecord): void {
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  rotateIfLarge(path);
+  appendFileSync(path, `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 });
+  chmodSync(path, 0o600);
+}