@gotgenes/pi-permission-model-judge 1.1.0 → 1.1.1

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [1.1.1](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.1.0...pi-permission-model-judge-v1.1.1) (2026-07-22)
+
+
+### Bug Fixes
+
+* **pi-permission-model-judge:** force a structured verdict tool call in the model review ([5a89bba](https://github.com/gotgenes/pi-packages/commit/5a89bba94b6493790a664e87cad98514253927fe)), closes [#628](https://github.com/gotgenes/pi-packages/issues/628)
+
+
+### Documentation
+
+* **pi-permission-model-judge:** describe the forced verdict tool call ([1a2b940](https://github.com/gotgenes/pi-packages/commit/1a2b9404ad06b083f96b455df0a66e934b3f5eab))
+
 ## [1.1.0](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.0.2...pi-permission-model-judge-v1.1.0) (2026-07-21)
 
 

package/README.md

@@ -39,9 +39,9 @@ Once an ask matches a `typoPattern` — the case that *should* reach the model
 | `modelId`        | `<provider>/<model>`.                                                                                                               |
 | `latencyMs`      | Model-call wall-clock in ms, or `null` when no call was made.                                                                       |
 | `verdict`        | `"deny"` or `"defer"`.                                                                                                              |
-| `deferReason`    | `null` on a deny, else one of `model-unresolved` / `auth-failed` / `parse-failed` / `non-deny-verdict` / `timeout` / `call-failed`. |
+| `deferReason`    | `null` on a deny, else one of `model-unresolved` / `auth-failed` / `no-tool-call` / `non-deny-verdict` / `timeout` / `call-failed`. |
 
-Cheaper events go to pi-permission-system's **debug** log, and only when its `debugLog` toggle is on: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer) and `model_judge.model_reply` (the model's raw reply text).
+Cheaper events go to pi-permission-system's **debug** log, and only when its `debugLog` toggle is on: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer) and `model_judge.model_reply` (the verdict tool-call arguments as JSON, or the model's text when it emitted no tool call).
 A non-`external_directory` ask is not logged — it is not this link's concern.
 
 Because every pattern-matched ask leaves a positive record, a misconfiguration that silently defers every path (an auth failure, an unresolved model) shows up as a run of `deferReason` entries rather than an empty log.

package/config/config.example.json

@@ -2,7 +2,7 @@
   "$schema": "https://raw.githubusercontent.com/gotgenes/pi-packages/main/packages/pi-permission-model-judge/schemas/model-judge.schema.json",
   "provider": "anthropic",
   "model": "claude-haiku-4-5",
-  "instructions": "You review a filesystem path a tool is about to access outside the working directory. Two common typos both point at the wrong location. (1) A doubled package segment — a package name repeated around `packages/`, for example `pi-permission-system/packages/pi-permission-system/…`, where the correct path names the package only once under the repository's `packages/` directory. (2) A dropped repository prefix — a path that jumps straight to a package directory such as `development/pi/pi-permission-system/…`, where the correct path routes through the repository's `pi-packages/packages/` directory (for example `development/pi/pi-packages/packages/pi-permission-system/…`). When the path shows either typo, deny it and give the single corrected absolute path in the reason so the caller can retry against the right location. If the path looks intentional or you are unsure, defer. Reply with strict JSON: {\"verdict\":\"deny\",\"reason\":\"…\"} to reject, or {\"verdict\":\"defer\"} otherwise.",
+  "instructions": "You review a filesystem path a tool is about to access outside the working directory. Two common typos both point at the wrong location. (1) A doubled package segment — a package name repeated around `packages/`, for example `pi-permission-system/packages/pi-permission-system/…`, where the correct path names the package only once under the repository's `packages/` directory. (2) A dropped repository prefix — a path that jumps straight to a package directory such as `development/pi/pi-permission-system/…`, where the correct path routes through the repository's `pi-packages/packages/` directory (for example `development/pi/pi-packages/packages/pi-permission-system/…`). When the path shows either typo, deny it and give the single corrected absolute path in the reason so the caller can retry against the right location. If the path looks intentional or you are unsure, defer.",
   "typoPatterns": [
     "([^/]+)/packages/\\1(/|$)",
     "development/pi/(?!pi-packages/)pi-[^/]+(/|$)"

package/docs/configuration.md

@@ -49,8 +49,8 @@ If the model does not resolve (wrong id, no credentials), the reviewer records a
 ### `instructions`
 
 This is the model's system prompt.
-Describe the typo shape you care about and instruct the model to reply with the wrong segment and the correct location, so the invoking agent can self-correct.
-The reviewer expects the model to answer with strict JSON — `{"verdict":"deny","reason":"…"}` to reject, `{"verdict":"defer"}` otherwise — and any other reply defers.
+Describe the typo shape you care about and instruct the model to give the wrong segment and the correct location, so the invoking agent can self-correct.
+The reviewer forces the model to call a single `report_verdict` tool, so the verdict is read from structured tool-call arguments rather than parsed from free text — a `deny` verdict with a `reason` rejects the path, and anything else (including no tool call) defers.
 
 ### `typoPatterns`
 
@@ -110,9 +110,9 @@ The **review** log (`~/.pi/agent/extensions/pi-permission-system/logs/pi-permiss
 | `modelId`        | `<provider>/<model>`.                                                                                                        |
 | `latencyMs`      | Model-call wall-clock in milliseconds, or `null` when no call was made.                                                      |
 | `verdict`        | `"deny"` or `"defer"`.                                                                                                       |
-| `deferReason`    | `null` on a deny, else `model-unresolved` / `auth-failed` / `parse-failed` / `non-deny-verdict` / `timeout` / `call-failed`. |
+| `deferReason`    | `null` on a deny, else `model-unresolved` / `auth-failed` / `no-tool-call` / `non-deny-verdict` / `timeout` / `call-failed`. |
 
-The **debug** log (same directory, `pi-permission-system-debug.jsonl`, only when pi-permission-system's `debugLog` is on) carries the verbose and cheap-path detail: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer before the model stage), `model_judge.model_reply` (the model's raw reply text), and `model_judge.invalid_patterns` (skipped `typoPatterns`).
+The **debug** log (same directory, `pi-permission-system-debug.jsonl`, only when pi-permission-system's `debugLog` is on) carries the verbose and cheap-path detail: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer before the model stage), `model_judge.model_reply` (the verdict tool-call arguments as JSON, or the model's text when it emitted no tool call), and `model_judge.invalid_patterns` (skipped `typoPatterns`).
 A non-`external_directory` ask is not logged at all.
 
-To diagnose "the judge defers everything," read the review log for `model_judge.decision` entries and inspect `deferReason`: an empty result means no ask ever matched a pattern, `auth-failed` / `model-unresolved` means it is misconfigured, and `non-deny-verdict` means the model saw the path and chose not to deny.
+To diagnose "the judge defers everything," read the review log for `model_judge.decision` entries and inspect `deferReason`: an empty result means no ask ever matched a pattern, `auth-failed` / `model-unresolved` means it is misconfigured, `non-deny-verdict` means the model saw the path and chose not to deny, and `no-tool-call` means the model replied without calling the verdict tool (rare, since the tool is forced).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotgenes/pi-permission-model-judge",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Deny-first typo-path model judge — a pi-permission-system Authorizer chain link",
   "type": "module",
   "imports": {

package/src/model-review.ts

@@ -12,6 +12,8 @@ import type {
   Context,
   Model,
   TextContent,
+  Tool,
+  ToolCall,
 } from "@earendil-works/pi-ai";
 import type { AuthorizerVerdict } from "@gotgenes/pi-permission-system";
 
@@ -21,6 +23,39 @@ import type { ModelJudgeConfig } from "./config-schema";
 export const GENERIC_TEACHING_REASON =
   "This looks like a mistyped path. Verify the correct location before retrying.";
 
+/**
+ * The single tool the model is forced to call. Forcing it (`toolChoice: "any"`)
+ * removes free-text JSON parsing by construction — the verdict arrives as
+ * structured `arguments`, so a Markdown fence or a prose preamble can no longer
+ * cost a verdict.
+ *
+ * The Anthropic provider reads only `parameters.properties` / `parameters.required`,
+ * so a plain JSON-Schema object is correct at runtime; the `as unknown as Tool`
+ * bridge satisfies the `TSchema`-typed `parameters` field without a `typebox`
+ * dependency.
+ */
+const VERDICT_TOOL = {
+  name: "report_verdict",
+  description:
+    "Report whether the path is a mistyped path to reject (deny) or should be deferred to the human (defer).",
+  parameters: {
+    type: "object",
+    properties: {
+      verdict: {
+        type: "string",
+        enum: ["deny", "defer"],
+        description: "deny a mistyped path; defer anything else",
+      },
+      reason: {
+        type: "string",
+        description:
+          "Why the path is wrong and the correct location (required when denying)",
+      },
+    },
+    required: ["verdict"],
+  },
+} as unknown as Tool;
+
 /**
  * The injected model-completion seam — structurally the `complete` export of
  * `@earendil-works/pi-ai`. Injected so tests substitute a fake.
@@ -32,6 +67,7 @@ export type CompleteFn = (
     signal?: AbortSignal;
     apiKey?: string;
     headers?: Record<string, string>;
+    toolChoice?: string;
   },
 ) => Promise<AssistantMessage>;
 
@@ -62,13 +98,14 @@ export interface ReviewPathInputs {
 
 /**
  * Why a model call defers, distinct enough to diagnose from the decision trail:
- * the reply parsed but was not JSON (`parse-failed`), was valid JSON but not a
- * `deny` (`non-deny-verdict`), the call was aborted at `timeoutMs` (`timeout`),
- * or `complete` threw for any other reason (`call-failed` — the honest superset
- * that catches, e.g., a 401 slipping past pre-call auth resolution).
+ * the reply carried no tool call to read (`no-tool-call`), the tool call named a
+ * verdict other than `deny` (`non-deny-verdict`), the call was aborted at
+ * `timeoutMs` (`timeout`), or `complete` threw for any other reason
+ * (`call-failed` — the honest superset that catches, e.g., a 401 slipping past
+ * pre-call auth resolution).
  */
 export type ModelCallDeferReason =
-  | "parse-failed"
+  | "no-tool-call"
   | "non-deny-verdict"
   | "timeout"
   | "call-failed";
@@ -76,8 +113,9 @@ export type ModelCallDeferReason =
 /**
  * The full result of a model review: the verdict plus the observability the
  * decision trail records. `deferReason` is set iff the verdict is `defer` and a
- * model call was made; `rawReply` carries the assistant text when a reply
- * arrived (absent on a timeout/throw before any reply).
+ * model call was made; `rawReply` carries the tool-call arguments as JSON when a
+ * tool call arrived, or the assistant text on a `no-tool-call` defer (absent on
+ * a timeout/throw before any reply).
  */
 export interface ReviewOutcome {
   verdict: AuthorizerVerdict;
@@ -104,6 +142,7 @@ export async function reviewPath(
   try {
     const context: Context = {
       systemPrompt: inputs.config.instructions,
+      tools: [VERDICT_TOOL],
       messages: [
         {
           role: "user",
@@ -116,8 +155,9 @@ export async function reviewPath(
       signal: controller.signal,
       apiKey: inputs.apiKey,
       headers: inputs.headers,
+      toolChoice: "any",
     });
-    return parseOutcome(extractText(reply).trim(), Date.now() - startedAt);
+    return readToolCallOutcome(reply, Date.now() - startedAt);
   } catch {
     return {
       verdict: { kind: "defer" },
@@ -129,35 +169,41 @@ export async function reviewPath(
   }
 }
 
-/** The user-turn prompt: hand the model the path and the required reply shape. */
+/** The user-turn prompt: hand the model the path and point it at the tool. */
 function renderReviewPrompt(path: string): string {
   return [
     "A tool is about to access this path outside the working directory:",
     "",
     path,
     "",
-    'Reply with strict JSON. To reject a mistyped path: {"verdict":"deny","reason":"<why it is wrong and the correct location>"}.',
-    'Otherwise: {"verdict":"defer"}.',
+    'Call report_verdict. Use verdict "deny" with a reason naming the correct location if this is a mistyped path; otherwise use verdict "defer".',
   ].join("\n");
 }
 
 /**
- * Map the assistant reply text to an outcome; anything but a clean `deny`
- * defers with the reason that distinguishes it.
+ * Map the forced tool call to an outcome; anything but a clean `deny` defers
+ * with the reason that distinguishes it. The tool call is read by position (the
+ * first one), not by name — under OAuth the provider rewrites the registered
+ * name, so the reply's tool-call name cannot be relied on.
  */
-function parseOutcome(rawReply: string, latencyMs: number): ReviewOutcome {
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(rawReply);
-  } catch {
+function readToolCallOutcome(
+  reply: AssistantMessage,
+  latencyMs: number,
+): ReviewOutcome {
+  const call = reply.content.find(
+    (part): part is ToolCall => part.type === "toolCall",
+  );
+  if (!call) {
     return {
       verdict: { kind: "defer" },
-      deferReason: "parse-failed",
+      deferReason: "no-tool-call",
       latencyMs,
-      rawReply,
+      rawReply: extractText(reply),
     };
   }
-  if (!isRecord(parsed) || parsed.verdict !== "deny") {
+  const args = call.arguments;
+  const rawReply = JSON.stringify(args);
+  if (args.verdict !== "deny") {
     return {
       verdict: { kind: "defer" },
       deferReason: "non-deny-verdict",
@@ -166,8 +212,8 @@ function parseOutcome(rawReply: string, latencyMs: number): ReviewOutcome {
     };
   }
   const reason =
-    typeof parsed.reason === "string" && parsed.reason.length > 0
-      ? parsed.reason
+    typeof args.reason === "string" && args.reason.length > 0
+      ? args.reason
       : GENERIC_TEACHING_REASON;
   return { verdict: { kind: "deny", reason }, latencyMs, rawReply };
 }
@@ -179,7 +225,3 @@ function extractText(reply: AssistantMessage): string {
     .map((part) => part.text)
     .join("");
 }
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null;
-}