@gotgenes/pi-permission-model-judge 1.0.2 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,31 @@
1
1
  # Changelog
2
2
 
3
+ ## [1.1.1](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.1.0...pi-permission-model-judge-v1.1.1) (2026-07-22)
4
+
5
+
6
+ ### Bug Fixes
7
+
8
+ * **pi-permission-model-judge:** force a structured verdict tool call in the model review ([5a89bba](https://github.com/gotgenes/pi-packages/commit/5a89bba94b6493790a664e87cad98514253927fe)), closes [#628](https://github.com/gotgenes/pi-packages/issues/628)
9
+
10
+
11
+ ### Documentation
12
+
13
+ * **pi-permission-model-judge:** describe the forced verdict tool call ([1a2b940](https://github.com/gotgenes/pi-packages/commit/1a2b9404ad06b083f96b455df0a66e934b3f5eab))
14
+
15
+ ## [1.1.0](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.0.2...pi-permission-model-judge-v1.1.0) (2026-07-21)
16
+
17
+
18
+ ### Features
19
+
20
+ * **pi-permission-model-judge:** distinguish model-call defer reasons in reviewPath ([9098f46](https://github.com/gotgenes/pi-packages/commit/9098f465d64dfee3994132bd87a71ad40152b259))
21
+ * **pi-permission-model-judge:** record the decision trail to the permission review log ([e3964ae](https://github.com/gotgenes/pi-packages/commit/e3964ae6ca443d043db4e735ad69df90bac11c0d))
22
+ * **pi-permission-model-judge:** return the matched typo pattern ([4db29fb](https://github.com/gotgenes/pi-packages/commit/4db29fbf098c8d3c853539b408316ee2a8fffe54))
23
+
24
+
25
+ ### Documentation
26
+
27
+ * **pi-permission-model-judge:** document the decision trail ([335a7b5](https://github.com/gotgenes/pi-packages/commit/335a7b5b0f40650760d8597f511f32dc2116bf81))
28
+
3
29
  ## [1.0.2](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.0.1...pi-permission-model-judge-v1.0.2) (2026-07-21)
4
30
 
5
31
 
package/README.md CHANGED
@@ -23,6 +23,29 @@ The reviewer runs a short, cheap decision on each ask and defers at the first mi
23
23
  It is fail-safe by construction: a missing model, invalid config, model timeout, unparseable reply, or an unsure verdict all resolve to `defer`.
24
24
  Deferring means the ask falls through to the normal permission prompt — this extension only ever *removes* a hand-denial, never grants access (it emits no `allow`).
25
25
 
26
+ ## What it records
27
+
28
+ Every review the link performs leaves a trail in pi-permission-system's shared review log (`~/.pi/agent/extensions/pi-permission-system/logs/pi-permission-system-permission-review.jsonl`), so you can answer "did the judge run, did it reach the model, and why did it defer?"
29
+ without guesswork.
30
+
31
+ Once an ask matches a `typoPattern` — the case that *should* reach the model — the link writes one `model_judge.decision` entry recording the outcome:
32
+
33
+ | Field | Meaning |
34
+ | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
35
+ | `requestId` | Joins to the `permission_request.*` entries for the same ask. |
36
+ | `path` | The candidate path reviewed. |
37
+ | `matchedPattern` | The `typoPatterns` entry (as you wrote it) that matched. |
38
+ | `modelCalled` | `false` when the model or its auth did not resolve. |
39
+ | `modelId` | `<provider>/<model>`. |
40
+ | `latencyMs` | Model-call wall-clock in ms, or `null` when no call was made. |
41
+ | `verdict` | `"deny"` or `"defer"`. |
42
+ | `deferReason` | `null` on a deny, else one of `model-unresolved` / `auth-failed` / `no-tool-call` / `non-deny-verdict` / `timeout` / `call-failed`. |
43
+
44
+ Cheaper events go to pi-permission-system's **debug** log, and only when its `debugLog` toggle is on: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer) and `model_judge.model_reply` (the verdict tool-call arguments as JSON, or the model's text when it emitted no tool call).
45
+ A non-`external_directory` ask is not logged — it is not this link's concern.
46
+
47
+ Because every pattern-matched ask leaves a positive record, a misconfiguration that silently defers every path (an auth failure, an unresolved model) shows up as a run of `deferReason` entries rather than an empty log.
48
+
26
49
  ## Install
27
50
 
28
51
  ```bash
@@ -2,7 +2,7 @@
2
2
  "$schema": "https://raw.githubusercontent.com/gotgenes/pi-packages/main/packages/pi-permission-model-judge/schemas/model-judge.schema.json",
3
3
  "provider": "anthropic",
4
4
  "model": "claude-haiku-4-5",
5
- "instructions": "You review a filesystem path a tool is about to access outside the working directory. Two common typos both point at the wrong location. (1) A doubled package segment — a package name repeated around `packages/`, for example `pi-permission-system/packages/pi-permission-system/…`, where the correct path names the package only once under the repository's `packages/` directory. (2) A dropped repository prefix — a path that jumps straight to a package directory such as `development/pi/pi-permission-system/…`, where the correct path routes through the repository's `pi-packages/packages/` directory (for example `development/pi/pi-packages/packages/pi-permission-system/…`). When the path shows either typo, deny it and give the single corrected absolute path in the reason so the caller can retry against the right location. If the path looks intentional or you are unsure, defer. Reply with strict JSON: {\"verdict\":\"deny\",\"reason\":\"…\"} to reject, or {\"verdict\":\"defer\"} otherwise.",
5
+ "instructions": "You review a filesystem path a tool is about to access outside the working directory. Two common typos both point at the wrong location. (1) A doubled package segment — a package name repeated around `packages/`, for example `pi-permission-system/packages/pi-permission-system/…`, where the correct path names the package only once under the repository's `packages/` directory. (2) A dropped repository prefix — a path that jumps straight to a package directory such as `development/pi/pi-permission-system/…`, where the correct path routes through the repository's `pi-packages/packages/` directory (for example `development/pi/pi-packages/packages/pi-permission-system/…`). When the path shows either typo, deny it and give the single corrected absolute path in the reason so the caller can retry against the right location. If the path looks intentional or you are unsure, defer.",
6
6
  "typoPatterns": [
7
7
  "([^/]+)/packages/\\1(/|$)",
8
8
  "development/pi/(?!pi-packages/)pi-[^/]+(/|$)"
@@ -44,19 +44,19 @@ The link decides nothing until you name it in `authorizerChain` (opt-in activati
44
44
  ### `provider` and `model`
45
45
 
46
46
  The reviewer resolves `provider` / `model` against the session's model registry at review time.
47
- If the model does not resolve (wrong id, no credentials), the reviewer logs a warning and defers — it never blocks an ask because the judge is misconfigured.
47
+ If the model does not resolve (wrong id, no credentials), the reviewer records a `model-unresolved` decision entry and defers — it never blocks an ask because the judge is misconfigured.
48
48
 
49
49
  ### `instructions`
50
50
 
51
51
  This is the model's system prompt.
52
- Describe the typo shape you care about and instruct the model to reply with the wrong segment and the correct location, so the invoking agent can self-correct.
53
- The reviewer expects the model to answer with strict JSON — `{"verdict":"deny","reason":"…"}` to reject, `{"verdict":"defer"}` otherwise and any other reply defers.
52
+ Describe the typo shape you care about and instruct the model to give the wrong segment and the correct location, so the invoking agent can self-correct.
53
+ The reviewer forces the model to call a single `report_verdict` tool, so the verdict is read from structured tool-call arguments rather than parsed from free text a `deny` verdict with a `reason` rejects the path, and anything else (including no tool call) defers.
54
54
 
55
55
  ### `typoPatterns`
56
56
 
57
57
  Each string is compiled with `new RegExp(pattern)` (no flags) and tested against the candidate path.
58
58
  Only a path that matches at least one pattern is sent to the model — this is the cost gate that keeps the reviewer from calling the model on every ask.
59
- An invalid regular expression is skipped with a warning.
59
+ An invalid regular expression is skipped and recorded in the debug log (`model_judge.invalid_patterns`).
60
60
  An empty list (the default) matches nothing, so the reviewer defers everything.
61
61
 
62
62
  Two typo shapes are worth featuring, each as its own pattern:
@@ -92,3 +92,27 @@ Every uncertain outcome defers, which sends the ask to the normal permission pro
92
92
  - an unresolved model, a timeout, an unparseable reply, or an unsure verdict.
93
93
 
94
94
  Deferring never grants access — this extension emits no `allow` verdict.
95
+
96
+ ## The decision trail
97
+
98
+ The reviewer records what it did to pi-permission-system's shared logs, keyed by `requestId` so an entry joins to the `permission_request.*` lines for the same ask.
99
+ This closes the observability gap that let an auth failure defer every path undetected: a silent defer is now a recorded defer with a reason.
100
+
101
+ The **review** log (`~/.pi/agent/extensions/pi-permission-system/logs/pi-permission-system-permission-review.jsonl`, on by default via pi-permission-system's `permissionReviewLog`) carries one `model_judge.decision` entry per **pattern-matched** ask — the case that should reach the model:
102
+
103
+ | Field | Meaning |
104
+ | ---------------- | ---------------------------------------------------------------------------------------------------------------------------- |
105
+ | `requestId` | The ask id, shared with the `permission_request.*` entries. |
106
+ | `surface` | Always `external_directory` (the only reviewed surface). |
107
+ | `path` | The candidate path reviewed. |
108
+ | `matchedPattern` | The `typoPatterns` entry, as written, that matched. |
109
+ | `modelCalled` | `false` for a `model-unresolved` / `auth-failed` defer. |
110
+ | `modelId` | `<provider>/<model>`. |
111
+ | `latencyMs` | Model-call wall-clock in milliseconds, or `null` when no call was made. |
112
+ | `verdict` | `"deny"` or `"defer"`. |
113
+ | `deferReason` | `null` on a deny, else `model-unresolved` / `auth-failed` / `no-tool-call` / `non-deny-verdict` / `timeout` / `call-failed`. |
114
+
115
+ The **debug** log (same directory, `pi-permission-system-debug.jsonl`, only when pi-permission-system's `debugLog` is on) carries the verbose and cheap-path detail: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer before the model stage), `model_judge.model_reply` (the verdict tool-call arguments as JSON, or the model's text when it emitted no tool call), and `model_judge.invalid_patterns` (skipped `typoPatterns`).
116
+ A non-`external_directory` ask is not logged at all.
117
+
118
+ To diagnose "the judge defers everything," read the review log for `model_judge.decision` entries and inspect `deferReason`: an empty result means no ask ever matched a pattern, `auth-failed` / `model-unresolved` means it is misconfigured, `non-deny-verdict` means the model saw the path and chose not to deny, and `no-tool-call` means the model replied without calling the verdict tool (rare, since the tool is forced).
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gotgenes/pi-permission-model-judge",
3
- "version": "1.0.2",
3
+ "version": "1.1.1",
4
4
  "description": "Deny-first typo-path model judge — a pi-permission-system Authorizer chain link",
5
5
  "type": "module",
6
6
  "imports": {
@@ -54,13 +54,13 @@
54
54
  "peerDependencies": {
55
55
  "@earendil-works/pi-ai": ">=0.79.0",
56
56
  "@earendil-works/pi-coding-agent": ">=0.79.0",
57
- "@gotgenes/pi-permission-system": ">=20.9.0"
57
+ "@gotgenes/pi-permission-system": ">=20.10.0"
58
58
  },
59
59
  "devDependencies": {
60
60
  "@biomejs/biome": "^2.4.16",
61
61
  "@earendil-works/pi-ai": "0.79.1",
62
62
  "@earendil-works/pi-coding-agent": "0.79.1",
63
- "@gotgenes/pi-permission-system": "20.9.0",
63
+ "@gotgenes/pi-permission-system": "20.10.0",
64
64
  "@types/node": "^22.15.3",
65
65
  "rumdl": "^0.2.10",
66
66
  "typescript": "^6.0.3",
package/src/extension.ts CHANGED
@@ -67,7 +67,6 @@ export function createModelJudgeExtension(
67
67
  getConfig: () => config,
68
68
  getRegistry: () => registry,
69
69
  complete,
70
- warn,
71
70
  });
72
71
  dispose = service.registerAuthorizer(LINK_NAME, authorize);
73
72
  }
@@ -12,6 +12,8 @@ import type {
12
12
  Context,
13
13
  Model,
14
14
  TextContent,
15
+ Tool,
16
+ ToolCall,
15
17
  } from "@earendil-works/pi-ai";
16
18
  import type { AuthorizerVerdict } from "@gotgenes/pi-permission-system";
17
19
 
@@ -21,6 +23,39 @@ import type { ModelJudgeConfig } from "./config-schema";
21
23
  export const GENERIC_TEACHING_REASON =
22
24
  "This looks like a mistyped path. Verify the correct location before retrying.";
23
25
 
26
+ /**
27
+ * The single tool the model is forced to call. Forcing it (`toolChoice: "any"`)
28
+ * removes free-text JSON parsing by construction — the verdict arrives as
29
+ * structured `arguments`, so a Markdown fence or a prose preamble can no longer
30
+ * cost a verdict.
31
+ *
32
+ * The Anthropic provider reads only `parameters.properties` / `parameters.required`,
33
+ * so a plain JSON-Schema object is correct at runtime; the `as unknown as Tool`
34
+ * bridge satisfies the `TSchema`-typed `parameters` field without a `typebox`
35
+ * dependency.
36
+ */
37
+ const VERDICT_TOOL = {
38
+ name: "report_verdict",
39
+ description:
40
+ "Report whether the path is a mistyped path to reject (deny) or should be deferred to the human (defer).",
41
+ parameters: {
42
+ type: "object",
43
+ properties: {
44
+ verdict: {
45
+ type: "string",
46
+ enum: ["deny", "defer"],
47
+ description: "deny a mistyped path; defer anything else",
48
+ },
49
+ reason: {
50
+ type: "string",
51
+ description:
52
+ "Why the path is wrong and the correct location (required when denying)",
53
+ },
54
+ },
55
+ required: ["verdict"],
56
+ },
57
+ } as unknown as Tool;
58
+
24
59
  /**
25
60
  * The injected model-completion seam — structurally the `complete` export of
26
61
  * `@earendil-works/pi-ai`. Injected so tests substitute a fake.
@@ -32,6 +67,7 @@ export type CompleteFn = (
32
67
  signal?: AbortSignal;
33
68
  apiKey?: string;
34
69
  headers?: Record<string, string>;
70
+ toolChoice?: string;
35
71
  },
36
72
  ) => Promise<AssistantMessage>;
37
73
 
@@ -61,21 +97,52 @@ export interface ReviewPathInputs {
61
97
  }
62
98
 
63
99
  /**
64
- * Review one candidate path with the model and return its verdict.
100
+ * Why a model call defers, distinct enough to diagnose from the decision trail:
101
+ * the reply carried no tool call to read (`no-tool-call`), the tool call named a
102
+ * verdict other than `deny` (`non-deny-verdict`), the call was aborted at
103
+ * `timeoutMs` (`timeout`), or `complete` threw for any other reason
104
+ * (`call-failed` — the honest superset that catches, e.g., a 401 slipping past
105
+ * pre-call auth resolution).
106
+ */
107
+ export type ModelCallDeferReason =
108
+ | "no-tool-call"
109
+ | "non-deny-verdict"
110
+ | "timeout"
111
+ | "call-failed";
112
+
113
+ /**
114
+ * The full result of a model review: the verdict plus the observability the
115
+ * decision trail records. `deferReason` is set iff the verdict is `defer` and a
116
+ * model call was made; `rawReply` carries the tool-call arguments as JSON when a
117
+ * tool call arrived, or the assistant text on a `no-tool-call` defer (absent on
118
+ * a timeout/throw before any reply).
119
+ */
120
+ export interface ReviewOutcome {
121
+ verdict: AuthorizerVerdict;
122
+ deferReason?: ModelCallDeferReason;
123
+ latencyMs: number;
124
+ rawReply?: string;
125
+ }
126
+
127
+ /**
128
+ * Review one candidate path with the model and return the structured outcome.
65
129
  *
66
130
  * The call is aborted after `config.timeoutMs`; an abort, a rejection, or any
67
- * non-`deny` reply yields `defer`.
131
+ * non-`deny` reply yields `defer` (more prompting, never less) with the reason
132
+ * annotated so the caller can record why.
68
133
  */
69
134
  export async function reviewPath(
70
135
  inputs: ReviewPathInputs,
71
- ): Promise<AuthorizerVerdict> {
136
+ ): Promise<ReviewOutcome> {
72
137
  const controller = new AbortController();
73
138
  const timer = setTimeout(() => {
74
139
  controller.abort();
75
140
  }, inputs.config.timeoutMs);
141
+ const startedAt = Date.now();
76
142
  try {
77
143
  const context: Context = {
78
144
  systemPrompt: inputs.config.instructions,
145
+ tools: [VERDICT_TOOL],
79
146
  messages: [
80
147
  {
81
148
  role: "user",
@@ -88,44 +155,67 @@ export async function reviewPath(
88
155
  signal: controller.signal,
89
156
  apiKey: inputs.apiKey,
90
157
  headers: inputs.headers,
158
+ toolChoice: "any",
91
159
  });
92
- return parseVerdict(reply);
160
+ return readToolCallOutcome(reply, Date.now() - startedAt);
93
161
  } catch {
94
- return { kind: "defer" };
162
+ return {
163
+ verdict: { kind: "defer" },
164
+ deferReason: controller.signal.aborted ? "timeout" : "call-failed",
165
+ latencyMs: Date.now() - startedAt,
166
+ };
95
167
  } finally {
96
168
  clearTimeout(timer);
97
169
  }
98
170
  }
99
171
 
100
- /** The user-turn prompt: hand the model the path and the required reply shape. */
172
+ /** The user-turn prompt: hand the model the path and point it at the tool. */
101
173
  function renderReviewPrompt(path: string): string {
102
174
  return [
103
175
  "A tool is about to access this path outside the working directory:",
104
176
  "",
105
177
  path,
106
178
  "",
107
- 'Reply with strict JSON. To reject a mistyped path: {"verdict":"deny","reason":"<why it is wrong and the correct location>"}.',
108
- 'Otherwise: {"verdict":"defer"}.',
179
+ 'Call report_verdict. Use verdict "deny" with a reason naming the correct location if this is a mistyped path; otherwise use verdict "defer".',
109
180
  ].join("\n");
110
181
  }
111
182
 
112
- /** Map the assistant reply to a verdict; anything but a clean `deny` defers. */
113
- function parseVerdict(reply: AssistantMessage): AuthorizerVerdict {
114
- const text = extractText(reply).trim();
115
- let parsed: unknown;
116
- try {
117
- parsed = JSON.parse(text);
118
- } catch {
119
- return { kind: "defer" };
183
+ /**
184
+ * Map the forced tool call to an outcome; anything but a clean `deny` defers
185
+ * with the reason that distinguishes it. The tool call is read by position (the
186
+ * first one), not by name — under OAuth the provider rewrites the registered
187
+ * name, so the reply's tool-call name cannot be relied on.
188
+ */
189
+ function readToolCallOutcome(
190
+ reply: AssistantMessage,
191
+ latencyMs: number,
192
+ ): ReviewOutcome {
193
+ const call = reply.content.find(
194
+ (part): part is ToolCall => part.type === "toolCall",
195
+ );
196
+ if (!call) {
197
+ return {
198
+ verdict: { kind: "defer" },
199
+ deferReason: "no-tool-call",
200
+ latencyMs,
201
+ rawReply: extractText(reply),
202
+ };
120
203
  }
121
- if (!isRecord(parsed) || parsed.verdict !== "deny") {
122
- return { kind: "defer" };
204
+ const args = call.arguments;
205
+ const rawReply = JSON.stringify(args);
206
+ if (args.verdict !== "deny") {
207
+ return {
208
+ verdict: { kind: "defer" },
209
+ deferReason: "non-deny-verdict",
210
+ latencyMs,
211
+ rawReply,
212
+ };
123
213
  }
124
214
  const reason =
125
- typeof parsed.reason === "string" && parsed.reason.length > 0
126
- ? parsed.reason
215
+ typeof args.reason === "string" && args.reason.length > 0
216
+ ? args.reason
127
217
  : GENERIC_TEACHING_REASON;
128
- return { kind: "deny", reason };
218
+ return { verdict: { kind: "deny", reason }, latencyMs, rawReply };
129
219
  }
130
220
 
131
221
  /** Concatenate the text parts of an assistant reply. */
@@ -135,7 +225,3 @@ function extractText(reply: AssistantMessage): string {
135
225
  .map((part) => part.text)
136
226
  .join("");
137
227
  }
138
-
139
- function isRecord(value: unknown): value is Record<string, unknown> {
140
- return typeof value === "object" && value !== null;
141
- }
@@ -7,9 +7,16 @@
7
7
  * without a model call.
8
8
  */
9
9
 
10
- /** Result of compiling a pattern list: usable regexes plus any that failed. */
10
+ /**
11
+ * Result of compiling a pattern list: usable regexes (with their original
12
+ * source strings, parallel to `regexes`) plus any that failed to compile.
13
+ * `patterns[i]` is the operator's literal config string for `regexes[i]`, kept
14
+ * so the decision trail records the pattern as written, not the slash-escaped
15
+ * `RegExp.source`.
16
+ */
11
17
  export interface CompiledTypoPatterns {
12
18
  regexes: RegExp[];
19
+ patterns: string[];
13
20
  invalidPatterns: string[];
14
21
  }
15
22
 
@@ -22,21 +29,28 @@ export function compileTypoPatterns(
22
29
  patterns: readonly string[],
23
30
  ): CompiledTypoPatterns {
24
31
  const regexes: RegExp[] = [];
32
+ const sources: string[] = [];
25
33
  const invalidPatterns: string[] = [];
26
34
  for (const pattern of patterns) {
27
35
  try {
28
36
  regexes.push(new RegExp(pattern));
37
+ sources.push(pattern);
29
38
  } catch {
30
39
  invalidPatterns.push(pattern);
31
40
  }
32
41
  }
33
- return { regexes, invalidPatterns };
42
+ return { regexes, patterns: sources, invalidPatterns };
34
43
  }
35
44
 
36
- /** Whether any compiled pattern matches `path`. */
37
- export function matchesAnyTypoPattern(
45
+ /**
46
+ * The source string of the first compiled pattern that matches `path`, or
47
+ * `undefined` if none do. The source (not the boolean) so the decision trail
48
+ * can record *which* pattern matched.
49
+ */
50
+ export function matchTypoPattern(
38
51
  path: string,
39
52
  compiled: CompiledTypoPatterns,
40
- ): boolean {
41
- return compiled.regexes.some((regex) => regex.test(path));
53
+ ): string | undefined {
54
+ const index = compiled.regexes.findIndex((regex) => regex.test(path));
55
+ return index === -1 ? undefined : compiled.patterns[index];
42
56
  }
@@ -4,17 +4,26 @@
4
4
  *
5
5
  * The decision runs top to bottom, deferring at the first miss so the cheap
6
6
  * gates short-circuit before any model call:
7
- * 1. surface is `external_directory` (else defer),
7
+ * 1. surface is `external_directory` (else defer, silently — not our surface),
8
8
  * 2. a candidate path is present (else defer),
9
9
  * 3. the path matches a configured typo pattern (else defer, no model call),
10
10
  * 4. the model confirms the typo (deny with a teaching reason) or defers.
11
11
  *
12
12
  * Every failure path defers — more prompting, never less (ADR 0007 invariant 2).
13
13
  * This slice never emits `allow`.
14
+ *
15
+ * Observability (issue #626): once an ask matches a typo pattern (the [#625]
16
+ * class — a candidate typo that *should* reach the model), the reviewer writes a
17
+ * positive `model_judge.decision` review entry recording the outcome and, on a
18
+ * defer, its reason — so a silent 100%-defer regression cannot hide again. The
19
+ * cheap short-circuits (`no-path`, `pattern-miss`) and the raw model reply go to
20
+ * the debug log; a non-`external_directory` surface is not logged at all.
14
21
  */
15
22
 
16
23
  import type {
17
24
  Authorizer,
25
+ AuthorizerLog,
26
+ AuthorizerVerdict,
18
27
  PromptPermissionDetails,
19
28
  } from "@gotgenes/pi-permission-system";
20
29
 
@@ -27,11 +36,31 @@ import {
27
36
  import {
28
37
  type CompiledTypoPatterns,
29
38
  compileTypoPatterns,
30
- matchesAnyTypoPattern,
39
+ matchTypoPattern,
31
40
  } from "./typo-patterns";
32
41
 
33
42
  const REVIEWED_SURFACE = "external_directory";
34
43
 
44
+ /** Review-log event: one positive decision record per pattern-matched ask. */
45
+ const DECISION_EVENT = "model_judge.decision";
46
+ /** Debug-log event: a cheap short-circuit before the model stage. */
47
+ const SHORT_CIRCUIT_EVENT = "model_judge.short_circuit";
48
+ /** Debug-log event: the raw model reply, gated behind `debugLog`. */
49
+ const MODEL_REPLY_EVENT = "model_judge.model_reply";
50
+ /** Debug-log event: `typoPatterns` entries that failed to compile. */
51
+ const INVALID_PATTERNS_EVENT = "model_judge.invalid_patterns";
52
+
53
+ /** A defer decided before the model call, still recorded positively. */
54
+ type PreModelDeferReason = "model-unresolved" | "auth-failed";
55
+
56
+ /** The shared fields of a `model_judge.decision` record, before the outcome. */
57
+ interface DecisionBase {
58
+ requestId: string;
59
+ path: string;
60
+ matchedPattern: string;
61
+ modelId: string;
62
+ }
63
+
35
64
  /** Collaborators for the reviewer, injected so the extension and tests wire them. */
36
65
  export interface TypoReviewerDeps {
37
66
  /** The loaded config, read live (absent until session config loads). */
@@ -40,21 +69,20 @@ export interface TypoReviewerDeps {
40
69
  getRegistry: () => ModelRegistryLike | undefined;
41
70
  /** The model-completion seam (production: `complete` from `@earendil-works/pi-ai`). */
42
71
  complete: CompleteFn;
43
- /** Optional warn sink for operational notices (unresolved model, bad patterns). */
44
- warn?: (message: string) => void;
45
72
  }
46
73
 
47
74
  /**
48
75
  * Build the `authorize` callback registered on the chain. The `query` argument
49
76
  * is unused in this slice — the deny-first reviewer decides from the path
50
- * pattern and the model, not from an engine re-query (slice 2 consumes it).
77
+ * pattern and the model, not from an engine re-query (slice 2 consumes it). The
78
+ * `log` argument is the injected review-log seam the decision trail records to.
51
79
  */
52
80
  export function createTypoReviewer(
53
81
  deps: TypoReviewerDeps,
54
82
  ): Authorizer["authorize"] {
55
- const compiledFor = memoizeCompiledPatterns(deps.warn);
83
+ const compiledFor = memoizeCompiledPatterns();
56
84
 
57
- return async (details) => {
85
+ return async (details, _query, log) => {
58
86
  const config = deps.getConfig();
59
87
  if (!config) {
60
88
  return { kind: "defer" };
@@ -62,29 +90,35 @@ export function createTypoReviewer(
62
90
  if (surfaceOf(details) !== REVIEWED_SURFACE) {
63
91
  return { kind: "defer" };
64
92
  }
93
+ const { requestId } = details;
65
94
  const path = pathOf(details);
66
95
  if (path === undefined) {
96
+ log.debug(SHORT_CIRCUIT_EVENT, { requestId, reason: "no-path" });
67
97
  return { kind: "defer" };
68
98
  }
69
- if (!matchesAnyTypoPattern(path, compiledFor(config))) {
99
+ const matchedPattern = matchTypoPattern(path, compiledFor(config, log));
100
+ if (matchedPattern === undefined) {
101
+ log.debug(SHORT_CIRCUIT_EVENT, {
102
+ requestId,
103
+ path,
104
+ reason: "pattern-miss",
105
+ });
70
106
  return { kind: "defer" };
71
107
  }
108
+
109
+ const modelId = `${config.provider}/${config.model}`;
110
+ const base: DecisionBase = { requestId, path, matchedPattern, modelId };
72
111
  const registry = deps.getRegistry();
73
112
  const model = registry?.find(config.provider, config.model);
74
113
  if (!registry || !model) {
75
- deps.warn?.(
76
- `model "${config.provider}/${config.model}" did not resolve; deferring`,
77
- );
78
- return { kind: "defer" };
114
+ return deferWith(log, base, "model-unresolved");
79
115
  }
80
116
  const auth = await registry.getApiKeyAndHeaders(model);
81
117
  if (!auth.ok) {
82
- deps.warn?.(
83
- `auth for "${config.provider}/${config.model}" did not resolve (${auth.error}); deferring`,
84
- );
85
- return { kind: "defer" };
118
+ return deferWith(log, base, "auth-failed");
86
119
  }
87
- return reviewPath({
120
+
121
+ const outcome = await reviewPath({
88
122
  path,
89
123
  config,
90
124
  model,
@@ -92,9 +126,52 @@ export function createTypoReviewer(
92
126
  apiKey: auth.apiKey,
93
127
  headers: auth.headers,
94
128
  });
129
+ if (outcome.rawReply !== undefined) {
130
+ log.debug(MODEL_REPLY_EVENT, {
131
+ requestId,
132
+ modelId,
133
+ rawReply: outcome.rawReply,
134
+ });
135
+ }
136
+ log.review(DECISION_EVENT, {
137
+ requestId,
138
+ surface: REVIEWED_SURFACE,
139
+ path,
140
+ matchedPattern,
141
+ modelCalled: true,
142
+ modelId,
143
+ latencyMs: outcome.latencyMs,
144
+ verdict: outcome.verdict.kind,
145
+ deferReason: outcome.deferReason ?? null,
146
+ });
147
+ return outcome.verdict;
95
148
  };
96
149
  }
97
150
 
151
+ /**
152
+ * Record a pre-model defer (`model-unresolved` / `auth-failed`) as a positive
153
+ * `model_judge.decision` entry and return the defer verdict — so the two
154
+ * model-resolution failures leave evidence on record, not a silent absence.
155
+ */
156
+ function deferWith(
157
+ log: AuthorizerLog,
158
+ base: DecisionBase,
159
+ deferReason: PreModelDeferReason,
160
+ ): AuthorizerVerdict {
161
+ log.review(DECISION_EVENT, {
162
+ requestId: base.requestId,
163
+ surface: REVIEWED_SURFACE,
164
+ path: base.path,
165
+ matchedPattern: base.matchedPattern,
166
+ modelCalled: false,
167
+ modelId: base.modelId,
168
+ latencyMs: null,
169
+ verdict: "defer",
170
+ deferReason,
171
+ });
172
+ return { kind: "defer" };
173
+ }
174
+
98
175
  /** The gate-authoritative surface, falling back to the display surface. */
99
176
  function surfaceOf(details: PromptPermissionDetails): string | undefined {
100
177
  return details.accessIntent?.surface ?? details.surface ?? undefined;
@@ -110,21 +187,22 @@ function pathOf(details: PromptPermissionDetails): string | undefined {
110
187
  * object is in effect — the config is stable per session, so this avoids
111
188
  * recompiling the regexes on every ask.
112
189
  */
113
- function memoizeCompiledPatterns(
114
- warn?: (message: string) => void,
115
- ): (config: ModelJudgeConfig) => CompiledTypoPatterns {
190
+ function memoizeCompiledPatterns(): (
191
+ config: ModelJudgeConfig,
192
+ log: AuthorizerLog,
193
+ ) => CompiledTypoPatterns {
116
194
  let cache:
117
195
  | { config: ModelJudgeConfig; compiled: CompiledTypoPatterns }
118
196
  | undefined;
119
- return (config) => {
197
+ return (config, log) => {
120
198
  if (cache?.config === config) {
121
199
  return cache.compiled;
122
200
  }
123
201
  const compiled = compileTypoPatterns(config.typoPatterns);
124
202
  if (compiled.invalidPatterns.length > 0) {
125
- warn?.(
126
- `ignoring invalid typoPatterns: ${compiled.invalidPatterns.join(", ")}`,
127
- );
203
+ log.debug(INVALID_PATTERNS_EVENT, {
204
+ invalidPatterns: compiled.invalidPatterns,
205
+ });
128
206
  }
129
207
  cache = { config, compiled };
130
208
  return compiled;