@gotgenes/pi-permission-model-judge 1.0.2 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +23 -0
- package/config/config.example.json +1 -1
- package/docs/configuration.md +28 -4
- package/package.json +3 -3
- package/src/extension.ts +0 -1
- package/src/model-review.ts +111 -25
- package/src/typo-patterns.ts +20 -6
- package/src/typo-reviewer.ts +102 -24
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.1.1](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.1.0...pi-permission-model-judge-v1.1.1) (2026-07-22)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* **pi-permission-model-judge:** force a structured verdict tool call in the model review ([5a89bba](https://github.com/gotgenes/pi-packages/commit/5a89bba94b6493790a664e87cad98514253927fe)), closes [#628](https://github.com/gotgenes/pi-packages/issues/628)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Documentation
|
|
12
|
+
|
|
13
|
+
* **pi-permission-model-judge:** describe the forced verdict tool call ([1a2b940](https://github.com/gotgenes/pi-packages/commit/1a2b9404ad06b083f96b455df0a66e934b3f5eab))
|
|
14
|
+
|
|
15
|
+
## [1.1.0](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.0.2...pi-permission-model-judge-v1.1.0) (2026-07-21)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
### Features
|
|
19
|
+
|
|
20
|
+
* **pi-permission-model-judge:** distinguish model-call defer reasons in reviewPath ([9098f46](https://github.com/gotgenes/pi-packages/commit/9098f465d64dfee3994132bd87a71ad40152b259))
|
|
21
|
+
* **pi-permission-model-judge:** record the decision trail to the permission review log ([e3964ae](https://github.com/gotgenes/pi-packages/commit/e3964ae6ca443d043db4e735ad69df90bac11c0d))
|
|
22
|
+
* **pi-permission-model-judge:** return the matched typo pattern ([4db29fb](https://github.com/gotgenes/pi-packages/commit/4db29fbf098c8d3c853539b408316ee2a8fffe54))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
### Documentation
|
|
26
|
+
|
|
27
|
+
* **pi-permission-model-judge:** document the decision trail ([335a7b5](https://github.com/gotgenes/pi-packages/commit/335a7b5b0f40650760d8597f511f32dc2116bf81))
|
|
28
|
+
|
|
3
29
|
## [1.0.2](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.0.1...pi-permission-model-judge-v1.0.2) (2026-07-21)
|
|
4
30
|
|
|
5
31
|
|
package/README.md
CHANGED
|
@@ -23,6 +23,29 @@ The reviewer runs a short, cheap decision on each ask and defers at the first mi
|
|
|
23
23
|
It is fail-safe by construction: a missing model, invalid config, model timeout, unparseable reply, or an unsure verdict all resolve to `defer`.
|
|
24
24
|
Deferring means the ask falls through to the normal permission prompt — this extension only ever *removes* a hand-denial, never grants access (it emits no `allow`).
|
|
25
25
|
|
|
26
|
+
## What it records
|
|
27
|
+
|
|
28
|
+
Every review the link performs leaves a trail in pi-permission-system's shared review log (`~/.pi/agent/extensions/pi-permission-system/logs/pi-permission-system-permission-review.jsonl`), so you can answer "did the judge run, did it reach the model, and why did it defer?"
|
|
29
|
+
without guesswork.
|
|
30
|
+
|
|
31
|
+
Once an ask matches a `typoPattern` — the case that *should* reach the model — the link writes one `model_judge.decision` entry recording the outcome:
|
|
32
|
+
|
|
33
|
+
| Field | Meaning |
|
|
34
|
+
| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
|
35
|
+
| `requestId` | Joins to the `permission_request.*` entries for the same ask. |
|
|
36
|
+
| `path` | The candidate path reviewed. |
|
|
37
|
+
| `matchedPattern` | The `typoPatterns` entry (as you wrote it) that matched. |
|
|
38
|
+
| `modelCalled` | `false` when the model or its auth did not resolve. |
|
|
39
|
+
| `modelId` | `<provider>/<model>`. |
|
|
40
|
+
| `latencyMs` | Model-call wall-clock in ms, or `null` when no call was made. |
|
|
41
|
+
| `verdict` | `"deny"` or `"defer"`. |
|
|
42
|
+
| `deferReason` | `null` on a deny, else one of `model-unresolved` / `auth-failed` / `no-tool-call` / `non-deny-verdict` / `timeout` / `call-failed`. |
|
|
43
|
+
|
|
44
|
+
Cheaper events go to pi-permission-system's **debug** log, and only when its `debugLog` toggle is on: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer) and `model_judge.model_reply` (the verdict tool-call arguments as JSON, or the model's text when it emitted no tool call).
|
|
45
|
+
A non-`external_directory` ask is not logged — it is not this link's concern.
|
|
46
|
+
|
|
47
|
+
Because every pattern-matched ask leaves a positive record, a misconfiguration that silently defers every path (an auth failure, an unresolved model) shows up as a run of `deferReason` entries rather than an empty log.
|
|
48
|
+
|
|
26
49
|
## Install
|
|
27
50
|
|
|
28
51
|
```bash
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://raw.githubusercontent.com/gotgenes/pi-packages/main/packages/pi-permission-model-judge/schemas/model-judge.schema.json",
|
|
3
3
|
"provider": "anthropic",
|
|
4
4
|
"model": "claude-haiku-4-5",
|
|
5
|
-
"instructions": "You review a filesystem path a tool is about to access outside the working directory. Two common typos both point at the wrong location. (1) A doubled package segment — a package name repeated around `packages/`, for example `pi-permission-system/packages/pi-permission-system/…`, where the correct path names the package only once under the repository's `packages/` directory. (2) A dropped repository prefix — a path that jumps straight to a package directory such as `development/pi/pi-permission-system/…`, where the correct path routes through the repository's `pi-packages/packages/` directory (for example `development/pi/pi-packages/packages/pi-permission-system/…`). When the path shows either typo, deny it and give the single corrected absolute path in the reason so the caller can retry against the right location. If the path looks intentional or you are unsure, defer.
|
|
5
|
+
"instructions": "You review a filesystem path a tool is about to access outside the working directory. Two common typos both point at the wrong location. (1) A doubled package segment — a package name repeated around `packages/`, for example `pi-permission-system/packages/pi-permission-system/…`, where the correct path names the package only once under the repository's `packages/` directory. (2) A dropped repository prefix — a path that jumps straight to a package directory such as `development/pi/pi-permission-system/…`, where the correct path routes through the repository's `pi-packages/packages/` directory (for example `development/pi/pi-packages/packages/pi-permission-system/…`). When the path shows either typo, deny it and give the single corrected absolute path in the reason so the caller can retry against the right location. If the path looks intentional or you are unsure, defer.",
|
|
6
6
|
"typoPatterns": [
|
|
7
7
|
"([^/]+)/packages/\\1(/|$)",
|
|
8
8
|
"development/pi/(?!pi-packages/)pi-[^/]+(/|$)"
|
package/docs/configuration.md
CHANGED
|
@@ -44,19 +44,19 @@ The link decides nothing until you name it in `authorizerChain` (opt-in activati
|
|
|
44
44
|
### `provider` and `model`
|
|
45
45
|
|
|
46
46
|
The reviewer resolves `provider` / `model` against the session's model registry at review time.
|
|
47
|
-
If the model does not resolve (wrong id, no credentials), the reviewer
|
|
47
|
+
If the model does not resolve (wrong id, no credentials), the reviewer records a `model-unresolved` decision entry and defers — it never blocks an ask because the judge is misconfigured.
|
|
48
48
|
|
|
49
49
|
### `instructions`
|
|
50
50
|
|
|
51
51
|
This is the model's system prompt.
|
|
52
|
-
Describe the typo shape you care about and instruct the model to
|
|
53
|
-
The reviewer
|
|
52
|
+
Describe the typo shape you care about and instruct the model to give the wrong segment and the correct location, so the invoking agent can self-correct.
|
|
53
|
+
The reviewer forces the model to call a single `report_verdict` tool, so the verdict is read from structured tool-call arguments rather than parsed from free text — a `deny` verdict with a `reason` rejects the path, and anything else (including no tool call) defers.
|
|
54
54
|
|
|
55
55
|
### `typoPatterns`
|
|
56
56
|
|
|
57
57
|
Each string is compiled with `new RegExp(pattern)` (no flags) and tested against the candidate path.
|
|
58
58
|
Only a path that matches at least one pattern is sent to the model — this is the cost gate that keeps the reviewer from calling the model on every ask.
|
|
59
|
-
An invalid regular expression is skipped
|
|
59
|
+
An invalid regular expression is skipped and recorded in the debug log (`model_judge.invalid_patterns`).
|
|
60
60
|
An empty list (the default) matches nothing, so the reviewer defers everything.
|
|
61
61
|
|
|
62
62
|
Two typo shapes are worth featuring, each as its own pattern:
|
|
@@ -92,3 +92,27 @@ Every uncertain outcome defers, which sends the ask to the normal permission pro
|
|
|
92
92
|
- an unresolved model, a timeout, an unparseable reply, or an unsure verdict.
|
|
93
93
|
|
|
94
94
|
Deferring never grants access — this extension emits no `allow` verdict.
|
|
95
|
+
|
|
96
|
+
## The decision trail
|
|
97
|
+
|
|
98
|
+
The reviewer records what it did to pi-permission-system's shared logs, keyed by `requestId` so an entry joins to the `permission_request.*` lines for the same ask.
|
|
99
|
+
This closes the observability gap that let an auth failure defer every path undetected: a silent defer is now a recorded defer with a reason.
|
|
100
|
+
|
|
101
|
+
The **review** log (`~/.pi/agent/extensions/pi-permission-system/logs/pi-permission-system-permission-review.jsonl`, on by default via pi-permission-system's `permissionReviewLog`) carries one `model_judge.decision` entry per **pattern-matched** ask — the case that should reach the model:
|
|
102
|
+
|
|
103
|
+
| Field | Meaning |
|
|
104
|
+
| ---------------- | ---------------------------------------------------------------------------------------------------------------------------- |
|
|
105
|
+
| `requestId` | The ask id, shared with the `permission_request.*` entries. |
|
|
106
|
+
| `surface` | Always `external_directory` (the only reviewed surface). |
|
|
107
|
+
| `path` | The candidate path reviewed. |
|
|
108
|
+
| `matchedPattern` | The `typoPatterns` entry, as written, that matched. |
|
|
109
|
+
| `modelCalled` | `false` for a `model-unresolved` / `auth-failed` defer. |
|
|
110
|
+
| `modelId` | `<provider>/<model>`. |
|
|
111
|
+
| `latencyMs` | Model-call wall-clock in milliseconds, or `null` when no call was made. |
|
|
112
|
+
| `verdict` | `"deny"` or `"defer"`. |
|
|
113
|
+
| `deferReason` | `null` on a deny, else `model-unresolved` / `auth-failed` / `no-tool-call` / `non-deny-verdict` / `timeout` / `call-failed`. |
|
|
114
|
+
|
|
115
|
+
The **debug** log (same directory, `pi-permission-system-debug.jsonl`, only when pi-permission-system's `debugLog` is on) carries the verbose and cheap-path detail: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer before the model stage), `model_judge.model_reply` (the verdict tool-call arguments as JSON, or the model's text when it emitted no tool call), and `model_judge.invalid_patterns` (skipped `typoPatterns`).
|
|
116
|
+
A non-`external_directory` ask is not logged at all.
|
|
117
|
+
|
|
118
|
+
To diagnose "the judge defers everything," read the review log for `model_judge.decision` entries and inspect `deferReason`: an empty result means no ask ever matched a pattern, `auth-failed` / `model-unresolved` means it is misconfigured, `non-deny-verdict` means the model saw the path and chose not to deny, and `no-tool-call` means the model replied without calling the verdict tool (rare, since the tool is forced).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotgenes/pi-permission-model-judge",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "Deny-first typo-path model judge — a pi-permission-system Authorizer chain link",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"imports": {
|
|
@@ -54,13 +54,13 @@
|
|
|
54
54
|
"peerDependencies": {
|
|
55
55
|
"@earendil-works/pi-ai": ">=0.79.0",
|
|
56
56
|
"@earendil-works/pi-coding-agent": ">=0.79.0",
|
|
57
|
-
"@gotgenes/pi-permission-system": ">=20.
|
|
57
|
+
"@gotgenes/pi-permission-system": ">=20.10.0"
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
60
|
"@biomejs/biome": "^2.4.16",
|
|
61
61
|
"@earendil-works/pi-ai": "0.79.1",
|
|
62
62
|
"@earendil-works/pi-coding-agent": "0.79.1",
|
|
63
|
-
"@gotgenes/pi-permission-system": "20.
|
|
63
|
+
"@gotgenes/pi-permission-system": "20.10.0",
|
|
64
64
|
"@types/node": "^22.15.3",
|
|
65
65
|
"rumdl": "^0.2.10",
|
|
66
66
|
"typescript": "^6.0.3",
|
package/src/extension.ts
CHANGED
package/src/model-review.ts
CHANGED
|
@@ -12,6 +12,8 @@ import type {
|
|
|
12
12
|
Context,
|
|
13
13
|
Model,
|
|
14
14
|
TextContent,
|
|
15
|
+
Tool,
|
|
16
|
+
ToolCall,
|
|
15
17
|
} from "@earendil-works/pi-ai";
|
|
16
18
|
import type { AuthorizerVerdict } from "@gotgenes/pi-permission-system";
|
|
17
19
|
|
|
@@ -21,6 +23,39 @@ import type { ModelJudgeConfig } from "./config-schema";
|
|
|
21
23
|
export const GENERIC_TEACHING_REASON =
|
|
22
24
|
"This looks like a mistyped path. Verify the correct location before retrying.";
|
|
23
25
|
|
|
26
|
+
/**
|
|
27
|
+
* The single tool the model is forced to call. Forcing it (`toolChoice: "any"`)
|
|
28
|
+
* removes free-text JSON parsing by construction — the verdict arrives as
|
|
29
|
+
* structured `arguments`, so a Markdown fence or a prose preamble can no longer
|
|
30
|
+
* cost a verdict.
|
|
31
|
+
*
|
|
32
|
+
* The Anthropic provider reads only `parameters.properties` / `parameters.required`,
|
|
33
|
+
* so a plain JSON-Schema object is correct at runtime; the `as unknown as Tool`
|
|
34
|
+
* bridge satisfies the `TSchema`-typed `parameters` field without a `typebox`
|
|
35
|
+
* dependency.
|
|
36
|
+
*/
|
|
37
|
+
const VERDICT_TOOL = {
|
|
38
|
+
name: "report_verdict",
|
|
39
|
+
description:
|
|
40
|
+
"Report whether the path is a mistyped path to reject (deny) or should be deferred to the human (defer).",
|
|
41
|
+
parameters: {
|
|
42
|
+
type: "object",
|
|
43
|
+
properties: {
|
|
44
|
+
verdict: {
|
|
45
|
+
type: "string",
|
|
46
|
+
enum: ["deny", "defer"],
|
|
47
|
+
description: "deny a mistyped path; defer anything else",
|
|
48
|
+
},
|
|
49
|
+
reason: {
|
|
50
|
+
type: "string",
|
|
51
|
+
description:
|
|
52
|
+
"Why the path is wrong and the correct location (required when denying)",
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
required: ["verdict"],
|
|
56
|
+
},
|
|
57
|
+
} as unknown as Tool;
|
|
58
|
+
|
|
24
59
|
/**
|
|
25
60
|
* The injected model-completion seam — structurally the `complete` export of
|
|
26
61
|
* `@earendil-works/pi-ai`. Injected so tests substitute a fake.
|
|
@@ -32,6 +67,7 @@ export type CompleteFn = (
|
|
|
32
67
|
signal?: AbortSignal;
|
|
33
68
|
apiKey?: string;
|
|
34
69
|
headers?: Record<string, string>;
|
|
70
|
+
toolChoice?: string;
|
|
35
71
|
},
|
|
36
72
|
) => Promise<AssistantMessage>;
|
|
37
73
|
|
|
@@ -61,21 +97,52 @@ export interface ReviewPathInputs {
|
|
|
61
97
|
}
|
|
62
98
|
|
|
63
99
|
/**
|
|
64
|
-
*
|
|
100
|
+
* Why a model call defers, distinct enough to diagnose from the decision trail:
|
|
101
|
+
* the reply carried no tool call to read (`no-tool-call`), the tool call named a
|
|
102
|
+
* verdict other than `deny` (`non-deny-verdict`), the call was aborted at
|
|
103
|
+
* `timeoutMs` (`timeout`), or `complete` threw for any other reason
|
|
104
|
+
* (`call-failed` — the honest superset that catches, e.g., a 401 slipping past
|
|
105
|
+
* pre-call auth resolution).
|
|
106
|
+
*/
|
|
107
|
+
export type ModelCallDeferReason =
|
|
108
|
+
| "no-tool-call"
|
|
109
|
+
| "non-deny-verdict"
|
|
110
|
+
| "timeout"
|
|
111
|
+
| "call-failed";
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The full result of a model review: the verdict plus the observability the
|
|
115
|
+
* decision trail records. `deferReason` is set iff the verdict is `defer` and a
|
|
116
|
+
* model call was made; `rawReply` carries the tool-call arguments as JSON when a
|
|
117
|
+
* tool call arrived, or the assistant text on a `no-tool-call` defer (absent on
|
|
118
|
+
* a timeout/throw before any reply).
|
|
119
|
+
*/
|
|
120
|
+
export interface ReviewOutcome {
|
|
121
|
+
verdict: AuthorizerVerdict;
|
|
122
|
+
deferReason?: ModelCallDeferReason;
|
|
123
|
+
latencyMs: number;
|
|
124
|
+
rawReply?: string;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Review one candidate path with the model and return the structured outcome.
|
|
65
129
|
*
|
|
66
130
|
* The call is aborted after `config.timeoutMs`; an abort, a rejection, or any
|
|
67
|
-
* non-`deny` reply yields `defer
|
|
131
|
+
* non-`deny` reply yields `defer` (more prompting, never less) with the reason
|
|
132
|
+
* annotated so the caller can record why.
|
|
68
133
|
*/
|
|
69
134
|
export async function reviewPath(
|
|
70
135
|
inputs: ReviewPathInputs,
|
|
71
|
-
): Promise<
|
|
136
|
+
): Promise<ReviewOutcome> {
|
|
72
137
|
const controller = new AbortController();
|
|
73
138
|
const timer = setTimeout(() => {
|
|
74
139
|
controller.abort();
|
|
75
140
|
}, inputs.config.timeoutMs);
|
|
141
|
+
const startedAt = Date.now();
|
|
76
142
|
try {
|
|
77
143
|
const context: Context = {
|
|
78
144
|
systemPrompt: inputs.config.instructions,
|
|
145
|
+
tools: [VERDICT_TOOL],
|
|
79
146
|
messages: [
|
|
80
147
|
{
|
|
81
148
|
role: "user",
|
|
@@ -88,44 +155,67 @@ export async function reviewPath(
|
|
|
88
155
|
signal: controller.signal,
|
|
89
156
|
apiKey: inputs.apiKey,
|
|
90
157
|
headers: inputs.headers,
|
|
158
|
+
toolChoice: "any",
|
|
91
159
|
});
|
|
92
|
-
return
|
|
160
|
+
return readToolCallOutcome(reply, Date.now() - startedAt);
|
|
93
161
|
} catch {
|
|
94
|
-
return {
|
|
162
|
+
return {
|
|
163
|
+
verdict: { kind: "defer" },
|
|
164
|
+
deferReason: controller.signal.aborted ? "timeout" : "call-failed",
|
|
165
|
+
latencyMs: Date.now() - startedAt,
|
|
166
|
+
};
|
|
95
167
|
} finally {
|
|
96
168
|
clearTimeout(timer);
|
|
97
169
|
}
|
|
98
170
|
}
|
|
99
171
|
|
|
100
|
-
/** The user-turn prompt: hand the model the path and
|
|
172
|
+
/** The user-turn prompt: hand the model the path and point it at the tool. */
|
|
101
173
|
function renderReviewPrompt(path: string): string {
|
|
102
174
|
return [
|
|
103
175
|
"A tool is about to access this path outside the working directory:",
|
|
104
176
|
"",
|
|
105
177
|
path,
|
|
106
178
|
"",
|
|
107
|
-
'
|
|
108
|
-
'Otherwise: {"verdict":"defer"}.',
|
|
179
|
+
'Call report_verdict. Use verdict "deny" with a reason naming the correct location if this is a mistyped path; otherwise use verdict "defer".',
|
|
109
180
|
].join("\n");
|
|
110
181
|
}
|
|
111
182
|
|
|
112
|
-
/**
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
183
|
+
/**
|
|
184
|
+
* Map the forced tool call to an outcome; anything but a clean `deny` defers
|
|
185
|
+
* with the reason that distinguishes it. The tool call is read by position (the
|
|
186
|
+
* first one), not by name — under OAuth the provider rewrites the registered
|
|
187
|
+
* name, so the reply's tool-call name cannot be relied on.
|
|
188
|
+
*/
|
|
189
|
+
function readToolCallOutcome(
|
|
190
|
+
reply: AssistantMessage,
|
|
191
|
+
latencyMs: number,
|
|
192
|
+
): ReviewOutcome {
|
|
193
|
+
const call = reply.content.find(
|
|
194
|
+
(part): part is ToolCall => part.type === "toolCall",
|
|
195
|
+
);
|
|
196
|
+
if (!call) {
|
|
197
|
+
return {
|
|
198
|
+
verdict: { kind: "defer" },
|
|
199
|
+
deferReason: "no-tool-call",
|
|
200
|
+
latencyMs,
|
|
201
|
+
rawReply: extractText(reply),
|
|
202
|
+
};
|
|
120
203
|
}
|
|
121
|
-
|
|
122
|
-
|
|
204
|
+
const args = call.arguments;
|
|
205
|
+
const rawReply = JSON.stringify(args);
|
|
206
|
+
if (args.verdict !== "deny") {
|
|
207
|
+
return {
|
|
208
|
+
verdict: { kind: "defer" },
|
|
209
|
+
deferReason: "non-deny-verdict",
|
|
210
|
+
latencyMs,
|
|
211
|
+
rawReply,
|
|
212
|
+
};
|
|
123
213
|
}
|
|
124
214
|
const reason =
|
|
125
|
-
typeof
|
|
126
|
-
?
|
|
215
|
+
typeof args.reason === "string" && args.reason.length > 0
|
|
216
|
+
? args.reason
|
|
127
217
|
: GENERIC_TEACHING_REASON;
|
|
128
|
-
return { kind: "deny", reason };
|
|
218
|
+
return { verdict: { kind: "deny", reason }, latencyMs, rawReply };
|
|
129
219
|
}
|
|
130
220
|
|
|
131
221
|
/** Concatenate the text parts of an assistant reply. */
|
|
@@ -135,7 +225,3 @@ function extractText(reply: AssistantMessage): string {
|
|
|
135
225
|
.map((part) => part.text)
|
|
136
226
|
.join("");
|
|
137
227
|
}
|
|
138
|
-
|
|
139
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
140
|
-
return typeof value === "object" && value !== null;
|
|
141
|
-
}
|
package/src/typo-patterns.ts
CHANGED
|
@@ -7,9 +7,16 @@
|
|
|
7
7
|
* without a model call.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
/**
|
|
10
|
+
/**
|
|
11
|
+
* Result of compiling a pattern list: usable regexes (with their original
|
|
12
|
+
* source strings, parallel to `regexes`) plus any that failed to compile.
|
|
13
|
+
* `patterns[i]` is the operator's literal config string for `regexes[i]`, kept
|
|
14
|
+
* so the decision trail records the pattern as written, not the slash-escaped
|
|
15
|
+
* `RegExp.source`.
|
|
16
|
+
*/
|
|
11
17
|
export interface CompiledTypoPatterns {
|
|
12
18
|
regexes: RegExp[];
|
|
19
|
+
patterns: string[];
|
|
13
20
|
invalidPatterns: string[];
|
|
14
21
|
}
|
|
15
22
|
|
|
@@ -22,21 +29,28 @@ export function compileTypoPatterns(
|
|
|
22
29
|
patterns: readonly string[],
|
|
23
30
|
): CompiledTypoPatterns {
|
|
24
31
|
const regexes: RegExp[] = [];
|
|
32
|
+
const sources: string[] = [];
|
|
25
33
|
const invalidPatterns: string[] = [];
|
|
26
34
|
for (const pattern of patterns) {
|
|
27
35
|
try {
|
|
28
36
|
regexes.push(new RegExp(pattern));
|
|
37
|
+
sources.push(pattern);
|
|
29
38
|
} catch {
|
|
30
39
|
invalidPatterns.push(pattern);
|
|
31
40
|
}
|
|
32
41
|
}
|
|
33
|
-
return { regexes, invalidPatterns };
|
|
42
|
+
return { regexes, patterns: sources, invalidPatterns };
|
|
34
43
|
}
|
|
35
44
|
|
|
36
|
-
/**
|
|
37
|
-
|
|
45
|
+
/**
|
|
46
|
+
* The source string of the first compiled pattern that matches `path`, or
|
|
47
|
+
* `undefined` if none do. The source (not the boolean) so the decision trail
|
|
48
|
+
* can record *which* pattern matched.
|
|
49
|
+
*/
|
|
50
|
+
export function matchTypoPattern(
|
|
38
51
|
path: string,
|
|
39
52
|
compiled: CompiledTypoPatterns,
|
|
40
|
-
):
|
|
41
|
-
|
|
53
|
+
): string | undefined {
|
|
54
|
+
const index = compiled.regexes.findIndex((regex) => regex.test(path));
|
|
55
|
+
return index === -1 ? undefined : compiled.patterns[index];
|
|
42
56
|
}
|
package/src/typo-reviewer.ts
CHANGED
|
@@ -4,17 +4,26 @@
|
|
|
4
4
|
*
|
|
5
5
|
* The decision runs top to bottom, deferring at the first miss so the cheap
|
|
6
6
|
* gates short-circuit before any model call:
|
|
7
|
-
* 1. surface is `external_directory` (else defer),
|
|
7
|
+
* 1. surface is `external_directory` (else defer, silently — not our surface),
|
|
8
8
|
* 2. a candidate path is present (else defer),
|
|
9
9
|
* 3. the path matches a configured typo pattern (else defer, no model call),
|
|
10
10
|
* 4. the model confirms the typo (deny with a teaching reason) or defers.
|
|
11
11
|
*
|
|
12
12
|
* Every failure path defers — more prompting, never less (ADR 0007 invariant 2).
|
|
13
13
|
* This slice never emits `allow`.
|
|
14
|
+
*
|
|
15
|
+
* Observability (issue #626): once an ask matches a typo pattern (the [#625]
|
|
16
|
+
* class — a candidate typo that *should* reach the model), the reviewer writes a
|
|
17
|
+
* positive `model_judge.decision` review entry recording the outcome and, on a
|
|
18
|
+
* defer, its reason — so a silent 100%-defer regression cannot hide again. The
|
|
19
|
+
* cheap short-circuits (`no-path`, `pattern-miss`) and the raw model reply go to
|
|
20
|
+
* the debug log; a non-`external_directory` surface is not logged at all.
|
|
14
21
|
*/
|
|
15
22
|
|
|
16
23
|
import type {
|
|
17
24
|
Authorizer,
|
|
25
|
+
AuthorizerLog,
|
|
26
|
+
AuthorizerVerdict,
|
|
18
27
|
PromptPermissionDetails,
|
|
19
28
|
} from "@gotgenes/pi-permission-system";
|
|
20
29
|
|
|
@@ -27,11 +36,31 @@ import {
|
|
|
27
36
|
import {
|
|
28
37
|
type CompiledTypoPatterns,
|
|
29
38
|
compileTypoPatterns,
|
|
30
|
-
|
|
39
|
+
matchTypoPattern,
|
|
31
40
|
} from "./typo-patterns";
|
|
32
41
|
|
|
33
42
|
const REVIEWED_SURFACE = "external_directory";
|
|
34
43
|
|
|
44
|
+
/** Review-log event: one positive decision record per pattern-matched ask. */
|
|
45
|
+
const DECISION_EVENT = "model_judge.decision";
|
|
46
|
+
/** Debug-log event: a cheap short-circuit before the model stage. */
|
|
47
|
+
const SHORT_CIRCUIT_EVENT = "model_judge.short_circuit";
|
|
48
|
+
/** Debug-log event: the raw model reply, gated behind `debugLog`. */
|
|
49
|
+
const MODEL_REPLY_EVENT = "model_judge.model_reply";
|
|
50
|
+
/** Debug-log event: `typoPatterns` entries that failed to compile. */
|
|
51
|
+
const INVALID_PATTERNS_EVENT = "model_judge.invalid_patterns";
|
|
52
|
+
|
|
53
|
+
/** A defer decided before the model call, still recorded positively. */
|
|
54
|
+
type PreModelDeferReason = "model-unresolved" | "auth-failed";
|
|
55
|
+
|
|
56
|
+
/** The shared fields of a `model_judge.decision` record, before the outcome. */
|
|
57
|
+
interface DecisionBase {
|
|
58
|
+
requestId: string;
|
|
59
|
+
path: string;
|
|
60
|
+
matchedPattern: string;
|
|
61
|
+
modelId: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
35
64
|
/** Collaborators for the reviewer, injected so the extension and tests wire them. */
|
|
36
65
|
export interface TypoReviewerDeps {
|
|
37
66
|
/** The loaded config, read live (absent until session config loads). */
|
|
@@ -40,21 +69,20 @@ export interface TypoReviewerDeps {
|
|
|
40
69
|
getRegistry: () => ModelRegistryLike | undefined;
|
|
41
70
|
/** The model-completion seam (production: `complete` from `@earendil-works/pi-ai`). */
|
|
42
71
|
complete: CompleteFn;
|
|
43
|
-
/** Optional warn sink for operational notices (unresolved model, bad patterns). */
|
|
44
|
-
warn?: (message: string) => void;
|
|
45
72
|
}
|
|
46
73
|
|
|
47
74
|
/**
|
|
48
75
|
* Build the `authorize` callback registered on the chain. The `query` argument
|
|
49
76
|
* is unused in this slice — the deny-first reviewer decides from the path
|
|
50
|
-
* pattern and the model, not from an engine re-query (slice 2 consumes it).
|
|
77
|
+
* pattern and the model, not from an engine re-query (slice 2 consumes it). The
|
|
78
|
+
* `log` argument is the injected review-log seam the decision trail records to.
|
|
51
79
|
*/
|
|
52
80
|
export function createTypoReviewer(
|
|
53
81
|
deps: TypoReviewerDeps,
|
|
54
82
|
): Authorizer["authorize"] {
|
|
55
|
-
const compiledFor = memoizeCompiledPatterns(
|
|
83
|
+
const compiledFor = memoizeCompiledPatterns();
|
|
56
84
|
|
|
57
|
-
return async (details) => {
|
|
85
|
+
return async (details, _query, log) => {
|
|
58
86
|
const config = deps.getConfig();
|
|
59
87
|
if (!config) {
|
|
60
88
|
return { kind: "defer" };
|
|
@@ -62,29 +90,35 @@ export function createTypoReviewer(
|
|
|
62
90
|
if (surfaceOf(details) !== REVIEWED_SURFACE) {
|
|
63
91
|
return { kind: "defer" };
|
|
64
92
|
}
|
|
93
|
+
const { requestId } = details;
|
|
65
94
|
const path = pathOf(details);
|
|
66
95
|
if (path === undefined) {
|
|
96
|
+
log.debug(SHORT_CIRCUIT_EVENT, { requestId, reason: "no-path" });
|
|
67
97
|
return { kind: "defer" };
|
|
68
98
|
}
|
|
69
|
-
|
|
99
|
+
const matchedPattern = matchTypoPattern(path, compiledFor(config, log));
|
|
100
|
+
if (matchedPattern === undefined) {
|
|
101
|
+
log.debug(SHORT_CIRCUIT_EVENT, {
|
|
102
|
+
requestId,
|
|
103
|
+
path,
|
|
104
|
+
reason: "pattern-miss",
|
|
105
|
+
});
|
|
70
106
|
return { kind: "defer" };
|
|
71
107
|
}
|
|
108
|
+
|
|
109
|
+
const modelId = `${config.provider}/${config.model}`;
|
|
110
|
+
const base: DecisionBase = { requestId, path, matchedPattern, modelId };
|
|
72
111
|
const registry = deps.getRegistry();
|
|
73
112
|
const model = registry?.find(config.provider, config.model);
|
|
74
113
|
if (!registry || !model) {
|
|
75
|
-
|
|
76
|
-
`model "${config.provider}/${config.model}" did not resolve; deferring`,
|
|
77
|
-
);
|
|
78
|
-
return { kind: "defer" };
|
|
114
|
+
return deferWith(log, base, "model-unresolved");
|
|
79
115
|
}
|
|
80
116
|
const auth = await registry.getApiKeyAndHeaders(model);
|
|
81
117
|
if (!auth.ok) {
|
|
82
|
-
|
|
83
|
-
`auth for "${config.provider}/${config.model}" did not resolve (${auth.error}); deferring`,
|
|
84
|
-
);
|
|
85
|
-
return { kind: "defer" };
|
|
118
|
+
return deferWith(log, base, "auth-failed");
|
|
86
119
|
}
|
|
87
|
-
|
|
120
|
+
|
|
121
|
+
const outcome = await reviewPath({
|
|
88
122
|
path,
|
|
89
123
|
config,
|
|
90
124
|
model,
|
|
@@ -92,9 +126,52 @@ export function createTypoReviewer(
|
|
|
92
126
|
apiKey: auth.apiKey,
|
|
93
127
|
headers: auth.headers,
|
|
94
128
|
});
|
|
129
|
+
if (outcome.rawReply !== undefined) {
|
|
130
|
+
log.debug(MODEL_REPLY_EVENT, {
|
|
131
|
+
requestId,
|
|
132
|
+
modelId,
|
|
133
|
+
rawReply: outcome.rawReply,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
log.review(DECISION_EVENT, {
|
|
137
|
+
requestId,
|
|
138
|
+
surface: REVIEWED_SURFACE,
|
|
139
|
+
path,
|
|
140
|
+
matchedPattern,
|
|
141
|
+
modelCalled: true,
|
|
142
|
+
modelId,
|
|
143
|
+
latencyMs: outcome.latencyMs,
|
|
144
|
+
verdict: outcome.verdict.kind,
|
|
145
|
+
deferReason: outcome.deferReason ?? null,
|
|
146
|
+
});
|
|
147
|
+
return outcome.verdict;
|
|
95
148
|
};
|
|
96
149
|
}
|
|
97
150
|
|
|
151
|
+
/**
|
|
152
|
+
* Record a pre-model defer (`model-unresolved` / `auth-failed`) as a positive
|
|
153
|
+
* `model_judge.decision` entry and return the defer verdict — so the two
|
|
154
|
+
* model-resolution failures leave evidence on record, not a silent absence.
|
|
155
|
+
*/
|
|
156
|
+
function deferWith(
|
|
157
|
+
log: AuthorizerLog,
|
|
158
|
+
base: DecisionBase,
|
|
159
|
+
deferReason: PreModelDeferReason,
|
|
160
|
+
): AuthorizerVerdict {
|
|
161
|
+
log.review(DECISION_EVENT, {
|
|
162
|
+
requestId: base.requestId,
|
|
163
|
+
surface: REVIEWED_SURFACE,
|
|
164
|
+
path: base.path,
|
|
165
|
+
matchedPattern: base.matchedPattern,
|
|
166
|
+
modelCalled: false,
|
|
167
|
+
modelId: base.modelId,
|
|
168
|
+
latencyMs: null,
|
|
169
|
+
verdict: "defer",
|
|
170
|
+
deferReason,
|
|
171
|
+
});
|
|
172
|
+
return { kind: "defer" };
|
|
173
|
+
}
|
|
174
|
+
|
|
98
175
|
/** The gate-authoritative surface, falling back to the display surface. */
|
|
99
176
|
function surfaceOf(details: PromptPermissionDetails): string | undefined {
|
|
100
177
|
return details.accessIntent?.surface ?? details.surface ?? undefined;
|
|
@@ -110,21 +187,22 @@ function pathOf(details: PromptPermissionDetails): string | undefined {
|
|
|
110
187
|
* object is in effect — the config is stable per session, so this avoids
|
|
111
188
|
* recompiling the regexes on every ask.
|
|
112
189
|
*/
|
|
113
|
-
function memoizeCompiledPatterns(
|
|
114
|
-
|
|
115
|
-
|
|
190
|
+
function memoizeCompiledPatterns(): (
|
|
191
|
+
config: ModelJudgeConfig,
|
|
192
|
+
log: AuthorizerLog,
|
|
193
|
+
) => CompiledTypoPatterns {
|
|
116
194
|
let cache:
|
|
117
195
|
| { config: ModelJudgeConfig; compiled: CompiledTypoPatterns }
|
|
118
196
|
| undefined;
|
|
119
|
-
return (config) => {
|
|
197
|
+
return (config, log) => {
|
|
120
198
|
if (cache?.config === config) {
|
|
121
199
|
return cache.compiled;
|
|
122
200
|
}
|
|
123
201
|
const compiled = compileTypoPatterns(config.typoPatterns);
|
|
124
202
|
if (compiled.invalidPatterns.length > 0) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
);
|
|
203
|
+
log.debug(INVALID_PATTERNS_EVENT, {
|
|
204
|
+
invalidPatterns: compiled.invalidPatterns,
|
|
205
|
+
});
|
|
128
206
|
}
|
|
129
207
|
cache = { config, compiled };
|
|
130
208
|
return compiled;
|