@gotgenes/pi-permission-model-judge 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +29 -3
- package/config/config.example.json +5 -2
- package/docs/configuration.md +38 -5
- package/package.json +3 -3
- package/src/extension.ts +0 -1
- package/src/model-review.ts +75 -13
- package/src/typo-patterns.ts +20 -6
- package/src/typo-reviewer.ts +115 -22
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.1.0](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.0.2...pi-permission-model-judge-v1.1.0) (2026-07-21)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* **pi-permission-model-judge:** distinguish model-call defer reasons in reviewPath ([9098f46](https://github.com/gotgenes/pi-packages/commit/9098f465d64dfee3994132bd87a71ad40152b259))
|
|
9
|
+
* **pi-permission-model-judge:** record the decision trail to the permission review log ([e3964ae](https://github.com/gotgenes/pi-packages/commit/e3964ae6ca443d043db4e735ad69df90bac11c0d))
|
|
10
|
+
* **pi-permission-model-judge:** return the matched typo pattern ([4db29fb](https://github.com/gotgenes/pi-packages/commit/4db29fbf098c8d3c853539b408316ee2a8fffe54))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### Documentation
|
|
14
|
+
|
|
15
|
+
* **pi-permission-model-judge:** document the decision trail ([335a7b5](https://github.com/gotgenes/pi-packages/commit/335a7b5b0f40650760d8597f511f32dc2116bf81))
|
|
16
|
+
|
|
17
|
+
## [1.0.2](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.0.1...pi-permission-model-judge-v1.0.2) (2026-07-21)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Bug Fixes
|
|
21
|
+
|
|
22
|
+
* **pi-permission-model-judge:** forward resolved auth into the review model call ([9836020](https://github.com/gotgenes/pi-packages/commit/98360207d19a91edcf8e166b59dd1a9262349277)), closes [#625](https://github.com/gotgenes/pi-packages/issues/625)
|
|
23
|
+
* **pi-permission-model-judge:** resolve model auth before the review call ([4c90603](https://github.com/gotgenes/pi-packages/commit/4c906031144e18c7f6b60eac5f8607e271f090f2)), closes [#625](https://github.com/gotgenes/pi-packages/issues/625)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
### Documentation
|
|
27
|
+
|
|
28
|
+
* **pi-permission-model-judge:** feature the corrected dropped-prefix typo pattern ([13553d5](https://github.com/gotgenes/pi-packages/commit/13553d5c4ba8ba50a90688bd26e597a842f5f509)), closes [#625](https://github.com/gotgenes/pi-packages/issues/625)
|
|
29
|
+
|
|
3
30
|
## [1.0.1](https://github.com/gotgenes/pi-packages/compare/pi-permission-model-judge-v1.0.0...pi-permission-model-judge-v1.0.1) (2026-07-20)
|
|
4
31
|
|
|
5
32
|
|
package/README.md
CHANGED
|
@@ -23,6 +23,29 @@ The reviewer runs a short, cheap decision on each ask and defers at the first mi
|
|
|
23
23
|
It is fail-safe by construction: a missing model, invalid config, model timeout, unparseable reply, or an unsure verdict all resolve to `defer`.
|
|
24
24
|
Deferring means the ask falls through to the normal permission prompt — this extension only ever *removes* a hand-denial, never grants access (it emits no `allow`).
|
|
25
25
|
|
|
26
|
+
## What it records
|
|
27
|
+
|
|
28
|
+
Every review the link performs leaves a trail in pi-permission-system's shared review log (`~/.pi/agent/extensions/pi-permission-system/logs/pi-permission-system-permission-review.jsonl`), so you can answer "did the judge run, did it reach the model, and why did it defer?"
|
|
29
|
+
without guesswork.
|
|
30
|
+
|
|
31
|
+
Once an ask matches a `typoPattern` — the case that *should* reach the model — the link writes one `model_judge.decision` entry recording the outcome:
|
|
32
|
+
|
|
33
|
+
| Field | Meaning |
|
|
34
|
+
| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
|
35
|
+
| `requestId` | Joins to the `permission_request.*` entries for the same ask. |
|
|
36
|
+
| `path` | The candidate path reviewed. |
|
|
37
|
+
| `matchedPattern` | The `typoPatterns` entry (as you wrote it) that matched. |
|
|
38
|
+
| `modelCalled` | `false` when the model or its auth did not resolve. |
|
|
39
|
+
| `modelId` | `<provider>/<model>`. |
|
|
40
|
+
| `latencyMs` | Model-call wall-clock in ms, or `null` when no call was made. |
|
|
41
|
+
| `verdict` | `"deny"` or `"defer"`. |
|
|
42
|
+
| `deferReason` | `null` on a deny, else one of `model-unresolved` / `auth-failed` / `parse-failed` / `non-deny-verdict` / `timeout` / `call-failed`. |
|
|
43
|
+
|
|
44
|
+
Cheaper events go to pi-permission-system's **debug** log, and only when its `debugLog` toggle is on: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer) and `model_judge.model_reply` (the model's raw reply text).
|
|
45
|
+
A non-`external_directory` ask is not logged — it is not this link's concern.
|
|
46
|
+
|
|
47
|
+
Because every pattern-matched ask leaves a positive record, a misconfiguration that silently defers every path (an auth failure, an unresolved model) shows up as a run of `deferReason` entries rather than an empty log.
|
|
48
|
+
|
|
26
49
|
## Install
|
|
27
50
|
|
|
28
51
|
```bash
|
|
@@ -49,9 +72,12 @@ Two independent config files are involved — the safety policy lives in pi-perm
|
|
|
49
72
|
{
|
|
50
73
|
"provider": "anthropic",
|
|
51
74
|
"model": "claude-haiku-4-5",
|
|
52
|
-
"instructions": "Deny a path that repeats a package name around `packages
|
|
53
|
-
// Catches a doubled package segment
|
|
54
|
-
"typoPatterns": [
|
|
75
|
+
"instructions": "Deny a path that repeats a package name around `packages/`, or drops the repo's `pi-packages/packages/` prefix…",
|
|
76
|
+
// Catches a doubled package segment and a dropped repository prefix.
|
|
77
|
+
"typoPatterns": [
|
|
78
|
+
"([^/]+)/packages/\\1(/|$)",
|
|
79
|
+
"development/pi/(?!pi-packages/)pi-[^/]+(/|$)"
|
|
80
|
+
]
|
|
55
81
|
}
|
|
56
82
|
```
|
|
57
83
|
|
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
"$schema": "https://raw.githubusercontent.com/gotgenes/pi-packages/main/packages/pi-permission-model-judge/schemas/model-judge.schema.json",
|
|
3
3
|
"provider": "anthropic",
|
|
4
4
|
"model": "claude-haiku-4-5",
|
|
5
|
-
"instructions": "You review a filesystem path a tool is about to access outside the working directory.
|
|
6
|
-
"typoPatterns": [
|
|
5
|
+
"instructions": "You review a filesystem path a tool is about to access outside the working directory. Two common typos both point at the wrong location. (1) A doubled package segment — a package name repeated around `packages/`, for example `pi-permission-system/packages/pi-permission-system/…`, where the correct path names the package only once under the repository's `packages/` directory. (2) A dropped repository prefix — a path that jumps straight to a package directory such as `development/pi/pi-permission-system/…`, where the correct path routes through the repository's `pi-packages/packages/` directory (for example `development/pi/pi-packages/packages/pi-permission-system/…`). When the path shows either typo, deny it and give the single corrected absolute path in the reason so the caller can retry against the right location. If the path looks intentional or you are unsure, defer. Reply with strict JSON: {\"verdict\":\"deny\",\"reason\":\"…\"} to reject, or {\"verdict\":\"defer\"} otherwise.",
|
|
6
|
+
"typoPatterns": [
|
|
7
|
+
"([^/]+)/packages/\\1(/|$)",
|
|
8
|
+
"development/pi/(?!pi-packages/)pi-[^/]+(/|$)"
|
|
9
|
+
],
|
|
7
10
|
"timeoutMs": 5000
|
|
8
11
|
}
|
package/docs/configuration.md
CHANGED
|
@@ -44,7 +44,7 @@ The link decides nothing until you name it in `authorizerChain` (opt-in activati
|
|
|
44
44
|
### `provider` and `model`
|
|
45
45
|
|
|
46
46
|
The reviewer resolves `provider` / `model` against the session's model registry at review time.
|
|
47
|
-
If the model does not resolve (wrong id, no credentials), the reviewer
|
|
47
|
+
If the model does not resolve (wrong id, no credentials), the reviewer records a `model-unresolved` decision entry and defers — it never blocks an ask because the judge is misconfigured.
|
|
48
48
|
|
|
49
49
|
### `instructions`
|
|
50
50
|
|
|
@@ -56,18 +56,27 @@ The reviewer expects the model to answer with strict JSON — `{"verdict":"deny"
|
|
|
56
56
|
|
|
57
57
|
Each string is compiled with `new RegExp(pattern)` (no flags) and tested against the candidate path.
|
|
58
58
|
Only a path that matches at least one pattern is sent to the model — this is the cost gate that keeps the reviewer from calling the model on every ask.
|
|
59
|
-
An invalid regular expression is skipped
|
|
59
|
+
An invalid regular expression is skipped and recorded in the debug log (`model_judge.invalid_patterns`).
|
|
60
60
|
An empty list (the default) matches nothing, so the reviewer defers everything.
|
|
61
61
|
|
|
62
|
-
|
|
63
|
-
A backreference matches this for *any* package name in one pattern:
|
|
62
|
+
Two typo shapes are worth featuring, each as its own pattern:
|
|
64
63
|
|
|
65
64
|
```json
|
|
66
|
-
"typoPatterns": [
|
|
65
|
+
"typoPatterns": [
|
|
66
|
+
"([^/]+)/packages/\\1(/|$)",
|
|
67
|
+
"development/pi/(?!pi-packages/)pi-[^/]+(/|$)"
|
|
68
|
+
]
|
|
67
69
|
```
|
|
68
70
|
|
|
71
|
+
The first catches a **doubled package segment** — a package name repeated around `packages/`, e.g. `pi-permission-system/packages/pi-permission-system/…`.
|
|
69
72
|
The `\1` backreference requires the segment before and after `packages/` to be identical, so it catches `<name>/packages/<name>` for every `<name>` while leaving a correct `packages/<name>/…` path untouched. (In JSON the backslash is doubled — `\\1` — so the parsed string is the regex `\1`.)
|
|
70
73
|
|
|
74
|
+
The second catches a **dropped repository prefix** — a path that jumps straight to a package directory (`development/pi/pi-permission-system/…`) instead of routing through the repo's `pi-packages/packages/` directory.
|
|
75
|
+
The negative lookahead `(?!pi-packages/)` exempts the correct `pi-packages` path, and `pi-[^/]+` requires a package segment after `pi-` so the sibling monorepo at `~/development/pi/pi` is left alone.
|
|
76
|
+
|
|
77
|
+
Both patterns anchor the segment with `(/|$)`, not a bare trailing `/`, so a path that *ends* at the offending segment (`…/pi-permission-system`) still matches.
|
|
78
|
+
Adjust the `development/pi/…` prefix to wherever your checkouts live.
|
|
79
|
+
|
|
71
80
|
### `timeoutMs`
|
|
72
81
|
|
|
73
82
|
The model call is aborted after this many milliseconds, and an aborted call defers.
|
|
@@ -83,3 +92,27 @@ Every uncertain outcome defers, which sends the ask to the normal permission pro
|
|
|
83
92
|
- an unresolved model, a timeout, an unparseable reply, or an unsure verdict.
|
|
84
93
|
|
|
85
94
|
Deferring never grants access — this extension emits no `allow` verdict.
|
|
95
|
+
|
|
96
|
+
## The decision trail
|
|
97
|
+
|
|
98
|
+
The reviewer records what it did to pi-permission-system's shared logs, keyed by `requestId` so an entry joins to the `permission_request.*` lines for the same ask.
|
|
99
|
+
This closes the observability gap that let an auth failure defer every path undetected: a silent defer is now a recorded defer with a reason.
|
|
100
|
+
|
|
101
|
+
The **review** log (`~/.pi/agent/extensions/pi-permission-system/logs/pi-permission-system-permission-review.jsonl`, on by default via pi-permission-system's `permissionReviewLog`) carries one `model_judge.decision` entry per **pattern-matched** ask — the case that should reach the model:
|
|
102
|
+
|
|
103
|
+
| Field | Meaning |
|
|
104
|
+
| ---------------- | ---------------------------------------------------------------------------------------------------------------------------- |
|
|
105
|
+
| `requestId` | The ask id, shared with the `permission_request.*` entries. |
|
|
106
|
+
| `surface` | Always `external_directory` (the only reviewed surface). |
|
|
107
|
+
| `path` | The candidate path reviewed. |
|
|
108
|
+
| `matchedPattern` | The `typoPatterns` entry, as written, that matched. |
|
|
109
|
+
| `modelCalled` | `false` for a `model-unresolved` / `auth-failed` defer. |
|
|
110
|
+
| `modelId` | `<provider>/<model>`. |
|
|
111
|
+
| `latencyMs` | Model-call wall-clock in milliseconds, or `null` when no call was made. |
|
|
112
|
+
| `verdict` | `"deny"` or `"defer"`. |
|
|
113
|
+
| `deferReason` | `null` on a deny, else `model-unresolved` / `auth-failed` / `parse-failed` / `non-deny-verdict` / `timeout` / `call-failed`. |
|
|
114
|
+
|
|
115
|
+
The **debug** log (same directory, `pi-permission-system-debug.jsonl`, only when pi-permission-system's `debugLog` is on) carries the verbose and cheap-path detail: `model_judge.short_circuit` (a `no-path` or `pattern-miss` defer before the model stage), `model_judge.model_reply` (the model's raw reply text), and `model_judge.invalid_patterns` (skipped `typoPatterns`).
|
|
116
|
+
A non-`external_directory` ask is not logged at all.
|
|
117
|
+
|
|
118
|
+
To diagnose "the judge defers everything," read the review log for `model_judge.decision` entries and inspect `deferReason`: an empty result means no ask ever matched a pattern, `auth-failed` / `model-unresolved` means it is misconfigured, and `non-deny-verdict` means the model saw the path and chose not to deny.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotgenes/pi-permission-model-judge",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Deny-first typo-path model judge — a pi-permission-system Authorizer chain link",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"imports": {
|
|
@@ -54,13 +54,13 @@
|
|
|
54
54
|
"peerDependencies": {
|
|
55
55
|
"@earendil-works/pi-ai": ">=0.79.0",
|
|
56
56
|
"@earendil-works/pi-coding-agent": ">=0.79.0",
|
|
57
|
-
"@gotgenes/pi-permission-system": ">=20.
|
|
57
|
+
"@gotgenes/pi-permission-system": ">=20.10.0"
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
60
|
"@biomejs/biome": "^2.4.16",
|
|
61
61
|
"@earendil-works/pi-ai": "0.79.1",
|
|
62
62
|
"@earendil-works/pi-coding-agent": "0.79.1",
|
|
63
|
-
"@gotgenes/pi-permission-system": "20.
|
|
63
|
+
"@gotgenes/pi-permission-system": "20.10.0",
|
|
64
64
|
"@types/node": "^22.15.3",
|
|
65
65
|
"rumdl": "^0.2.10",
|
|
66
66
|
"typescript": "^6.0.3",
|
package/src/extension.ts
CHANGED
package/src/model-review.ts
CHANGED
|
@@ -28,12 +28,26 @@ export const GENERIC_TEACHING_REASON =
|
|
|
28
28
|
export type CompleteFn = (
|
|
29
29
|
model: Model<any>,
|
|
30
30
|
context: Context,
|
|
31
|
-
options?: {
|
|
31
|
+
options?: {
|
|
32
|
+
signal?: AbortSignal;
|
|
33
|
+
apiKey?: string;
|
|
34
|
+
headers?: Record<string, string>;
|
|
35
|
+
},
|
|
32
36
|
) => Promise<AssistantMessage>;
|
|
33
37
|
|
|
38
|
+
/**
|
|
39
|
+
* The auth resolved for a model call — structurally the `ResolvedRequestAuth`
|
|
40
|
+
* of the core `ModelRegistry`, redeclared here because that type is not
|
|
41
|
+
* re-exported from `@earendil-works/pi-coding-agent`.
|
|
42
|
+
*/
|
|
43
|
+
export type ResolvedRequestAuth =
|
|
44
|
+
| { ok: true; apiKey?: string; headers?: Record<string, string> }
|
|
45
|
+
| { ok: false; error: string };
|
|
46
|
+
|
|
34
47
|
/** The narrow model-registry projection the reviewer needs (ISP). */
|
|
35
48
|
export interface ModelRegistryLike {
|
|
36
49
|
find(provider: string, modelId: string): Model<any> | undefined;
|
|
50
|
+
getApiKeyAndHeaders(model: Model<any>): Promise<ResolvedRequestAuth>;
|
|
37
51
|
}
|
|
38
52
|
|
|
39
53
|
/** Inputs for a single path review. */
|
|
@@ -42,21 +56,51 @@ export interface ReviewPathInputs {
|
|
|
42
56
|
config: ModelJudgeConfig;
|
|
43
57
|
model: Model<any>;
|
|
44
58
|
complete: CompleteFn;
|
|
59
|
+
apiKey?: string;
|
|
60
|
+
headers?: Record<string, string>;
|
|
45
61
|
}
|
|
46
62
|
|
|
47
63
|
/**
|
|
48
|
-
*
|
|
64
|
+
* Why a model call defers, distinct enough to diagnose from the decision trail:
|
|
65
|
+
* the reply parsed but was not JSON (`parse-failed`), was valid JSON but not a
|
|
66
|
+
* `deny` (`non-deny-verdict`), the call was aborted at `timeoutMs` (`timeout`),
|
|
67
|
+
* or `complete` threw for any other reason (`call-failed` — the honest superset
|
|
68
|
+
* that catches, e.g., a 401 slipping past pre-call auth resolution).
|
|
69
|
+
*/
|
|
70
|
+
export type ModelCallDeferReason =
|
|
71
|
+
| "parse-failed"
|
|
72
|
+
| "non-deny-verdict"
|
|
73
|
+
| "timeout"
|
|
74
|
+
| "call-failed";
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The full result of a model review: the verdict plus the observability the
|
|
78
|
+
* decision trail records. `deferReason` is set iff the verdict is `defer` and a
|
|
79
|
+
* model call was made; `rawReply` carries the assistant text when a reply
|
|
80
|
+
* arrived (absent on a timeout/throw before any reply).
|
|
81
|
+
*/
|
|
82
|
+
export interface ReviewOutcome {
|
|
83
|
+
verdict: AuthorizerVerdict;
|
|
84
|
+
deferReason?: ModelCallDeferReason;
|
|
85
|
+
latencyMs: number;
|
|
86
|
+
rawReply?: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Review one candidate path with the model and return the structured outcome.
|
|
49
91
|
*
|
|
50
92
|
* The call is aborted after `config.timeoutMs`; an abort, a rejection, or any
|
|
51
|
-
* non-`deny` reply yields `defer
|
|
93
|
+
* non-`deny` reply yields `defer` (more prompting, never less) with the reason
|
|
94
|
+
* annotated so the caller can record why.
|
|
52
95
|
*/
|
|
53
96
|
export async function reviewPath(
|
|
54
97
|
inputs: ReviewPathInputs,
|
|
55
|
-
): Promise<
|
|
98
|
+
): Promise<ReviewOutcome> {
|
|
56
99
|
const controller = new AbortController();
|
|
57
100
|
const timer = setTimeout(() => {
|
|
58
101
|
controller.abort();
|
|
59
102
|
}, inputs.config.timeoutMs);
|
|
103
|
+
const startedAt = Date.now();
|
|
60
104
|
try {
|
|
61
105
|
const context: Context = {
|
|
62
106
|
systemPrompt: inputs.config.instructions,
|
|
@@ -70,10 +114,16 @@ export async function reviewPath(
|
|
|
70
114
|
};
|
|
71
115
|
const reply = await inputs.complete(inputs.model, context, {
|
|
72
116
|
signal: controller.signal,
|
|
117
|
+
apiKey: inputs.apiKey,
|
|
118
|
+
headers: inputs.headers,
|
|
73
119
|
});
|
|
74
|
-
return
|
|
120
|
+
return parseOutcome(extractText(reply).trim(), Date.now() - startedAt);
|
|
75
121
|
} catch {
|
|
76
|
-
return {
|
|
122
|
+
return {
|
|
123
|
+
verdict: { kind: "defer" },
|
|
124
|
+
deferReason: controller.signal.aborted ? "timeout" : "call-failed",
|
|
125
|
+
latencyMs: Date.now() - startedAt,
|
|
126
|
+
};
|
|
77
127
|
} finally {
|
|
78
128
|
clearTimeout(timer);
|
|
79
129
|
}
|
|
@@ -91,23 +141,35 @@ function renderReviewPrompt(path: string): string {
|
|
|
91
141
|
].join("\n");
|
|
92
142
|
}
|
|
93
143
|
|
|
94
|
-
/**
|
|
95
|
-
|
|
96
|
-
|
|
144
|
+
/**
|
|
145
|
+
* Map the assistant reply text to an outcome; anything but a clean `deny`
|
|
146
|
+
* defers with the reason that distinguishes it.
|
|
147
|
+
*/
|
|
148
|
+
function parseOutcome(rawReply: string, latencyMs: number): ReviewOutcome {
|
|
97
149
|
let parsed: unknown;
|
|
98
150
|
try {
|
|
99
|
-
parsed = JSON.parse(
|
|
151
|
+
parsed = JSON.parse(rawReply);
|
|
100
152
|
} catch {
|
|
101
|
-
return {
|
|
153
|
+
return {
|
|
154
|
+
verdict: { kind: "defer" },
|
|
155
|
+
deferReason: "parse-failed",
|
|
156
|
+
latencyMs,
|
|
157
|
+
rawReply,
|
|
158
|
+
};
|
|
102
159
|
}
|
|
103
160
|
if (!isRecord(parsed) || parsed.verdict !== "deny") {
|
|
104
|
-
return {
|
|
161
|
+
return {
|
|
162
|
+
verdict: { kind: "defer" },
|
|
163
|
+
deferReason: "non-deny-verdict",
|
|
164
|
+
latencyMs,
|
|
165
|
+
rawReply,
|
|
166
|
+
};
|
|
105
167
|
}
|
|
106
168
|
const reason =
|
|
107
169
|
typeof parsed.reason === "string" && parsed.reason.length > 0
|
|
108
170
|
? parsed.reason
|
|
109
171
|
: GENERIC_TEACHING_REASON;
|
|
110
|
-
return { kind: "deny", reason };
|
|
172
|
+
return { verdict: { kind: "deny", reason }, latencyMs, rawReply };
|
|
111
173
|
}
|
|
112
174
|
|
|
113
175
|
/** Concatenate the text parts of an assistant reply. */
|
package/src/typo-patterns.ts
CHANGED
|
@@ -7,9 +7,16 @@
|
|
|
7
7
|
* without a model call.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
/**
|
|
10
|
+
/**
|
|
11
|
+
* Result of compiling a pattern list: usable regexes (with their original
|
|
12
|
+
* source strings, parallel to `regexes`) plus any that failed to compile.
|
|
13
|
+
* `patterns[i]` is the operator's literal config string for `regexes[i]`, kept
|
|
14
|
+
* so the decision trail records the pattern as written, not the slash-escaped
|
|
15
|
+
* `RegExp.source`.
|
|
16
|
+
*/
|
|
11
17
|
export interface CompiledTypoPatterns {
|
|
12
18
|
regexes: RegExp[];
|
|
19
|
+
patterns: string[];
|
|
13
20
|
invalidPatterns: string[];
|
|
14
21
|
}
|
|
15
22
|
|
|
@@ -22,21 +29,28 @@ export function compileTypoPatterns(
|
|
|
22
29
|
patterns: readonly string[],
|
|
23
30
|
): CompiledTypoPatterns {
|
|
24
31
|
const regexes: RegExp[] = [];
|
|
32
|
+
const sources: string[] = [];
|
|
25
33
|
const invalidPatterns: string[] = [];
|
|
26
34
|
for (const pattern of patterns) {
|
|
27
35
|
try {
|
|
28
36
|
regexes.push(new RegExp(pattern));
|
|
37
|
+
sources.push(pattern);
|
|
29
38
|
} catch {
|
|
30
39
|
invalidPatterns.push(pattern);
|
|
31
40
|
}
|
|
32
41
|
}
|
|
33
|
-
return { regexes, invalidPatterns };
|
|
42
|
+
return { regexes, patterns: sources, invalidPatterns };
|
|
34
43
|
}
|
|
35
44
|
|
|
36
|
-
/**
|
|
37
|
-
|
|
45
|
+
/**
|
|
46
|
+
* The source string of the first compiled pattern that matches `path`, or
|
|
47
|
+
* `undefined` if none do. The source (not the boolean) so the decision trail
|
|
48
|
+
* can record *which* pattern matched.
|
|
49
|
+
*/
|
|
50
|
+
export function matchTypoPattern(
|
|
38
51
|
path: string,
|
|
39
52
|
compiled: CompiledTypoPatterns,
|
|
40
|
-
):
|
|
41
|
-
|
|
53
|
+
): string | undefined {
|
|
54
|
+
const index = compiled.regexes.findIndex((regex) => regex.test(path));
|
|
55
|
+
return index === -1 ? undefined : compiled.patterns[index];
|
|
42
56
|
}
|
package/src/typo-reviewer.ts
CHANGED
|
@@ -4,17 +4,26 @@
|
|
|
4
4
|
*
|
|
5
5
|
* The decision runs top to bottom, deferring at the first miss so the cheap
|
|
6
6
|
* gates short-circuit before any model call:
|
|
7
|
-
* 1. surface is `external_directory` (else defer),
|
|
7
|
+
* 1. surface is `external_directory` (else defer, silently — not our surface),
|
|
8
8
|
* 2. a candidate path is present (else defer),
|
|
9
9
|
* 3. the path matches a configured typo pattern (else defer, no model call),
|
|
10
10
|
* 4. the model confirms the typo (deny with a teaching reason) or defers.
|
|
11
11
|
*
|
|
12
12
|
* Every failure path defers — more prompting, never less (ADR 0007 invariant 2).
|
|
13
13
|
* This slice never emits `allow`.
|
|
14
|
+
*
|
|
15
|
+
* Observability (issue #626): once an ask matches a typo pattern (the [#625]
|
|
16
|
+
* class — a candidate typo that *should* reach the model), the reviewer writes a
|
|
17
|
+
* positive `model_judge.decision` review entry recording the outcome and, on a
|
|
18
|
+
* defer, its reason — so a silent 100%-defer regression cannot hide again. The
|
|
19
|
+
* cheap short-circuits (`no-path`, `pattern-miss`) and the raw model reply go to
|
|
20
|
+
* the debug log; a non-`external_directory` surface is not logged at all.
|
|
14
21
|
*/
|
|
15
22
|
|
|
16
23
|
import type {
|
|
17
24
|
Authorizer,
|
|
25
|
+
AuthorizerLog,
|
|
26
|
+
AuthorizerVerdict,
|
|
18
27
|
PromptPermissionDetails,
|
|
19
28
|
} from "@gotgenes/pi-permission-system";
|
|
20
29
|
|
|
@@ -27,11 +36,31 @@ import {
|
|
|
27
36
|
import {
|
|
28
37
|
type CompiledTypoPatterns,
|
|
29
38
|
compileTypoPatterns,
|
|
30
|
-
|
|
39
|
+
matchTypoPattern,
|
|
31
40
|
} from "./typo-patterns";
|
|
32
41
|
|
|
33
42
|
const REVIEWED_SURFACE = "external_directory";
|
|
34
43
|
|
|
44
|
+
/** Review-log event: one positive decision record per pattern-matched ask. */
|
|
45
|
+
const DECISION_EVENT = "model_judge.decision";
|
|
46
|
+
/** Debug-log event: a cheap short-circuit before the model stage. */
|
|
47
|
+
const SHORT_CIRCUIT_EVENT = "model_judge.short_circuit";
|
|
48
|
+
/** Debug-log event: the raw model reply, gated behind `debugLog`. */
|
|
49
|
+
const MODEL_REPLY_EVENT = "model_judge.model_reply";
|
|
50
|
+
/** Debug-log event: `typoPatterns` entries that failed to compile. */
|
|
51
|
+
const INVALID_PATTERNS_EVENT = "model_judge.invalid_patterns";
|
|
52
|
+
|
|
53
|
+
/** A defer decided before the model call, still recorded positively. */
|
|
54
|
+
type PreModelDeferReason = "model-unresolved" | "auth-failed";
|
|
55
|
+
|
|
56
|
+
/** The shared fields of a `model_judge.decision` record, before the outcome. */
|
|
57
|
+
interface DecisionBase {
|
|
58
|
+
requestId: string;
|
|
59
|
+
path: string;
|
|
60
|
+
matchedPattern: string;
|
|
61
|
+
modelId: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
35
64
|
/** Collaborators for the reviewer, injected so the extension and tests wire them. */
|
|
36
65
|
export interface TypoReviewerDeps {
|
|
37
66
|
/** The loaded config, read live (absent until session config loads). */
|
|
@@ -40,21 +69,20 @@ export interface TypoReviewerDeps {
|
|
|
40
69
|
getRegistry: () => ModelRegistryLike | undefined;
|
|
41
70
|
/** The model-completion seam (production: `complete` from `@earendil-works/pi-ai`). */
|
|
42
71
|
complete: CompleteFn;
|
|
43
|
-
/** Optional warn sink for operational notices (unresolved model, bad patterns). */
|
|
44
|
-
warn?: (message: string) => void;
|
|
45
72
|
}
|
|
46
73
|
|
|
47
74
|
/**
|
|
48
75
|
* Build the `authorize` callback registered on the chain. The `query` argument
|
|
49
76
|
* is unused in this slice — the deny-first reviewer decides from the path
|
|
50
|
-
* pattern and the model, not from an engine re-query (slice 2 consumes it).
|
|
77
|
+
* pattern and the model, not from an engine re-query (slice 2 consumes it). The
|
|
78
|
+
* `log` argument is the injected review-log seam the decision trail records to.
|
|
51
79
|
*/
|
|
52
80
|
export function createTypoReviewer(
|
|
53
81
|
deps: TypoReviewerDeps,
|
|
54
82
|
): Authorizer["authorize"] {
|
|
55
|
-
const compiledFor = memoizeCompiledPatterns(
|
|
83
|
+
const compiledFor = memoizeCompiledPatterns();
|
|
56
84
|
|
|
57
|
-
return async (details) => {
|
|
85
|
+
return async (details, _query, log) => {
|
|
58
86
|
const config = deps.getConfig();
|
|
59
87
|
if (!config) {
|
|
60
88
|
return { kind: "defer" };
|
|
@@ -62,24 +90,88 @@ export function createTypoReviewer(
|
|
|
62
90
|
if (surfaceOf(details) !== REVIEWED_SURFACE) {
|
|
63
91
|
return { kind: "defer" };
|
|
64
92
|
}
|
|
93
|
+
const { requestId } = details;
|
|
65
94
|
const path = pathOf(details);
|
|
66
95
|
if (path === undefined) {
|
|
96
|
+
log.debug(SHORT_CIRCUIT_EVENT, { requestId, reason: "no-path" });
|
|
67
97
|
return { kind: "defer" };
|
|
68
98
|
}
|
|
69
|
-
|
|
99
|
+
const matchedPattern = matchTypoPattern(path, compiledFor(config, log));
|
|
100
|
+
if (matchedPattern === undefined) {
|
|
101
|
+
log.debug(SHORT_CIRCUIT_EVENT, {
|
|
102
|
+
requestId,
|
|
103
|
+
path,
|
|
104
|
+
reason: "pattern-miss",
|
|
105
|
+
});
|
|
70
106
|
return { kind: "defer" };
|
|
71
107
|
}
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
108
|
+
|
|
109
|
+
const modelId = `${config.provider}/${config.model}`;
|
|
110
|
+
const base: DecisionBase = { requestId, path, matchedPattern, modelId };
|
|
111
|
+
const registry = deps.getRegistry();
|
|
112
|
+
const model = registry?.find(config.provider, config.model);
|
|
113
|
+
if (!registry || !model) {
|
|
114
|
+
return deferWith(log, base, "model-unresolved");
|
|
78
115
|
}
|
|
79
|
-
|
|
116
|
+
const auth = await registry.getApiKeyAndHeaders(model);
|
|
117
|
+
if (!auth.ok) {
|
|
118
|
+
return deferWith(log, base, "auth-failed");
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const outcome = await reviewPath({
|
|
122
|
+
path,
|
|
123
|
+
config,
|
|
124
|
+
model,
|
|
125
|
+
complete: deps.complete,
|
|
126
|
+
apiKey: auth.apiKey,
|
|
127
|
+
headers: auth.headers,
|
|
128
|
+
});
|
|
129
|
+
if (outcome.rawReply !== undefined) {
|
|
130
|
+
log.debug(MODEL_REPLY_EVENT, {
|
|
131
|
+
requestId,
|
|
132
|
+
modelId,
|
|
133
|
+
rawReply: outcome.rawReply,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
log.review(DECISION_EVENT, {
|
|
137
|
+
requestId,
|
|
138
|
+
surface: REVIEWED_SURFACE,
|
|
139
|
+
path,
|
|
140
|
+
matchedPattern,
|
|
141
|
+
modelCalled: true,
|
|
142
|
+
modelId,
|
|
143
|
+
latencyMs: outcome.latencyMs,
|
|
144
|
+
verdict: outcome.verdict.kind,
|
|
145
|
+
deferReason: outcome.deferReason ?? null,
|
|
146
|
+
});
|
|
147
|
+
return outcome.verdict;
|
|
80
148
|
};
|
|
81
149
|
}
|
|
82
150
|
|
|
151
|
+
/**
|
|
152
|
+
* Record a pre-model defer (`model-unresolved` / `auth-failed`) as a positive
|
|
153
|
+
* `model_judge.decision` entry and return the defer verdict — so the two
|
|
154
|
+
* model-resolution failures leave evidence on record, not a silent absence.
|
|
155
|
+
*/
|
|
156
|
+
function deferWith(
|
|
157
|
+
log: AuthorizerLog,
|
|
158
|
+
base: DecisionBase,
|
|
159
|
+
deferReason: PreModelDeferReason,
|
|
160
|
+
): AuthorizerVerdict {
|
|
161
|
+
log.review(DECISION_EVENT, {
|
|
162
|
+
requestId: base.requestId,
|
|
163
|
+
surface: REVIEWED_SURFACE,
|
|
164
|
+
path: base.path,
|
|
165
|
+
matchedPattern: base.matchedPattern,
|
|
166
|
+
modelCalled: false,
|
|
167
|
+
modelId: base.modelId,
|
|
168
|
+
latencyMs: null,
|
|
169
|
+
verdict: "defer",
|
|
170
|
+
deferReason,
|
|
171
|
+
});
|
|
172
|
+
return { kind: "defer" };
|
|
173
|
+
}
|
|
174
|
+
|
|
83
175
|
/** The gate-authoritative surface, falling back to the display surface. */
|
|
84
176
|
function surfaceOf(details: PromptPermissionDetails): string | undefined {
|
|
85
177
|
return details.accessIntent?.surface ?? details.surface ?? undefined;
|
|
@@ -95,21 +187,22 @@ function pathOf(details: PromptPermissionDetails): string | undefined {
|
|
|
95
187
|
* object is in effect — the config is stable per session, so this avoids
|
|
96
188
|
* recompiling the regexes on every ask.
|
|
97
189
|
*/
|
|
98
|
-
function memoizeCompiledPatterns(
|
|
99
|
-
|
|
100
|
-
|
|
190
|
+
function memoizeCompiledPatterns(): (
|
|
191
|
+
config: ModelJudgeConfig,
|
|
192
|
+
log: AuthorizerLog,
|
|
193
|
+
) => CompiledTypoPatterns {
|
|
101
194
|
let cache:
|
|
102
195
|
| { config: ModelJudgeConfig; compiled: CompiledTypoPatterns }
|
|
103
196
|
| undefined;
|
|
104
|
-
return (config) => {
|
|
197
|
+
return (config, log) => {
|
|
105
198
|
if (cache?.config === config) {
|
|
106
199
|
return cache.compiled;
|
|
107
200
|
}
|
|
108
201
|
const compiled = compileTypoPatterns(config.typoPatterns);
|
|
109
202
|
if (compiled.invalidPatterns.length > 0) {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
);
|
|
203
|
+
log.debug(INVALID_PATTERNS_EVENT, {
|
|
204
|
+
invalidPatterns: compiled.invalidPatterns,
|
|
205
|
+
});
|
|
113
206
|
}
|
|
114
207
|
cache = { config, compiled };
|
|
115
208
|
return compiled;
|