@protokol11/pi-judgment 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +63 -2
- package/extensions/commands.ts +143 -0
- package/extensions/core/cp-corpus.ts +68 -0
- package/extensions/core/cp-gate.ts +192 -0
- package/extensions/core/cp-review.ts +76 -0
- package/extensions/core/gate-hook.ts +234 -0
- package/extensions/core/hash.ts +10 -0
- package/extensions/core/judge.ts +194 -0
- package/extensions/core/ledger.ts +153 -0
- package/extensions/core/lint.ts +29 -0
- package/extensions/core/policy.ts +105 -0
- package/extensions/core/prompt.ts +24 -0
- package/extensions/core/resolve-plan.ts +24 -0
- package/extensions/core/scope.ts +234 -0
- package/extensions/core/session-chain.ts +99 -0
- package/extensions/core/stats.ts +118 -0
- package/extensions/core/types.ts +93 -0
- package/extensions/explorer-state.ts +126 -0
- package/extensions/explorer.ts +324 -0
- package/extensions/index.ts +376 -0
- package/extensions/test/annotate-warmup.test.mjs +176 -0
- package/extensions/test/cp-corpus.test.mjs +78 -0
- package/extensions/test/cp-gate-hook.test.mjs +204 -0
- package/extensions/test/cp-gate.test.mjs +153 -0
- package/extensions/test/cp-review.test.mjs +99 -0
- package/extensions/test/explorer-state.test.mjs +146 -0
- package/extensions/test/explorer.test.mjs +495 -0
- package/extensions/test/fixtures.mjs +18 -0
- package/extensions/test/judge.test.mjs +139 -0
- package/extensions/test/ledger.test.mjs +185 -0
- package/extensions/test/lint.test.mjs +25 -0
- package/extensions/test/policy.test.mjs +107 -0
- package/extensions/test/refusal.test.mjs +107 -0
- package/extensions/test/scope.test.mjs +241 -0
- package/extensions/test/session-chain.test.mjs +145 -0
- package/extensions/test/stats.test.mjs +306 -0
- package/extensions/tools/annotate-warmup.mjs +292 -0
- package/package.json +40 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 protokol11
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,64 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @protokol11/pi-judgment
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Typed, calibrated decisions for pi agents: a `judge` tool, an append-only ledger, a reliability curve, and a scope-lock gate with dual-judge review.
|
|
4
|
+
|
|
5
|
+
## Why
|
|
6
|
+
|
|
7
|
+
Agents get autonomy through a prompt. This extension grounds autonomy in evidence instead:
|
|
8
|
+
|
|
9
|
+
1. Every agent decision is recorded as a typed judgment with an explicit confidence.
|
|
10
|
+
2. Resolved outcomes build a calibration curve per model. The curve decides when a model's probability is trustworthy enough to act on.
|
|
11
|
+
3. Escalation to a human is a priced third action, not a fallback. Below the minimum sample size, every decision escalates: a cold-start ledger has no evidence, so the human decides.
|
|
12
|
+
|
|
13
|
+
## How it works
|
|
14
|
+
|
|
15
|
+
Mental model: `judge(state, questions)` asks typed questions and returns answers with a probability. Each answer lands in an append-only ledger. Later, `resolve_judgment` records whether the claim turned out true. The calibration view compares predicted vs. observed accuracy per confidence bin.
|
|
16
|
+
|
|
17
|
+
```text
|
|
18
|
+
agent ── judge(state, questions) ──► lint questions (free: no model call on bad input)
|
|
19
|
+
│
|
|
20
|
+
▼
|
|
21
|
+
model answers noul / choice / score
|
|
22
|
+
(yes/no + probability, or option, or score)
|
|
23
|
+
│
|
|
24
|
+
▼
|
|
25
|
+
append-only ledger: .pi/judgments.jsonl
|
|
26
|
+
judgment rows + resolution + refusal events, folded on read
|
|
27
|
+
│
|
|
28
|
+
resolve_judgment(id, outcome) ▼
|
|
29
|
+
reliability curve: predicted vs. observed per bin
|
|
30
|
+
/judgments → Tab: calibration pane
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Scope-lock gate: `write` and `edit` are blocked until a scope judgment is authorized. A deterministic rule predicate runs first (an explicit user trigger pins p, no model call). If no trigger is present, a semantic judge scores the residue of the conversation. A degraded judge never degrades to allow: it asks the human. Authorization is bound to the session that declared it and is inherited by subagents through the session chain (a child reads the root project's ledger).
|
|
34
|
+
|
|
35
|
+
Decision policy (`decide`): act yes, act no, or escalate — the policy picks the cheapest action, and a tie escalates to the human. Below `minSample` resolved records it always escalates.
|
|
36
|
+
|
|
37
|
+
- `judge(state, questions, model?)` — typed questions (`noul`, `choice`, `score`), linted before any model call; bad questions cost nothing.
|
|
38
|
+
- `resolve_judgment(judgment_id, outcome)` — records ground truth; unresolved judgments never count toward calibration.
|
|
39
|
+
- `scope_lock(declared_scope)` — authorizes file mutations for the session; call before the first `write`/`edit`.
|
|
40
|
+
- `/judgments` — full-screen explorer over the ledger: browse, filter by model and provenance, resolve with `y`/`n`; `Tab` switches to the calibration pane. Outside the TUI it prints a one-line stats summary.
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
Requires Node 22 or later. For the pi coding agent:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pi install npm:@protokol11/pi-judgment
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Or add the package to `packages` in your pi settings. See pi.dev for the extension API.
|
|
51
|
+
|
|
52
|
+
## Optional integration: cp-review gate
|
|
53
|
+
|
|
54
|
+
The gate reviews `write`/`edit` payloads against CPs (composable principles) from a trained classifier. By default it reads the corpus at `~/.agents/skills/principles-render/rendered/cp-corpus.json`. That file comes from the `principles-render` skill in the `principles` package: https://github.com/protokol11/principles
|
|
55
|
+
|
|
56
|
+
Without a corpus, or when the classifier is unavailable, the gate fails loudly with `classifier/corpus unavailable` and asks for human review. There is no silent fallback.
|
|
57
|
+
|
|
58
|
+
## Honest status
|
|
59
|
+
|
|
60
|
+
Working. 16 test files, 142 tests passing. The cp gate caches the corpus per process; a session-level cache is not implemented.
|
|
61
|
+
|
|
62
|
+
## License
|
|
63
|
+
|
|
64
|
+
MIT
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// commands.ts — thin wiring for the /judgments command. No logic: read ledger,
|
|
2
|
+
// hand state to the pure renderer, map keys to state transitions. The state
|
|
3
|
+
// builder and the y/n decision live in pure modules (explorer-state.ts,
|
|
4
|
+
// core/resolve-plan.ts) so `node --test` can drive them without the pi UI layer.
|
|
5
|
+
// Prior art: session-manager/commands.ts (ctx.ui.custom + keyDispatch, V008).
|
|
6
|
+
import type { ExtensionAPI, Theme } from "@earendil-works/pi-coding-agent";
|
|
7
|
+
import { Key, matchesKey, truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
import { CONFIG_DIR_NAME } from "@earendil-works/pi-coding-agent";
|
|
10
|
+
import { appendResolution, readJudgments } from "./core/ledger.ts";
|
|
11
|
+
import { judgmentStats, nextProvenance } from "./core/stats.ts";
|
|
12
|
+
import { planResolution } from "./core/resolve-plan.ts";
|
|
13
|
+
import { nextGroup, openingGroup, toExplorerState, type ExplorerViewState } from "./explorer-state.ts";
|
|
14
|
+
import { bodyFor, renderJudgments } from "./explorer.ts";
|
|
15
|
+
|
|
16
|
+
type ViewState = ExplorerViewState;
|
|
17
|
+
|
|
18
|
+
function applyTheme(theme: Theme, s: { text: string; color?: string }): string {
|
|
19
|
+
if (!s.color || s.color === "default") return s.text;
|
|
20
|
+
return theme.fg(s.color as Parameters<Theme["fg"]>[0], s.text);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function registerJudgmentsCommand(pi: ExtensionAPI): void {
|
|
24
|
+
pi.registerCommand("judgments", {
|
|
25
|
+
description: "Full-screen explorer over the judgment ledger: calibration, browse, resolve",
|
|
26
|
+
handler: async (_args, ctx) => {
|
|
27
|
+
const ledgerPath = join(ctx.cwd, CONFIG_DIR_NAME, "judgments.jsonl");
|
|
28
|
+
if (ctx.mode !== "tui" || !ctx.hasUI) {
|
|
29
|
+
const stats = judgmentStats(readJudgments(ledgerPath));
|
|
30
|
+
ctx.ui.notify(
|
|
31
|
+
`judgments: ${stats.total} records, ${stats.resolved} resolved, ${stats.pending} pending (open in TUI for the explorer)`,
|
|
32
|
+
"info"
|
|
33
|
+
);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const v: ViewState = {
|
|
38
|
+
view: "ledger",
|
|
39
|
+
modelFilter: null,
|
|
40
|
+
provenanceFilter: "judge",
|
|
41
|
+
pendingOnly: true,
|
|
42
|
+
questionGroup: null, // null = open on openingGroup(); `g` fills it in
|
|
43
|
+
selected: 0,
|
|
44
|
+
message: null,
|
|
45
|
+
help: false,
|
|
46
|
+
height: 24,
|
|
47
|
+
records: readJudgments(ledgerPath),
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
await ctx.ui.custom<string | null>(
|
|
51
|
+
(tui, theme, _kb, done) => ({
|
|
52
|
+
render(width: number): string[] {
|
|
53
|
+
// Reserve two columns for the side borders and two rows for the top
|
|
54
|
+
// and bottom borders, so the box stays inside the overlay bounds.
|
|
55
|
+
const rows = Math.max(10, Math.floor(tui.terminal.rows * 0.85));
|
|
56
|
+
const height = Math.max(8, rows - 3);
|
|
57
|
+
v.height = height; // paging matches the window the renderer just laid out
|
|
58
|
+
const inner = Math.max(1, width - 2);
|
|
59
|
+
const frame = (s: string) => theme.fg("border", s);
|
|
60
|
+
const body = renderJudgments({ ...toExplorerState(v, height, width) }).map((line) =>
|
|
61
|
+
line.map((s) => applyTheme(theme, s)).join("")
|
|
62
|
+
);
|
|
63
|
+
const out: string[] = [frame(`╭${"─".repeat(inner)}╮`)];
|
|
64
|
+
for (const line of body) {
|
|
65
|
+
const clipped = truncateToWidth(line, inner, "");
|
|
66
|
+
out.push(frame("│") + clipped + " ".repeat(Math.max(0, inner - visibleWidth(clipped))) + frame("│"));
|
|
67
|
+
}
|
|
68
|
+
out.push(frame(`╰${"─".repeat(inner)}╯`));
|
|
69
|
+
return out;
|
|
70
|
+
},
|
|
71
|
+
invalidate() {},
|
|
72
|
+
handleInput(data: string) {
|
|
73
|
+
v.message = null;
|
|
74
|
+
if (matchesKey(data, Key.escape) || data === "q") {
|
|
75
|
+
done(null);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const state = toExplorerState(v, v.height, 0);
|
|
79
|
+
const last = Math.max(0, state.rows.length - 1);
|
|
80
|
+
const clamp = (n: number) => Math.max(0, Math.min(last, n));
|
|
81
|
+
if (matchesKey(data, Key.up) || matchesKey(data, Key.down)) {
|
|
82
|
+
v.selected = clamp(v.selected + (matchesKey(data, Key.up) ? -1 : 1));
|
|
83
|
+
} else if (matchesKey(data, Key.pageUp) || matchesKey(data, Key.pageDown)) {
|
|
84
|
+
const page = bodyFor(v.height);
|
|
85
|
+
v.selected = clamp(v.selected + (matchesKey(data, Key.pageUp) ? -page : page));
|
|
86
|
+
} else if (matchesKey(data, Key.home)) {
|
|
87
|
+
v.selected = 0;
|
|
88
|
+
} else if (matchesKey(data, Key.end)) {
|
|
89
|
+
v.selected = last;
|
|
90
|
+
} else if (data === "\t") {
|
|
91
|
+
v.view = v.view === "ledger" ? "calibration" : "ledger";
|
|
92
|
+
} else if (matchesKey(data, Key.question)) {
|
|
93
|
+
v.help = !v.help;
|
|
94
|
+
} else if (data === "f") {
|
|
95
|
+
v.pendingOnly = !v.pendingOnly;
|
|
96
|
+
v.selected = 0;
|
|
97
|
+
} else if (data === "m") {
|
|
98
|
+
const models = judgmentStats(v.records).models;
|
|
99
|
+
const i = v.modelFilter ? models.indexOf(v.modelFilter) : -1;
|
|
100
|
+
v.modelFilter = i + 1 < models.length ? models[i + 1] : null;
|
|
101
|
+
v.selected = 0;
|
|
102
|
+
} else if (data === "p") {
|
|
103
|
+
// Cycle the provenances the ledger actually holds, so `code` and
|
|
104
|
+
// `classifier` rows are reachable; the curve stays one family at a
|
|
105
|
+
// time (mixed provenances would blur it).
|
|
106
|
+
v.provenanceFilter = nextProvenance(
|
|
107
|
+
v.records.map((r) => r.provenance),
|
|
108
|
+
v.provenanceFilter
|
|
109
|
+
);
|
|
110
|
+
v.selected = 0;
|
|
111
|
+
} else if (data === "g") {
|
|
112
|
+
// Cycle only the groups the current model + provenance filter leaves,
|
|
113
|
+
// so `g` never lands on a pane that would render another filter's rows.
|
|
114
|
+
const current = v.questionGroup ?? openingGroup(v.records, v.modelFilter, v.provenanceFilter);
|
|
115
|
+
v.questionGroup = nextGroup(v.records, current, v.modelFilter, v.provenanceFilter);
|
|
116
|
+
v.selected = 0;
|
|
117
|
+
} else if ((data === "y" || data === "n") && v.view === "ledger") {
|
|
118
|
+
// A verdict is only ever written for a row the reader can see: no
|
|
119
|
+
// write from the calibration pane and none on an already-resolved row.
|
|
120
|
+
const plan = planResolution(state.rows, v.selected);
|
|
121
|
+
if (plan.kind === "write") {
|
|
122
|
+
const row = plan.row;
|
|
123
|
+
const target = v.records.find((r) => r.ts === row.ts && r.question_id === row.question_id);
|
|
124
|
+
if (target) {
|
|
125
|
+
appendResolution(ledgerPath, target.judgment_id, data === "y", new Date().toISOString());
|
|
126
|
+
v.records = readJudgments(ledgerPath);
|
|
127
|
+
v.message = `resolved ${target.judgment_id} → ${data === "y"}`;
|
|
128
|
+
// The resolved row leaves the queue, so the cursor can point past
|
|
129
|
+
// the end; toExplorerState clamps `selected` into the new rows.
|
|
130
|
+
toExplorerState(v, v.height, 0);
|
|
131
|
+
}
|
|
132
|
+
} else if (plan.kind === "already-resolved") {
|
|
133
|
+
v.message = "already resolved";
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
tui.requestRender();
|
|
137
|
+
},
|
|
138
|
+
}),
|
|
139
|
+
{ overlay: true, overlayOptions: { anchor: "center", width: "90%", maxHeight: "85%", margin: 1 } }
|
|
140
|
+
);
|
|
141
|
+
},
|
|
142
|
+
});
|
|
143
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// core/cp-corpus.ts — the CP corpus the gate reviews diffs against.
|
|
2
|
+
//
|
|
3
|
+
// Source of truth: principles-render emits rendered/cp-corpus.json from the
|
|
4
|
+
// to-violations catalog (render.mjs owns YAML parsing; DRY). This module only
|
|
5
|
+
// validates a JSON string into typed entries — no fs, no pi import.
|
|
6
|
+
import type { Question } from "./types.ts";
|
|
7
|
+
|
|
8
|
+
export const CP_SEVERITIES = ["must", "should"] as const;
|
|
9
|
+
export type CpSeverity = (typeof CP_SEVERITIES)[number];
|
|
10
|
+
|
|
11
|
+
export interface CpEntry {
|
|
12
|
+
id: string;
|
|
13
|
+
severity: CpSeverity;
|
|
14
|
+
statement: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface CpCorpus {
|
|
18
|
+
generated_from?: string;
|
|
19
|
+
cps: CpEntry[];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Parse and validate a corpus JSON string. Throws naming the offending entry. */
|
|
23
|
+
export function parseCorpus(json: string): CpCorpus {
|
|
24
|
+
let raw: unknown;
|
|
25
|
+
try {
|
|
26
|
+
raw = JSON.parse(json);
|
|
27
|
+
} catch (e) {
|
|
28
|
+
throw new Error(`cp-corpus: invalid JSON (${(e as Error).message})`);
|
|
29
|
+
}
|
|
30
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
31
|
+
throw new Error("cp-corpus: top level must be an object");
|
|
32
|
+
}
|
|
33
|
+
const obj = raw as Record<string, unknown>;
|
|
34
|
+
if (!Array.isArray(obj.cps)) throw new Error("cp-corpus: missing cps array");
|
|
35
|
+
const cps: CpEntry[] = obj.cps.map((entry, i) => {
|
|
36
|
+
const where = `cps[${i}]`;
|
|
37
|
+
if (!entry || typeof entry !== "object") throw new Error(`cp-corpus: ${where} not an object`);
|
|
38
|
+
const e = entry as Record<string, unknown>;
|
|
39
|
+
if (typeof e.id !== "string" || !e.id) throw new Error(`cp-corpus: ${where} missing id`);
|
|
40
|
+
if (!CP_SEVERITIES.includes(e.severity as CpSeverity)) {
|
|
41
|
+
throw new Error(`cp-corpus: ${e.id} bad severity ${String(e.severity)}`);
|
|
42
|
+
}
|
|
43
|
+
if (typeof e.statement !== "string" || !e.statement.trim()) {
|
|
44
|
+
throw new Error(`cp-corpus: ${e.id} missing statement`);
|
|
45
|
+
}
|
|
46
|
+
return { id: e.id, severity: e.severity as CpSeverity, statement: e.statement };
|
|
47
|
+
});
|
|
48
|
+
const seen = new Set<string>();
|
|
49
|
+
for (const cp of cps) {
|
|
50
|
+
if (seen.has(cp.id)) throw new Error(`cp-corpus: duplicate id ${cp.id}`);
|
|
51
|
+
seen.add(cp.id);
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
generated_from: typeof obj.generated_from === "string" ? obj.generated_from : undefined,
|
|
55
|
+
cps,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function mustCps(corpus: CpCorpus): CpEntry[] {
|
|
60
|
+
return corpus.cps.filter((c) => c.severity === "must");
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function shouldCps(corpus: CpCorpus): CpEntry[] {
|
|
64
|
+
return corpus.cps.filter((c) => c.severity === "should");
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Kept import-side: Question is the wire shape buildQuestions emits. */
|
|
68
|
+
export type { Question };
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
// core/cp-gate.ts — pure cp-review gate: questions in, verdicts out.
|
|
2
|
+
//
|
|
3
|
+
// The trained cell speaks the S1 wire: state {diff, cp_texts} with one
|
|
4
|
+
// cp-review:CP-XX noul question per should-CP. The drilled operating point
|
|
5
|
+
// (sess-drill-rlcd-operating-point) is applied per call here and only here:
|
|
6
|
+
// violated = P(yes) >= CP_GATE_THRESHOLD. decide() and scope-lock stay
|
|
7
|
+
// untouched. No fs, no model, no pi import.
|
|
8
|
+
import { CP_REVIEW_PREFIX } from "./cp-review.ts";
|
|
9
|
+
import type { CpEntry } from "./cp-corpus.ts";
|
|
10
|
+
import type { Answer, Question } from "./types.ts";
|
|
11
|
+
|
|
12
|
+
/** Drilled BA operating point; per-call, not baked into decide(). */
|
|
13
|
+
export const CP_GATE_THRESHOLD = 0.081149;
|
|
14
|
+
|
|
15
|
+
/** One gate review: the diff under edit plus the CP texts it is judged against. */
|
|
16
|
+
export interface CpReviewState {
|
|
17
|
+
diff: string;
|
|
18
|
+
cp_texts: Record<string, string>;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface GateQuestion {
|
|
22
|
+
questions: Record<string, Question>;
|
|
23
|
+
state: CpReviewState;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Build the classifier request for one diff. Instruction shape matches the
|
|
28
|
+
* cell's training wire verbatim (calibration-warmup corpus) with the CP id
|
|
29
|
+
* substituted — the cell was trained on this exact phrasing.
|
|
30
|
+
*/
|
|
31
|
+
export function buildQuestions(diff: string, cps: CpEntry[]): GateQuestion {
|
|
32
|
+
const questions: Record<string, Question> = {};
|
|
33
|
+
const cp_texts: Record<string, string> = {};
|
|
34
|
+
for (const cp of cps) {
|
|
35
|
+
cp_texts[cp.id] = cp.statement;
|
|
36
|
+
questions[`${CP_REVIEW_PREFIX}${cp.id}`] = {
|
|
37
|
+
type: "noul",
|
|
38
|
+
instructions:
|
|
39
|
+
`Same diff in state.diff. Full principle text in state.cp_texts under key ${cp.id}. ` +
|
|
40
|
+
`Does this diff violate principle ${cp.id}? yes/no with probability.`,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
return { questions, state: { diff, cp_texts } };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface Verdict {
|
|
47
|
+
cp_id: string;
|
|
48
|
+
/** P(violation) recovered from the answer; finite, in [0,1]. */
|
|
49
|
+
p: number;
|
|
50
|
+
violated: boolean;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface VerdictError {
|
|
54
|
+
cp_id: string;
|
|
55
|
+
error: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface GateVerdicts {
|
|
59
|
+
verdicts: Verdict[];
|
|
60
|
+
errors: VerdictError[];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function sanitizeConfidence(raw: unknown): number | null {
|
|
64
|
+
if (typeof raw !== "number" || !Number.isFinite(raw)) return null;
|
|
65
|
+
if (raw < 0 || raw > 1) return null;
|
|
66
|
+
return raw;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Turn judgment answers into verdicts at the operating point. `confidence` is
|
|
71
|
+
* P(recorded value) (ADR-0045), so P(yes) is reconstructed before the
|
|
72
|
+
* comparison: pYes = value === "yes" ? confidence : 1 - confidence. The
|
|
73
|
+
* verdict fills `p` with pYes — the same quantity the cell's raw probability
|
|
74
|
+
* used to carry — so thresholds, reported p values and sort order stay
|
|
75
|
+
* bit-for-bit identical to the pre-ADR-0045 records. Comparison is >= (prototype
|
|
76
|
+
* edge p == t → violated). Answers with non-finite or out-of-range confidence
|
|
77
|
+
* surface as errors, never as a clean verdict; so does a value that is neither
|
|
78
|
+
* "yes" nor "no", which names no direction to threshold on. An unanswered CP
|
|
79
|
+
* produces no row at all (HUMAN-JUDGMENT-FINAL: no row, not violated:false).
|
|
80
|
+
* Verdicts sort by p (pYes) desc.
|
|
81
|
+
*/
|
|
82
|
+
export function verdictsFromAnswers(
|
|
83
|
+
answers: Record<string, Answer>,
|
|
84
|
+
threshold: number = CP_GATE_THRESHOLD
|
|
85
|
+
): GateVerdicts {
|
|
86
|
+
const verdicts: Verdict[] = [];
|
|
87
|
+
const errors: VerdictError[] = [];
|
|
88
|
+
for (const [id, a] of Object.entries(answers)) {
|
|
89
|
+
const cp_id = id.startsWith(CP_REVIEW_PREFIX) ? id.slice(CP_REVIEW_PREFIX.length) : id;
|
|
90
|
+
if (!a || typeof a !== "object") {
|
|
91
|
+
errors.push({ cp_id, error: "missing answer" });
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
const p = sanitizeConfidence(a.confidence);
|
|
95
|
+
if (p === null) {
|
|
96
|
+
errors.push({ cp_id, error: `bad confidence ${String(a.confidence)}` });
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
// Only a yes/no value names a direction. Anything else has no P(yes) to
|
|
100
|
+
// recover, so it errors out instead of silently inverting one.
|
|
101
|
+
if (a.value !== "yes" && a.value !== "no") {
|
|
102
|
+
errors.push({ cp_id, error: `unsupported value ${String(a.value)}` });
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
// confidence is P(value); the gate thresholds on P(yes), so invert "no".
|
|
106
|
+
const pYes = a.value === "yes" ? p : 1 - p;
|
|
107
|
+
verdicts.push({ cp_id, p: pYes, violated: pYes >= threshold });
|
|
108
|
+
}
|
|
109
|
+
verdicts.sort((x, y) => y.p - x.p);
|
|
110
|
+
return { verdicts, errors };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The must-CP slot. Predicates are synthesized by rule-lifecycle promotion
|
|
115
|
+
* (ADR-0034) and registered here; the registry ships empty — no deterministic
|
|
116
|
+
* predicates exist for the 31 must-CPs yet, and inventing them would be a lie.
|
|
117
|
+
* check returns a violation reason or null.
|
|
118
|
+
*/
|
|
119
|
+
export interface LintPredicate {
|
|
120
|
+
id: string;
|
|
121
|
+
check: (content: string) => string | null;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export const lintRegistry: LintPredicate[] = [];
|
|
125
|
+
|
|
126
|
+
/** Run registered must-lint predicates over the payload. Empty registry passes. */
|
|
127
|
+
export function runLints(content: string, registry: LintPredicate[] = lintRegistry): VerdictError[] {
|
|
128
|
+
const hits: VerdictError[] = [];
|
|
129
|
+
for (const pred of registry) {
|
|
130
|
+
const reason = pred.check(content);
|
|
131
|
+
if (reason !== null) hits.push({ cp_id: pred.id, error: reason });
|
|
132
|
+
}
|
|
133
|
+
return hits;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The hook's decision, planned pure from gate outputs. Typed union (CP-50):
|
|
138
|
+
* allow — proceed; notice — proceed, the caller surfaces why the gate stood
|
|
139
|
+
* down (oversized payload, own-infra failure); block — hard stop (lint hit,
|
|
140
|
+
* or verdict/escalation with no UI to ask); confirm — ask the human (verdict
|
|
141
|
+
* override or classifier-failure escalation).
|
|
142
|
+
*/
|
|
143
|
+
export type GateDecision =
|
|
144
|
+
| { action: "allow"; reason: string }
|
|
145
|
+
| { action: "notice"; reason: string }
|
|
146
|
+
| { action: "block"; reason: string; kind: "lint" | "no-ui" }
|
|
147
|
+
| { action: "confirm"; reason: string; violated: Verdict[] | null };
|
|
148
|
+
|
|
149
|
+
export interface GatePlanInput {
|
|
150
|
+
gateEnabled: boolean;
|
|
151
|
+
contentLength: number;
|
|
152
|
+
cap: number;
|
|
153
|
+
/** Own-infra failure (corpus unreadable/unparseable). Degrades open, loudly. */
|
|
154
|
+
infraError: string | null;
|
|
155
|
+
lintHits: VerdictError[];
|
|
156
|
+
/** Classifier unavailable or judge returned errors. Escalates, never a silent no. */
|
|
157
|
+
classifyError: string | null;
|
|
158
|
+
verdicts: Verdict[];
|
|
159
|
+
hasUI: boolean;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function planGateDecision(input: GatePlanInput): GateDecision {
|
|
163
|
+
if (!input.gateEnabled) return { action: "allow", reason: "cp-review gate off (CP_REVIEW_GATE=off)" };
|
|
164
|
+
if (input.contentLength > input.cap) {
|
|
165
|
+
return {
|
|
166
|
+
action: "notice",
|
|
167
|
+
reason: `cp-review gate: payload ${input.contentLength} chars exceeds ${input.cap}-char review cap; written UNREVIEWED (full-diff retrieval is a lab-side step)`,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
if (input.infraError) {
|
|
171
|
+
return { action: "notice", reason: `cp-review gate: own infra down (${input.infraError}); written UNREVIEWED` };
|
|
172
|
+
}
|
|
173
|
+
if (input.lintHits.length) {
|
|
174
|
+
const names = input.lintHits.map((h) => `${h.cp_id}: ${h.error}`).join("; ");
|
|
175
|
+
return { action: "block", kind: "lint", reason: `cp-review gate (must-CP lint): ${names}` };
|
|
176
|
+
}
|
|
177
|
+
if (input.classifyError) {
|
|
178
|
+
const reason = `cp-review gate: classifier failure — ${input.classifyError}. No guess recorded; a human decides.`;
|
|
179
|
+
return input.hasUI
|
|
180
|
+
? { action: "confirm", reason, violated: null }
|
|
181
|
+
: { action: "block", kind: "no-ui", reason: `${reason} (no UI: request human review)` };
|
|
182
|
+
}
|
|
183
|
+
const violated = input.verdicts.filter((v) => v.violated);
|
|
184
|
+
if (violated.length) {
|
|
185
|
+
const names = violated.map((v) => `${v.cp_id} (p=${v.p.toFixed(3)})`).join(", ");
|
|
186
|
+
const reason = `cp-review gate: suspected violation(s) at t=${CP_GATE_THRESHOLD}: ${names}. Confirm to override.`;
|
|
187
|
+
return input.hasUI
|
|
188
|
+
? { action: "confirm", reason, violated }
|
|
189
|
+
: { action: "block", kind: "no-ui", reason: `${reason} (no UI: request human review)` };
|
|
190
|
+
}
|
|
191
|
+
return { action: "allow", reason: "cp-review gate: clean" };
|
|
192
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// core/cp-review.ts — the pure cp-review request/answer contract.
|
|
2
|
+
//
|
|
3
|
+
// The RLCD cell speaks the classifier wire: one `bool` question with
|
|
4
|
+
// criteria {true,false} answers with {type:"bool", probability}. The judgment
|
|
5
|
+
// engine speaks `noul` questions and {value, confidence} answers. This module
|
|
6
|
+
// is the only place that translates between the two, and the only place that
|
|
7
|
+
// names the request shape. No model, no fs, no pi import.
|
|
8
|
+
import type { Answer, Question } from "./types.ts";
|
|
9
|
+
|
|
10
|
+
/** Question-id prefix that marks a request for the trained cp-review cell. */
|
|
11
|
+
export const CP_REVIEW_PREFIX = "cp-review:";
|
|
12
|
+
|
|
13
|
+
export function isCpReviewQuestion(id: string): boolean {
|
|
14
|
+
return id.startsWith(CP_REVIEW_PREFIX);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* All-or-nothing route: a request goes to the classifier only when every
|
|
19
|
+
* question is a cp-review question. A mixed request keeps the generative path,
|
|
20
|
+
* so one state_hash is never split across two models.
|
|
21
|
+
*/
|
|
22
|
+
export function isCpReviewRequest(questions: Record<string, { type: string }>): boolean {
|
|
23
|
+
const ids = Object.keys(questions);
|
|
24
|
+
return ids.length > 0 && ids.every(isCpReviewQuestion);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Classifier wire question: fixed yes/no criteria, per the S1 prototype. */
|
|
28
|
+
export interface ClassifierBoolQuestion {
|
|
29
|
+
type: "bool";
|
|
30
|
+
instructions: string;
|
|
31
|
+
criteria: { true: string; false: string };
|
|
32
|
+
/** Per-question CP key (cp-review questions only): lets the sidecar align the CP
|
|
33
|
+
* text without parsing the id string. See ADR-0039. */
|
|
34
|
+
cp_key?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Map judgment `noul` questions to the classifier `bool` wire shape. */
|
|
38
|
+
export function toClassifierQuestions(questions: Record<string, Question>): Record<string, ClassifierBoolQuestion> {
|
|
39
|
+
const out: Record<string, ClassifierBoolQuestion> = {};
|
|
40
|
+
for (const [id, q] of Object.entries(questions)) {
|
|
41
|
+
out[id] = {
|
|
42
|
+
type: "bool",
|
|
43
|
+
instructions: q.instructions,
|
|
44
|
+
criteria: { true: "yes", false: "no" },
|
|
45
|
+
};
|
|
46
|
+
// The CP key is derived from the single id-format source (CP_REVIEW_PREFIX);
|
|
47
|
+
// the sidecar reads the field and never parses the id (ADR-0039).
|
|
48
|
+
if (isCpReviewQuestion(id)) out[id].cp_key = id.slice(CP_REVIEW_PREFIX.length);
|
|
49
|
+
}
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Classifier wire answer; only `bool` carries a probability. */
|
|
54
|
+
export interface ClassifierBoolAnswer {
|
|
55
|
+
type: string;
|
|
56
|
+
probability?: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Map classifier answers back to judgment answers. The value is the point
|
|
61
|
+
* guess at the 0.5 boundary; `confidence` is P(recorded value) — the raw
|
|
62
|
+
* probability for "yes", its complement for "no" (ADR-0045). A caller that
|
|
63
|
+
* thresholds on P(yes), like the cp-review gate, must invert explicitly.
|
|
64
|
+
*/
|
|
65
|
+
export function toJudgmentAnswers(answers: Record<string, ClassifierBoolAnswer>): Record<string, Answer> {
|
|
66
|
+
const out: Record<string, Answer> = {};
|
|
67
|
+
for (const [id, a] of Object.entries(answers)) {
|
|
68
|
+
const p = typeof a.probability === "number" && Number.isFinite(a.probability) ? a.probability : 0.5;
|
|
69
|
+
const clamped = Math.min(1, Math.max(0, p));
|
|
70
|
+
out[id] =
|
|
71
|
+
clamped >= 0.5
|
|
72
|
+
? { value: "yes", confidence: clamped }
|
|
73
|
+
: { value: "no", confidence: 1 - clamped };
|
|
74
|
+
}
|
|
75
|
+
return out;
|
|
76
|
+
}
|