@kealthas-dev/opencode-llm-review-gate 1.1.3 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/llm-review-gate.ts +85 -35
- package/package.json +1 -1
package/README.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# llm-review-gate
|
|
2
|
+
|
|
3
|
+
Gates `bash` tool calls behind an LLM safety review. Before a command runs it is sent to a hidden, throwaway opencode session that answers ALLOW or BLOCK; this sits on top of opencode's own permission config, it does not replace it.
|
|
4
|
+
|
|
5
|
+
- Package: `@kealthas-dev/opencode-llm-review-gate`, source `llm-review-gate.ts`.
|
|
6
|
+
- **Opt-in.** The plugin line is commented out in `deploy/opencode.json.example` because it adds a hidden model call before every `bash` command and can block commands. Uncomment it to enable (SETUP.md step 5).
|
|
7
|
+
- Fails closed: a review error or timeout blocks the command (`FAIL_OPEN_ON_ERROR = false` in the source).
|
|
8
|
+
- Never reviews its own review session's calls, so there is no recursion.
|
|
9
|
+
|
|
10
|
+
## Why it is built this way
|
|
11
|
+
|
|
12
|
+
- **A dedicated `review-gate` agent** (defined in `deploy/opencode.json.example`, no `prompt` override, every tool denied except `review_verdict`). A per-request `system` field is appended after an agent's configured `prompt`, not substituted for it (checked against opencode's `packages/opencode/src/session/llm/request.ts`). Without a dedicated agent the reviewer would inherit the full coding-agent persona from `system-prompt.txt` and answer with chat or tool calls instead of a verdict.
|
|
13
|
+
- **One throwaway session per review**, created right before the prompt and deleted right after. A session is a stateful conversation; reusing one would grow its history by a command-and-verdict turn per review, inflating context and cost and contradicting the "judge this one command in isolation" framing. `session.create`/`session.delete` are metadata calls, not model calls.
|
|
14
|
+
- **The verdict is a real function call.** The plugin registers a `review_verdict(allow, reason?)` tool; `execute()` stores the arguments keyed by the review's session ID and `review()` reads them back after `session.prompt()` resolves. Plugin-registered tools are visible to every agent, so `deploy/opencode.json.example` denies it globally and allows it only for `review-gate`. If the model never calls it, `review()` falls back to parsing ALLOW/BLOCK from the text.
|
|
15
|
+
- It is the only plugin with a runtime dependency (`zod`, for the tool's argument schema); the others import only types.
|
|
16
|
+
|
|
17
|
+
## Tests and limits
|
|
18
|
+
|
|
19
|
+
- `tests/unit/llm-review-gate.test.mjs` drives it with a fake opencode client: both verdict paths and their precedence, skipped non-bash tools, the recursion guard, fail-closed behavior, the `review-gate` agent name, the tool name matching the permission config, one session per review. This covers the plugin's decision logic, not opencode's real dispatch.
|
|
20
|
+
- That throwing inside `tool.execute.before` really stops the tool is checked against a live server in `tests/capability-probes/hooks.probe.mjs`.
|
|
21
|
+
- Not verified against the real Qwen model: whether it reliably calls `review_verdict` rather than falling back to text.
|
package/llm-review-gate.ts
CHANGED
|
@@ -19,11 +19,18 @@ import type { Plugin, ToolContext, ToolResult } from "@opencode-ai/plugin";
|
|
|
19
19
|
// Extend this to gate more tools (e.g. "edit", "webfetch").
|
|
20
20
|
const GATED_TOOLS = new Set(["bash"]);
|
|
21
21
|
|
|
22
|
-
// If the review call fails (server down, network error, timeout)
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
|
|
22
|
+
// If the review call fails (server down, network error, timeout) or the
|
|
23
|
+
// model's reply can't be parsed as ALLOW/BLOCK: false blocks the command,
|
|
24
|
+
// since this plugin's whole job is to be a safety net and "couldn't get an
|
|
25
|
+
// opinion" isn't a clean bill of health. The cost: on this repo's
|
|
26
|
+
// self-hosted-model deployment (see docs/deployment-environment.md), a
|
|
27
|
+
// hiccup on that model server blocks every bash call on the machine until
|
|
28
|
+
// it recovers - including the ones that would help debug why review is
|
|
29
|
+
// down. Accepted (2026-09-29, Franco): this plugin isn't left running by
|
|
30
|
+
// default, only turned on situationally, so that blast radius stays
|
|
31
|
+
// bounded to sessions that opted into it. Flip to true to trade the
|
|
32
|
+
// safety margin away for availability instead.
|
|
33
|
+
const FAIL_OPEN_ON_ERROR = false;
|
|
27
34
|
|
|
28
35
|
const REVIEW_TIMEOUT_MS = 30_000;
|
|
29
36
|
|
|
@@ -57,6 +64,9 @@ async function logReview(entry: Record<string, unknown>) {
|
|
|
57
64
|
// one plugin actually needing the package installed. `zod` alone (already
|
|
58
65
|
// needed for the schema) avoids that.
|
|
59
66
|
const VERDICT_TOOL_NAME = "review_verdict";
|
|
67
|
+
// Cleaned up by tool.execute.before's own finally, not review()'s - see the
|
|
68
|
+
// comment there for why: a verdict recorded just before a timeout/late error
|
|
69
|
+
// must still be readable after review() itself has already thrown.
|
|
60
70
|
const pendingVerdicts = new Map<string, { allow: boolean; reason?: string }>();
|
|
61
71
|
|
|
62
72
|
const reviewVerdictArgs = {
|
|
@@ -94,12 +104,13 @@ export const LlmReviewGate: Plugin = async ({ client }) => {
|
|
|
94
104
|
// cheap metadata calls, not model calls, so no second LLM round trip.
|
|
95
105
|
const reviewSessionIDs = new Set<string>();
|
|
96
106
|
|
|
97
|
-
async function review(command: string) {
|
|
107
|
+
async function review(command: string, onSessionCreated?: (sessionID: string) => void) {
|
|
98
108
|
const created = await client.session.create({
|
|
99
109
|
body: { title: "llm-review-gate (internal, safe to delete)" },
|
|
100
110
|
throwOnError: true,
|
|
101
111
|
});
|
|
102
112
|
const sessionID = created.data.id;
|
|
113
|
+
onSessionCreated?.(sessionID); // lets the caller check pendingVerdicts for this session if this call ends up racing a timeout
|
|
103
114
|
reviewSessionIDs.add(sessionID); // never review this session's own tool calls (no self-recursion)
|
|
104
115
|
try {
|
|
105
116
|
const res = await client.session.prompt({
|
|
@@ -143,7 +154,6 @@ export const LlmReviewGate: Plugin = async ({ client }) => {
|
|
|
143
154
|
}
|
|
144
155
|
return { verdict: "unclear" as const, raw: text };
|
|
145
156
|
} finally {
|
|
146
|
-
pendingVerdicts.delete(sessionID);
|
|
147
157
|
reviewSessionIDs.delete(sessionID);
|
|
148
158
|
await client.session.delete({ path: { id: sessionID } }).catch(() => {
|
|
149
159
|
// best-effort cleanup; a leaked internal session is harmless clutter, not a correctness problem
|
|
@@ -160,44 +170,84 @@ export const LlmReviewGate: Plugin = async ({ client }) => {
|
|
|
160
170
|
if (reviewSessionIDs.has(input.sessionID)) return; // never review the review session's own calls
|
|
161
171
|
|
|
162
172
|
const command = input.tool === "bash" ? output.args?.command : JSON.stringify(output.args);
|
|
173
|
+
let reviewSessionID: string | undefined;
|
|
174
|
+
let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
|
|
163
175
|
|
|
164
|
-
let result;
|
|
165
176
|
try {
|
|
166
|
-
result
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
177
|
+
let result;
|
|
178
|
+
try {
|
|
179
|
+
result = await Promise.race([
|
|
180
|
+
review(command, (id) => { reviewSessionID = id; }),
|
|
181
|
+
new Promise<never>((_, reject) => {
|
|
182
|
+
timeoutHandle = setTimeout(() => reject(new Error("review timed out")), REVIEW_TIMEOUT_MS);
|
|
183
|
+
}),
|
|
184
|
+
]);
|
|
185
|
+
} catch (e) {
|
|
186
|
+
// The model may have already called review_verdict - recording a
|
|
187
|
+
// real verdict in pendingVerdicts - before this race was lost to a
|
|
188
|
+
// timeout, or before a late error (e.g. a dropped connection right
|
|
189
|
+
// after the tool call). Either way that's a real answer, not a
|
|
190
|
+
// "couldn't get an opinion" situation, so it must win over the
|
|
191
|
+
// FAIL_OPEN_ON_ERROR default below.
|
|
192
|
+
const lateVerdict = reviewSessionID ? pendingVerdicts.get(reviewSessionID) : undefined;
|
|
193
|
+
if (lateVerdict) {
|
|
194
|
+
await logReview({
|
|
195
|
+
tool: input.tool,
|
|
196
|
+
sessionID: input.sessionID,
|
|
197
|
+
callID: input.callID,
|
|
198
|
+
command,
|
|
199
|
+
verdict: lateVerdict.allow ? "allow" : "block",
|
|
200
|
+
raw: `[late verdict after "${errorMessage(e)}"] allow=${lateVerdict.allow} reason=${lateVerdict.reason ?? ""}`,
|
|
201
|
+
});
|
|
202
|
+
if (!lateVerdict.allow) {
|
|
203
|
+
throw new Error(`Blocked by LLM review: ${lateVerdict.reason || "blocked by LLM review"}`);
|
|
204
|
+
}
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const decision = FAIL_OPEN_ON_ERROR ? "allow (fail-open)" : "block (fail-closed)";
|
|
209
|
+
await logReview({
|
|
210
|
+
tool: input.tool,
|
|
211
|
+
sessionID: input.sessionID,
|
|
212
|
+
callID: input.callID,
|
|
213
|
+
command,
|
|
214
|
+
error: errorMessage(e),
|
|
215
|
+
decision,
|
|
216
|
+
});
|
|
217
|
+
if (FAIL_OPEN_ON_ERROR) return;
|
|
218
|
+
throw new Error(`llm-review-gate: review unavailable, blocking (fail-closed): ${errorMessage(e)}`);
|
|
219
|
+
}
|
|
220
|
+
|
|
172
221
|
await logReview({
|
|
173
222
|
tool: input.tool,
|
|
174
223
|
sessionID: input.sessionID,
|
|
175
224
|
callID: input.callID,
|
|
176
225
|
command,
|
|
177
|
-
|
|
178
|
-
|
|
226
|
+
verdict: result.verdict,
|
|
227
|
+
raw: result.raw,
|
|
179
228
|
});
|
|
180
|
-
if (FAIL_OPEN_ON_ERROR) return;
|
|
181
|
-
throw new Error(`llm-review-gate: review unavailable, blocking (fail-closed): ${errorMessage(e)}`);
|
|
182
|
-
}
|
|
183
229
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
230
|
+
if (result.verdict === "block") {
|
|
231
|
+
throw new Error(`Blocked by LLM review: ${result.reason}`);
|
|
232
|
+
}
|
|
233
|
+
if (result.verdict === "unclear" && !FAIL_OPEN_ON_ERROR) {
|
|
234
|
+
throw new Error(`llm-review-gate: could not parse review verdict, blocking (fail-closed). Raw: ${result.raw.slice(0, 200)}`);
|
|
235
|
+
}
|
|
236
|
+
// "allow", or "unclear" while fail-open: fall through to whatever
|
|
237
|
+
// the config's own allow/ask/deny tier would normally do.
|
|
238
|
+
} finally {
|
|
239
|
+
// Promise.race doesn't cancel its losing side - without this, a
|
|
240
|
+
// review that resolves quickly still leaves this timer running for
|
|
241
|
+
// the full REVIEW_TIMEOUT_MS, uselessly rejecting an already-settled
|
|
242
|
+
// promise once it fires (harmless, but it's a real dangling timer:
|
|
243
|
+
// reproduced by it keeping this plugin's test process alive for a
|
|
244
|
+
// full 30s per test despite every individual test finishing in
|
|
245
|
+
// milliseconds).
|
|
246
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
247
|
+
// The only place pendingVerdicts is cleaned up - see the comment on
|
|
248
|
+
// its declaration for why review() itself no longer does this.
|
|
249
|
+
if (reviewSessionID) pendingVerdicts.delete(reviewSessionID);
|
|
198
250
|
}
|
|
199
|
-
// "allow", or "unclear" while fail-open: fall through to whatever
|
|
200
|
-
// the config's own allow/ask/deny tier would normally do.
|
|
201
251
|
},
|
|
202
252
|
};
|
|
203
253
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kealthas-dev/opencode-llm-review-gate",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "llm-review-gate.ts opencode plugin (gates bash tool calls behind an LLM safety review before they run, via a real review_verdict function-call tool with a text-parsing fallback), published to npm so opencode's own npm-plugin loader can install it directly from a bare package name in opencode.json's plugin array - see CLAUDE.md. Written in TypeScript against @opencode-ai/plugin's Plugin type (type-only) plus a real `zod` runtime dependency for the tool's argument schema - the one plugin in this repo with an actual runtime dependency, unlike hook-logger.ts/system-prompt-tools.ts's zero-dependency design. Independently installable from hook-logger.ts - not bundled together.",
|
|
6
6
|
"main": "llm-review-gate.ts",
|