opencode-goal-plugin 0.4.7 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -3
- package/CONTRIBUTING.md +8 -7
- package/README.md +98 -13
- package/SECURITY.md +29 -1
- package/demo/README.md +86 -0
- package/demo/opencode.json +11 -0
- package/demo/package.json +10 -0
- package/demo/src/math.js +3 -0
- package/demo/test/math.test.js +11 -0
- package/docs/providers.md +84 -0
- package/index.d.ts +344 -0
- package/package.json +15 -5
- package/scripts/behavior-benchmark.mjs +272 -0
- package/scripts/packed-host-contract.mjs +160 -0
- package/scripts/verify.mjs +125 -0
- package/src/completion-claim.js +127 -0
- package/src/goal-plugin.js +604 -137
- package/src/goal-tool-result.js +18 -0
- package/src/native-agent-config.js +69 -0
- package/src/opencode-session-api.js +100 -0
- package/src/persistence-lease.js +82 -0
package/index.d.ts
ADDED
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Type declarations for opencode-goal-plugin.
|
|
3
|
+
*
|
|
4
|
+
* These describe the plugin-level configuration object accepted in
|
|
5
|
+
* `opencode.json` under `plugin: [["opencode-goal-plugin", { ... }]]`,
|
|
6
|
+
* and the shape of the module's exports.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Verdict returned by a completion auditor (built-in or custom). See
|
|
11
|
+
* {@link GoalPluginOptions.auditor} and {@link GoalPluginOptions.completionAudit}.
|
|
12
|
+
*/
|
|
13
|
+
export interface CompletionAuditVerdict {
|
|
14
|
+
/** `true` to archive the goal as achieved; `false` to reject the completion. */
|
|
15
|
+
approved: boolean
|
|
16
|
+
/** Human-readable reason, surfaced in the goal's status when rejected. */
|
|
17
|
+
reason?: string
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Arguments passed to a custom {@link GoalPluginOptions.auditor} function. */
|
|
21
|
+
export interface CompletionAuditContext {
|
|
22
|
+
/** The goal being audited (objective, budget usage, checkpoints, etc.). */
|
|
23
|
+
goal: unknown
|
|
24
|
+
/** The OpenCode session ID the goal belongs to. */
|
|
25
|
+
sessionID: string
|
|
26
|
+
/** The assistant's latest response text, containing the `[goal:evidence]`/`[goal:complete]` claim. */
|
|
27
|
+
latestText: string
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Options for the built-in child-session completion auditor (`completionAudit: true`). */
|
|
31
|
+
export interface CompletionAuditorOptions {
|
|
32
|
+
/**
|
|
33
|
+
* How long, in milliseconds, the built-in auditor waits for a verdict from
|
|
34
|
+
* its child OpenCode session. A timeout rejects the audit and pauses the
|
|
35
|
+
* goal. Operational failures follow {@link failurePolicy}.
|
|
36
|
+
* @default 120000
|
|
37
|
+
*/
|
|
38
|
+
timeoutMs?: number
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Result used when the child-session API is unavailable, malformed, throws,
|
|
42
|
+
* or times out. Semantic rejection or an invalid verdict always rejects.
|
|
43
|
+
* `"approve"` is an explicit compatibility escape hatch.
|
|
44
|
+
* @default "reject"
|
|
45
|
+
*/
|
|
46
|
+
failurePolicy?: "reject" | "approve"
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Configuration options for opencode-goal-plugin. All fields are optional;
|
|
51
|
+
* unset fields fall back to the plugin's built-in defaults. These act as
|
|
52
|
+
* the default limits for every goal set in a session, and most of the
|
|
53
|
+
* budget/behavior fields can be overridden per-goal via `/goal` command
|
|
54
|
+
* flags (e.g. `--max-turns`, `--success`, `--mode`).
|
|
55
|
+
*/
|
|
56
|
+
export interface GoalPluginOptions {
|
|
57
|
+
/**
|
|
58
|
+
* OpenCode session SDK argument shape. PluginInput currently supplies the
|
|
59
|
+
* legacy generated client; set `"flat"` when embedding with the v2 SDK.
|
|
60
|
+
* The compatibility adapter remembers the successful shape per operation.
|
|
61
|
+
* @default "legacy"
|
|
62
|
+
*/
|
|
63
|
+
sdkShape?: "legacy" | "flat"
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Maximum number of auto-continue turns sent toward a goal before it is
|
|
67
|
+
* stopped for exceeding limits. Overridable per-goal with `--max-turns`.
|
|
68
|
+
* @default 10
|
|
69
|
+
*/
|
|
70
|
+
maxTurns?: number
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Maximum wall-clock duration, in milliseconds, a goal may run before it
|
|
74
|
+
* is stopped for exceeding limits. Overridable per-goal with
|
|
75
|
+
* `--max-duration-ms` or `--max-minutes`.
|
|
76
|
+
* @default 900000
|
|
77
|
+
*/
|
|
78
|
+
maxDurationMs?: number
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Maximum context token budget a goal may consume before it is stopped
|
|
82
|
+
* for exceeding limits. Overridable per-goal with `--max-tokens` or the
|
|
83
|
+
* `--budget` shorthand (accepts a `k`/`m` suffix, e.g. `100k`, `1.5m`).
|
|
84
|
+
* @default 200000
|
|
85
|
+
*/
|
|
86
|
+
maxTokens?: number
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Minimum delay, in milliseconds, enforced between consecutive
|
|
90
|
+
* auto-continue prompts. Overridable per-goal with `--cooldown-ms`.
|
|
91
|
+
* @default 1500
|
|
92
|
+
*/
|
|
93
|
+
minDelayMs?: number
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* How many recent session messages to scan when looking for the latest
|
|
97
|
+
* assistant turn before auto-continuing. Higher values make long,
|
|
98
|
+
* tool-heavy sessions less likely to lose the most recent assistant
|
|
99
|
+
* response.
|
|
100
|
+
* @default 50
|
|
101
|
+
*/
|
|
102
|
+
maxRecentMessages?: number
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Output token floor below which a turn is considered "low-output" for
|
|
106
|
+
* no-progress detection. Overridable per-goal with
|
|
107
|
+
* `--no-progress-threshold`.
|
|
108
|
+
* @default 50
|
|
109
|
+
*/
|
|
110
|
+
noProgressTokenThreshold?: number
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Grace window for low-output stalls: the goal is paused only after this
|
|
114
|
+
* many consecutive stalled low-output turns, rather than on the first
|
|
115
|
+
* one. Overridable per-goal with `--no-progress-turns`.
|
|
116
|
+
* @default 2
|
|
117
|
+
*/
|
|
118
|
+
noProgressTurnsBeforePause?: number
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Grace window for tool-free continuation turns (a "talk only" turn that
|
|
122
|
+
* calls no tool). Complements the no-progress check by catching
|
|
123
|
+
* self-chat loops that still produce output. Overridable per-goal with
|
|
124
|
+
* `--no-tool-turns`.
|
|
125
|
+
* @default 2
|
|
126
|
+
*/
|
|
127
|
+
noToolCallTurnsBeforePause?: number
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Fraction (between 0 and 1, exclusive) of any budget (turns, duration,
|
|
131
|
+
* or tokens) at which the plugin sends a one-time "wrap up" prompt
|
|
132
|
+
* nudging the model to finish before the hard limit is hit.
|
|
133
|
+
* @default 0.8
|
|
134
|
+
*/
|
|
135
|
+
budgetWrapupRatio?: number
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Number of remaining auto-continue turns at which a limit-approaching
|
|
139
|
+
* warning is included in status output.
|
|
140
|
+
* @default 3
|
|
141
|
+
*/
|
|
142
|
+
warnTurnsRemaining?: number
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Remaining duration, in milliseconds, at which a limit-approaching
|
|
146
|
+
* warning is included in status output.
|
|
147
|
+
* @default 60000
|
|
148
|
+
*/
|
|
149
|
+
warnDurationMsRemaining?: number
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Remaining context tokens at which a limit-approaching warning is
|
|
153
|
+
* included in status output.
|
|
154
|
+
* @default 25000
|
|
155
|
+
*/
|
|
156
|
+
warnTokensRemaining?: number
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Maximum number of consecutive prompt failures (e.g. transport errors
|
|
160
|
+
* sending the auto-continue prompt, or repeated missing-evidence /
|
|
161
|
+
* missing-blocker format violations) tolerated before the goal is
|
|
162
|
+
* stopped.
|
|
163
|
+
* @default 3
|
|
164
|
+
*/
|
|
165
|
+
maxPromptFailures?: number
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Whether to persist active/backgrounded goals and recent goal results
|
|
169
|
+
* to disk so they survive a restart. Recovered active goals are loaded
|
|
170
|
+
* in a paused state. Set to `false` for purely in-memory behavior (this
|
|
171
|
+
* also disables the lifecycle ledger).
|
|
172
|
+
* @default true
|
|
173
|
+
*/
|
|
174
|
+
persistState?: boolean
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Filesystem path where persisted goal state is written when
|
|
178
|
+
* `persistState` is enabled. Overrides both the project-local default
|
|
179
|
+
* and the `OPENCODE_GOAL_STATE_PATH` environment variable.
|
|
180
|
+
* @default "<cwd>/.opencode/goals/state.json"
|
|
181
|
+
*/
|
|
182
|
+
stateFilePath?: string
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Filesystem path for the append-only lifecycle ledger
|
|
186
|
+
* (`<event> per line`, used to reconstruct active goals if the main
|
|
187
|
+
* state file is missing or corrupted).
|
|
188
|
+
* @default "<stateFilePath>.ledger.jsonl"
|
|
189
|
+
*/
|
|
190
|
+
ledgerFilePath?: string
|
|
191
|
+
|
|
192
|
+
/** Maximum bytes in one lifecycle-ledger generation. @default 2097152 */
|
|
193
|
+
ledgerMaxBytes?: number
|
|
194
|
+
|
|
195
|
+
/** Number of rotated lifecycle-ledger generations to retain (0-10). @default 3 */
|
|
196
|
+
ledgerRetentionFiles?: number
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* How long, in milliseconds, a completed goal's summary remains
|
|
200
|
+
* available through `/goal status` after the goal leaves active memory.
|
|
201
|
+
* @default 604800000
|
|
202
|
+
*/
|
|
203
|
+
resultRetentionMs?: number
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Maximum number of completed-goal summaries retained in process memory
|
|
207
|
+
* before the oldest ones are evicted.
|
|
208
|
+
* @default 200
|
|
209
|
+
*/
|
|
210
|
+
maxStoredResults?: number
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* The slash command the plugin owns. Set to e.g. `"objective"` to drive
|
|
214
|
+
* the workflow with `/objective` instead of `/goal`; a leading slash is
|
|
215
|
+
* tolerated and stripped. Remember to register the matching command
|
|
216
|
+
* name in your OpenCode `command` config.
|
|
217
|
+
* @default "goal"
|
|
218
|
+
*/
|
|
219
|
+
commandName?: string
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Whether the plugin installs its `command.execute.before` hook at all.
|
|
223
|
+
* Set to `false` if you only want the auto-continue/persistence
|
|
224
|
+
* behavior driven programmatically (e.g. via {@link registerTools})
|
|
225
|
+
* and don't want the plugin to own a slash command.
|
|
226
|
+
* @default true
|
|
227
|
+
*/
|
|
228
|
+
registerCommand?: boolean
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Whether the plugin registers the agent-facing goal tools
|
|
232
|
+
* (canonical `goal_status`, `goal_set`, `goal_pause`, `goal_resume`,
|
|
233
|
+
* `goal_block`, `goal_complete`, plus legacy `get_goal`,
|
|
234
|
+
* `get_goal_history`, `set_goal`, `update_goal`, `clear_goal`).
|
|
235
|
+
* Canonical tools return versioned JSON envelopes. Requires the optional `@opencode-ai/plugin` peer
|
|
236
|
+
* dependency; when it is absent, tool registration is silently skipped
|
|
237
|
+
* and the command/event hooks still work.
|
|
238
|
+
* @default true
|
|
239
|
+
*/
|
|
240
|
+
registerTools?: boolean
|
|
241
|
+
|
|
242
|
+
/** Register collision-safe native `goal` and `goal-verify` agents through OpenCode's config hook. */
|
|
243
|
+
registerAgents?: boolean
|
|
244
|
+
|
|
245
|
+
/** Name of the native primary goal agent. @default "goal" */
|
|
246
|
+
goalAgentName?: string
|
|
247
|
+
|
|
248
|
+
/** Name of the native read-only verifier subagent. @default "goal-verify" */
|
|
249
|
+
verifierAgentName?: string
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Enables the built-in child-session completion auditor: before a
|
|
253
|
+
* `[goal:complete]` is archived, the plugin spawns an independent
|
|
254
|
+
* OpenCode session to verify the completion against the goal and
|
|
255
|
+
* workspace. Ignored if {@link auditor} is also set (the custom
|
|
256
|
+
* auditor takes precedence). Tune the built-in auditor with
|
|
257
|
+
* {@link auditorOptions}.
|
|
258
|
+
* @default false
|
|
259
|
+
*/
|
|
260
|
+
completionAudit?: boolean
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Supply a custom completion auditor instead of the built-in
|
|
264
|
+
* child-session one. Takes precedence over `completionAudit: true`.
|
|
265
|
+
* A verdict of `{ approved: false }` pauses the goal (stop reason
|
|
266
|
+
* `"audit rejected"`) instead of archiving it. A thrown error is
|
|
267
|
+
* treated as a rejection (fail closed).
|
|
268
|
+
*/
|
|
269
|
+
auditor?: (context: CompletionAuditContext) => Promise<CompletionAuditVerdict>
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Tuning options for the built-in child-session auditor. Ignored when
|
|
273
|
+
* a custom {@link auditor} is supplied.
|
|
274
|
+
*/
|
|
275
|
+
auditorOptions?: CompletionAuditorOptions
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Whether the plugin announces completion/blocked audits (an
|
|
279
|
+
* audit-start and an audit-result message) instead of running silently.
|
|
280
|
+
* @default true
|
|
281
|
+
*/
|
|
282
|
+
auditMessages?: boolean
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Custom sink for audit announcements. Defaults to routing through
|
|
286
|
+
* OpenCode's structured log (`client.app.log`). Provide this to route
|
|
287
|
+
* audit messages elsewhere, e.g. into the live conversation.
|
|
288
|
+
*/
|
|
289
|
+
auditMessenger?: (sessionID: string, text: string) => Promise<void>
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* OpenCode plugin hook map returned by the plugin's `server` factory.
|
|
294
|
+
* Matches OpenCode's plugin hook contract; kept loose (`unknown`
|
|
295
|
+
* input/output) since hook payload shapes are defined by OpenCode itself,
|
|
296
|
+
* not by this package.
|
|
297
|
+
*/
|
|
298
|
+
export interface GoalPluginHooks {
|
|
299
|
+
/** Registers collision-safe native goal and verifier agents. */
|
|
300
|
+
config: (config: unknown) => Promise<void>
|
|
301
|
+
/** Omitted entirely when {@link GoalPluginOptions.registerCommand} is `false`. */
|
|
302
|
+
"command.execute.before"?: (input: unknown, output: unknown) => Promise<void>
|
|
303
|
+
event: (input: unknown) => Promise<void>
|
|
304
|
+
"experimental.chat.system.transform": (input: unknown, output: unknown) => Promise<void>
|
|
305
|
+
"experimental.compaction.autocontinue": (input: unknown, output: unknown) => Promise<void>
|
|
306
|
+
"experimental.session.compacting": (input: unknown, output: unknown) => Promise<void>
|
|
307
|
+
/**
|
|
308
|
+
* Agent-facing tool definitions, present only when
|
|
309
|
+
* {@link GoalPluginOptions.registerTools} is enabled (default) and the
|
|
310
|
+
* optional `@opencode-ai/plugin` peer dependency is installed.
|
|
311
|
+
*/
|
|
312
|
+
tool?: Record<string, unknown>
|
|
313
|
+
/** Cancels pending continuation work and releases this plugin instance. */
|
|
314
|
+
dispose: () => Promise<void>
|
|
315
|
+
[hook: string]: unknown
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* The plugin's `server` factory. OpenCode calls this with a client bound
|
|
320
|
+
* to the running session and the resolved plugin options from
|
|
321
|
+
* `opencode.json`.
|
|
322
|
+
*/
|
|
323
|
+
export function GoalPlugin(
|
|
324
|
+
context: {
|
|
325
|
+
client: unknown
|
|
326
|
+
/** OpenCode's resolved project directory for this plugin instance. */
|
|
327
|
+
directory?: string
|
|
328
|
+
/** OpenCode's resolved worktree directory for this plugin instance. */
|
|
329
|
+
worktree?: string
|
|
330
|
+
},
|
|
331
|
+
options?: GoalPluginOptions,
|
|
332
|
+
): Promise<GoalPluginHooks>
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* Default export consumed by OpenCode's plugin loader:
|
|
336
|
+
* `{ "opencode-goal-plugin": { ... } }` in `opencode.json` resolves `id`
|
|
337
|
+
* and calls `server` to obtain the plugin's hooks.
|
|
338
|
+
*/
|
|
339
|
+
declare const goalPlugin: {
|
|
340
|
+
id: "opencode-goal-plugin"
|
|
341
|
+
server: typeof GoalPlugin
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
export default goalPlugin
|
package/package.json
CHANGED
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-goal-plugin",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.6.0",
|
|
4
|
+
"description": "Durable, guarded goal workflows for OpenCode.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/goal-plugin.js",
|
|
7
|
+
"types": "./index.d.ts",
|
|
7
8
|
"exports": {
|
|
8
9
|
".": "./src/goal-plugin.js",
|
|
9
10
|
"./server": "./src/goal-plugin.js"
|
|
10
11
|
},
|
|
12
|
+
"bin": {
|
|
13
|
+
"opencode-goal-plugin": "./scripts/verify.mjs"
|
|
14
|
+
},
|
|
11
15
|
"files": [
|
|
12
16
|
"src",
|
|
13
17
|
"scripts",
|
|
14
18
|
"examples",
|
|
19
|
+
"demo",
|
|
20
|
+
"docs",
|
|
21
|
+
"index.d.ts",
|
|
15
22
|
"README.md",
|
|
16
23
|
"CHANGELOG.md",
|
|
17
24
|
"CONTRIBUTING.md",
|
|
@@ -20,9 +27,12 @@
|
|
|
20
27
|
".nvmrc"
|
|
21
28
|
],
|
|
22
29
|
"scripts": {
|
|
23
|
-
"test": "node --test",
|
|
24
|
-
"test:coverage": "node --test --experimental-test-coverage",
|
|
30
|
+
"test": "node --test test/*.test.js",
|
|
31
|
+
"test:coverage": "node --test --experimental-test-coverage test/*.test.js",
|
|
25
32
|
"smoke": "node scripts/smoke-command-hook.mjs",
|
|
33
|
+
"smoke:packed-host": "node scripts/packed-host-contract.mjs",
|
|
34
|
+
"benchmark:behavior": "node scripts/behavior-benchmark.mjs",
|
|
35
|
+
"verify": "node scripts/verify.mjs",
|
|
26
36
|
"check": "node -c src/goal-plugin.js && npm test",
|
|
27
37
|
"pack:check": "npm pack --dry-run"
|
|
28
38
|
},
|
|
@@ -33,7 +43,7 @@
|
|
|
33
43
|
"goal"
|
|
34
44
|
],
|
|
35
45
|
"peerDependencies": {
|
|
36
|
-
"@opencode-ai/plugin": ">=1"
|
|
46
|
+
"@opencode-ai/plugin": ">=1.17.15 <2"
|
|
37
47
|
},
|
|
38
48
|
"peerDependenciesMeta": {
|
|
39
49
|
"@opencode-ai/plugin": {
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import assert from "node:assert/strict"
|
|
4
|
+
import { promises as fs } from "node:fs"
|
|
5
|
+
import { tmpdir } from "node:os"
|
|
6
|
+
import { join } from "node:path"
|
|
7
|
+
import { GoalPlugin } from "../src/goal-plugin.js"
|
|
8
|
+
|
|
9
|
+
const startedAt = performance.now()
|
|
10
|
+
const temporaryDirectories = []
|
|
11
|
+
|
|
12
|
+
function assistantMessage(sessionID, text, id = `assistant-${sessionID}`) {
|
|
13
|
+
return {
|
|
14
|
+
info: {
|
|
15
|
+
id,
|
|
16
|
+
role: "assistant",
|
|
17
|
+
sessionID,
|
|
18
|
+
tokens: { input: 20, output: 120, reasoning: 0 },
|
|
19
|
+
},
|
|
20
|
+
parts: [{ type: "text", text }],
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function createHost(messageForSession = () => "Working with tools.") {
|
|
25
|
+
const prompts = []
|
|
26
|
+
const notices = []
|
|
27
|
+
return {
|
|
28
|
+
prompts,
|
|
29
|
+
notices,
|
|
30
|
+
client: {
|
|
31
|
+
app: { log: async () => {} },
|
|
32
|
+
session: {
|
|
33
|
+
messages: async ({ path }) => ({
|
|
34
|
+
data: [assistantMessage(path.id, messageForSession(path.id))],
|
|
35
|
+
}),
|
|
36
|
+
promptAsync: async (input) => {
|
|
37
|
+
prompts.push(input)
|
|
38
|
+
return {}
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function promptCharacters(prompts) {
|
|
46
|
+
return prompts.reduce(
|
|
47
|
+
(total, prompt) => total + (prompt?.body?.parts || []).reduce(
|
|
48
|
+
(partTotal, part) => partTotal + (typeof part?.text === "string" ? part.text.length : 0),
|
|
49
|
+
0,
|
|
50
|
+
),
|
|
51
|
+
0,
|
|
52
|
+
)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function makeDirectory(label) {
|
|
56
|
+
const directory = await fs.mkdtemp(join(tmpdir(), `goal-benchmark-${label}-`))
|
|
57
|
+
temporaryDirectories.push(directory)
|
|
58
|
+
return directory
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function createHooks(host, options = {}) {
|
|
62
|
+
return GoalPlugin(
|
|
63
|
+
{ client: host.client, directory: await makeDirectory("workspace") },
|
|
64
|
+
{
|
|
65
|
+
persistState: false,
|
|
66
|
+
registerTools: false,
|
|
67
|
+
registerAgents: false,
|
|
68
|
+
minDelayMs: 1,
|
|
69
|
+
noProgressTokenThreshold: 1,
|
|
70
|
+
noProgressTurnsBeforePause: 10,
|
|
71
|
+
noToolCallTurnsBeforePause: 2,
|
|
72
|
+
...options,
|
|
73
|
+
},
|
|
74
|
+
)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function goalCommand(hooks, sessionID, argumentsText) {
|
|
78
|
+
const output = { parts: [] }
|
|
79
|
+
await hooks["command.execute.before"](
|
|
80
|
+
{ command: "goal", sessionID, arguments: argumentsText },
|
|
81
|
+
output,
|
|
82
|
+
)
|
|
83
|
+
return output.parts.map((part) => part.text || "").join("\n")
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function idle(hooks, sessionID, id) {
|
|
87
|
+
await hooks.event({
|
|
88
|
+
event: {
|
|
89
|
+
id,
|
|
90
|
+
type: "session.status",
|
|
91
|
+
properties: { sessionID, status: { type: "idle" } },
|
|
92
|
+
},
|
|
93
|
+
})
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function scenario(name, points, run) {
|
|
97
|
+
const scenarioStartedAt = performance.now()
|
|
98
|
+
try {
|
|
99
|
+
const telemetry = await run()
|
|
100
|
+
return {
|
|
101
|
+
name,
|
|
102
|
+
passed: true,
|
|
103
|
+
points,
|
|
104
|
+
durationMs: Number((performance.now() - scenarioStartedAt).toFixed(2)),
|
|
105
|
+
...telemetry,
|
|
106
|
+
}
|
|
107
|
+
} catch (error) {
|
|
108
|
+
return {
|
|
109
|
+
name,
|
|
110
|
+
passed: false,
|
|
111
|
+
points: 0,
|
|
112
|
+
possiblePoints: points,
|
|
113
|
+
durationMs: Number((performance.now() - scenarioStartedAt).toFixed(2)),
|
|
114
|
+
error: error instanceof Error ? error.message : String(error),
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const results = []
|
|
120
|
+
|
|
121
|
+
results.push(await scenario("verified-success", 20, async () => {
|
|
122
|
+
const sessionID = "benchmark-success"
|
|
123
|
+
const host = createHost(() => "Tests pass.\n[goal:evidence] npm test: 210/210\n[goal:complete]")
|
|
124
|
+
const hooks = await createHooks(host, {
|
|
125
|
+
auditor: async () => ({ approved: true, reason: "evidence independently accepted" }),
|
|
126
|
+
})
|
|
127
|
+
await goalCommand(hooks, sessionID, "ship a verified release")
|
|
128
|
+
await idle(hooks, sessionID, "success-idle")
|
|
129
|
+
const status = await goalCommand(hooks, sessionID, "status")
|
|
130
|
+
assert.match(status, /State: achieved/)
|
|
131
|
+
await hooks.dispose()
|
|
132
|
+
return {
|
|
133
|
+
continuationPrompts: host.prompts.length,
|
|
134
|
+
continuationCharacters: promptCharacters(host.prompts),
|
|
135
|
+
status: "archived",
|
|
136
|
+
}
|
|
137
|
+
}))
|
|
138
|
+
|
|
139
|
+
results.push(await scenario("false-completion", 20, async () => {
|
|
140
|
+
const sessionID = "benchmark-false-completion"
|
|
141
|
+
const host = createHost(() => "Looks done.\n[goal:evidence] guessed from source\n[goal:complete]")
|
|
142
|
+
const hooks = await createHooks(host, {
|
|
143
|
+
auditor: async () => ({ approved: false, reason: "no executed verification" }),
|
|
144
|
+
})
|
|
145
|
+
await goalCommand(hooks, sessionID, "do not accept an unverified claim")
|
|
146
|
+
await idle(hooks, sessionID, "false-idle")
|
|
147
|
+
const status = await goalCommand(hooks, sessionID, "status")
|
|
148
|
+
assert.match(status, /audit rejected/i)
|
|
149
|
+
assert.doesNotMatch(status, /No active goal/)
|
|
150
|
+
await hooks.dispose()
|
|
151
|
+
return {
|
|
152
|
+
continuationPrompts: host.prompts.length,
|
|
153
|
+
continuationCharacters: promptCharacters(host.prompts),
|
|
154
|
+
status: "rejected",
|
|
155
|
+
}
|
|
156
|
+
}))
|
|
157
|
+
|
|
158
|
+
results.push(await scenario("loop-circuit-breaker", 15, async () => {
|
|
159
|
+
const sessionID = "benchmark-loop"
|
|
160
|
+
let turn = 0
|
|
161
|
+
const host = createHost(() => `Still discussing the work, turn ${turn++}.`)
|
|
162
|
+
const hooks = await createHooks(host)
|
|
163
|
+
await goalCommand(hooks, sessionID, "stop self-chat loops")
|
|
164
|
+
await idle(hooks, sessionID, "loop-1")
|
|
165
|
+
await idle(hooks, sessionID, "loop-2")
|
|
166
|
+
await idle(hooks, sessionID, "loop-3")
|
|
167
|
+
const status = await goalCommand(hooks, sessionID, "status")
|
|
168
|
+
assert.match(status, /no tool calls|self-chat loop/i)
|
|
169
|
+
assert.equal(host.prompts.length, 2)
|
|
170
|
+
await hooks.dispose()
|
|
171
|
+
return {
|
|
172
|
+
continuationPrompts: host.prompts.length,
|
|
173
|
+
continuationCharacters: promptCharacters(host.prompts),
|
|
174
|
+
status: "paused",
|
|
175
|
+
}
|
|
176
|
+
}))
|
|
177
|
+
|
|
178
|
+
results.push(await scenario("human-interruption", 15, async () => {
|
|
179
|
+
const sessionID = "benchmark-interruption"
|
|
180
|
+
const host = createHost()
|
|
181
|
+
const hooks = await createHooks(host)
|
|
182
|
+
await goalCommand(hooks, sessionID, "respect explicit interruption")
|
|
183
|
+
await hooks.event({
|
|
184
|
+
event: {
|
|
185
|
+
type: "session.error",
|
|
186
|
+
properties: {
|
|
187
|
+
sessionID,
|
|
188
|
+
error: { name: "MessageAbortedError", message: "aborted by user" },
|
|
189
|
+
},
|
|
190
|
+
},
|
|
191
|
+
})
|
|
192
|
+
await idle(hooks, sessionID, "interruption-idle")
|
|
193
|
+
assert.equal(host.prompts.length, 0)
|
|
194
|
+
assert.match(await goalCommand(hooks, sessionID, "status"), /abort|paused|stopped/i)
|
|
195
|
+
await hooks.dispose()
|
|
196
|
+
return { continuationPrompts: 0, status: "paused" }
|
|
197
|
+
}))
|
|
198
|
+
|
|
199
|
+
results.push(await scenario("compaction-continuity", 15, async () => {
|
|
200
|
+
const sessionID = "benchmark-compaction"
|
|
201
|
+
const host = createHost()
|
|
202
|
+
const hooks = await createHooks(host)
|
|
203
|
+
await goalCommand(hooks, sessionID, "preserve the objective across compaction")
|
|
204
|
+
const output = { context: [] }
|
|
205
|
+
await hooks["experimental.session.compacting"]({ sessionID }, output)
|
|
206
|
+
assert.equal(output.context.length, 1)
|
|
207
|
+
assert.match(output.context[0], /preserve the objective across compaction/)
|
|
208
|
+
assert.ok(output.context[0].length < 2_000, "compaction context exceeded token-efficient size cap")
|
|
209
|
+
await hooks.dispose()
|
|
210
|
+
return {
|
|
211
|
+
contextCharacters: output.context[0].length,
|
|
212
|
+
estimatedContextTokens: Math.ceil(output.context[0].length / 4),
|
|
213
|
+
status: "preserved",
|
|
214
|
+
}
|
|
215
|
+
}))
|
|
216
|
+
|
|
217
|
+
results.push(await scenario("restart-recovery", 15, async () => {
|
|
218
|
+
const sessionID = "benchmark-restart"
|
|
219
|
+
const directory = await makeDirectory("restart")
|
|
220
|
+
const stateFilePath = join(directory, "state.json")
|
|
221
|
+
const host = createHost()
|
|
222
|
+
const first = await GoalPlugin(
|
|
223
|
+
{ client: host.client, directory },
|
|
224
|
+
{ persistState: true, stateFilePath, registerTools: false, registerAgents: false, minDelayMs: 1 },
|
|
225
|
+
)
|
|
226
|
+
await goalCommand(first, sessionID, "recover safely after restart")
|
|
227
|
+
await first.dispose()
|
|
228
|
+
const second = await GoalPlugin(
|
|
229
|
+
{ client: host.client, directory },
|
|
230
|
+
{ persistState: true, stateFilePath, registerTools: false, registerAgents: false, minDelayMs: 1 },
|
|
231
|
+
)
|
|
232
|
+
const status = await goalCommand(second, sessionID, "status")
|
|
233
|
+
assert.match(status, /Recovered persisted goal state|recovered after restart/i)
|
|
234
|
+
await idle(second, sessionID, "restart-idle")
|
|
235
|
+
assert.equal(host.prompts.length, 0, "recovered goals must not resume without user consent")
|
|
236
|
+
const stateBytes = (await fs.stat(stateFilePath)).size
|
|
237
|
+
await second.dispose()
|
|
238
|
+
return { continuationPrompts: 0, persistedStateBytes: stateBytes, status: "recovered-paused" }
|
|
239
|
+
}))
|
|
240
|
+
|
|
241
|
+
const score = results.reduce((total, result) => total + result.points, 0)
|
|
242
|
+
const possibleScore = 100
|
|
243
|
+
const continuationCharacters = results.reduce(
|
|
244
|
+
(total, result) => total + (result.continuationCharacters || 0),
|
|
245
|
+
0,
|
|
246
|
+
)
|
|
247
|
+
const report = {
|
|
248
|
+
schemaVersion: 1,
|
|
249
|
+
benchmark: "opencode-goal-plugin-behavior",
|
|
250
|
+
score,
|
|
251
|
+
possibleScore,
|
|
252
|
+
passed: score === possibleScore,
|
|
253
|
+
durationMs: Number((performance.now() - startedAt).toFixed(2)),
|
|
254
|
+
efficiency: {
|
|
255
|
+
totalContinuationPrompts: results.reduce(
|
|
256
|
+
(total, result) => total + (result.continuationPrompts || 0),
|
|
257
|
+
0,
|
|
258
|
+
),
|
|
259
|
+
continuationCharacters,
|
|
260
|
+
estimatedContinuationTokens: Math.ceil(continuationCharacters / 4),
|
|
261
|
+
modelCalls: 0,
|
|
262
|
+
externalRequests: 0,
|
|
263
|
+
},
|
|
264
|
+
scenarios: results,
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
for (const directory of temporaryDirectories) {
|
|
268
|
+
await fs.rm(directory, { recursive: true, force: true })
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
|
|
272
|
+
if (!report.passed) process.exitCode = 1
|