@alecsibilia/luca 14.0.0-alpha.21 → 14.0.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/board/package.json +1 -1
- package/board/server/luca-version.ts +1 -1
- package/engine/agents/agent-launcher.ts +7 -2
- package/engine/agents/claude-launcher-helpers.ts +6 -1
- package/engine/agents/claude-launcher.ts +3 -12
- package/engine/agents/claude-options.ts +3 -6
- package/engine/agents/launch-session.ts +5 -9
- package/engine/agents/role-effort.ts +55 -0
- package/engine/agents/scripted-launcher.ts +17 -3
- package/engine/config/config-fields.ts +3 -2
- package/engine/config/engine-config-schema.ts +42 -4
- package/engine/config/engine-config.ts +2 -0
- package/engine/core/agent-turn.ts +34 -11
- package/engine/core/execute-final-review.ts +11 -2
- package/engine/core/execute-memory.ts +7 -0
- package/engine/guards/helper-rules.ts +6 -0
- package/engine/index.ts +1 -1
- package/engine/jev/jev-jobs.ts +57 -10
- package/engine/jev/jev-schemas.ts +3 -1
- package/engine/journal/agent-entries.ts +5 -1
- package/engine/models/model-score-facts.ts +273 -0
- package/engine/models/model-score-rules.ts +35 -0
- package/engine/models/model-score.ts +169 -246
- package/engine/testing/effort-pick-fixtures.ts +139 -0
- package/engine/testing/model-score-fixtures.ts +300 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
package/board/package.json
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** The Luca version of this board, stamped when the package was packed. */
|
|
2
|
-
export const LUCA_VERSION: string = "14.0.0-alpha.
|
|
2
|
+
export const LUCA_VERSION: string = "14.0.0-alpha.22"
|
|
@@ -3,7 +3,7 @@ import { z } from 'zod'
|
|
|
3
3
|
import type { AgentRole } from './role-results'
|
|
4
4
|
import type { ProgressListener } from './turn-progress'
|
|
5
5
|
|
|
6
|
-
import type { EngineConfig } from '../config/engine-config'
|
|
6
|
+
import type { EffortLevel, EngineConfig } from '../config/engine-config'
|
|
7
7
|
|
|
8
8
|
/** One model's tokens in one agent turn. */
|
|
9
9
|
export const ModelTokensSchema = z.object({
|
|
@@ -173,6 +173,11 @@ export type AgentLauncher = {
|
|
|
173
173
|
* follow-ups run on it too.
|
|
174
174
|
*/
|
|
175
175
|
model: string
|
|
176
|
+
/**
|
|
177
|
+
* The effort the agent runs at (#659), whatever its model. The
|
|
178
|
+
* session's follow-ups run at it too.
|
|
179
|
+
*/
|
|
180
|
+
effort: EffortLevel
|
|
176
181
|
/**
|
|
177
182
|
* Hears the turn's live progress (#628) each time it changes, before
|
|
178
183
|
* the turn ends, for the engine's live file. Left out, nothing hears it.
|
|
@@ -183,7 +188,7 @@ export type AgentLauncher = {
|
|
|
183
188
|
* Sends a follow-up message to an agent session that is still open, such
|
|
184
189
|
* as a failed gate's output in a fix loop, and returns the session's new
|
|
185
190
|
* result. The agent keeps everything it knew from its earlier turns, the
|
|
186
|
-
* guards it was launched with, and its
|
|
191
|
+
* guards it was launched with, its model and its effort. A session the launcher doesn't know,
|
|
187
192
|
* or one that has closed, is an engine failure with `session_gone`.
|
|
188
193
|
*
|
|
189
194
|
* @example
|
|
@@ -10,9 +10,10 @@ import { afterEach, beforeEach } from 'bun:test'
|
|
|
10
10
|
import type { AgentMessaging, AgentTurn } from './agent-launcher'
|
|
11
11
|
import { createClaudeLauncher, type AgentQuery } from './claude-launcher'
|
|
12
12
|
import { CLAUDE_MODEL } from './claude-options'
|
|
13
|
+
import { REVIEW_EFFORT } from './role-effort'
|
|
13
14
|
import type { AgentRole } from './role-results'
|
|
14
15
|
|
|
15
|
-
import type { EngineConfig } from '../config/engine-config'
|
|
16
|
+
import type { EffortLevel, EngineConfig } from '../config/engine-config'
|
|
16
17
|
|
|
17
18
|
/**
|
|
18
19
|
* Helpers shared by the launcher tests (`claude-launcher.test.ts` and the
|
|
@@ -199,12 +200,14 @@ export const repoPerTest = () => {
|
|
|
199
200
|
export const launch = ({
|
|
200
201
|
query,
|
|
201
202
|
model,
|
|
203
|
+
effort,
|
|
202
204
|
role,
|
|
203
205
|
turn_timeout_ms,
|
|
204
206
|
messaging,
|
|
205
207
|
}: {
|
|
206
208
|
query: AgentQuery
|
|
207
209
|
model?: string
|
|
210
|
+
effort?: EffortLevel
|
|
208
211
|
role?: AgentRole
|
|
209
212
|
turn_timeout_ms?: number
|
|
210
213
|
messaging?: AgentMessaging
|
|
@@ -222,6 +225,8 @@ export const launch = ({
|
|
|
222
225
|
config: CONFIG,
|
|
223
226
|
messaging: messaging ?? null,
|
|
224
227
|
model: model ?? CLAUDE_MODEL,
|
|
228
|
+
// One effort for every role unless given, as the default reviewer's.
|
|
229
|
+
effort: effort ?? REVIEW_EFFORT,
|
|
225
230
|
})
|
|
226
231
|
|
|
227
232
|
export const IMPLEMENTER_DONE = {
|
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
query as sdkQuery,
|
|
3
|
-
type EffortLevel,
|
|
4
|
-
} from '@anthropic-ai/claude-agent-sdk'
|
|
1
|
+
import { query as sdkQuery } from '@anthropic-ai/claude-agent-sdk'
|
|
5
2
|
|
|
6
3
|
import type { AgentLauncher } from './agent-launcher'
|
|
7
|
-
import { AGENT_EFFORT } from './claude-options'
|
|
8
4
|
import { createLaunch, type AgentQuery } from './launch-session'
|
|
9
5
|
import { createSessionPool } from './session-pool'
|
|
10
6
|
|
|
@@ -25,7 +21,7 @@ export const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60_000
|
|
|
25
21
|
* every guard on. Per launch it:
|
|
26
22
|
*
|
|
27
23
|
* 1. refuses any model but `opus`, `sonnet` or `haiku` before starting
|
|
28
|
-
* anything; follow-ups stay on their session's model;
|
|
24
|
+
* anything; follow-ups stay on their session's model, and at its effort;
|
|
29
25
|
* 2. builds the role's locked-down options (`agentOptions`) with the guard
|
|
30
26
|
* hook (honoring `may_edit_tests`), the sandbox, and a clean environment;
|
|
31
27
|
* with `messaging` (test-writers and implementers), the `luca` server's
|
|
@@ -55,20 +51,17 @@ export const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60_000
|
|
|
55
51
|
*
|
|
56
52
|
* @example
|
|
57
53
|
* const launcher = createClaudeLauncher({})
|
|
58
|
-
* const turn = await launcher.launch({ role: 'implementer', ticket: 11, prompt, cwd, may_edit_tests: false, config, messaging, model: 'opus' })
|
|
54
|
+
* const turn = await launcher.launch({ role: 'implementer', ticket: 11, prompt, cwd, may_edit_tests: false, config, messaging, model: 'opus', effort: 'medium' })
|
|
59
55
|
* // ... later, in a fix loop:
|
|
60
56
|
* if (turn.ok) await launcher.followUp({ session_id: turn.session_id, role: 'implementer', ticket: 11, message, cwd, config })
|
|
61
57
|
* await launcher.closeAll()
|
|
62
58
|
*/
|
|
63
59
|
export const createClaudeLauncher = ({
|
|
64
|
-
effort,
|
|
65
60
|
query,
|
|
66
61
|
claude_path,
|
|
67
62
|
turn_timeout_ms,
|
|
68
63
|
idle_timeout_ms,
|
|
69
64
|
}: {
|
|
70
|
-
/** Defaults to `AGENT_EFFORT`, the same for every role. */
|
|
71
|
-
effort?: EffortLevel
|
|
72
65
|
query?: AgentQuery
|
|
73
66
|
claude_path?: string
|
|
74
67
|
/** Defaults to `DEFAULT_TURN_TIMEOUT_MS`. */
|
|
@@ -76,7 +69,6 @@ export const createClaudeLauncher = ({
|
|
|
76
69
|
/** Defaults to `DEFAULT_IDLE_TIMEOUT_MS`. */
|
|
77
70
|
idle_timeout_ms?: number
|
|
78
71
|
}): AgentLauncher & { closeAll: () => Promise<void> } => {
|
|
79
|
-
const useEffort = effort ?? AGENT_EFFORT
|
|
80
72
|
const startQuery: AgentQuery = query ?? sdkQuery
|
|
81
73
|
const timeoutMs = turn_timeout_ms ?? DEFAULT_TURN_TIMEOUT_MS
|
|
82
74
|
const idleMs = idle_timeout_ms ?? DEFAULT_IDLE_TIMEOUT_MS
|
|
@@ -86,7 +78,6 @@ export const createClaudeLauncher = ({
|
|
|
86
78
|
})
|
|
87
79
|
const { sessions, close, startTurn, runTurn, closeAll } = pool
|
|
88
80
|
const launch = createLaunch({
|
|
89
|
-
effort: useEffort,
|
|
90
81
|
start_query: startQuery,
|
|
91
82
|
claude_path,
|
|
92
83
|
turn_timeout_ms: timeoutMs,
|
|
@@ -33,9 +33,6 @@ export const MODEL_NAMES = ModelNameSchema.options
|
|
|
33
33
|
/** The model reviewers run on, and every agent the decision model doesn't route (#603). */
|
|
34
34
|
export const CLAUDE_MODEL = DEFAULT_MODEL
|
|
35
35
|
|
|
36
|
-
/** Every role runs at the same effort level. */
|
|
37
|
-
export const AGENT_EFFORT: EffortLevel = 'high'
|
|
38
|
-
|
|
39
36
|
/** How many model turns one agent gets before its try ends. */
|
|
40
37
|
export const MAX_TURNS: Record<AgentRole, number> = {
|
|
41
38
|
'test-writer': 80,
|
|
@@ -161,7 +158,7 @@ export const resultJsonSchema = ({
|
|
|
161
158
|
}
|
|
162
159
|
|
|
163
160
|
/**
|
|
164
|
-
* The SDK options for one agent: its model at
|
|
161
|
+
* The SDK options for one agent: its model at its launch's effort, the
|
|
165
162
|
* role's instructions, its result schema, `dontAsk` with only its own
|
|
166
163
|
* pre-approved calls, the guard hook, the sandbox, a clean environment, and
|
|
167
164
|
* nothing loaded from the machine: no settings, MCP servers, or skills.
|
|
@@ -172,7 +169,7 @@ export const resultJsonSchema = ({
|
|
|
172
169
|
* hooks.
|
|
173
170
|
*
|
|
174
171
|
* @example
|
|
175
|
-
* const options = agentOptions({ role, may_edit_tests, cwd, model: CLAUDE_MODEL, effort:
|
|
172
|
+
* const options = agentOptions({ role, may_edit_tests, cwd, model: CLAUDE_MODEL, effort: 'xhigh', claude_path, config, common_git_dir, home, env, guard_hook, luca_server: null, delivery_hook: null })
|
|
176
173
|
*/
|
|
177
174
|
export const agentOptions = ({
|
|
178
175
|
role,
|
|
@@ -238,7 +235,7 @@ export const agentOptions = ({
|
|
|
238
235
|
tools: rules.tools,
|
|
239
236
|
allowedTools: rules.allowed,
|
|
240
237
|
disallowedTools: rules.disallowed,
|
|
241
|
-
// The one helper every role may start (#627), always on Haiku.
|
|
238
|
+
// The one helper every role may start (#627), always on Haiku at low effort.
|
|
242
239
|
agents: { [HELPER_TYPE]: SEARCHER },
|
|
243
240
|
settingSources: [],
|
|
244
241
|
strictMcpConfig: true,
|
|
@@ -8,11 +8,7 @@ import { randomUUID } from 'node:crypto'
|
|
|
8
8
|
import { realpathSync } from 'node:fs'
|
|
9
9
|
import { homedir } from 'node:os'
|
|
10
10
|
|
|
11
|
-
import type {
|
|
12
|
-
EffortLevel,
|
|
13
|
-
Options,
|
|
14
|
-
SDKUserMessage,
|
|
15
|
-
} from '@anthropic-ai/claude-agent-sdk'
|
|
11
|
+
import type { Options, SDKUserMessage } from '@anthropic-ai/claude-agent-sdk'
|
|
16
12
|
|
|
17
13
|
import {
|
|
18
14
|
AgentSessionSchema,
|
|
@@ -54,16 +50,14 @@ export type AgentQuery = (params: {
|
|
|
54
50
|
* and runs the session's first turn through the pool.
|
|
55
51
|
*
|
|
56
52
|
* @example
|
|
57
|
-
* const launch = createLaunch({
|
|
53
|
+
* const launch = createLaunch({ start_query, claude_path, turn_timeout_ms, pool })
|
|
58
54
|
*/
|
|
59
55
|
export const createLaunch = ({
|
|
60
|
-
effort: useEffort,
|
|
61
56
|
start_query: startQuery,
|
|
62
57
|
claude_path,
|
|
63
58
|
turn_timeout_ms: timeoutMs,
|
|
64
59
|
pool,
|
|
65
60
|
}: {
|
|
66
|
-
effort: EffortLevel
|
|
67
61
|
start_query: AgentQuery
|
|
68
62
|
claude_path: string | undefined
|
|
69
63
|
turn_timeout_ms: number
|
|
@@ -80,6 +74,7 @@ export const createLaunch = ({
|
|
|
80
74
|
config,
|
|
81
75
|
messaging,
|
|
82
76
|
model,
|
|
77
|
+
effort,
|
|
83
78
|
on_progress,
|
|
84
79
|
}) => {
|
|
85
80
|
const refused = checkLaunchModel({ model })
|
|
@@ -115,7 +110,8 @@ export const createLaunch = ({
|
|
|
115
110
|
may_edit_tests,
|
|
116
111
|
cwd: worktree,
|
|
117
112
|
model,
|
|
118
|
-
|
|
113
|
+
// Kept in the session's options, so its follow-ups run at it too.
|
|
114
|
+
effort,
|
|
119
115
|
claude_path: claudePath,
|
|
120
116
|
config,
|
|
121
117
|
common_git_dir: gitDir,
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure: the effort each role runs at (#659), whatever model it runs on.
|
|
3
|
+
* Writers on tickets and the learner run at the config's `models.effort`,
|
|
4
|
+
* medium by default (#660); the reviewers, and the last writers before the
|
|
5
|
+
* pull request, at xhigh. Helpers run at their own definition's effort
|
|
6
|
+
* (`HELPER_EFFORT`).
|
|
7
|
+
*/
|
|
8
|
+
import type { AgentRole } from './role-results'
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
modelsOf,
|
|
12
|
+
ModelsConfigSchema,
|
|
13
|
+
type EffortLevel,
|
|
14
|
+
type EngineConfig,
|
|
15
|
+
} from '../config/engine-config'
|
|
16
|
+
|
|
17
|
+
/** The effort the reviewers, and the final review's fixers, always run at. */
|
|
18
|
+
export const REVIEW_EFFORT: EffortLevel = 'xhigh'
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Each role's effort on ticket work (and the learner's), with no
|
|
22
|
+
* `models.effort` set: the writers' and the learner's are the schema's
|
|
23
|
+
* defaults.
|
|
24
|
+
*/
|
|
25
|
+
export const ROLE_EFFORT: Record<AgentRole, EffortLevel> = {
|
|
26
|
+
...ModelsConfigSchema.parse({}).effort,
|
|
27
|
+
'ticket-reviewer': REVIEW_EFFORT,
|
|
28
|
+
'architecture-lens': REVIEW_EFFORT,
|
|
29
|
+
'simplification-lens': REVIEW_EFFORT,
|
|
30
|
+
'security-lens': REVIEW_EFFORT,
|
|
31
|
+
'integration-lens': REVIEW_EFFORT,
|
|
32
|
+
'rules-lens': REVIEW_EFFORT,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The effort a fresh agent launches at: its role's, as the run's config
|
|
37
|
+
* sets it, or `REVIEW_EFFORT` for a final review's fixer. A follow-up runs
|
|
38
|
+
* at its session's instead.
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* roleEffort({ role: 'implementer', config }) // 'medium'
|
|
42
|
+
* roleEffort({ role: 'implementer', config, final_fix: true }) // 'xhigh'
|
|
43
|
+
*/
|
|
44
|
+
export const roleEffort = ({
|
|
45
|
+
role,
|
|
46
|
+
config,
|
|
47
|
+
final_fix = false,
|
|
48
|
+
}: {
|
|
49
|
+
role: AgentRole
|
|
50
|
+
config: EngineConfig
|
|
51
|
+
final_fix?: boolean
|
|
52
|
+
}): EffortLevel =>
|
|
53
|
+
final_fix
|
|
54
|
+
? REVIEW_EFFORT
|
|
55
|
+
: { ...ROLE_EFFORT, ...modelsOf({ config }).effort }[role]
|
|
@@ -10,6 +10,8 @@ import type {
|
|
|
10
10
|
} from './agent-launcher'
|
|
11
11
|
import type { AgentRole } from './role-results'
|
|
12
12
|
|
|
13
|
+
import type { EffortLevel } from '../config/engine-config'
|
|
14
|
+
|
|
13
15
|
/**
|
|
14
16
|
* The engine's tools, as a scripted agent calls them. They reach the
|
|
15
17
|
* messaging the session was launched with; a reviewer has none, so its
|
|
@@ -71,6 +73,11 @@ export type ScriptedCall = {
|
|
|
71
73
|
* for a session the launcher doesn't know.
|
|
72
74
|
*/
|
|
73
75
|
model: string | null
|
|
76
|
+
/**
|
|
77
|
+
* The effort a launch asked for (#659); a follow-up's is its session's,
|
|
78
|
+
* `null` for a session the launcher doesn't know.
|
|
79
|
+
*/
|
|
80
|
+
effort: EffortLevel | null
|
|
74
81
|
/** What each of the turn's tool calls handed over, in order. */
|
|
75
82
|
delivered: string[]
|
|
76
83
|
}
|
|
@@ -107,10 +114,14 @@ export const createScriptedLauncher = ({
|
|
|
107
114
|
}): ScriptedLauncher => {
|
|
108
115
|
const remaining = [...turns]
|
|
109
116
|
const calls: ScriptedCall[] = []
|
|
110
|
-
/** Each open session's messaging and
|
|
117
|
+
/** Each open session's messaging, model and effort, kept for its follow-ups. */
|
|
111
118
|
const sessions = new Map<
|
|
112
119
|
string,
|
|
113
|
-
{
|
|
120
|
+
{
|
|
121
|
+
messaging: AgentMessaging | null
|
|
122
|
+
model: string
|
|
123
|
+
effort: EffortLevel
|
|
124
|
+
}
|
|
114
125
|
>()
|
|
115
126
|
|
|
116
127
|
const toolsFor = ({
|
|
@@ -192,9 +203,10 @@ export const createScriptedLauncher = ({
|
|
|
192
203
|
may_edit_tests,
|
|
193
204
|
messaging,
|
|
194
205
|
model,
|
|
206
|
+
effort,
|
|
195
207
|
}) => {
|
|
196
208
|
const session_id = `scripted-${role}-${ticket}-${calls.length + 1}`
|
|
197
|
-
sessions.set(session_id, { messaging, model })
|
|
209
|
+
sessions.set(session_id, { messaging, model, effort })
|
|
198
210
|
const call: ScriptedCall = {
|
|
199
211
|
kind: 'launch',
|
|
200
212
|
role,
|
|
@@ -203,6 +215,7 @@ export const createScriptedLauncher = ({
|
|
|
203
215
|
session_id,
|
|
204
216
|
may_edit_tests,
|
|
205
217
|
model,
|
|
218
|
+
effort,
|
|
206
219
|
delivered: [],
|
|
207
220
|
}
|
|
208
221
|
calls.push(call)
|
|
@@ -218,6 +231,7 @@ export const createScriptedLauncher = ({
|
|
|
218
231
|
prompt: message,
|
|
219
232
|
session_id,
|
|
220
233
|
model: open?.model ?? null,
|
|
234
|
+
effort: open?.effort ?? null,
|
|
221
235
|
delivered: [],
|
|
222
236
|
}
|
|
223
237
|
calls.push(call)
|
|
@@ -84,8 +84,9 @@ export const decisionModelOf = ({
|
|
|
84
84
|
config.decision_model?.model ?? DecisionModelConfigSchema.parse({}).model
|
|
85
85
|
|
|
86
86
|
/**
|
|
87
|
-
* The run's model floors and confidence bar: the config's
|
|
88
|
-
* the schema's defaults (every floor `opus
|
|
87
|
+
* The run's model floors, writers' effort and confidence bar: the config's
|
|
88
|
+
* `models`, else the schema's defaults (every floor `opus`, every effort
|
|
89
|
+
* `medium`).
|
|
89
90
|
*
|
|
90
91
|
* @example
|
|
91
92
|
* modelsOf({ config }).floors['test-writer'] // 'opus'
|
|
@@ -71,6 +71,20 @@ export type ModelName = z.infer<typeof ModelNameSchema>
|
|
|
71
71
|
/** The model reviewers run on, and every fallback of the model pick (#603). */
|
|
72
72
|
export const DEFAULT_MODEL: ModelName = ModelNameSchema.enum.opus
|
|
73
73
|
|
|
74
|
+
/**
|
|
75
|
+
* How hard an agent thinks before it acts (#659), lowest first: the levels
|
|
76
|
+
* the SDK's `effort` option takes.
|
|
77
|
+
*/
|
|
78
|
+
export const EffortLevelSchema = z.enum([
|
|
79
|
+
'low',
|
|
80
|
+
'medium',
|
|
81
|
+
'high',
|
|
82
|
+
'xhigh',
|
|
83
|
+
'max',
|
|
84
|
+
])
|
|
85
|
+
|
|
86
|
+
export type EffortLevel = z.infer<typeof EffortLevelSchema>
|
|
87
|
+
|
|
74
88
|
/** The roles with a model floor (#603); every other role always runs Opus. */
|
|
75
89
|
export const ROUTED_ROLES = ['test-writer', 'implementer', 'learner'] as const
|
|
76
90
|
|
|
@@ -93,12 +107,33 @@ const ModelFloorsSchema = z.strictObject(
|
|
|
93
107
|
)
|
|
94
108
|
|
|
95
109
|
/**
|
|
96
|
-
* The
|
|
97
|
-
*
|
|
98
|
-
*
|
|
110
|
+
* The effort each writer runs at on tickets, and the learner's (#660); a
|
|
111
|
+
* role left out is `medium`. An effort for any other role, a reviewer or a
|
|
112
|
+
* lens, is refused: they, and the final review's fixers, always run at
|
|
113
|
+
* xhigh.
|
|
114
|
+
*/
|
|
115
|
+
const RoleEffortsSchema = z.strictObject(
|
|
116
|
+
{
|
|
117
|
+
'test-writer': EffortLevelSchema.default('medium'),
|
|
118
|
+
implementer: EffortLevelSchema.default('medium'),
|
|
119
|
+
learner: EffortLevelSchema.default('medium'),
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
error: (issue) =>
|
|
123
|
+
issue.code === 'unrecognized_keys'
|
|
124
|
+
? `models.effort can't set ${issue.keys.join(', ')}: reviewers and the final review's fixers always run at xhigh.`
|
|
125
|
+
: undefined,
|
|
126
|
+
}
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The config's `models` block (#603): each routed role's model floor, each
|
|
131
|
+
* writer's effort (#660), and how sure the decision model must be
|
|
132
|
+
* (`confidence_bar`) before its model pick is used.
|
|
99
133
|
*/
|
|
100
134
|
export const ModelsConfigSchema = z.object({
|
|
101
135
|
floors: ModelFloorsSchema.prefault({}),
|
|
136
|
+
effort: RoleEffortsSchema.prefault({}),
|
|
102
137
|
confidence_bar: z.number().min(0).max(1).default(0.8),
|
|
103
138
|
})
|
|
104
139
|
|
|
@@ -223,7 +258,10 @@ export const EngineConfigSchema = z.object({
|
|
|
223
258
|
muninn: MuninnConfigSchema.optional(),
|
|
224
259
|
/** The decision model shadow mode asks; left out, Clef. */
|
|
225
260
|
decision_model: DecisionModelConfigSchema.optional(),
|
|
226
|
-
/**
|
|
261
|
+
/**
|
|
262
|
+
* Each routed role's model floor, each writer's effort, and the
|
|
263
|
+
* confidence bar; left out, every role runs Opus, and writers at medium.
|
|
264
|
+
*/
|
|
227
265
|
models: ModelsConfigSchema.optional(),
|
|
228
266
|
/**
|
|
229
267
|
* The repo's run budget in tokens; left out, the engine's default
|
|
@@ -30,6 +30,7 @@ export {
|
|
|
30
30
|
export {
|
|
31
31
|
DecisionModelConfigSchema,
|
|
32
32
|
DEFAULT_MODEL,
|
|
33
|
+
EffortLevelSchema,
|
|
33
34
|
ENGINE_CONFIG_FILE,
|
|
34
35
|
EngineConfigSchema,
|
|
35
36
|
ModelNameSchema,
|
|
@@ -39,6 +40,7 @@ export {
|
|
|
39
40
|
TestResultsSchema,
|
|
40
41
|
} from './engine-config-schema'
|
|
41
42
|
export type {
|
|
43
|
+
EffortLevel,
|
|
42
44
|
EngineConfig,
|
|
43
45
|
ModelName,
|
|
44
46
|
ModelsConfig,
|
|
@@ -9,12 +9,13 @@ import { doChores } from './engine-chores'
|
|
|
9
9
|
import { closeSessions } from './session-close'
|
|
10
10
|
|
|
11
11
|
import type { AgentTurn } from '../agents/agent-launcher'
|
|
12
|
+
import { roleEffort } from '../agents/role-effort'
|
|
12
13
|
import {
|
|
13
14
|
parseRoleResult,
|
|
14
15
|
type AgentRole,
|
|
15
16
|
type RoleResult,
|
|
16
17
|
} from '../agents/role-results'
|
|
17
|
-
import { DEFAULT_MODEL } from '../config/engine-config'
|
|
18
|
+
import { DEFAULT_MODEL, type EffortLevel } from '../config/engine-config'
|
|
18
19
|
import { describeViolations } from '../guards/after-turn-check'
|
|
19
20
|
import { guardRoleOf } from '../guards/role-rules'
|
|
20
21
|
import { enforceAfterTurn, snapshotWorktree } from '../guards/worktree-state'
|
|
@@ -239,17 +240,24 @@ const sessionOf = (record: JournalRecord): string | null => {
|
|
|
239
240
|
}
|
|
240
241
|
}
|
|
241
242
|
|
|
242
|
-
/**
|
|
243
|
-
|
|
243
|
+
/**
|
|
244
|
+
* The model a session launched on, and why (`null` before #603), and the
|
|
245
|
+
* effort it runs at (`null` before #659).
|
|
246
|
+
*/
|
|
247
|
+
export type SessionModel = {
|
|
248
|
+
model: string
|
|
249
|
+
model_reason: ModelReason | null
|
|
250
|
+
effort: EffortLevel | null
|
|
251
|
+
}
|
|
244
252
|
|
|
245
253
|
/**
|
|
246
|
-
* The model a session launched on, and
|
|
254
|
+
* The model a session launched on, why, and its effort, from the journal: the
|
|
247
255
|
* `agent_started` of the launch the session's first record came after (same
|
|
248
256
|
* ticket and role), or of an earlier follow-up to it. `null` if the journal
|
|
249
257
|
* names none, as in journals from before #600; every agent ran Opus then.
|
|
250
258
|
*
|
|
251
259
|
* @example
|
|
252
|
-
* sessionModel({ records: journal.read(), session_id }) // { model: 'opus', model_reason: 'locked' }
|
|
260
|
+
* sessionModel({ records: journal.read(), session_id }) // { model: 'opus', model_reason: 'locked', effort: 'xhigh' }
|
|
253
261
|
*/
|
|
254
262
|
export const sessionModel = ({
|
|
255
263
|
records,
|
|
@@ -282,11 +290,14 @@ export const sessionModel = ({
|
|
|
282
290
|
record.content.follow_up_of === null
|
|
283
291
|
))
|
|
284
292
|
if (started?.kind !== 'agent_started') return null
|
|
285
|
-
const { model, model_reason } = started.content
|
|
286
|
-
return model === null ? null : { model, model_reason }
|
|
293
|
+
const { model, model_reason, effort } = started.content
|
|
294
|
+
return model === null ? null : { model, model_reason, effort }
|
|
287
295
|
}
|
|
288
296
|
|
|
289
|
-
/**
|
|
297
|
+
/**
|
|
298
|
+
* A follow-up's model and effort: its session's, else Opus at an unknown
|
|
299
|
+
* effort, as before #600.
|
|
300
|
+
*/
|
|
290
301
|
export const followUpModel = ({
|
|
291
302
|
records,
|
|
292
303
|
session_id,
|
|
@@ -297,6 +308,7 @@ export const followUpModel = ({
|
|
|
297
308
|
sessionModel({ records, session_id }) ?? {
|
|
298
309
|
model: DEFAULT_MODEL,
|
|
299
310
|
model_reason: null,
|
|
311
|
+
effort: null,
|
|
300
312
|
}
|
|
301
313
|
|
|
302
314
|
export const launchAgent = async ({
|
|
@@ -309,7 +321,8 @@ export const launchAgent = async ({
|
|
|
309
321
|
const { ticket, role, prompt, may_edit_tests } = action
|
|
310
322
|
// The decision model's answer, journaled just before this step, picks
|
|
311
323
|
// a routed role's model; a reviewer is locked on Opus. A redo keeps
|
|
312
|
-
// the model its first try journaled.
|
|
324
|
+
// the model its first try journaled. The role's effort applies on any
|
|
325
|
+
// model (#659).
|
|
313
326
|
const { model, model_reason } = launchPick({
|
|
314
327
|
records: context.journal.read(),
|
|
315
328
|
state: context.state,
|
|
@@ -318,11 +331,19 @@ export const launchAgent = async ({
|
|
|
318
331
|
role,
|
|
319
332
|
step: context.step,
|
|
320
333
|
})
|
|
334
|
+
const effort = roleEffort({ role, config: context.config })
|
|
321
335
|
context.journal.append({
|
|
322
336
|
kind: 'agent_started',
|
|
323
337
|
ticket,
|
|
324
338
|
role,
|
|
325
|
-
content: {
|
|
339
|
+
content: {
|
|
340
|
+
role,
|
|
341
|
+
prompt,
|
|
342
|
+
follow_up_of: null,
|
|
343
|
+
model,
|
|
344
|
+
model_reason,
|
|
345
|
+
effort,
|
|
346
|
+
},
|
|
326
347
|
})
|
|
327
348
|
await runTurn({
|
|
328
349
|
context,
|
|
@@ -344,6 +365,7 @@ export const launchAgent = async ({
|
|
|
344
365
|
role,
|
|
345
366
|
}),
|
|
346
367
|
model,
|
|
368
|
+
effort,
|
|
347
369
|
}),
|
|
348
370
|
})
|
|
349
371
|
}
|
|
@@ -360,7 +382,7 @@ export const followUpAgent = async ({
|
|
|
360
382
|
value: context.state.snapshot?.tickets[ticket],
|
|
361
383
|
what: `snapshot of #${ticket}`,
|
|
362
384
|
})
|
|
363
|
-
const { model, model_reason } = followUpModel({
|
|
385
|
+
const { model, model_reason, effort } = followUpModel({
|
|
364
386
|
records: context.journal.read(),
|
|
365
387
|
session_id,
|
|
366
388
|
})
|
|
@@ -374,6 +396,7 @@ export const followUpAgent = async ({
|
|
|
374
396
|
follow_up_of: session_id,
|
|
375
397
|
model,
|
|
376
398
|
model_reason,
|
|
399
|
+
effort,
|
|
377
400
|
},
|
|
378
401
|
})
|
|
379
402
|
await runTurn({
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
} from './execute-build'
|
|
16
16
|
import { closeSessions, openSessionsIn } from './session-close'
|
|
17
17
|
|
|
18
|
+
import { roleEffort } from '../agents/role-effort'
|
|
18
19
|
import { hasSlowTests } from '../config/engine-config'
|
|
19
20
|
import type { EngineCommit } from '../git/git-adapter'
|
|
20
21
|
import { LOCKED_PICK } from '../jev/model-pick'
|
|
@@ -137,6 +138,7 @@ export const executeFinalReviewAction = async ({
|
|
|
137
138
|
case 'launch_lens': {
|
|
138
139
|
const { lens, role, prompt } = action
|
|
139
140
|
const round = state.final_review.round
|
|
141
|
+
const effort = roleEffort({ role, config })
|
|
140
142
|
journal.append({
|
|
141
143
|
kind: 'lens_started',
|
|
142
144
|
ticket: null,
|
|
@@ -152,6 +154,7 @@ export const executeFinalReviewAction = async ({
|
|
|
152
154
|
prompt,
|
|
153
155
|
follow_up_of: null,
|
|
154
156
|
...LOCKED_PICK,
|
|
157
|
+
effort,
|
|
155
158
|
},
|
|
156
159
|
})
|
|
157
160
|
const result = await runTurn({
|
|
@@ -171,6 +174,7 @@ export const executeFinalReviewAction = async ({
|
|
|
171
174
|
// Lenses are reviewers: no agent messages.
|
|
172
175
|
messaging: null,
|
|
173
176
|
model: LOCKED_PICK.model,
|
|
177
|
+
effort,
|
|
174
178
|
}),
|
|
175
179
|
})
|
|
176
180
|
if (result === null || !('verdict' in result.result)) return
|
|
@@ -206,7 +210,9 @@ export const executeFinalReviewAction = async ({
|
|
|
206
210
|
return
|
|
207
211
|
case 'launch_final_fixer': {
|
|
208
212
|
const { role, prompt, may_edit_tests } = action
|
|
209
|
-
// The last fixes before the PR are never routed: always Opus
|
|
213
|
+
// The last fixes before the PR are never routed: always Opus,
|
|
214
|
+
// and at the reviewers' effort, so the last code is the cleanest.
|
|
215
|
+
const effort = roleEffort({ role, config, final_fix: true })
|
|
210
216
|
journal.append({
|
|
211
217
|
kind: 'agent_started',
|
|
212
218
|
ticket: null,
|
|
@@ -216,6 +222,7 @@ export const executeFinalReviewAction = async ({
|
|
|
216
222
|
prompt,
|
|
217
223
|
follow_up_of: null,
|
|
218
224
|
...LOCKED_PICK,
|
|
225
|
+
effort,
|
|
219
226
|
},
|
|
220
227
|
})
|
|
221
228
|
await runTurn({
|
|
@@ -235,13 +242,14 @@ export const executeFinalReviewAction = async ({
|
|
|
235
242
|
// Every ticket is over, so nobody is left to talk to.
|
|
236
243
|
messaging: null,
|
|
237
244
|
model: LOCKED_PICK.model,
|
|
245
|
+
effort,
|
|
238
246
|
}),
|
|
239
247
|
})
|
|
240
248
|
return
|
|
241
249
|
}
|
|
242
250
|
case 'follow_up_final_fixer': {
|
|
243
251
|
const { role, session_id, message } = action
|
|
244
|
-
const { model, model_reason } = followUpModel({
|
|
252
|
+
const { model, model_reason, effort } = followUpModel({
|
|
245
253
|
records: journal.read(),
|
|
246
254
|
session_id,
|
|
247
255
|
})
|
|
@@ -255,6 +263,7 @@ export const executeFinalReviewAction = async ({
|
|
|
255
263
|
follow_up_of: session_id,
|
|
256
264
|
model,
|
|
257
265
|
model_reason,
|
|
266
|
+
effort,
|
|
258
267
|
},
|
|
259
268
|
})
|
|
260
269
|
await runTurn({
|