@alecsibilia/luca 14.0.0-alpha.26 → 14.0.0-alpha.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/board/client/model-line.tsx +14 -3
- package/board/package.json +1 -1
- package/board/server/luca-version.ts +1 -1
- package/board/shared/board-state-run.ts +1 -1
- package/engine/agents/agent-launcher.ts +10 -1
- package/engine/agents/claude-messages.ts +2 -0
- package/engine/agents/claude-options.ts +7 -18
- package/engine/agents/launch-session.ts +2 -0
- package/engine/agents/refactor-update-prompts.ts +62 -0
- package/engine/agents/role-turns.ts +23 -0
- package/engine/agents/scripted-launcher.ts +4 -0
- package/engine/agents/session-pool.ts +1 -0
- package/engine/config/engine-config-schema.ts +85 -42
- package/engine/config/engine-config.ts +3 -0
- package/engine/config/test-commands.ts +35 -0
- package/engine/core/agent-turn.ts +32 -8
- package/engine/core/build-action.ts +17 -0
- package/engine/core/build-launch.ts +29 -9
- package/engine/core/decide-final-review.ts +30 -12
- package/engine/core/decide-red-green.ts +7 -5
- package/engine/core/decide-refactor-update.ts +109 -0
- package/engine/core/evidence-text.ts +13 -3
- package/engine/core/execute-build.ts +8 -0
- package/engine/core/execute-final-review.ts +3 -1
- package/engine/core/execute-memory.ts +5 -0
- package/engine/core/execute-red-check.ts +21 -0
- package/engine/core/execute-refactor-update.ts +107 -0
- package/engine/core/final-review-action.ts +11 -2
- package/engine/core/session-close.ts +10 -2
- package/engine/gates/gate-runner.ts +29 -4
- package/engine/gates/gate-schemas.ts +4 -1
- package/engine/gates/own-slow-check.ts +147 -0
- package/engine/gates/red-check.ts +44 -0
- package/engine/index.ts +1 -1
- package/engine/jev/model-pick.ts +115 -17
- package/engine/journal/agent-entries.ts +5 -2
- package/engine/journal/final-review-replay.ts +17 -1
- package/engine/journal/final-review-state.ts +10 -0
- package/engine/journal/journal-entry-schema.ts +6 -0
- package/engine/journal/journal-kinds.ts +2 -0
- package/engine/journal/journal-record-schema.ts +6 -0
- package/engine/journal/refactor-update-entries.ts +45 -0
- package/engine/journal/ticket-progress.ts +32 -0
- package/engine/journal/ticket-replay.ts +18 -0
- package/engine/journal/ticket-result-replay.ts +36 -1
- package/engine/journal/ticket-retry-replay.ts +5 -2
- package/engine/models/model-score-facts.ts +37 -16
- package/engine/models/model-score-sides.ts +80 -0
- package/engine/models/model-score.ts +146 -88
- package/engine/testing/effort-pick-fixtures.ts +7 -8
- package/engine/testing/model-score-fixtures.ts +30 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -36,8 +36,8 @@ const capital = ({ text }: { text: string }): string =>
|
|
|
36
36
|
* doesn't know.
|
|
37
37
|
*
|
|
38
38
|
* @example
|
|
39
|
-
* clefLine({ model: { role: 'implementer', name: 'opus', reason: 'unsure', answer: { role: 'implementer', pick: 'sonnet', confidence: 0.
|
|
40
|
-
* // { short: '0.
|
|
39
|
+
* clefLine({ model: { role: 'implementer', name: 'opus', reason: 'unsure', answer: { role: 'implementer', pick: 'sonnet', confidence: 0.36 } }, bar: 0.5 })
|
|
40
|
+
* // { short: '0.36 split → stayed on opus', full: 'Clef picked sonnet but was only 0.36 sure (split: under 0.5), so Luca stayed on opus.', confidence: 0.36, sharp: false }
|
|
41
41
|
*/
|
|
42
42
|
export const clefLine = ({
|
|
43
43
|
model,
|
|
@@ -54,10 +54,11 @@ export const clefLine = ({
|
|
|
54
54
|
const sure = confidence?.toFixed(2) ?? ''
|
|
55
55
|
const judged = `${word}: ${sharp ? 'at or over' : 'under'} ${bar}`
|
|
56
56
|
const plain = { confidence: null, sharp: false }
|
|
57
|
+
const picked = `${confidence === null ? 'clef' : `${sure} ${word}`} → ${name}`
|
|
57
58
|
switch (reason) {
|
|
58
59
|
case 'decision_model':
|
|
59
60
|
return {
|
|
60
|
-
short:
|
|
61
|
+
short: picked,
|
|
61
62
|
full:
|
|
62
63
|
confidence === null
|
|
63
64
|
? `Clef picked ${name}, so Luca ran the ${role} on it.`
|
|
@@ -65,6 +66,16 @@ export const clefLine = ({
|
|
|
65
66
|
confidence,
|
|
66
67
|
sharp,
|
|
67
68
|
}
|
|
69
|
+
case 'trial':
|
|
70
|
+
return {
|
|
71
|
+
short: `${picked} (trial)`,
|
|
72
|
+
full:
|
|
73
|
+
confidence === null
|
|
74
|
+
? `Clef picked ${name}, and Luca ran the ${role} on it in a model trial.`
|
|
75
|
+
: `Clef picked ${name} and was ${sure} sure (${judged}), and Luca ran the ${role} on it in a model trial.`,
|
|
76
|
+
confidence,
|
|
77
|
+
sharp,
|
|
78
|
+
}
|
|
68
79
|
case 'unsure':
|
|
69
80
|
return {
|
|
70
81
|
short: `${confidence === null ? 'unsure' : `${sure} ${word}`} → stayed on ${name}`,
|
package/board/package.json
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** The Luca version of this board, stamped when the package was packed. */
|
|
2
|
-
export const LUCA_VERSION: string = "14.0.0-alpha.
|
|
2
|
+
export const LUCA_VERSION: string = "14.0.0-alpha.28"
|
|
@@ -94,7 +94,7 @@ export const DEFAULT_RUN_BUDGET_TOKENS = 9_000_000
|
|
|
94
94
|
* `engine/src/config/engine-config-schema.ts`): the decision model is sharp at or
|
|
95
95
|
* over it, split under it. Kept here for the same reason.
|
|
96
96
|
*/
|
|
97
|
-
export const DEFAULT_CONFIDENCE_BAR = 0.
|
|
97
|
+
export const DEFAULT_CONFIDENCE_BAR = 0.5
|
|
98
98
|
|
|
99
99
|
export const EngineEndedSchema = z.object({
|
|
100
100
|
ok: z.boolean(),
|
|
@@ -103,7 +103,9 @@ export type LauncherFailure = z.infer<typeof LauncherFailureSchema>
|
|
|
103
103
|
* of the model session, for the journal. `session_gone` marks a follow-up
|
|
104
104
|
* that found its session closed or unknown (#521): no agent ran, and the
|
|
105
105
|
* engine hands the follow-up's message to a fresh agent. `out_of_turns`
|
|
106
|
-
* marks a turn that reached its role's turn limit (#543).
|
|
106
|
+
* marks a turn that reached its role's turn limit (#543). `session_closed`
|
|
107
|
+
* marks a turn interrupted for running past its time limit (#685): the
|
|
108
|
+
* launcher closed its session, so no follow-up can reach it.
|
|
107
109
|
*/
|
|
108
110
|
export type AgentTurn =
|
|
109
111
|
| {
|
|
@@ -120,6 +122,7 @@ export type AgentTurn =
|
|
|
120
122
|
session?: AgentSession
|
|
121
123
|
session_gone?: boolean
|
|
122
124
|
out_of_turns?: boolean
|
|
125
|
+
session_closed?: boolean
|
|
123
126
|
}
|
|
124
127
|
|
|
125
128
|
/**
|
|
@@ -178,6 +181,12 @@ export type AgentLauncher = {
|
|
|
178
181
|
* session's follow-ups run at it too.
|
|
179
182
|
*/
|
|
180
183
|
effort: EffortLevel
|
|
184
|
+
/**
|
|
185
|
+
* The turn limit for this launch (#684), in place of its role's: a
|
|
186
|
+
* lens whose last try ran out of turns gets twice its role's. Left
|
|
187
|
+
* out, the role's limit.
|
|
188
|
+
*/
|
|
189
|
+
max_turns?: number
|
|
181
190
|
/**
|
|
182
191
|
* Hears the turn's live progress (#628) each time it changes, before
|
|
183
192
|
* the turn ends, for the engine's live file. Left out, nothing hears it.
|
|
@@ -105,6 +105,8 @@ export type Failed = {
|
|
|
105
105
|
error: string
|
|
106
106
|
/** The turn reached its role's turn limit (#543). */
|
|
107
107
|
out_of_turns?: boolean
|
|
108
|
+
/** The turn ran past its time limit, and its session was closed (#685). */
|
|
109
|
+
session_closed?: boolean
|
|
108
110
|
}
|
|
109
111
|
|
|
110
112
|
export type Outcome = { ok: true; structured_output: unknown } | Failed
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
TicketReviewResultSchema,
|
|
17
17
|
type AgentRole,
|
|
18
18
|
} from './role-results'
|
|
19
|
+
import { MAX_TURNS } from './role-turns'
|
|
19
20
|
|
|
20
21
|
import {
|
|
21
22
|
DEFAULT_MODEL,
|
|
@@ -27,29 +28,14 @@ import { HELPER_TYPE, SEARCHER } from '../guards/helper-rules'
|
|
|
27
28
|
import { guardRoleOf, permissionRules } from '../guards/role-rules'
|
|
28
29
|
import { sandboxSettings } from '../guards/sandbox-settings'
|
|
29
30
|
|
|
31
|
+
export { MAX_TURNS } from './role-turns'
|
|
32
|
+
|
|
30
33
|
/** The models an agent may launch on, by short name (`ModelNameSchema`). */
|
|
31
34
|
export const MODEL_NAMES = ModelNameSchema.options
|
|
32
35
|
|
|
33
36
|
/** The model reviewers run on, and every agent the decision model doesn't route (#603). */
|
|
34
37
|
export const CLAUDE_MODEL = DEFAULT_MODEL
|
|
35
38
|
|
|
36
|
-
/**
|
|
37
|
-
* How many model turns one agent gets before its try ends. The integration
|
|
38
|
-
* lens reads how every ticket fits together on the whole branch, so it gets
|
|
39
|
-
* more than the other lenses.
|
|
40
|
-
*/
|
|
41
|
-
export const MAX_TURNS: Record<AgentRole, number> = {
|
|
42
|
-
'test-writer': 80,
|
|
43
|
-
implementer: 150,
|
|
44
|
-
'ticket-reviewer': 60,
|
|
45
|
-
'architecture-lens': 60,
|
|
46
|
-
'simplification-lens': 60,
|
|
47
|
-
'security-lens': 60,
|
|
48
|
-
'integration-lens': 100,
|
|
49
|
-
'rules-lens': 60,
|
|
50
|
-
learner: 40,
|
|
51
|
-
}
|
|
52
|
-
|
|
53
39
|
/**
|
|
54
40
|
* Why a model may not launch an agent, or `null` if it may: exactly one of
|
|
55
41
|
* `MODEL_NAMES`. Every other short name is refused, since some (such as
|
|
@@ -181,6 +167,7 @@ export const agentOptions = ({
|
|
|
181
167
|
cwd,
|
|
182
168
|
model,
|
|
183
169
|
effort,
|
|
170
|
+
max_turns,
|
|
184
171
|
claude_path,
|
|
185
172
|
config,
|
|
186
173
|
common_git_dir,
|
|
@@ -199,6 +186,8 @@ export const agentOptions = ({
|
|
|
199
186
|
cwd: string
|
|
200
187
|
model: string
|
|
201
188
|
effort: EffortLevel
|
|
189
|
+
/** The launch's own turn limit (#684), in place of the role's; left out, the role's. */
|
|
190
|
+
max_turns?: number
|
|
202
191
|
/** The real path of the installed `claude` binary. */
|
|
203
192
|
claude_path: string
|
|
204
193
|
config: EngineConfig
|
|
@@ -264,7 +253,7 @@ export const agentOptions = ({
|
|
|
264
253
|
// Streamed chunks only feed the live file's heartbeat (#631); the
|
|
265
254
|
// launcher keeps nothing of them.
|
|
266
255
|
includePartialMessages: true,
|
|
267
|
-
maxTurns: MAX_TURNS[role],
|
|
256
|
+
maxTurns: max_turns ?? MAX_TURNS[role],
|
|
268
257
|
hooks: {
|
|
269
258
|
PreToolUse: [{ hooks: [guard_hook] }],
|
|
270
259
|
...(delivery_hook === null
|
|
@@ -80,6 +80,7 @@ export const createLaunch = ({
|
|
|
80
80
|
messaging,
|
|
81
81
|
model,
|
|
82
82
|
effort,
|
|
83
|
+
max_turns,
|
|
83
84
|
on_progress,
|
|
84
85
|
}) => {
|
|
85
86
|
const refused = checkLaunchModel({ model })
|
|
@@ -117,6 +118,7 @@ export const createLaunch = ({
|
|
|
117
118
|
model,
|
|
118
119
|
// Kept in the session's options, so its follow-ups run at it too.
|
|
119
120
|
effort,
|
|
121
|
+
max_turns,
|
|
120
122
|
claude_path: claudePath,
|
|
121
123
|
config,
|
|
122
124
|
common_git_dir: gitDir,
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The texts of a refactor test update (#686): the section a fresh
|
|
3
|
+
* test-writer gets in place of the rejoin's test update section, the
|
|
4
|
+
* follow-up the implementer gets once it answered, and what the
|
|
5
|
+
* ticket-reviewer is told once it closed.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The section a test-writer gets when a refactor ticket's implementer sent
|
|
10
|
+
* a test back as bad: why, and that it may change only that file.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* refactorUpdateSection({ file: 'src/rows.test.ts', reason: 'It pins the old order.' })
|
|
14
|
+
*/
|
|
15
|
+
export const refactorUpdateSection = ({
|
|
16
|
+
file,
|
|
17
|
+
reason,
|
|
18
|
+
}: {
|
|
19
|
+
file: string
|
|
20
|
+
reason: string
|
|
21
|
+
}): string =>
|
|
22
|
+
[
|
|
23
|
+
"## A refactor ticket's test was sent back as bad",
|
|
24
|
+
`The implementer says this test is wrong:\n\n- File: ${file}\n- Reason: ${reason || 'no reason given'}`,
|
|
25
|
+
`This is a refactor ticket. Change only ${file}, and only so it checks what the ticket's criteria say.`,
|
|
26
|
+
"The implementer's work is already in the worktree, so the test may pass now. That is fine here.",
|
|
27
|
+
].join('\n\n')
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The follow-up a refactor ticket's implementer gets once the test-writer
|
|
31
|
+
* fixed the test it sent back.
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* refactorTestFixedMessage({ file: 'src/rows.test.ts' })
|
|
35
|
+
*/
|
|
36
|
+
export const refactorTestFixedMessage = ({ file }: { file: string }): string =>
|
|
37
|
+
`The test-writer changed ${file} so it checks what the ticket's criteria say. Carry on with the refactor. You may follow renames into test files, but don't change what a test checks.`
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The section a refactor ticket's reviewer gets once a refactor test update
|
|
41
|
+
* fixed a test: which file, why, and that only that change to what a test
|
|
42
|
+
* checks is expected.
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* refactorFixedReviewSection({ fixed: [{ file: 'src/rows.test.ts', reason: 'It pins the old order.' }] })
|
|
46
|
+
*/
|
|
47
|
+
export const refactorFixedReviewSection = ({
|
|
48
|
+
fixed,
|
|
49
|
+
}: {
|
|
50
|
+
fixed: { file: string; reason: string }[]
|
|
51
|
+
}): string =>
|
|
52
|
+
[
|
|
53
|
+
'## Tests a test-writer changed on purpose',
|
|
54
|
+
"The implementer sent these tests back as bad, and a test-writer changed them so they check what the ticket's criteria say. The engine checked it changed no other test file.",
|
|
55
|
+
fixed
|
|
56
|
+
.map(
|
|
57
|
+
({ file, reason }) =>
|
|
58
|
+
`- File: ${file}\n Reason: ${reason || 'no reason given'}`
|
|
59
|
+
)
|
|
60
|
+
.join('\n'),
|
|
61
|
+
"This change to what these tests check is expected: judge it against the ticket's criteria, not as a behavior change. Any other change to what a test checks is still a blocker.",
|
|
62
|
+
].join('\n\n')
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure: how many model turns each role gets, whatever launcher runs it. A
|
|
3
|
+
* launch's own `max_turns` (#684) replaces its role's.
|
|
4
|
+
*/
|
|
5
|
+
import type { AgentRole } from './role-results'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* How many model turns one agent gets before its try ends. The integration
|
|
9
|
+
* lens reads how every ticket fits together on the whole branch, and the
|
|
10
|
+
* rules lens reads every rule file against it (#684), so they get more than
|
|
11
|
+
* the other lenses.
|
|
12
|
+
*/
|
|
13
|
+
export const MAX_TURNS: Record<AgentRole, number> = {
|
|
14
|
+
'test-writer': 80,
|
|
15
|
+
implementer: 150,
|
|
16
|
+
'ticket-reviewer': 60,
|
|
17
|
+
'architecture-lens': 60,
|
|
18
|
+
'simplification-lens': 60,
|
|
19
|
+
'security-lens': 60,
|
|
20
|
+
'integration-lens': 100,
|
|
21
|
+
'rules-lens': 100,
|
|
22
|
+
learner: 40,
|
|
23
|
+
}
|
|
@@ -78,6 +78,8 @@ export type ScriptedCall = {
|
|
|
78
78
|
* `null` for a session the launcher doesn't know.
|
|
79
79
|
*/
|
|
80
80
|
effort: EffortLevel | null
|
|
81
|
+
/** The turn limit a launch asked for (#684); left out when it asked for none. */
|
|
82
|
+
max_turns?: number
|
|
81
83
|
/** What each of the turn's tool calls handed over, in order. */
|
|
82
84
|
delivered: string[]
|
|
83
85
|
}
|
|
@@ -204,6 +206,7 @@ export const createScriptedLauncher = ({
|
|
|
204
206
|
messaging,
|
|
205
207
|
model,
|
|
206
208
|
effort,
|
|
209
|
+
max_turns,
|
|
207
210
|
}) => {
|
|
208
211
|
const session_id = `scripted-${role}-${ticket}-${calls.length + 1}`
|
|
209
212
|
sessions.set(session_id, { messaging, model, effort })
|
|
@@ -216,6 +219,7 @@ export const createScriptedLauncher = ({
|
|
|
216
219
|
may_edit_tests,
|
|
217
220
|
model,
|
|
218
221
|
effort,
|
|
222
|
+
...(max_turns === undefined ? {} : { max_turns }),
|
|
219
223
|
delivered: [],
|
|
220
224
|
}
|
|
221
225
|
calls.push(call)
|
|
@@ -51,46 +51,84 @@ const JunitPathSchema = z.string().min(1).refine(isFileInsideRepo, {
|
|
|
51
51
|
'must be a file inside the repo, relative to its root: not an absolute path, not one that leaves the repo with "..", and not ending in "/"',
|
|
52
52
|
})
|
|
53
53
|
|
|
54
|
+
/** The text a slow command's `run_files` must hold, where the files go (#683). */
|
|
55
|
+
export const RUN_FILES_PLACEHOLDER = '{files}'
|
|
56
|
+
|
|
54
57
|
/**
|
|
55
|
-
* One entry of a `checks.test` list: a command, or a
|
|
56
|
-
* A `junit` command names the JUnit files it writes in
|
|
57
|
-
* one path or a list; no other kind may have one.
|
|
58
|
+
* One entry of a `checks.test` or `checks.slow_test` list: a command, or a
|
|
59
|
+
* command and its kind. A `junit` command names the JUnit files it writes in
|
|
60
|
+
* `junit_path` (#668), one path or a list; no other kind may have one. A slow
|
|
61
|
+
* entry may also name the test files it owns (`file_patterns`) and how to run
|
|
62
|
+
* some of them (`run_files`, #683); they come together, and a `checks.test`
|
|
63
|
+
* entry may have neither.
|
|
58
64
|
*/
|
|
59
|
-
const
|
|
60
|
-
z.
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
65
|
+
const testCommandEntrySchema = ({ slow }: { slow: boolean }) =>
|
|
66
|
+
z.union([
|
|
67
|
+
z.string().min(1),
|
|
68
|
+
z
|
|
69
|
+
.object({
|
|
70
|
+
run: z.string().min(1),
|
|
71
|
+
results: TestResultsSchema.optional(),
|
|
72
|
+
junit_path: z
|
|
73
|
+
.union([JunitPathSchema, z.array(JunitPathSchema).min(1)])
|
|
74
|
+
.optional(),
|
|
75
|
+
file_patterns: z.array(z.string().min(1)).min(1).optional(),
|
|
76
|
+
run_files: z.string().min(1).optional(),
|
|
77
|
+
})
|
|
78
|
+
.refine(
|
|
79
|
+
({ file_patterns, run_files }) =>
|
|
80
|
+
slow ||
|
|
81
|
+
(file_patterns === undefined && run_files === undefined),
|
|
82
|
+
{
|
|
83
|
+
message:
|
|
84
|
+
'checks.test: file_patterns and run_files belong only on checks.slow_test entries.',
|
|
85
|
+
}
|
|
86
|
+
)
|
|
87
|
+
.refine(
|
|
88
|
+
({ file_patterns, run_files }) =>
|
|
89
|
+
!slow ||
|
|
90
|
+
(file_patterns === undefined) === (run_files === undefined),
|
|
91
|
+
{
|
|
92
|
+
message:
|
|
93
|
+
'checks.slow_test: file_patterns and run_files come together; this entry has only one of them.',
|
|
94
|
+
}
|
|
95
|
+
)
|
|
96
|
+
.refine(
|
|
97
|
+
({ run_files }) =>
|
|
98
|
+
!slow ||
|
|
99
|
+
run_files === undefined ||
|
|
100
|
+
run_files.includes(RUN_FILES_PLACEHOLDER),
|
|
101
|
+
{
|
|
102
|
+
message: `checks.slow_test: run_files must contain ${RUN_FILES_PLACEHOLDER}.`,
|
|
103
|
+
path: ['run_files'],
|
|
104
|
+
}
|
|
105
|
+
)
|
|
106
|
+
.refine(
|
|
107
|
+
({ results, junit_path }) =>
|
|
108
|
+
results !== 'junit' || junit_path !== undefined,
|
|
109
|
+
{
|
|
110
|
+
message:
|
|
111
|
+
'a "junit" test command needs junit_path: the JUnit files it writes, relative to the repo root',
|
|
112
|
+
path: ['junit_path'],
|
|
113
|
+
}
|
|
114
|
+
)
|
|
115
|
+
.refine(
|
|
116
|
+
({ results, junit_path }) =>
|
|
117
|
+
results === 'junit' || junit_path === undefined,
|
|
118
|
+
{
|
|
119
|
+
message:
|
|
120
|
+
'junit_path is only for a test command with "results": "junit"',
|
|
121
|
+
path: ['junit_path'],
|
|
122
|
+
}
|
|
123
|
+
),
|
|
124
|
+
])
|
|
88
125
|
|
|
89
|
-
/**
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
126
|
+
/**
|
|
127
|
+
* A test command setting: one command, or a list of them (see
|
|
128
|
+
* `testCommands`). Only a slow one's entries may own files (#683).
|
|
129
|
+
*/
|
|
130
|
+
const testCommandsSchema = ({ slow }: { slow: boolean }) =>
|
|
131
|
+
z.union([z.string().min(1), z.array(testCommandEntrySchema({ slow }))])
|
|
94
132
|
|
|
95
133
|
/** The config's `muninn` block. */
|
|
96
134
|
export const MuninnConfigSchema = z.object({
|
|
@@ -178,12 +216,15 @@ const RoleEffortsSchema = z.strictObject(
|
|
|
178
216
|
/**
|
|
179
217
|
* The config's `models` block (#603): each routed role's model floor, each
|
|
180
218
|
* writer's effort (#660), and how sure the decision model must be
|
|
181
|
-
* (`confidence_bar`) before its model pick is used
|
|
219
|
+
* (`confidence_bar`) before its model pick is used, and the share of its
|
|
220
|
+
* unsure but cheaper picks that run anyway in a model trial (`trial_share`,
|
|
221
|
+
* #698).
|
|
182
222
|
*/
|
|
183
223
|
export const ModelsConfigSchema = z.object({
|
|
184
224
|
floors: ModelFloorsSchema.prefault({}),
|
|
185
225
|
effort: RoleEffortsSchema.prefault({}),
|
|
186
|
-
confidence_bar: z.number().min(0).max(1).default(0.
|
|
226
|
+
confidence_bar: z.number().min(0).max(1).default(0.5),
|
|
227
|
+
trial_share: z.number().min(0).max(1).default(0),
|
|
187
228
|
})
|
|
188
229
|
|
|
189
230
|
export type ModelsConfig = z.infer<typeof ModelsConfigSchema>
|
|
@@ -204,13 +245,15 @@ export type ModelsConfig = z.infer<typeof ModelsConfigSchema>
|
|
|
204
245
|
export const EngineConfigSchema = z.object({
|
|
205
246
|
checks: z
|
|
206
247
|
.object({
|
|
207
|
-
test:
|
|
248
|
+
test: testCommandsSchema({ slow: false }).optional(),
|
|
208
249
|
/**
|
|
209
250
|
* The slow tests (#563): the tests `test` leaves out, run only at
|
|
210
|
-
* the final review. Same shape as `test
|
|
251
|
+
* the final review. Same shape as `test`, and an entry may also
|
|
252
|
+
* own some test files and run them in a ticket's gates
|
|
253
|
+
* (`file_patterns`, `run_files`, #683). Left out, nothing
|
|
211
254
|
* changes.
|
|
212
255
|
*/
|
|
213
|
-
slow_test:
|
|
256
|
+
slow_test: testCommandsSchema({ slow: true }).optional(),
|
|
214
257
|
types: z.string().min(1).optional(),
|
|
215
258
|
lint: z.string().min(1).optional(),
|
|
216
259
|
})
|
|
@@ -37,6 +37,7 @@ export {
|
|
|
37
37
|
ModelsConfigSchema,
|
|
38
38
|
MuninnConfigSchema,
|
|
39
39
|
ROUTED_ROLES,
|
|
40
|
+
RUN_FILES_PLACEHOLDER,
|
|
40
41
|
TestResultsSchema,
|
|
41
42
|
} from './engine-config-schema'
|
|
42
43
|
export type {
|
|
@@ -55,8 +56,10 @@ export {
|
|
|
55
56
|
hasSlowTests,
|
|
56
57
|
isBunTestCommand,
|
|
57
58
|
junitCommands,
|
|
59
|
+
ownSlowCommands,
|
|
58
60
|
slowTestCommands,
|
|
59
61
|
testCommands,
|
|
62
|
+
type OwnSlowCommand,
|
|
60
63
|
type TestCommand,
|
|
61
64
|
} from './test-commands'
|
|
62
65
|
|
|
@@ -77,6 +77,41 @@ export const slowTestCommands = ({
|
|
|
77
77
|
config: EngineConfig
|
|
78
78
|
}): TestCommand[] => commandsOf(config.checks.slow_test)
|
|
79
79
|
|
|
80
|
+
/**
|
|
81
|
+
* A slow test command that runs some of its files (#683): the test files it
|
|
82
|
+
* owns, and the command with `{files}` where they go.
|
|
83
|
+
*/
|
|
84
|
+
export type OwnSlowCommand = { file_patterns: string[]; run_files: string }
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The slow test commands with `run_files` (#683), in config order: a
|
|
88
|
+
* ticket's gates run its own files that match `file_patterns` through them.
|
|
89
|
+
*
|
|
90
|
+
* @example
|
|
91
|
+
* ownSlowCommands({ config: { ...config, checks: { test: 'bun test src', slow_test: [{ run: 'bun run ui', file_patterns: ['ui/**'], run_files: 'bun run ui-files {files}' }] } } })
|
|
92
|
+
* // [{ file_patterns: ['ui/**'], run_files: 'bun run ui-files {files}' }]
|
|
93
|
+
*/
|
|
94
|
+
export const ownSlowCommands = ({
|
|
95
|
+
config,
|
|
96
|
+
}: {
|
|
97
|
+
config: EngineConfig
|
|
98
|
+
}): OwnSlowCommand[] => {
|
|
99
|
+
const value = config.checks.slow_test
|
|
100
|
+
if (value === undefined || typeof value === 'string') return []
|
|
101
|
+
return value.flatMap((entry) =>
|
|
102
|
+
typeof entry === 'string' ||
|
|
103
|
+
entry.file_patterns === undefined ||
|
|
104
|
+
entry.run_files === undefined
|
|
105
|
+
? []
|
|
106
|
+
: [
|
|
107
|
+
{
|
|
108
|
+
file_patterns: entry.file_patterns,
|
|
109
|
+
run_files: entry.run_files,
|
|
110
|
+
},
|
|
111
|
+
]
|
|
112
|
+
)
|
|
113
|
+
}
|
|
114
|
+
|
|
80
115
|
/** Whether the config has a slow test command (#563). */
|
|
81
116
|
export const hasSlowTests = ({ config }: { config: EngineConfig }): boolean =>
|
|
82
117
|
slowTestCommands({ config }).length > 0
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
* ended, journaled. Also the `launch_agent` and `follow_up_agent` steps,
|
|
4
4
|
* which start a turn.
|
|
5
5
|
*/
|
|
6
|
+
import { basename } from 'node:path'
|
|
7
|
+
|
|
6
8
|
import { need, ticketWorktree, type BuildContext } from './build-context'
|
|
7
9
|
import { mayEditTests, type BuildAction } from './decide-build'
|
|
8
10
|
import { doChores } from './engine-chores'
|
|
@@ -19,7 +21,11 @@ import { DEFAULT_MODEL, type EffortLevel } from '../config/engine-config'
|
|
|
19
21
|
import { describeViolations } from '../guards/after-turn-check'
|
|
20
22
|
import { guardRoleOf } from '../guards/role-rules'
|
|
21
23
|
import { enforceAfterTurn, snapshotWorktree } from '../guards/worktree-state'
|
|
22
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
launchPick,
|
|
26
|
+
type ModelReason,
|
|
27
|
+
type TrialGroup,
|
|
28
|
+
} from '../jev/model-pick'
|
|
23
29
|
import type { AgentFailure, JournalRecord } from '../journal/journal-record'
|
|
24
30
|
import type { ReplayedWorktree } from '../journal/replay'
|
|
25
31
|
import { sessionSignal } from '../limits/plan-signals'
|
|
@@ -66,7 +72,9 @@ const failTurn = ({
|
|
|
66
72
|
* every launcher alike; what others changed in the shared `.git` meanwhile
|
|
67
73
|
* is journaled as `shared_git_changed`. Then the result is judged by its
|
|
68
74
|
* structured output. Each failed turn is journaled once as `agent_failed`
|
|
69
|
-
* with how it failed; the decision step picks what happens next. A
|
|
75
|
+
* with how it failed; the decision step picks what happens next. A turn the
|
|
76
|
+
* launcher interrupted for its time limit then journals its session closed
|
|
77
|
+
* (#685), since the launcher closed it. A writer's
|
|
70
78
|
* result then gets its chores done (`doChores`, #618); one the engine could
|
|
71
79
|
* not do fails the turn as a `chore` failure. A launcher stop journals
|
|
72
80
|
* `run_stopped` and ends the run. A turn the plan cut off (a rejected limit,
|
|
@@ -177,6 +185,16 @@ export const runTurn = async ({
|
|
|
177
185
|
session_gone,
|
|
178
186
|
out_of_turns,
|
|
179
187
|
})
|
|
188
|
+
// A timed-out turn's session was closed by the launcher (#685):
|
|
189
|
+
// journal the close, so the next try is a fresh agent.
|
|
190
|
+
if (!turn.ok && turn.session_closed === true && session_id !== null) {
|
|
191
|
+
context.journal.append({
|
|
192
|
+
kind: 'agent_session_closed',
|
|
193
|
+
ticket,
|
|
194
|
+
role,
|
|
195
|
+
content: { role, session_id },
|
|
196
|
+
})
|
|
197
|
+
}
|
|
180
198
|
return null
|
|
181
199
|
}
|
|
182
200
|
if (violations.length > 0) {
|
|
@@ -241,13 +259,15 @@ const sessionOf = (record: JournalRecord): string | null => {
|
|
|
241
259
|
}
|
|
242
260
|
|
|
243
261
|
/**
|
|
244
|
-
* The model a session launched on, and why (`null` before #603),
|
|
245
|
-
* effort it runs at (`null` before #659)
|
|
262
|
+
* The model a session launched on, and why (`null` before #603), the
|
|
263
|
+
* effort it runs at (`null` before #659), and its model trial group
|
|
264
|
+
* (`null` when not eligible, or before #698).
|
|
246
265
|
*/
|
|
247
266
|
export type SessionModel = {
|
|
248
267
|
model: string
|
|
249
268
|
model_reason: ModelReason | null
|
|
250
269
|
effort: EffortLevel | null
|
|
270
|
+
trial_group: TrialGroup | null
|
|
251
271
|
}
|
|
252
272
|
|
|
253
273
|
/**
|
|
@@ -290,8 +310,8 @@ export const sessionModel = ({
|
|
|
290
310
|
record.content.follow_up_of === null
|
|
291
311
|
))
|
|
292
312
|
if (started?.kind !== 'agent_started') return null
|
|
293
|
-
const { model, model_reason, effort } = started.content
|
|
294
|
-
return model === null ? null : { model, model_reason, effort }
|
|
313
|
+
const { model, model_reason, effort, trial_group } = started.content
|
|
314
|
+
return model === null ? null : { model, model_reason, effort, trial_group }
|
|
295
315
|
}
|
|
296
316
|
|
|
297
317
|
/**
|
|
@@ -309,6 +329,7 @@ export const followUpModel = ({
|
|
|
309
329
|
model: DEFAULT_MODEL,
|
|
310
330
|
model_reason: null,
|
|
311
331
|
effort: null,
|
|
332
|
+
trial_group: null,
|
|
312
333
|
}
|
|
313
334
|
|
|
314
335
|
export const launchAgent = async ({
|
|
@@ -323,10 +344,11 @@ export const launchAgent = async ({
|
|
|
323
344
|
// a routed role's model; a reviewer is locked on Opus. A redo keeps
|
|
324
345
|
// the model its first try journaled. The role's effort applies on any
|
|
325
346
|
// model (#659).
|
|
326
|
-
const { model, model_reason } = launchPick({
|
|
347
|
+
const { model, model_reason, trial_group } = launchPick({
|
|
327
348
|
records: context.journal.read(),
|
|
328
349
|
state: context.state,
|
|
329
350
|
config: context.config,
|
|
351
|
+
run_id: basename(context.run_dir),
|
|
330
352
|
ticket,
|
|
331
353
|
role,
|
|
332
354
|
step: context.step,
|
|
@@ -343,6 +365,7 @@ export const launchAgent = async ({
|
|
|
343
365
|
model,
|
|
344
366
|
model_reason,
|
|
345
367
|
effort,
|
|
368
|
+
trial_group,
|
|
346
369
|
},
|
|
347
370
|
})
|
|
348
371
|
await runTurn({
|
|
@@ -382,7 +405,7 @@ export const followUpAgent = async ({
|
|
|
382
405
|
value: context.state.snapshot?.tickets[ticket],
|
|
383
406
|
what: `snapshot of #${ticket}`,
|
|
384
407
|
})
|
|
385
|
-
const { model, model_reason, effort } = followUpModel({
|
|
408
|
+
const { model, model_reason, effort, trial_group } = followUpModel({
|
|
386
409
|
records: context.journal.read(),
|
|
387
410
|
session_id,
|
|
388
411
|
})
|
|
@@ -397,6 +420,7 @@ export const followUpAgent = async ({
|
|
|
397
420
|
model,
|
|
398
421
|
model_reason,
|
|
399
422
|
effort,
|
|
423
|
+
trial_group,
|
|
400
424
|
},
|
|
401
425
|
})
|
|
402
426
|
await runTurn({
|