@hecer/yoke 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/CHANGELOG.md +18 -3
- package/README.md +87 -21
- package/bench/README.md +46 -14
- package/bench/RESULTS.md +108 -11
- package/bench/analyze-routing-study.mjs +96 -0
- package/bench/fixtures/routing-queue/.yoke/config.yaml +30 -0
- package/bench/fixtures/routing-queue/.yoke/prd.yaml +24 -0
- package/bench/fixtures/routing-queue/bench-verify.mjs +9 -0
- package/bench/fixtures/routing-queue/package.json +9 -0
- package/bench/fixtures/routing-queue/src/task-queue.mjs +1 -0
- package/bench/fixtures/routing-queue/tests/STORY-1.test.mjs +68 -0
- package/bench/fixtures/routing-queue/tests/STORY-2.test.mjs +72 -0
- package/bench/results/routing-queue-codex-routing-off-2026-08-01T21-44-00.json +46 -0
- package/bench/results/routing-queue-codex-routing-on-2026-08-01T21-50-31.json +109 -0
- package/bench/results/yoke-codex-study-codex-routing-off-2026-08-02T07-58-56.json +65 -0
- package/bench/results/yoke-codex-study-codex-routing-off-2026-08-02T08-08-48.json +65 -0
- package/bench/results/yoke-codex-study-codex-routing-off-2026-08-02T08-34-45.json +65 -0
- package/bench/results/yoke-codex-study-codex-routing-on-2026-08-02T07-51-59.json +168 -0
- package/bench/results/yoke-codex-study-codex-routing-on-2026-08-02T08-18-59.json +168 -0
- package/bench/results/yoke-codex-study-codex-routing-on-2026-08-02T08-25-25.json +168 -0
- package/bench/results/yoke-large-codex-routing-off-2026-08-01T22-58-01.json +54 -0
- package/bench/results/yoke-large-codex-routing-on-2026-08-01T23-09-39.json +133 -0
- package/bench/run-large.mjs +188 -0
- package/bench/run.mjs +42 -21
- package/dist/agents/providers.js +21 -4
- package/dist/agents/telemetry.js +46 -7
- package/dist/cli.js +7 -5
- package/dist/loop/decision.js +3 -1
- package/dist/loop/loop.js +10 -0
- package/dist/loop/reporter.js +9 -0
- package/dist/loop/run-command.js +35 -7
- package/dist/loop/runner.js +68 -9
- package/dist/retrofit/config.js +21 -0
- package/dist/review/command.js +2 -2
- package/dist/routing/registry.js +92 -0
- package/dist/routing/router.js +195 -0
- package/dist/setup/command.js +23 -1
- package/gemini-extension.json +1 -1
- package/package.json +6 -4
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Controlled routing A/B against a full repository snapshot kept outside this repo.
|
|
3
|
+
import { spawn, spawnSync } from 'node:child_process'
|
|
4
|
+
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
|
|
5
|
+
import { basename, dirname, join, resolve } from 'node:path'
|
|
6
|
+
import { fileURLToPath } from 'node:url'
|
|
7
|
+
import { parse } from 'yaml'
|
|
8
|
+
import { validateResult } from './result-schema.mjs'
|
|
9
|
+
|
|
10
|
+
const benchDir = dirname(fileURLToPath(import.meta.url))
|
|
11
|
+
const repoRoot = dirname(benchDir)
|
|
12
|
+
const cli = join(repoRoot, 'dist', 'cli.js')
|
|
13
|
+
const args = Object.fromEntries(process.argv.slice(2).filter(arg => arg.startsWith('--')).map(arg => {
|
|
14
|
+
const [key, ...value] = arg.slice(2).split('=')
|
|
15
|
+
return [key, value.length ? value.join('=') : true]
|
|
16
|
+
}))
|
|
17
|
+
|
|
18
|
+
const seed = args.seed ? resolve(String(args.seed)) : null
|
|
19
|
+
const routing = String(args.routing ?? '')
|
|
20
|
+
const runRoot = resolve(String(args['run-root'] ?? join(benchDir, '.runs')))
|
|
21
|
+
if (!seed || !existsSync(seed) || !['on', 'off'].includes(routing)) {
|
|
22
|
+
console.error('usage: node bench/run-large.mjs --seed=<dir> --routing=<on|off> [--run-root=<dir>] [--max=4] [--timeout=15] [--label=note]')
|
|
23
|
+
process.exit(2)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let acceptanceManifest = null
|
|
27
|
+
try { acceptanceManifest = JSON.parse(readFileSync(join(seed, 'bench-acceptance.json'), 'utf8')) } catch { /* legacy large seed */ }
|
|
28
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)
|
|
29
|
+
const fixture = String(acceptanceManifest?.fixture ?? 'yoke-large')
|
|
30
|
+
const fixtureVersion = String(acceptanceManifest?.fixtureVersion ?? 'yoke-large@1')
|
|
31
|
+
const runDir = join(runRoot, `${fixture}-codex-routing-${routing}-${stamp}`)
|
|
32
|
+
mkdirSync(runDir, { recursive: true })
|
|
33
|
+
const excluded = new Set(['.git', 'node_modules', 'dist', '.worktrees'])
|
|
34
|
+
cpSync(seed, runDir, {
|
|
35
|
+
recursive: true,
|
|
36
|
+
filter: source => source === seed || !excluded.has(basename(source)),
|
|
37
|
+
})
|
|
38
|
+
symlinkSync(join(repoRoot, 'node_modules'), join(runDir, 'node_modules'), 'junction')
|
|
39
|
+
|
|
40
|
+
const git = (...gitArgs) => {
|
|
41
|
+
const result = spawnSync('git', ['-C', runDir, ...gitArgs], { encoding: 'utf8' })
|
|
42
|
+
if (result.status !== 0) throw new Error(`git ${gitArgs.join(' ')} failed: ${result.stderr}`)
|
|
43
|
+
}
|
|
44
|
+
git('init', '-q')
|
|
45
|
+
git('-c', 'user.name=bench', '-c', 'user.email=bench@yoke', 'add', '-A')
|
|
46
|
+
git('-c', 'user.name=bench', '-c', 'user.email=bench@yoke', 'commit', '-q', '-m', 'bench: full Yoke baseline')
|
|
47
|
+
|
|
48
|
+
const env = {
|
|
49
|
+
...process.env,
|
|
50
|
+
YOKE_NO_UPDATE_CHECK: '1',
|
|
51
|
+
YOKE_REGISTRY_DIR: join(runRoot, '.routing-registries', basename(runDir)),
|
|
52
|
+
}
|
|
53
|
+
for (const key of Object.keys(env)) if (key.startsWith('CLAUDE_CODE') || key === 'CLAUDECODE') delete env[key]
|
|
54
|
+
|
|
55
|
+
const events = []
|
|
56
|
+
const started = Date.now()
|
|
57
|
+
console.error(`[bench-large] routing=${routing} -> ${runDir}`)
|
|
58
|
+
const child = spawn(process.execPath, [
|
|
59
|
+
cli, 'loop', 'run', runDir, '--json', '--runner=codex',
|
|
60
|
+
`--max=${Number(args.max ?? 4)}`, `--timeout=${Number(args.timeout ?? 15)}`,
|
|
61
|
+
routing === 'on' ? '--routing' : '--no-routing', '--unsafe',
|
|
62
|
+
], { env, stdio: ['ignore', 'pipe', 'inherit'] })
|
|
63
|
+
let buffer = ''
|
|
64
|
+
child.stdout.on('data', chunk => {
|
|
65
|
+
buffer += chunk
|
|
66
|
+
let newline
|
|
67
|
+
while ((newline = buffer.indexOf('\n')) >= 0) {
|
|
68
|
+
const line = buffer.slice(0, newline).trim()
|
|
69
|
+
buffer = buffer.slice(newline + 1)
|
|
70
|
+
if (!line) continue
|
|
71
|
+
try { events.push({ at: Date.now(), ...JSON.parse(line) }) } catch { /* provider noise is not evidence */ }
|
|
72
|
+
}
|
|
73
|
+
})
|
|
74
|
+
const exitCode = await new Promise(resolveExit => child.on('close', resolveExit))
|
|
75
|
+
const wallClockMs = Date.now() - started
|
|
76
|
+
|
|
77
|
+
const build = spawnSync(process.execPath, [join(repoRoot, 'node_modules', 'typescript', 'bin', 'tsc')], { cwd: runDir, env, encoding: 'utf8', timeout: 10 * 60_000 })
|
|
78
|
+
const tests = spawnSync(process.execPath, [join(repoRoot, 'node_modules', 'vitest', 'vitest.mjs'), 'run'], { cwd: runDir, env, encoding: 'utf8', timeout: 10 * 60_000 })
|
|
79
|
+
const hiddenTargets = acceptanceManifest?.acceptanceTests?.map(test => [String(test.source), String(test.hidden)]) ?? [
|
|
80
|
+
['tests/routing/large-registry-status.test.ts', 'tests/routing/benchmark-original-status.test.ts'],
|
|
81
|
+
['tests/routing/large-fallback.test.ts', 'tests/routing/benchmark-original-fallback.test.ts'],
|
|
82
|
+
]
|
|
83
|
+
for (const [source, target] of hiddenTargets) {
|
|
84
|
+
mkdirSync(dirname(join(runDir, target)), { recursive: true })
|
|
85
|
+
cpSync(join(seed, source), join(runDir, target))
|
|
86
|
+
}
|
|
87
|
+
const hiddenTests = spawnSync(process.execPath, [
|
|
88
|
+
join(repoRoot, 'node_modules', 'vitest', 'vitest.mjs'), 'run',
|
|
89
|
+
...hiddenTargets.map(([, target]) => target), '--maxWorkers=1',
|
|
90
|
+
], { cwd: runDir, env, encoding: 'utf8', timeout: 10 * 60_000 })
|
|
91
|
+
for (const [, target] of hiddenTargets) rmSync(join(runDir, target), { force: true })
|
|
92
|
+
const finalTestsPass = build.status === 0 && tests.status === 0 && hiddenTests.status === 0
|
|
93
|
+
if (!finalTestsPass) {
|
|
94
|
+
console.error('[bench-large] independent final verification failed')
|
|
95
|
+
console.error(build.error ?? build.stderr ?? build.stdout)
|
|
96
|
+
console.error(tests.error ?? tests.stderr ?? tests.stdout)
|
|
97
|
+
console.error(hiddenTests.error ?? hiddenTests.stderr ?? hiddenTests.stdout)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const prd = parse(readFileSync(join(runDir, '.yoke', 'prd.yaml'), 'utf8'))
|
|
101
|
+
const storyIds = Array.isArray(prd) ? prd.map(story => String(story.id)) : []
|
|
102
|
+
const firstSeen = {}
|
|
103
|
+
for (const event of events) if (event.story && !(event.story in firstSeen)) firstSeen[event.story] = event.at
|
|
104
|
+
const stories = storyIds.map((id, index) => {
|
|
105
|
+
const start = firstSeen[id]
|
|
106
|
+
const next = storyIds.slice(index + 1).map(candidate => firstSeen[candidate]).find(value => value !== undefined)
|
|
107
|
+
return {
|
|
108
|
+
id,
|
|
109
|
+
durationMs: start === undefined ? null : (next ?? started + wallClockMs) - start,
|
|
110
|
+
iterations: new Set(events.filter(event => event.story === id).map(event => event.iteration)).size,
|
|
111
|
+
finalTestsPass,
|
|
112
|
+
}
|
|
113
|
+
})
|
|
114
|
+
const last = events.at(-1) ?? {}
|
|
115
|
+
let status = {}
|
|
116
|
+
try { status = JSON.parse(readFileSync(join(runDir, '.yoke', 'loop-status.json'), 'utf8')) } catch { /* early refusal */ }
|
|
117
|
+
|
|
118
|
+
const sourceStats = root => {
|
|
119
|
+
let files = 0
|
|
120
|
+
let lines = 0
|
|
121
|
+
const walk = dir => {
|
|
122
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
123
|
+
const path = join(dir, entry.name)
|
|
124
|
+
if (entry.isDirectory()) walk(path)
|
|
125
|
+
else if (/\.(ts|js|mjs)$/.test(entry.name)) {
|
|
126
|
+
files++
|
|
127
|
+
lines += readFileSync(path, 'utf8').split(/\r?\n/).filter(line => line.trim()).length
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
walk(root)
|
|
132
|
+
return { files, lines }
|
|
133
|
+
}
|
|
134
|
+
const stats = sourceStats(join(runDir, 'src'))
|
|
135
|
+
const config = parse(readFileSync(join(runDir, '.yoke', 'config.yaml'), 'utf8'))
|
|
136
|
+
const inputTokens = Number(status.tokens?.inputTokens ?? 0)
|
|
137
|
+
const cachedInputTokens = Number(status.tokens?.cachedInputTokens ?? 0)
|
|
138
|
+
const testCount = Number((tests.stdout.match(/Tests\s+(\d+)\s+passed/) ?? [])[1] ?? 0) || null
|
|
139
|
+
const result = {
|
|
140
|
+
schemaVersion: 1,
|
|
141
|
+
fixtureVersion,
|
|
142
|
+
runner: 'codex',
|
|
143
|
+
sampleLabel: String(args.label ?? `large-routing-${routing}`),
|
|
144
|
+
permissionProfile: 'unsafe',
|
|
145
|
+
yokeVersion: JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8')).version,
|
|
146
|
+
fixture,
|
|
147
|
+
routing,
|
|
148
|
+
startedAt: new Date(started).toISOString(),
|
|
149
|
+
wallClockMs,
|
|
150
|
+
exitCode,
|
|
151
|
+
finalState: last.state ?? null,
|
|
152
|
+
verdict: exitCode === 0 && finalTestsPass ? 'completed' : 'blocked',
|
|
153
|
+
blocker: exitCode === 0 && finalTestsPass ? null : (status.reason ?? 'loop or independent final verification failed'),
|
|
154
|
+
conflicts: events.filter(event => /conflict/i.test(String(event.reason ?? event.summary ?? ''))).length,
|
|
155
|
+
iterations: stories.reduce((sum, story) => sum + story.iterations, 0),
|
|
156
|
+
finalTestsPass,
|
|
157
|
+
progress: last.progress ?? null,
|
|
158
|
+
usageAvailable: Number(status.tokens?.inputTokens ?? 0) + Number(status.tokens?.outputTokens ?? 0) > 0,
|
|
159
|
+
modelAvailable: Boolean(status.tokens?.model || config?.runner?.model),
|
|
160
|
+
requestedParentModel: config?.runner?.model ?? null,
|
|
161
|
+
tokens: status.tokens ?? null,
|
|
162
|
+
modelCalls: status.tokens?.calls ?? [],
|
|
163
|
+
tokenBreakdown: status.tokens ? {
|
|
164
|
+
inputTokens,
|
|
165
|
+
cachedInputTokens,
|
|
166
|
+
freshInputTokens: Math.max(0, inputTokens - cachedInputTokens),
|
|
167
|
+
cacheWriteInputTokens: Number(status.tokens.cacheWriteInputTokens ?? 0),
|
|
168
|
+
outputTokens: Number(status.tokens.outputTokens ?? 0),
|
|
169
|
+
reasoningOutputTokens: Number(status.tokens.reasoningOutputTokens ?? 0),
|
|
170
|
+
...(typeof status.tokens.totalCostUsd === 'number' ? { reportedCostUsd: status.tokens.totalCostUsd } : {}),
|
|
171
|
+
} : null,
|
|
172
|
+
stories,
|
|
173
|
+
srcLoc: stats.lines,
|
|
174
|
+
sourceFiles: stats.files,
|
|
175
|
+
finalVerification: {
|
|
176
|
+
buildExitCode: build.status,
|
|
177
|
+
testExitCode: tests.status,
|
|
178
|
+
finalTestCount: testCount,
|
|
179
|
+
originalAcceptanceTestsExitCode: hiddenTests.status,
|
|
180
|
+
originalAcceptanceTestCount: Number(acceptanceManifest?.acceptanceTestCount ?? 4),
|
|
181
|
+
},
|
|
182
|
+
}
|
|
183
|
+
validateResult(result)
|
|
184
|
+
mkdirSync(join(benchDir, 'results'), { recursive: true })
|
|
185
|
+
const output = join(benchDir, 'results', `${fixture}-codex-routing-${routing}-${stamp}.json`)
|
|
186
|
+
writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`)
|
|
187
|
+
console.error(`[bench-large] done: ${output}`)
|
|
188
|
+
console.log(JSON.stringify(result, null, 2))
|
package/bench/run.mjs
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Yoke benchmark harness.
|
|
3
3
|
//
|
|
4
|
-
// node bench/run.mjs --runner=claude [--
|
|
4
|
+
// node bench/run.mjs --runner=claude [--fixture=string-kit] [--routing=on|off|config]
|
|
5
5
|
//
|
|
6
6
|
// Copies the fixture into bench/.runs/<runner>-<stamp>, git-inits it, then drives
|
|
7
7
|
// `yoke loop run --json` and measures from the OUTSIDE (the loop itself records no
|
|
8
|
-
// durations): per-story wall-clock from NDJSON event timestamps, tokens/model from
|
|
9
|
-
//
|
|
8
|
+
// durations): per-story wall-clock from NDJSON event timestamps, tokens/model from
|
|
9
|
+
// provider telemetry when available, and quality as the fixture's own
|
|
10
10
|
// pre-written tests — run per story AFTER the loop finishes, on the final tree.
|
|
11
11
|
import { spawn, spawnSync } from 'node:child_process'
|
|
12
|
-
import { cpSync, mkdirSync, readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs'
|
|
12
|
+
import { cpSync, existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs'
|
|
13
13
|
import { join, dirname } from 'node:path'
|
|
14
14
|
import { fileURLToPath } from 'node:url'
|
|
15
|
-
import { validateResult } from './result-schema.mjs'
|
|
15
|
+
import { validateResult } from './result-schema.mjs'
|
|
16
|
+
import { parse } from 'yaml'
|
|
16
17
|
|
|
17
18
|
const benchDir = dirname(fileURLToPath(import.meta.url))
|
|
18
19
|
const repoRoot = dirname(benchDir)
|
|
@@ -26,16 +27,28 @@ const args = Object.fromEntries(
|
|
|
26
27
|
)
|
|
27
28
|
const runner = args.runner
|
|
28
29
|
if (!['claude', 'codex', 'gemini'].includes(runner)) {
|
|
29
|
-
console.error('usage: node bench/run.mjs --runner=<claude|codex|gemini> [--max=6] [--timeout=10] [--label=note]')
|
|
30
|
+
console.error('usage: node bench/run.mjs --runner=<claude|codex|gemini> [--fixture=string-kit] [--routing=on|off|config] [--run-root=path] [--unsafe] [--max=6] [--timeout=10] [--label=note]')
|
|
30
31
|
process.exit(2)
|
|
31
32
|
}
|
|
32
|
-
const max = Number(args.max ?? 6)
|
|
33
|
-
const timeout = Number(args.timeout ?? 10)
|
|
34
|
-
|
|
35
|
-
const
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
33
|
+
const max = Number(args.max ?? 6)
|
|
34
|
+
const timeout = Number(args.timeout ?? 10)
|
|
35
|
+
const fixture = String(args.fixture ?? 'string-kit')
|
|
36
|
+
const fixtureDir = join(benchDir, 'fixtures', fixture)
|
|
37
|
+
const routing = String(args.routing ?? 'config')
|
|
38
|
+
if (!existsSync(fixtureDir)) {
|
|
39
|
+
console.error(`unknown fixture: ${fixture}`)
|
|
40
|
+
process.exit(2)
|
|
41
|
+
}
|
|
42
|
+
if (!['on', 'off', 'config'].includes(routing)) {
|
|
43
|
+
console.error('routing must be on, off, or config')
|
|
44
|
+
process.exit(2)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)
|
|
48
|
+
const runRoot = args['run-root'] ? String(args['run-root']) : join(benchDir, '.runs')
|
|
49
|
+
const runDir = join(runRoot, `${fixture}-${runner}-routing-${routing}-${stamp}`)
|
|
50
|
+
mkdirSync(runDir, { recursive: true })
|
|
51
|
+
cpSync(fixtureDir, runDir, { recursive: true })
|
|
39
52
|
|
|
40
53
|
const git = (...a) => {
|
|
41
54
|
const r = spawnSync('git', ['-C', runDir, ...a], { encoding: 'utf8' })
|
|
@@ -49,10 +62,12 @@ git('-c', 'user.name=bench', '-c', 'user.email=bench@yoke', 'commit', '-q', '-m'
|
|
|
49
62
|
const env = { ...process.env }
|
|
50
63
|
for (const k of Object.keys(env)) if (k.startsWith('CLAUDE_CODE') || k === 'CLAUDECODE') delete env[k]
|
|
51
64
|
|
|
52
|
-
console.error(`[bench] ${runner} → ${runDir}`)
|
|
65
|
+
console.error(`[bench] ${runner} · fixture=${fixture} · routing=${routing} → ${runDir}`)
|
|
53
66
|
const t0 = Date.now()
|
|
54
67
|
const events = []
|
|
55
|
-
const
|
|
68
|
+
const routingFlag = routing === 'on' ? ['--routing'] : routing === 'off' ? ['--no-routing'] : []
|
|
69
|
+
const permissionFlag = args.unsafe ? ['--unsafe'] : []
|
|
70
|
+
const child = spawn(process.execPath, [cli, 'loop', 'run', runDir, '--json', `--runner=${runner}`, `--max=${max}`, `--timeout=${timeout}`, ...routingFlag, ...permissionFlag], {
|
|
56
71
|
env, stdio: ['ignore', 'pipe', 'inherit'],
|
|
57
72
|
})
|
|
58
73
|
let buf = ''
|
|
@@ -70,7 +85,10 @@ const exitCode = await new Promise(res => child.on('close', res))
|
|
|
70
85
|
const wallClockMs = Date.now() - t0
|
|
71
86
|
|
|
72
87
|
// Per-story duration: first event mentioning the story -> first event mentioning the next story (or end).
|
|
73
|
-
const
|
|
88
|
+
const prd = parse(readFileSync(join(runDir, '.yoke', 'prd.yaml'), 'utf8'))
|
|
89
|
+
const storyIds = Array.isArray(prd) ? prd.map(story => String(story.id)) : []
|
|
90
|
+
const benchConfig = parse(readFileSync(join(runDir, '.yoke', 'config.yaml'), 'utf8'))
|
|
91
|
+
const requestedParentModel = benchConfig?.runner?.model ?? null
|
|
74
92
|
const firstSeen = {}
|
|
75
93
|
for (const e of events) if (e.story && !(e.story in firstSeen)) firstSeen[e.story] = e.at
|
|
76
94
|
const stories = storyIds.map((id, idx) => {
|
|
@@ -96,12 +114,13 @@ const loc = (dir) => readdirSync(dir).reduce((n, f) => {
|
|
|
96
114
|
|
|
97
115
|
const result = {
|
|
98
116
|
schemaVersion: 1,
|
|
99
|
-
fixtureVersion:
|
|
117
|
+
fixtureVersion: `${fixture}@1`,
|
|
100
118
|
runner,
|
|
101
119
|
sampleLabel: String(args.label ?? `${runner}-${stamp}`),
|
|
102
120
|
permissionProfile: args.unsafe ? 'unsafe' : 'safe',
|
|
103
121
|
yokeVersion: JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8')).version,
|
|
104
|
-
fixture
|
|
122
|
+
fixture,
|
|
123
|
+
routing,
|
|
105
124
|
startedAt: new Date(t0).toISOString(),
|
|
106
125
|
wallClockMs,
|
|
107
126
|
exitCode,
|
|
@@ -113,15 +132,17 @@ const result = {
|
|
|
113
132
|
finalTestsPass: stories.every(story => story.finalTestsPass),
|
|
114
133
|
progress: last.progress ?? null,
|
|
115
134
|
usageAvailable: Number(status.tokens?.inputTokens ?? 0) + Number(status.tokens?.outputTokens ?? 0) > 0,
|
|
116
|
-
modelAvailable: typeof status.tokens?.model === 'string' && status.tokens.model !== '<synthetic>',
|
|
117
|
-
|
|
135
|
+
modelAvailable: (typeof status.tokens?.model === 'string' && status.tokens.model !== '<synthetic>') || requestedParentModel !== null,
|
|
136
|
+
requestedParentModel,
|
|
137
|
+
tokens: status.tokens ?? null,
|
|
138
|
+
modelCalls: status.tokens?.calls ?? [],
|
|
118
139
|
stories,
|
|
119
140
|
srcLoc: loc(join(runDir, 'src')),
|
|
120
141
|
}
|
|
121
142
|
validateResult(result)
|
|
122
143
|
|
|
123
144
|
mkdirSync(join(benchDir, 'results'), { recursive: true })
|
|
124
|
-
const out = join(benchDir, 'results', `${runner}-${stamp}.json`)
|
|
145
|
+
const out = join(benchDir, 'results', `${fixture}-${runner}-routing-${routing}-${stamp}.json`)
|
|
125
146
|
writeFileSync(out, JSON.stringify(result, null, 2) + '\n')
|
|
126
147
|
console.error(`[bench] done: ${out}`)
|
|
127
148
|
console.log(JSON.stringify(result, null, 2))
|
package/dist/agents/providers.js
CHANGED
|
@@ -14,10 +14,27 @@ const argsFor = (agent, permissions) => {
|
|
|
14
14
|
return ['exec', '--full-auto', '--json'];
|
|
15
15
|
}
|
|
16
16
|
if (permissions === 'unsafe')
|
|
17
|
-
return ['--yolo'
|
|
17
|
+
return ['--yolo'];
|
|
18
18
|
const approval = permissions === 'read-only' ? 'plan' : 'auto_edit';
|
|
19
|
-
return ['--approval-mode', approval, '--sandbox'
|
|
19
|
+
return ['--approval-mode', approval, '--sandbox'];
|
|
20
20
|
};
|
|
21
|
-
export function buildProviderInvocation(agent, prompt, cwd, permissions = 'safe') {
|
|
22
|
-
|
|
21
|
+
export function buildProviderInvocation(agent, prompt, cwd, permissions = 'safe', selection = {}) {
|
|
22
|
+
const args = argsFor(agent, permissions);
|
|
23
|
+
if (selection.model)
|
|
24
|
+
args.push('--model', selection.model);
|
|
25
|
+
if (selection.reasoningEffort) {
|
|
26
|
+
if (agent === 'claude')
|
|
27
|
+
args.push('--effort', selection.reasoningEffort);
|
|
28
|
+
else if (agent === 'codex')
|
|
29
|
+
args.push('--config', `model_reasoning_effort=${selection.reasoningEffort}`);
|
|
30
|
+
}
|
|
31
|
+
if (agent === 'codex' && selection.nativeMultiAgent === false)
|
|
32
|
+
args.push('--disable', 'multi_agent');
|
|
33
|
+
if (selection.bare) {
|
|
34
|
+
if (agent === 'codex')
|
|
35
|
+
args.push('--ignore-user-config');
|
|
36
|
+
else if (agent === 'claude')
|
|
37
|
+
args.push('--bare');
|
|
38
|
+
}
|
|
39
|
+
return { command: agent, args, input: prompt, cwd };
|
|
23
40
|
}
|
package/dist/agents/telemetry.js
CHANGED
|
@@ -1,30 +1,69 @@
|
|
|
1
1
|
const finite = (value) => typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
2
2
|
export function parseProviderTelemetry(agent, lines) {
|
|
3
3
|
let inputTokens;
|
|
4
|
+
let cachedInputTokens;
|
|
5
|
+
let cacheWriteInputTokens;
|
|
4
6
|
let outputTokens;
|
|
7
|
+
let reasoningOutputTokens;
|
|
8
|
+
let totalCostUsd;
|
|
5
9
|
let model;
|
|
6
10
|
for (const line of lines) {
|
|
7
|
-
let
|
|
11
|
+
let parsed;
|
|
8
12
|
try {
|
|
9
|
-
|
|
13
|
+
parsed = JSON.parse(line);
|
|
10
14
|
}
|
|
11
15
|
catch {
|
|
12
16
|
continue;
|
|
13
17
|
}
|
|
18
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed))
|
|
19
|
+
continue;
|
|
20
|
+
const event = parsed;
|
|
14
21
|
const message = event.message && typeof event.message === 'object' ? event.message : undefined;
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
|
|
22
|
+
const stats = event.stats && typeof event.stats === 'object' ? event.stats : undefined;
|
|
23
|
+
const usage = (event.usage && typeof event.usage === 'object'
|
|
24
|
+
? event.usage
|
|
25
|
+
: message?.usage && typeof message.usage === 'object'
|
|
26
|
+
? message.usage
|
|
27
|
+
: stats?.usage && typeof stats.usage === 'object'
|
|
28
|
+
? stats.usage
|
|
29
|
+
: stats);
|
|
30
|
+
const models = stats?.models && typeof stats.models === 'object' ? stats.models : undefined;
|
|
31
|
+
const firstModel = models ? Object.entries(models)[0] : undefined;
|
|
32
|
+
const modelUsage = firstModel?.[1] && typeof firstModel[1] === 'object' ? firstModel[1] : undefined;
|
|
33
|
+
const nestedModelTokens = modelUsage?.tokens && typeof modelUsage.tokens === 'object' ? modelUsage.tokens : undefined;
|
|
34
|
+
const source = nestedModelTokens ?? modelUsage ?? usage;
|
|
35
|
+
const inValue = finite(source?.input_tokens ?? source?.inputTokens ?? source?.prompt_tokens ?? source?.promptTokenCount ?? source?.input);
|
|
36
|
+
const cachedValue = finite(source?.cached_input_tokens ?? source?.cache_read_input_tokens ?? source?.cachedInputTokens ?? source?.cachedContentTokenCount ?? source?.cached);
|
|
37
|
+
const cacheWriteValue = finite(source?.cache_write_input_tokens ?? source?.cache_creation_input_tokens ?? source?.cacheWriteInputTokens ?? source?.cacheWrite);
|
|
38
|
+
const outValue = finite(source?.output_tokens ?? source?.outputTokens ?? source?.completion_tokens ?? source?.candidatesTokenCount ?? source?.output);
|
|
39
|
+
const reasoningValue = finite(source?.reasoning_output_tokens ?? source?.reasoningOutputTokens ?? source?.thoughtsTokenCount ?? source?.thoughts);
|
|
18
40
|
if (inValue !== undefined)
|
|
19
41
|
inputTokens = inValue;
|
|
42
|
+
if (cachedValue !== undefined)
|
|
43
|
+
cachedInputTokens = cachedValue;
|
|
44
|
+
if (cacheWriteValue !== undefined)
|
|
45
|
+
cacheWriteInputTokens = cacheWriteValue;
|
|
20
46
|
if (outValue !== undefined)
|
|
21
47
|
outputTokens = outValue;
|
|
22
|
-
|
|
48
|
+
if (reasoningValue !== undefined)
|
|
49
|
+
reasoningOutputTokens = reasoningValue;
|
|
50
|
+
const costValue = finite(event.total_cost_usd ?? event.totalCostUsd ?? stats?.total_cost_usd);
|
|
51
|
+
if (costValue !== undefined)
|
|
52
|
+
totalCostUsd = costValue;
|
|
53
|
+
const eventModel = event.model ?? message?.model ?? firstModel?.[0];
|
|
23
54
|
if (typeof eventModel === 'string' && eventModel)
|
|
24
55
|
model = eventModel;
|
|
25
56
|
}
|
|
26
57
|
if (inputTokens === undefined && outputTokens === undefined)
|
|
27
58
|
return { usageAvailable: false };
|
|
28
|
-
const tokens = {
|
|
59
|
+
const tokens = {
|
|
60
|
+
inputTokens: inputTokens ?? 0,
|
|
61
|
+
...(cachedInputTokens !== undefined ? { cachedInputTokens } : {}),
|
|
62
|
+
...(cacheWriteInputTokens !== undefined ? { cacheWriteInputTokens } : {}),
|
|
63
|
+
outputTokens: outputTokens ?? 0,
|
|
64
|
+
...(reasoningOutputTokens !== undefined ? { reasoningOutputTokens } : {}),
|
|
65
|
+
...(totalCostUsd !== undefined ? { totalCostUsd } : {}),
|
|
66
|
+
...(model ? { model } : {}),
|
|
67
|
+
};
|
|
29
68
|
return { usageAvailable: true, tokens };
|
|
30
69
|
}
|
package/dist/cli.js
CHANGED
|
@@ -86,10 +86,11 @@ export function main(argv) {
|
|
|
86
86
|
return 1;
|
|
87
87
|
}
|
|
88
88
|
const loop = rest.includes('--loop') ? true : rest.includes('--no-loop') ? false : undefined;
|
|
89
|
+
const routing = rest.includes('--routing') ? true : rest.includes('--no-routing') ? false : undefined;
|
|
89
90
|
return runSetup(targetDir, {
|
|
90
91
|
host: hostArg, agents, runner: runnerArg,
|
|
91
92
|
codeGraph: graphArg,
|
|
92
|
-
loop, decisionPolicy: policyArg,
|
|
93
|
+
loop, routing, decisionPolicy: policyArg,
|
|
93
94
|
interactive: rest.includes('--yes') ? false : undefined,
|
|
94
95
|
});
|
|
95
96
|
}
|
|
@@ -254,8 +255,8 @@ export function main(argv) {
|
|
|
254
255
|
return 1;
|
|
255
256
|
}
|
|
256
257
|
const maxArg = rest.find(a => a.startsWith('--max='));
|
|
257
|
-
const rawMax = maxArg ? Number(maxArg.slice('--max='.length)) :
|
|
258
|
-
if (!Number.
|
|
258
|
+
const rawMax = maxArg ? Number(maxArg.slice('--max='.length)) : undefined;
|
|
259
|
+
if (rawMax !== undefined && (!Number.isInteger(rawMax) || rawMax <= 0)) {
|
|
259
260
|
console.error(`Invalid --max value: ${maxArg}`);
|
|
260
261
|
return 1;
|
|
261
262
|
}
|
|
@@ -286,6 +287,7 @@ export function main(argv) {
|
|
|
286
287
|
return 1;
|
|
287
288
|
}
|
|
288
289
|
const json = rest.includes('--json');
|
|
290
|
+
const routing = rest.includes('--routing') ? true : rest.includes('--no-routing') ? false : undefined;
|
|
289
291
|
const toArg = rest.find(a => a.startsWith('--timeout='));
|
|
290
292
|
let timeoutMinutes;
|
|
291
293
|
if (toArg) {
|
|
@@ -306,9 +308,9 @@ export function main(argv) {
|
|
|
306
308
|
console.error(`Invalid --decision-policy value: ${dpArg} (expected auto|critical)`);
|
|
307
309
|
return 1;
|
|
308
310
|
}
|
|
309
|
-
return runLoopCommand(targetDir, { maxIterations: rawMax, agent, isolate, parallel, reviewer, review, allowSelfReview, timeoutMinutes, json, onAmbiguity: oaArg, decisionPolicy: dpArg, permissions });
|
|
311
|
+
return runLoopCommand(targetDir, { maxIterations: rawMax, agent, isolate, parallel, reviewer, review, allowSelfReview, timeoutMinutes, json, routing, onAmbiguity: oaArg, decisionPolicy: dpArg, permissions });
|
|
310
312
|
}
|
|
311
|
-
console.log('usage: yoke loop <on|off|status|decision|answer|resume [--discard]|cleanup [--remove-worktrees] [--discard-stale-recovery]|run [--max=N] [--parallel=N] [--runner=<claude|codex|gemini>] [--reviewer=<claude|codex|gemini>] [--review] [--allow-self-review] [--isolate] [--unsafe] [--timeout=<minutes>] [--decision-policy=<auto|critical>] [--json]> [targetDir]');
|
|
313
|
+
console.log('usage: yoke loop <on|off|status|decision|answer|resume [--discard]|cleanup [--remove-worktrees] [--discard-stale-recovery]|run [--max=N] [--parallel=N] [--runner=<claude|codex|gemini>] [--reviewer=<claude|codex|gemini>] [--review] [--allow-self-review] [--routing|--no-routing] [--isolate] [--unsafe] [--timeout=<minutes>] [--decision-policy=<auto|critical>] [--json]> [targetDir]');
|
|
312
314
|
return 1;
|
|
313
315
|
}
|
|
314
316
|
case 'new': {
|
package/dist/loop/decision.js
CHANGED
|
@@ -58,7 +58,8 @@ export const DecisionResumeSchema = z.object({
|
|
|
58
58
|
storyId: singleLine(128),
|
|
59
59
|
requestId: z.string().regex(/^[a-f0-9]{16}$/u),
|
|
60
60
|
answered: z.boolean().optional(),
|
|
61
|
-
|
|
61
|
+
/** Absent preserves the default unlimited run across a critical-decision resume. */
|
|
62
|
+
maxIterations: z.number().int().positive().optional(),
|
|
62
63
|
agent: z.enum(['claude', 'codex', 'gemini']).optional(),
|
|
63
64
|
isolate: z.boolean().optional(),
|
|
64
65
|
reviewer: z.enum(['claude', 'codex', 'gemini']).optional(),
|
|
@@ -70,6 +71,7 @@ export const DecisionResumeSchema = z.object({
|
|
|
70
71
|
decisionPolicy: z.enum(['auto', 'critical']).optional(),
|
|
71
72
|
permissions: z.enum(['safe', 'unsafe', 'read-only']).optional(),
|
|
72
73
|
parallel: z.number().int().positive().optional(),
|
|
74
|
+
routing: z.boolean().optional(),
|
|
73
75
|
projectId: z.string().regex(/^[a-f0-9]{16}$/u).optional(),
|
|
74
76
|
baseCommit: z.string().regex(/^[a-f0-9]{40,64}$/u).optional(),
|
|
75
77
|
prdHash: z.string().regex(/^[a-f0-9]{64}$/u).optional(),
|
package/dist/loop/loop.js
CHANGED
|
@@ -138,6 +138,7 @@ export function runLoop(opts) {
|
|
|
138
138
|
reporter.phase('verifying');
|
|
139
139
|
const verdict = runGate(opts.verify, wt, story.id);
|
|
140
140
|
if (!verdict.passed) {
|
|
141
|
+
result.routing?.recordOutcome(false);
|
|
141
142
|
const base = result.success
|
|
142
143
|
? `story ${story.id} did not verify: ${verdict.summary}`
|
|
143
144
|
: `story ${story.id} runner failed (${result.summary}) and verify is red: ${verdict.summary}`;
|
|
@@ -149,6 +150,7 @@ export function runLoop(opts) {
|
|
|
149
150
|
reporter.phase('perf');
|
|
150
151
|
const perfVerdict = runGate(opts.perf, wt, story.id);
|
|
151
152
|
if (!perfVerdict.passed) {
|
|
153
|
+
result.routing?.recordOutcome(false);
|
|
152
154
|
const reason = blockReason(`story ${story.id} exceeded its performance budget: ${perfVerdict.summary}`, opts.targetDir, opts.git);
|
|
153
155
|
reporter.blocked(reason);
|
|
154
156
|
return { status: 'blocked', iterations, reason, finalProgress: progress(stories) };
|
|
@@ -158,6 +160,7 @@ export function runLoop(opts) {
|
|
|
158
160
|
reporter.phase('audit');
|
|
159
161
|
const auditVerdict = runGate(opts.audit, wt, story.id);
|
|
160
162
|
if (!auditVerdict.passed) {
|
|
163
|
+
result.routing?.recordOutcome(false);
|
|
161
164
|
const reason = blockReason(`story ${story.id} failed security audit: ${auditVerdict.summary}`, opts.targetDir, opts.git);
|
|
162
165
|
reporter.blocked(reason);
|
|
163
166
|
return { status: 'blocked', iterations, reason, finalProgress: progress(stories) };
|
|
@@ -170,11 +173,13 @@ export function runLoop(opts) {
|
|
|
170
173
|
reporter.phase('reviewing');
|
|
171
174
|
const reviewResult = opts.review({ targetDir: wt, story });
|
|
172
175
|
if (!reviewResult.success) {
|
|
176
|
+
result.routing?.recordOutcome(false);
|
|
173
177
|
const reason = blockReason(`story ${story.id} rejected in review: ${reviewResult.summary}`, opts.targetDir, opts.git);
|
|
174
178
|
reporter.blocked(reason);
|
|
175
179
|
return { status: 'blocked', iterations, reason, finalProgress: progress(stories) };
|
|
176
180
|
}
|
|
177
181
|
}
|
|
182
|
+
result.routing?.recordOutcome(true);
|
|
178
183
|
// The worktree is a checkout of committed HEAD, so the agent above reads
|
|
179
184
|
// context from HEAD's .yoke/context — commit context changes for --isolate
|
|
180
185
|
// to honour them. We write the decision here so `integrate` carries it back.
|
|
@@ -234,6 +239,7 @@ export function runLoop(opts) {
|
|
|
234
239
|
reporter.phase('verifying');
|
|
235
240
|
const verdict = runGate(opts.verify, opts.targetDir, story.id);
|
|
236
241
|
if (!verdict.passed) {
|
|
242
|
+
result.routing?.recordOutcome(false);
|
|
237
243
|
const base = result.success
|
|
238
244
|
? `story ${story.id} did not verify: ${verdict.summary}`
|
|
239
245
|
: `story ${story.id} runner failed (${result.summary}) and verify is red: ${verdict.summary}`;
|
|
@@ -250,6 +256,7 @@ export function runLoop(opts) {
|
|
|
250
256
|
reporter.phase('perf');
|
|
251
257
|
const perfVerdict = runGate(opts.perf, opts.targetDir, story.id);
|
|
252
258
|
if (!perfVerdict.passed) {
|
|
259
|
+
result.routing?.recordOutcome(false);
|
|
253
260
|
const reason = blockReason(`story ${story.id} exceeded its performance budget: ${perfVerdict.summary}`, opts.targetDir, opts.git);
|
|
254
261
|
reporter.blocked(reason);
|
|
255
262
|
return { status: 'blocked', iterations, reason, finalProgress: progress(stories) };
|
|
@@ -259,6 +266,7 @@ export function runLoop(opts) {
|
|
|
259
266
|
reporter.phase('audit');
|
|
260
267
|
const auditVerdict = runGate(opts.audit, opts.targetDir, story.id);
|
|
261
268
|
if (!auditVerdict.passed) {
|
|
269
|
+
result.routing?.recordOutcome(false);
|
|
262
270
|
const reason = blockReason(`story ${story.id} failed security audit: ${auditVerdict.summary}`, opts.targetDir, opts.git);
|
|
263
271
|
reporter.blocked(reason);
|
|
264
272
|
return { status: 'blocked', iterations, reason, finalProgress: progress(stories) };
|
|
@@ -271,6 +279,7 @@ export function runLoop(opts) {
|
|
|
271
279
|
reporter.phase('reviewing');
|
|
272
280
|
const reviewResult = opts.review({ targetDir: opts.targetDir, story });
|
|
273
281
|
if (!reviewResult.success) {
|
|
282
|
+
result.routing?.recordOutcome(false);
|
|
274
283
|
const reason = blockReason(`story ${story.id} rejected in review: ${reviewResult.summary}`, opts.targetDir, opts.git);
|
|
275
284
|
reporter.blocked(reason);
|
|
276
285
|
return {
|
|
@@ -281,6 +290,7 @@ export function runLoop(opts) {
|
|
|
281
290
|
};
|
|
282
291
|
}
|
|
283
292
|
}
|
|
293
|
+
result.routing?.recordOutcome(true);
|
|
284
294
|
reporter.phase('committing');
|
|
285
295
|
const dec = appendDecision(contextDir(opts.targetDir), {
|
|
286
296
|
storyId: story.id,
|
package/dist/loop/reporter.js
CHANGED
|
@@ -159,10 +159,19 @@ export function makeReporter(dir, opts = {}, now = () => new Date()) {
|
|
|
159
159
|
},
|
|
160
160
|
addTokens(usage) {
|
|
161
161
|
const model = usage.model ?? tokens?.model;
|
|
162
|
+
const cachedInputTokens = (tokens?.cachedInputTokens ?? 0) + (usage.cachedInputTokens ?? 0);
|
|
163
|
+
const cacheWriteInputTokens = (tokens?.cacheWriteInputTokens ?? 0) + (usage.cacheWriteInputTokens ?? 0);
|
|
164
|
+
const reasoningOutputTokens = (tokens?.reasoningOutputTokens ?? 0) + (usage.reasoningOutputTokens ?? 0);
|
|
165
|
+
const totalCostUsd = (tokens?.totalCostUsd ?? 0) + (usage.totalCostUsd ?? 0);
|
|
162
166
|
tokens = {
|
|
163
167
|
inputTokens: (tokens?.inputTokens ?? 0) + usage.inputTokens,
|
|
168
|
+
...((tokens?.cachedInputTokens !== undefined || usage.cachedInputTokens !== undefined) ? { cachedInputTokens } : {}),
|
|
169
|
+
...((tokens?.cacheWriteInputTokens !== undefined || usage.cacheWriteInputTokens !== undefined) ? { cacheWriteInputTokens } : {}),
|
|
164
170
|
outputTokens: (tokens?.outputTokens ?? 0) + usage.outputTokens,
|
|
171
|
+
...((tokens?.reasoningOutputTokens !== undefined || usage.reasoningOutputTokens !== undefined) ? { reasoningOutputTokens } : {}),
|
|
172
|
+
...((tokens?.totalCostUsd !== undefined || usage.totalCostUsd !== undefined) ? { totalCostUsd } : {}),
|
|
165
173
|
...(model ? { model } : {}),
|
|
174
|
+
...((tokens?.calls || usage.calls) ? { calls: [...(tokens?.calls ?? []), ...(usage.calls ?? [])] } : {}),
|
|
166
175
|
};
|
|
167
176
|
},
|
|
168
177
|
};
|