amicus 4.2.1 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +14 -1
- package/README.md +7 -4
- package/bin/amicus.js +5 -0
- package/package.json +1 -1
- package/schemas/council-run-live.schema.json +33 -0
- package/schemas/event.schema.json +15 -0
- package/schemas/progress.schema.json +24 -0
- package/schemas/run-live.schema.json +15 -0
- package/schemas/spend.schema.json +26 -1
- package/schemas/wave-live.schema.json +15 -0
- package/src/cli-handlers-council-run.js +61 -5
- package/src/cli-handlers-run.js +26 -0
- package/src/cli-handlers-spend.js +62 -27
- package/src/cli-handlers-watch.js +89 -0
- package/src/cli.js +58 -1
- package/src/council/run-chair.js +10 -2
- package/src/council/run-debate.js +5 -1
- package/src/council/run-launch.js +14 -1
- package/src/council/run-stages.js +13 -0
- package/src/council/run.js +32 -4
- package/src/headless.js +9 -1
- package/src/mcp-council-awareness.js +46 -1
- package/src/mcp-council-run.js +28 -4
- package/src/mcp-notify.js +54 -0
- package/src/mcp-server.js +51 -1
- package/src/mcp-spend.js +125 -0
- package/src/mcp-tools.js +39 -0
- package/src/mcp-wait.js +28 -2
- package/src/observe/events.js +156 -0
- package/src/observe/follow.js +26 -0
- package/src/observe/live-doc.js +38 -0
- package/src/observe/on-complete.js +117 -0
- package/src/observe/watch-render.js +149 -0
- package/src/sidecar/continue.js +32 -0
- package/src/sidecar/fallback-chains.js +65 -0
- package/src/sidecar/fanout-leg-fallback.js +189 -0
- package/src/sidecar/fanout-leg.js +58 -26
- package/src/sidecar/fanout-retry.js +208 -0
- package/src/sidecar/fanout-validate.js +42 -4
- package/src/sidecar/fanout.js +50 -30
- package/src/sidecar/progress.js +5 -0
- package/src/sidecar/resume.js +12 -0
- package/src/sidecar/start.js +13 -1
- package/src/spend-query.js +104 -0
- package/src/utils/error-classify.js +31 -0
- package/src/utils/model-tiers.js +1 -1
- package/src/utils/spend-ledger.js +24 -1
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
// src/observe/on-complete.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module observe/on-complete
|
|
6
|
+
* CLI --on-complete exec hook (spec 5.3). The command is user-authored on the
|
|
7
|
+
* command line of THIS invocation — the same trust level as typing it into the
|
|
8
|
+
* shell. Amicus never sources hook commands from config/briefings/model output
|
|
9
|
+
* (D8 withdrawn) and never interpolates anything into the command string. The
|
|
10
|
+
* payload rides via ENVIRONMENT only, ids/paths only — never model-generated
|
|
11
|
+
* text — so untrusted previews can never enter a user's shell pipeline. The
|
|
12
|
+
* hook can never change the run's exit code, docs, or events. CLI-only: MCP
|
|
13
|
+
* never gets exec (Task 15 gives MCP a notify-only hook).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const HOOK_TIMEOUT_MS = Number(process.env.AMICUS_HOOK_TIMEOUT_MS) || 60000;
|
|
17
|
+
|
|
18
|
+
/** ids/paths ONLY (spec 5.3). Every value is a string; cost/paths default ''. */
|
|
19
|
+
function buildHookEnv(info) {
|
|
20
|
+
return {
|
|
21
|
+
AMICUS_TASK_ID: String(info.taskId || ''),
|
|
22
|
+
AMICUS_TYPE: String(info.type || ''),
|
|
23
|
+
AMICUS_STATUS: String(info.status || ''),
|
|
24
|
+
AMICUS_EXIT_CODE: String(info.exitCode !== null && info.exitCode !== undefined ? info.exitCode : ''),
|
|
25
|
+
AMICUS_RESULT_FILE: String(info.resultFile || ''),
|
|
26
|
+
AMICUS_EVENTS_FILE: String(info.eventsFile || ''),
|
|
27
|
+
AMICUS_COST: String(info.cost || ''),
|
|
28
|
+
AMICUS_PROJECT: String(info.project || ''),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Fire the exec hook once at terminal state. Never throws/rejects; a non-zero
|
|
34
|
+
* exit or timeout is a warning only — the run's exit code/docs/events are
|
|
35
|
+
* never touched by this function.
|
|
36
|
+
*/
|
|
37
|
+
function runOnComplete(cmd, info, deps = {}) {
|
|
38
|
+
const spawn = deps.spawn || require('child_process').spawn;
|
|
39
|
+
const logger = deps.logger || require('../utils/logger').logger;
|
|
40
|
+
const timeoutMs = deps.timeoutMs || HOOK_TIMEOUT_MS;
|
|
41
|
+
return new Promise((resolve) => {
|
|
42
|
+
let child;
|
|
43
|
+
try {
|
|
44
|
+
child = spawn(cmd, {
|
|
45
|
+
shell: true, cwd: info.project || process.cwd(),
|
|
46
|
+
env: { ...process.env, ...buildHookEnv(info) },
|
|
47
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
48
|
+
});
|
|
49
|
+
} catch (e) {
|
|
50
|
+
logger.warn('on-complete hook failed to spawn (run unaffected)', { error: e.message });
|
|
51
|
+
return resolve();
|
|
52
|
+
}
|
|
53
|
+
let done = false;
|
|
54
|
+
const finish = () => { if (!done) { done = true; clearTimeout(timer); resolve(); } };
|
|
55
|
+
const timer = setTimeout(() => {
|
|
56
|
+
logger.warn('on-complete hook timed out — killing (run unaffected)', { timeoutMs });
|
|
57
|
+
try { child.kill(); } catch { /* already gone */ }
|
|
58
|
+
finish();
|
|
59
|
+
}, timeoutMs);
|
|
60
|
+
if (child.stdout) { child.stdout.on('data', (d) => process.stderr.write(d)); }
|
|
61
|
+
if (child.stderr) { child.stderr.on('data', (d) => process.stderr.write(d)); }
|
|
62
|
+
child.on('error', (e) => { logger.warn('on-complete hook error (run unaffected)', { error: e.message }); finish(); });
|
|
63
|
+
child.on('close', (code) => {
|
|
64
|
+
if (code && code !== 0) { logger.warn('on-complete hook exited non-zero (run unaffected)', { code }); }
|
|
65
|
+
finish();
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Thin fire helper for fanout's two terminal sites (normal finalize + the
|
|
72
|
+
* all-legs-failed short-circuit). No-ops on a falsy/non-string cmd so call
|
|
73
|
+
* sites stay one line with no guard of their own.
|
|
74
|
+
*/
|
|
75
|
+
async function fireWaveOnComplete(cmd, wave, { waveId, waveDir, wavePath, exitCode, project }, deps) {
|
|
76
|
+
if (!cmd || typeof cmd !== 'string') { return; }
|
|
77
|
+
try {
|
|
78
|
+
const path = require('path');
|
|
79
|
+
const { formatCost } = require('../utils/pricing');
|
|
80
|
+
const { EVENTS_FILE } = require('./events');
|
|
81
|
+
await runOnComplete(cmd, {
|
|
82
|
+
taskId: waveId, type: 'wave', status: wave.status, exitCode,
|
|
83
|
+
resultFile: wavePath, eventsFile: path.join(waveDir, EVENTS_FILE),
|
|
84
|
+
cost: wave.usage && wave.usage.cost ? formatCost(wave.usage.cost) : '',
|
|
85
|
+
project,
|
|
86
|
+
}, deps);
|
|
87
|
+
} catch (e) {
|
|
88
|
+
const logger = (deps && deps.logger) || require('../utils/logger').logger;
|
|
89
|
+
logger.warn('on-complete hook assembly failed (run unaffected)', { error: e.message });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Thin fire helper for council run.js's single finalize choke point. No-ops
|
|
95
|
+
* on a falsy/non-string cmd so the call site stays one line.
|
|
96
|
+
*/
|
|
97
|
+
async function fireCouncilOnComplete(cmd, run, { runId, runDir, exitCode, project }, deps) {
|
|
98
|
+
if (!cmd || typeof cmd !== 'string') { return; }
|
|
99
|
+
try {
|
|
100
|
+
const path = require('path');
|
|
101
|
+
const { formatCost } = require('../utils/pricing');
|
|
102
|
+
const { EVENTS_FILE } = require('./events');
|
|
103
|
+
await runOnComplete(cmd, {
|
|
104
|
+
taskId: runId, type: 'council-run', status: run.status, exitCode,
|
|
105
|
+
resultFile: path.join(runDir, 'run.json'), eventsFile: path.join(runDir, EVENTS_FILE),
|
|
106
|
+
cost: run.usage && run.usage.cost ? formatCost(run.usage.cost) : '',
|
|
107
|
+
project,
|
|
108
|
+
}, deps);
|
|
109
|
+
} catch (e) {
|
|
110
|
+
const logger = (deps && deps.logger) || require('../utils/logger').logger;
|
|
111
|
+
logger.warn('on-complete hook assembly failed (run unaffected)', { error: e.message });
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
module.exports = {
|
|
116
|
+
buildHookEnv, runOnComplete, fireWaveOnComplete, fireCouncilOnComplete, HOOK_TIMEOUT_MS,
|
|
117
|
+
};
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// src/observe/watch-render.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module observe/watch-render
|
|
6
|
+
* Pure renderers + the poll loop for `amicus watch` (spec 5.1). Renderers are
|
|
7
|
+
* pure functions over the composed live doc (wave-progress.js precedent) so the
|
|
8
|
+
* table adds no testing burden beyond string assertions. The loop reads only
|
|
9
|
+
* the data layer: handlers.amicus_status (Surface C) each interval + the events
|
|
10
|
+
* tail (Surface B) for milestone lines. TTY -> in-place refresh table (ANSI
|
|
11
|
+
* erase-line + cursor-up, NO alternate screen — scrollback preserved);
|
|
12
|
+
* non-TTY/--plain -> milestone log lines; --json -> NDJSON.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const { formatCost } = require('../utils/pricing');
|
|
16
|
+
// Single source of truth for "what statuses are terminal" across the
|
|
17
|
+
// observability layer (live-doc.js's markLive uses the SAME set to decide
|
|
18
|
+
// when to stamp view:'live') — redefining it here would risk the two
|
|
19
|
+
// modules drifting, which would make this loop spin forever on a run
|
|
20
|
+
// live-doc already considers finished (or exit early on one still running).
|
|
21
|
+
const { TERMINAL } = require('./live-doc');
|
|
22
|
+
// Single source of truth for the wave status -> exit code mapping (same
|
|
23
|
+
// drift risk as TERMINAL above) — mapExitCode's non-passthrough branch
|
|
24
|
+
// delegates here instead of hand-rolling the complete/partial/else mapping.
|
|
25
|
+
const { waveExitCode } = require('../utils/result-schema');
|
|
26
|
+
|
|
27
|
+
const DASH = '—';
|
|
28
|
+
const legCost = (leg) => (leg.usage && leg.usage.cost ? formatCost(leg.usage.cost) : DASH);
|
|
29
|
+
const legTokens = (leg) => (leg.usage && leg.usage.tokens ? `${leg.usage.tokens.input || 0}/${leg.usage.tokens.output || 0}` : DASH);
|
|
30
|
+
const truncate = (s, n) => { const t = String(s || ''); return t.length > n ? t.slice(0, n - 1) + '…' : t; };
|
|
31
|
+
|
|
32
|
+
const STAGE_MARK = { complete: '✓', running: '▶', pending: '·' };
|
|
33
|
+
|
|
34
|
+
/** The in-place refresh block for a composed wave/council/solo doc. */
|
|
35
|
+
function renderTable(doc, width = 100) {
|
|
36
|
+
const cost = doc.usage && doc.usage.cost ? formatCost(doc.usage.cost) : DASH;
|
|
37
|
+
const head = `${doc.taskId || doc.runId} ${doc.status} ${doc.elapsed || ''} ` +
|
|
38
|
+
(typeof doc.legsTotal === 'number' ? `legs ${doc.legsComplete}/${doc.legsTotal} ` : '') +
|
|
39
|
+
`cost ${cost}`;
|
|
40
|
+
const lines = [head];
|
|
41
|
+
if (Array.isArray(doc.stages)) { // council stage checklist
|
|
42
|
+
lines.push(doc.stages.map((s) => `${STAGE_MARK[s.status] || STAGE_MARK.pending} ${s.name}`).join(' '));
|
|
43
|
+
}
|
|
44
|
+
for (const leg of (doc.legs || [])) {
|
|
45
|
+
const flag = leg.stalled ? ' ⏳stalled' : '';
|
|
46
|
+
lines.push(
|
|
47
|
+
` ${String(leg.model || leg.taskId).padEnd(26)} ${String(leg.phase || leg.status).padEnd(11)} ` +
|
|
48
|
+
`${String(leg.messages || 0).toString().padStart(3)}msg ${legTokens(leg).padStart(11)} ${legCost(leg).padStart(9)} | ` +
|
|
49
|
+
`${truncate(leg.latestPreview, Math.max(10, width - 70))}${flag}`
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
return lines.join('\n');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Non-TTY / --plain milestone lines + a periodic one-line rollup. */
|
|
56
|
+
function renderPlainLines(events, doc) {
|
|
57
|
+
const lines = (events || []).map((e) => {
|
|
58
|
+
switch (e.event) {
|
|
59
|
+
case 'wave-started': return `[wave-started] ${e.id} models=${(e.models || []).join(',')}`;
|
|
60
|
+
case 'leg-started': return `[leg-started] ${e.legId} ${e.model}`;
|
|
61
|
+
case 'leg-fallback': return `[leg-fallback] ${e.legId} ${e.fromModel} -> ${e.toModel} (${e.reason})`;
|
|
62
|
+
case 'leg-terminal': return `[leg-terminal] ${e.legId} ${e.model} ${e.status}`;
|
|
63
|
+
case 'wave-terminal': return `[wave-terminal] ${e.id} ${e.status} exit=${e.exitCode}`;
|
|
64
|
+
case 'run-started': return `[run-started] ${e.id} bench=${(e.bench || []).join(',')}`;
|
|
65
|
+
case 'stage-started': return `[stage-started] ${e.stage}`;
|
|
66
|
+
case 'stage-terminal': return `[stage-terminal] ${e.stage} ${e.status}`;
|
|
67
|
+
case 'run-terminal': return `[run-terminal] ${e.id} ${e.status} exit=${e.exitCode}`;
|
|
68
|
+
default: return `[${e.event}] ${e.id || ''}`;
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
if (doc) {
|
|
72
|
+
const cost = doc.usage && doc.usage.cost ? formatCost(doc.usage.cost) : DASH;
|
|
73
|
+
lines.push(`… ${doc.status} ${typeof doc.legsTotal === 'number' ? `${doc.legsComplete}/${doc.legsTotal} legs ` : ''}cost ${cost}`);
|
|
74
|
+
}
|
|
75
|
+
return lines;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Exit mapping (spec 5.1). Council passes through its recorded exitCode. */
|
|
79
|
+
function mapExitCode(doc) {
|
|
80
|
+
if (doc && typeof doc.exitCode === 'number') { return doc.exitCode; }
|
|
81
|
+
if (!doc) { return 1; }
|
|
82
|
+
return waveExitCode(doc.status);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Stable-stringify diff: emit the composed doc only when it changed. */
|
|
86
|
+
function emitJsonChange(doc, prevText) {
|
|
87
|
+
const text = JSON.stringify(doc);
|
|
88
|
+
return text === prevText ? { emit: false } : { emit: true, text };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* The watch poll loop. DI-injected clock/status/tail for testability.
|
|
93
|
+
* @returns {Promise<number>} exit code
|
|
94
|
+
*/
|
|
95
|
+
async function runWatchLoop(target, args, project, deps = {}) {
|
|
96
|
+
const intervalSec = Math.max(0.5, Number(args.interval) || 2);
|
|
97
|
+
const statusFn = deps.statusFn || ((id, p) => require('../mcp-server').handlers.amicus_status({ taskId: id }, p));
|
|
98
|
+
const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
99
|
+
const isTTY = deps.isTTY !== undefined ? deps.isTTY : process.stdout.isTTY;
|
|
100
|
+
const { createEventTail, EVENTS_FILE } = require('./events');
|
|
101
|
+
const { getSessionDir } = require('../session-manager');
|
|
102
|
+
const path = require('path');
|
|
103
|
+
const eventsFile = target.kind === 'council'
|
|
104
|
+
? path.join(target.runDir, EVENTS_FILE)
|
|
105
|
+
: path.join(getSessionDir(project, target.id), EVENTS_FILE);
|
|
106
|
+
const tail = createEventTail(eventsFile);
|
|
107
|
+
let prevJson = null;
|
|
108
|
+
let prevRollup = null;
|
|
109
|
+
let lastLineCount = 0;
|
|
110
|
+
|
|
111
|
+
for (;;) {
|
|
112
|
+
const res = await statusFn(target.id, project);
|
|
113
|
+
let doc;
|
|
114
|
+
try { doc = JSON.parse(res.content[0].text); } catch { doc = null; }
|
|
115
|
+
const events = tail.poll();
|
|
116
|
+
const isTerminal = doc && TERMINAL.has(doc.status);
|
|
117
|
+
if (args.json) {
|
|
118
|
+
for (const e of events) { process.stdout.write(JSON.stringify(e) + '\n'); }
|
|
119
|
+
if (doc) { const c = emitJsonChange(doc, prevJson); if (c.emit) { process.stdout.write(c.text + '\n'); prevJson = c.text; } }
|
|
120
|
+
} else if (isTTY && !args.plain) {
|
|
121
|
+
if (lastLineCount) { process.stdout.write(`\x1b[${lastLineCount}A\x1b[0J`); }
|
|
122
|
+
const block = renderTable(doc || { status: 'unknown', legs: [] }, process.stdout.columns || 100);
|
|
123
|
+
process.stdout.write(block + '\n');
|
|
124
|
+
lastLineCount = block.split('\n').length;
|
|
125
|
+
} else {
|
|
126
|
+
// Milestone event lines: the tail only yields new events, so these are
|
|
127
|
+
// always fresh — print every tick, unthrottled.
|
|
128
|
+
for (const line of renderPlainLines(events, null)) { process.stdout.write(line + '\n'); }
|
|
129
|
+
// Rollup line: change-only (mirrors the --json path above), so a
|
|
130
|
+
// multi-minute --plain watch doesn't spam an identical line every
|
|
131
|
+
// interval. Always printed on the terminal tick so the final state
|
|
132
|
+
// is never silently swallowed.
|
|
133
|
+
if (doc) {
|
|
134
|
+
const rollup = renderPlainLines([], doc)[0];
|
|
135
|
+
if (rollup !== prevRollup || isTerminal) {
|
|
136
|
+
process.stdout.write(rollup + '\n');
|
|
137
|
+
prevRollup = rollup;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
if (isTerminal) {
|
|
142
|
+
if (args.json) { process.stdout.write(JSON.stringify(doc) + '\n'); }
|
|
143
|
+
return mapExitCode(doc);
|
|
144
|
+
}
|
|
145
|
+
await sleep(intervalSec * 1000);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
module.exports = { renderTable, renderPlainLines, mapExitCode, emitJsonChange, runWatchLoop, DASH };
|
package/src/sidecar/continue.js
CHANGED
|
@@ -116,6 +116,26 @@ function createContinueSessionMetadata(taskId, project, options, oldTaskId) {
|
|
|
116
116
|
return sessionDir;
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
+
/**
|
|
120
|
+
* Resolve a reopened session's usage, write it onto metadata, and append one
|
|
121
|
+
* attributed ledger row. Mirrors start.js's finalize (the only sites that
|
|
122
|
+
* dropped usage - BACKLOG.md:280). Best-effort ledger append; never throws.
|
|
123
|
+
* @returns {{usage: object|null}}
|
|
124
|
+
*/
|
|
125
|
+
function finalizeSpendForReopen({ taskId, model, mode, op, result, status, project, metadata }, ctx = {}) {
|
|
126
|
+
const { resolveUsage } = require('../utils/pricing');
|
|
127
|
+
const usage = result && result.usage ? resolveUsage({ model, usageTotals: result.usage }) : null;
|
|
128
|
+
if (usage) {
|
|
129
|
+
metadata.usage = usage; // buildRunResult surfaces metadata.usage into the --json doc for free
|
|
130
|
+
try {
|
|
131
|
+
const { appendSpend } = require('../utils/spend-ledger');
|
|
132
|
+
const gateway = metadata.gateway || (String(model).startsWith('openrouter/') ? 'openrouter' : 'direct');
|
|
133
|
+
appendSpend({ taskId, model, mode, usage, op, status, project, gateway }, ctx);
|
|
134
|
+
} catch { /* best-effort */ }
|
|
135
|
+
}
|
|
136
|
+
return { usage };
|
|
137
|
+
}
|
|
138
|
+
|
|
119
139
|
/**
|
|
120
140
|
* Continue from a previous sidecar session - Spec Reference: §4.4, §8.5
|
|
121
141
|
* @returns {Promise<number>} process exit code
|
|
@@ -246,6 +266,17 @@ async function continueSidecar(options) {
|
|
|
246
266
|
} else {
|
|
247
267
|
finalizeSession(sessionDir, summary, project, meta, { quietStdout: json, status: terminal.status });
|
|
248
268
|
}
|
|
269
|
+
// v4.3: attribute continue spend (C9/E4). Reload meta, write usage + append a
|
|
270
|
+
// ledger row (status: statusFromResult, matching start.js — not terminal.status).
|
|
271
|
+
{
|
|
272
|
+
const { statusFromResult } = require('../utils/result-schema');
|
|
273
|
+
const reloaded = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
|
|
274
|
+
const { usage } = finalizeSpendForReopen({
|
|
275
|
+
taskId: newTaskId, model, mode: headless ? 'headless' : 'interactive',
|
|
276
|
+
op: 'continue', result, status: statusFromResult(result), project, metadata: reloaded,
|
|
277
|
+
});
|
|
278
|
+
if (usage) { writeFileAtomic(metaPath, JSON.stringify(reloaded, null, 2), { mode: 0o600 }); }
|
|
279
|
+
}
|
|
249
280
|
|
|
250
281
|
if (json) {
|
|
251
282
|
const { buildRunResult } = require('../utils/result-schema');
|
|
@@ -261,5 +292,6 @@ module.exports = {
|
|
|
261
292
|
loadPreviousSession,
|
|
262
293
|
buildContinuationContext,
|
|
263
294
|
createContinueSessionMetadata,
|
|
295
|
+
finalizeSpendForReopen,
|
|
264
296
|
continueSidecar
|
|
265
297
|
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// src/sidecar/fallback-chains.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module fallback-chains
|
|
6
|
+
* Opt-in cheaper-model fallback chains (spec 6.2, resolved Q2). Default OFF;
|
|
7
|
+
* per-run --fallback/--no-fallback overrides config. Explicit chains win;
|
|
8
|
+
* otherwise a default chain is DERIVED from model-tiers by walking DOWN the
|
|
9
|
+
* failed model's vendor tiers (frontier -> balanced -> economy), keeping only
|
|
10
|
+
* entries CHEAPER than the failed model. Gateway-only + unknown vendors get
|
|
11
|
+
* no default chain (the leg fails as today). Chain entries may be aliases or
|
|
12
|
+
* full ids, including openrouter/... (so "same model, other gateway" is
|
|
13
|
+
* expressible).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const { resolveTier, TIER_ORDER } = require('../utils/model-tiers');
|
|
17
|
+
|
|
18
|
+
const DEFAULT_MAX_SUBSTITUTIONS = 2;
|
|
19
|
+
|
|
20
|
+
/** Merge user config `fallbacks` with the per-run flag (flag wins). */
|
|
21
|
+
function resolveFallbackConfig({ flagFallback, config } = {}) {
|
|
22
|
+
const fb = (config && config.fallbacks) || {};
|
|
23
|
+
let enabled = fb.enabled === true;
|
|
24
|
+
if (flagFallback === true) { enabled = true; }
|
|
25
|
+
if (flagFallback === false) { enabled = false; }
|
|
26
|
+
return {
|
|
27
|
+
enabled,
|
|
28
|
+
maxSubstitutions: Number.isInteger(fb.maxSubstitutions) ? fb.maxSubstitutions : DEFAULT_MAX_SUBSTITUTIONS,
|
|
29
|
+
chains: fb.chains || {},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Vendor segment for tier lookup; a bare alias returns itself (config key). */
|
|
34
|
+
function vendorOf(model) {
|
|
35
|
+
let id = String(model || '');
|
|
36
|
+
if (id.startsWith('openrouter/')) { id = id.slice('openrouter/'.length); }
|
|
37
|
+
const slash = id.indexOf('/');
|
|
38
|
+
return slash > 0 ? id.slice(0, slash) : id;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Explicit chain, else the tier-walk default. A substitute must be cheaper. */
|
|
42
|
+
function deriveChain(model, { config, catalog } = {}) {
|
|
43
|
+
const chains = (config && config.chains) || {};
|
|
44
|
+
// explicit chain keyed by the bare alias OR the vendor
|
|
45
|
+
const key = chains[model] ? model : (chains[vendorOf(model)] ? vendorOf(model) : null);
|
|
46
|
+
if (key) { return chains[key].slice(); }
|
|
47
|
+
|
|
48
|
+
// tier-walk default: frontier -> balanced -> economy (most -> least capable)
|
|
49
|
+
const vendor = vendorOf(model);
|
|
50
|
+
const ordered = [];
|
|
51
|
+
for (const tier of [...TIER_ORDER].reverse()) {
|
|
52
|
+
const id = resolveTier(vendor, tier, catalog || []);
|
|
53
|
+
if (id && !ordered.includes(id)) { ordered.push(id); } // do NOT drop the failed model here
|
|
54
|
+
}
|
|
55
|
+
// Keep only tiers CHEAPER than the failed model (strictly after its position
|
|
56
|
+
// in the most->least-capable ladder). This drops the failed model AND
|
|
57
|
+
// everything more capable — a substitute must be cheaper (spec 6.2). If the
|
|
58
|
+
// failed model is not a current tier pick (e.g. a slightly-stale id), fall
|
|
59
|
+
// back to the full vendor ladder minus the exact model (best-effort;
|
|
60
|
+
// bounded misclassification cost).
|
|
61
|
+
const failedIdx = ordered.indexOf(model);
|
|
62
|
+
return failedIdx === -1 ? ordered.filter((id) => id !== model) : ordered.slice(failedIdx + 1);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
module.exports = { resolveFallbackConfig, deriveChain, vendorOf, DEFAULT_MAX_SUBSTITUTIONS };
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
// src/sidecar/fanout-leg-fallback.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module fanout-leg-fallback
|
|
6
|
+
* Cheaper-model fallback substitution (spec 6.2), split out of fanout-leg.js
|
|
7
|
+
* to keep both files ≤300 lines (mirrors the fanout.js/fanout-validate.js
|
|
8
|
+
* split). Owns the substitution loop + its two bookkeeping helpers.
|
|
9
|
+
* `runSingleAttempt` is lazy-required from ./fanout-leg (function-scoped, to
|
|
10
|
+
* avoid a load-time circular require — fanout-leg.js requires this module for
|
|
11
|
+
* its thin runLeg wrapper).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const fs = require('fs');
|
|
15
|
+
const { logger } = require('../utils/logger');
|
|
16
|
+
const { classifyLegError, isRetryable } = require('../utils/error-classify');
|
|
17
|
+
const { deriveChain } = require('./fallback-chains');
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Append ONE attributed ledger row for a single attempt (spec 6.2/7.1). At
|
|
21
|
+
* `attempt:0` this is byte-identical to today's pre-fallback appendSpend row
|
|
22
|
+
* (same fields, sourced from `leg.attempt`/`leg.substitutedFor`/
|
|
23
|
+
* `leg.retryOfWaveId`, all undefined -> omitted for a plain leg). A
|
|
24
|
+
* substitution (`attempt` param > 0, the LOOP's attempt index) overrides
|
|
25
|
+
* `row.attempt`/`row.substitutedFor` with the substitution's own values.
|
|
26
|
+
* Best-effort — never throws, never affects the leg. `deps.spendDir` (tests)
|
|
27
|
+
* routes the write to a scratch dir; production leaves it undefined.
|
|
28
|
+
*/
|
|
29
|
+
function recordAttemptSpend({ doc, leg, currentModel, legId, waveId, project, attempt, originalModel, routeGateway }, deps = {}) {
|
|
30
|
+
const usage = doc && doc.usage;
|
|
31
|
+
if (!usage) { return; }
|
|
32
|
+
try {
|
|
33
|
+
const { appendSpend } = deps.spendLedger || require('../utils/spend-ledger');
|
|
34
|
+
// Resolved legs carry the router's gateway on `leg.gateway` (attempt 0);
|
|
35
|
+
// a substitution carries the substitute's resolved gateway on
|
|
36
|
+
// `routeGateway` (threaded in by the loop, incl. v4.2 'local').
|
|
37
|
+
const gateway = routeGateway || (leg && leg.gateway) ||
|
|
38
|
+
(String(currentModel).startsWith('openrouter/') ? 'openrouter' : 'direct');
|
|
39
|
+
const row = {
|
|
40
|
+
taskId: legId, waveId, model: currentModel, mode: 'leg', usage,
|
|
41
|
+
op: 'leg', status: doc.status, gateway,
|
|
42
|
+
councilRunId: leg && leg.councilRunId, councilName: leg && leg.councilName,
|
|
43
|
+
project, attempt: leg && leg.attempt, substitutedFor: leg && leg.substitutedFor,
|
|
44
|
+
retryOfWaveId: leg && leg.retryOfWaveId,
|
|
45
|
+
};
|
|
46
|
+
if (attempt > 0) { row.attempt = attempt; row.substitutedFor = originalModel; }
|
|
47
|
+
appendSpend(row, deps.spendDir ? { dir: deps.spendDir } : undefined);
|
|
48
|
+
} catch { /* best-effort */ }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Fold token totals + cost amount across a leg's attempts[]. A single-attempt
|
|
53
|
+
* leg returns that attempt's usage verbatim (no behavior change for non-fallback
|
|
54
|
+
* legs). Multi-attempt sums tokens.* and cost.amount, tagging source 'mixed'
|
|
55
|
+
* when attempts' sources differ — matching formatCost's `~` behavior so a summed
|
|
56
|
+
* cost never reads as authoritative.
|
|
57
|
+
*/
|
|
58
|
+
function sumAttemptUsage(attempts) {
|
|
59
|
+
const withUsage = (attempts || []).filter(a => a.usage && a.usage.tokens);
|
|
60
|
+
if (withUsage.length === 0) { return null; }
|
|
61
|
+
if (withUsage.length === 1) { return withUsage[0].usage; }
|
|
62
|
+
const tokens = {};
|
|
63
|
+
let amount = 0;
|
|
64
|
+
let anyCost = false;
|
|
65
|
+
const sources = new Set();
|
|
66
|
+
for (const a of withUsage) {
|
|
67
|
+
for (const [k, v] of Object.entries(a.usage.tokens || {})) {
|
|
68
|
+
tokens[k] = (tokens[k] || 0) + (v || 0);
|
|
69
|
+
}
|
|
70
|
+
if (a.usage.cost && typeof a.usage.cost.amount === 'number') {
|
|
71
|
+
amount += a.usage.cost.amount;
|
|
72
|
+
anyCost = true;
|
|
73
|
+
if (a.usage.cost.source) { sources.add(a.usage.cost.source); }
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const cost = anyCost
|
|
77
|
+
? { amount, currency: 'USD', source: sources.size > 1 ? 'mixed' : (sources.values().next().value || 'reported') }
|
|
78
|
+
: null;
|
|
79
|
+
return { tokens, cost };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Per-$/Mtok hard-cap check for ONE fallback substitute candidate (spec
|
|
84
|
+
* §6.2). Minimal inline version of budget.js's checkBudget threshold, applied
|
|
85
|
+
* to a single model instead of a leg list. Unpriced (direct-provider,
|
|
86
|
+
* pricing:null) candidates are never treated as over-cap — unknown cost is
|
|
87
|
+
* not exceeded cost.
|
|
88
|
+
*/
|
|
89
|
+
function isOverBudgetSubstitute(modelId, maxCostPerMtok) {
|
|
90
|
+
const { lookupPricing } = require('../utils/pricing');
|
|
91
|
+
const { DEFAULT_MAX_COST_PER_MTOK } = require('./budget');
|
|
92
|
+
const pricing = lookupPricing(modelId);
|
|
93
|
+
if (!pricing) { return false; }
|
|
94
|
+
const cap = (typeof maxCostPerMtok === 'number' && maxCostPerMtok > 0) ? maxCostPerMtok : DEFAULT_MAX_COST_PER_MTOK;
|
|
95
|
+
return Math.max(pricing.prompt * 1e6, pricing.completion * 1e6) > cap;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* runLeg with opt-in cheaper-model substitution (spec 6.2). Runs the primary;
|
|
100
|
+
* on a classified rate-limit/overload failure with chain + budget remaining,
|
|
101
|
+
* substitutes the next chain model in the SAME leg dir under the SAME legId
|
|
102
|
+
* (conversation appends; metadata.model tracks the current model, modelInput
|
|
103
|
+
* stays the original). Best-effort bookkeeping (attempts/events/ledger) never
|
|
104
|
+
* fails the leg.
|
|
105
|
+
*/
|
|
106
|
+
async function runLegWithFallback(args, deps = {}) {
|
|
107
|
+
const { leg, legId, waveId, project, fallback, catalog } = args;
|
|
108
|
+
const runOnce = deps.runOnce || ((a) => require('./fanout-leg').runSingleAttempt(a, deps));
|
|
109
|
+
const resolveRoute = deps.resolveRoute ||
|
|
110
|
+
((r) => require('../utils/route-launch').resolveRouteForLaunch(r));
|
|
111
|
+
const { getSessionDir } = require('../session-manager');
|
|
112
|
+
const { appendEvent } = require('../observe/events');
|
|
113
|
+
const waveDir = getSessionDir(project, waveId);
|
|
114
|
+
// In production the wave dir already exists (fanout.js creates it before
|
|
115
|
+
// any leg launches); this is a defensive no-op there and only matters for
|
|
116
|
+
// a direct/unit-test caller of runLegWithFallback. appendEvent itself never
|
|
117
|
+
// creates directories (spec: best-effort, no side effects beyond the file).
|
|
118
|
+
try { fs.mkdirSync(waveDir, { recursive: true, mode: 0o700 }); } catch { /* best-effort */ }
|
|
119
|
+
|
|
120
|
+
const chain = deriveChain(leg.model, { config: { chains: fallback.chains }, catalog });
|
|
121
|
+
const attempts = [];
|
|
122
|
+
let currentModel = leg.model;
|
|
123
|
+
let currentGateway = leg.gateway;
|
|
124
|
+
let reasonClass = null;
|
|
125
|
+
let attempt = 0; // count of actual runs beyond the primary (bounded by maxSubstitutions)
|
|
126
|
+
let chainIdx = 0; // chain cursor: candidates tried OR skipped-for-budget, monotonic
|
|
127
|
+
let last;
|
|
128
|
+
|
|
129
|
+
for (;;) {
|
|
130
|
+
const startedAt = new Date().toISOString();
|
|
131
|
+
// `model` is threaded BOTH at the top level (test/injected runOnce fakes
|
|
132
|
+
// destructure it directly, e.g. `async ({ model }) => ...`) and nested
|
|
133
|
+
// under `leg.model` (the real runSingleAttempt only reads the latter).
|
|
134
|
+
last = await runOnce({ ...args, leg: { ...leg, model: currentModel }, model: currentModel,
|
|
135
|
+
attempt, substitutedFor: attempt > 0 ? leg.model : undefined });
|
|
136
|
+
attempts.push({ model: currentModel, status: last.status, usage: last.usage || null,
|
|
137
|
+
startedAt, completedAt: new Date().toISOString(), reason: last.reason || last.error || null });
|
|
138
|
+
|
|
139
|
+
// record this attempt's spend (one row per attempt — spec 6.2)
|
|
140
|
+
recordAttemptSpend({ doc: last, leg, currentModel, legId, waveId, project, attempt,
|
|
141
|
+
originalModel: leg.model, routeGateway: currentGateway }, deps);
|
|
142
|
+
|
|
143
|
+
if (last.status === 'complete') { break; }
|
|
144
|
+
// runSingleAttempt exposes the failure text on both `.reason` (its own
|
|
145
|
+
// alias) and `.error` (the buildRunResult field); an injected runOnce may
|
|
146
|
+
// set only one — read either.
|
|
147
|
+
const cls = classifyLegError(last.reason || last.error);
|
|
148
|
+
if (!fallback.enabled || !isRetryable(cls) || attempt >= fallback.maxSubstitutions || chainIdx >= chain.length) { break; }
|
|
149
|
+
|
|
150
|
+
// Walk the chain forward from chainIdx: an over-cap candidate is skipped
|
|
151
|
+
// (spec §6.2 per-$/Mtok guard) and the walk tries the NEXT entry; a
|
|
152
|
+
// route-resolution failure stops the whole loop outright (spec 8).
|
|
153
|
+
let substituteId = null;
|
|
154
|
+
let acceptedGateway = null;
|
|
155
|
+
let routeFailed = false;
|
|
156
|
+
while (chainIdx < chain.length) {
|
|
157
|
+
const next = chain[chainIdx];
|
|
158
|
+
chainIdx += 1;
|
|
159
|
+
let route;
|
|
160
|
+
try { route = await resolveRoute({ model: next, gatewayMode: undefined, source: 'fallback', allowSelection: false, validateModel: false }); }
|
|
161
|
+
catch { route = { kind: 'error' }; }
|
|
162
|
+
// RouteResult is keyed by `.kind` ('resolved'|'selection_required'|'error') — no `.ok` field.
|
|
163
|
+
if (!route || route.kind !== 'resolved') { routeFailed = true; break; }
|
|
164
|
+
const candidateId = route.executableId || route.model || next;
|
|
165
|
+
if (isOverBudgetSubstitute(candidateId, args.maxCostPerMtok)) {
|
|
166
|
+
logger.warn('Fallback substitute over the per-$/Mtok cap — trying the next chain entry', { legId, candidate: candidateId });
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
substituteId = candidateId;
|
|
170
|
+
acceptedGateway = route.gateway;
|
|
171
|
+
break;
|
|
172
|
+
}
|
|
173
|
+
if (routeFailed || !substituteId) { break; }
|
|
174
|
+
|
|
175
|
+
reasonClass = cls;
|
|
176
|
+
appendEvent(waveDir, { event: 'leg-fallback', id: waveId, legId,
|
|
177
|
+
fromModel: currentModel, toModel: substituteId, reason: cls, attempt: attempt + 1 });
|
|
178
|
+
currentModel = substituteId;
|
|
179
|
+
currentGateway = acceptedGateway;
|
|
180
|
+
attempt += 1;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const finalUsage = sumAttemptUsage(attempts);
|
|
184
|
+
const doc = { ...last, legId, model: currentModel, modelInput: leg.modelInput, usage: finalUsage, attempts };
|
|
185
|
+
if (attempt > 0) { doc.fallback = { from: leg.model, reason: reasonClass, attempts: attempt }; }
|
|
186
|
+
return doc;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
module.exports = { runLegWithFallback, recordAttemptSpend, sumAttemptUsage };
|