amicus 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +37 -1
- package/README.md +8 -5
- package/bin/amicus.js +5 -0
- package/package.json +1 -1
- package/schemas/council-run-live.schema.json +33 -0
- package/schemas/event.schema.json +15 -0
- package/schemas/progress.schema.json +24 -0
- package/schemas/run-live.schema.json +15 -0
- package/schemas/spend.schema.json +26 -1
- package/schemas/wave-live.schema.json +15 -0
- package/src/cli-handlers-council-run.js +61 -5
- package/src/cli-handlers-run.js +26 -0
- package/src/cli-handlers-spend.js +62 -27
- package/src/cli-handlers-watch.js +89 -0
- package/src/cli.js +58 -1
- package/src/council/run-chair.js +10 -2
- package/src/council/run-debate.js +5 -1
- package/src/council/run-launch.js +14 -1
- package/src/council/run-stages.js +13 -0
- package/src/council/run.js +32 -4
- package/src/headless.js +9 -1
- package/src/mcp-council-awareness.js +46 -1
- package/src/mcp-council-run.js +28 -4
- package/src/mcp-notify.js +54 -0
- package/src/mcp-server.js +51 -1
- package/src/mcp-spend.js +125 -0
- package/src/mcp-tools.js +39 -0
- package/src/mcp-wait.js +28 -2
- package/src/observe/events.js +156 -0
- package/src/observe/follow.js +26 -0
- package/src/observe/live-doc.js +38 -0
- package/src/observe/on-complete.js +117 -0
- package/src/observe/watch-render.js +149 -0
- package/src/sidecar/continue.js +32 -0
- package/src/sidecar/fallback-chains.js +65 -0
- package/src/sidecar/fanout-leg-fallback.js +189 -0
- package/src/sidecar/fanout-leg.js +58 -26
- package/src/sidecar/fanout-retry.js +208 -0
- package/src/sidecar/fanout-validate.js +42 -4
- package/src/sidecar/fanout.js +50 -30
- package/src/sidecar/progress.js +5 -0
- package/src/sidecar/resume.js +12 -0
- package/src/sidecar/start.js +13 -1
- package/src/spend-query.js +104 -0
- package/src/utils/api-key-store.js +7 -4
- package/src/utils/env-loader.js +0 -1
- package/src/utils/env-raw-store.js +13 -4
- package/src/utils/error-classify.js +31 -0
- package/src/utils/model-tiers.js +1 -1
- package/src/utils/spend-ledger.js +24 -1
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
/**
|
|
5
5
|
* @module fanout-leg
|
|
6
6
|
* Per-leg helpers extracted from fanout.js to keep both files ≤300 lines.
|
|
7
|
-
* Exports: legStatusFromResult, writeLegPatch, runLeg
|
|
7
|
+
* Exports: legStatusFromResult, writeLegPatch, runLeg, runSingleAttempt,
|
|
8
|
+
* buildRoutingFailureLeg (+ runLegWithFallback/recordAttemptSpend/
|
|
9
|
+
* sumAttemptUsage, re-exported from ./fanout-leg-fallback — split out to keep
|
|
10
|
+
* THIS file under the size gate; see that module for the substitution loop).
|
|
8
11
|
*/
|
|
9
12
|
|
|
10
13
|
const fs = require('fs');
|
|
@@ -61,29 +64,40 @@ function buildRoutingFailureLeg({ leg, legId, waveId, quiet }) {
|
|
|
61
64
|
}
|
|
62
65
|
|
|
63
66
|
/**
|
|
64
|
-
* Run
|
|
65
|
-
* leg finalize.
|
|
67
|
+
* Run ONE leg attempt end-to-end: session record -> runHeadless (shared
|
|
68
|
+
* server) -> leg finalize. Faithful extraction of the pre-fallback `runLeg`
|
|
69
|
+
* body — same setup, same never-throws try/finally, same leg-started/
|
|
70
|
+
* leg-terminal emits (with `follow` + the M4 own-try/catch), same `directory`
|
|
71
|
+
* threading into runHeadless. Never throws — always resolves to a run
|
|
72
|
+
* document. Does NOT append spend (the caller owns that: `runLeg` appends
|
|
73
|
+
* once; `runLegWithFallback` appends per attempt via recordAttemptSpend).
|
|
74
|
+
* Adds `.reason` (alias of buildRunResult's `.error`) and `.legId` so the
|
|
75
|
+
* fallback loop reads a stable shape without re-deriving them.
|
|
66
76
|
*/
|
|
67
|
-
async function
|
|
77
|
+
async function runSingleAttempt({ leg, legId, waveId, project, directory, follow, systemPrompt, userMessage, timeoutMs, agent, client, server, summaryLength, reasoning, quiet, foldNonce }) {
|
|
68
78
|
const { IdleWatchdog } = require('../utils/idle-watchdog');
|
|
69
79
|
const { markAborted } = require('../utils/session-abort');
|
|
70
80
|
const { runHeadless } = require('../headless');
|
|
71
81
|
const { SessionPaths, saveInitialContext } = require('./session-utils');
|
|
72
|
-
const { buildRunResult } = require('../utils/result-schema');
|
|
82
|
+
const { buildRunResult, durationBetween } = require('../utils/result-schema');
|
|
73
83
|
const { createSessionMetadata } = require('./start');
|
|
84
|
+
const { emitLegStarted, emitLegTerminal } = require('../observe/events');
|
|
85
|
+
const { getSessionDir } = require('../session-manager');
|
|
74
86
|
|
|
75
|
-
// Setup + run under ONE try so ANY throw (session record creation, initial
|
|
76
|
-
// context write, watchdog arm, or the poll loop itself) becomes an error run
|
|
77
|
-
// document — the wave still aggregates and writes wave.json. This function
|
|
78
|
-
// must NEVER throw / reject for a leg error (fanout.js relies on this in its
|
|
79
|
-
// Promise.all so one leg cannot sink the whole wave).
|
|
80
87
|
let legDir = null;
|
|
81
88
|
let watchdog = null;
|
|
82
89
|
let result;
|
|
90
|
+
// Captured inside the try below so a getSessionDir throw (invalid taskId)
|
|
91
|
+
// still becomes an error run document like every other setup failure; used
|
|
92
|
+
// again AFTER the try/finally closes to emit leg-terminal (M4: that emit is
|
|
93
|
+
// unguarded by the try, so it must never depend on something that could throw).
|
|
94
|
+
let waveDir = null;
|
|
83
95
|
try {
|
|
84
96
|
legDir = createSessionMetadata(legId, project, {
|
|
85
97
|
model: leg.model, prompt: userMessage, noUi: true, agent: agent || 'build',
|
|
86
98
|
});
|
|
99
|
+
waveDir = getSessionDir(project, waveId);
|
|
100
|
+
emitLegStarted(waveDir, waveId, legId, leg.model, leg.modelInput, follow);
|
|
87
101
|
writeLegPatch(legDir, { parentWave: waveId, modelInput: leg.modelInput });
|
|
88
102
|
saveInitialContext(legDir, systemPrompt, userMessage);
|
|
89
103
|
|
|
@@ -119,14 +133,6 @@ async function runLeg({ leg, legId, waveId, project, systemPrompt, userMessage,
|
|
|
119
133
|
const summary = result.summary || null;
|
|
120
134
|
const { resolveUsage } = require('../utils/pricing');
|
|
121
135
|
const usage = result && result.usage ? resolveUsage({ model: leg.model, usageTotals: result.usage }) : null;
|
|
122
|
-
// B24: cross-run spend ledger — one row per leg (mirrors start.js's single-run
|
|
123
|
-
// append). Best-effort; never let ledger bookkeeping affect the leg's own result.
|
|
124
|
-
if (usage) {
|
|
125
|
-
try {
|
|
126
|
-
const { appendSpend } = require('../utils/spend-ledger');
|
|
127
|
-
appendSpend({ taskId: legId, waveId, model: leg.model, mode: 'leg', usage });
|
|
128
|
-
} catch { /* best-effort */ }
|
|
129
|
-
}
|
|
130
136
|
// If setup threw before the session dir existed, there is nothing on disk to
|
|
131
137
|
// finalize — still resolve to an error run document so the wave aggregates.
|
|
132
138
|
const legPatch = {
|
|
@@ -137,21 +143,47 @@ async function runLeg({ leg, legId, waveId, project, systemPrompt, userMessage,
|
|
|
137
143
|
};
|
|
138
144
|
let finalMeta = legPatch;
|
|
139
145
|
if (legDir) {
|
|
140
|
-
if (summary) {
|
|
141
|
-
fs.writeFileSync(SessionPaths.summaryFile(legDir), summary, { mode: 0o600 });
|
|
142
|
-
}
|
|
146
|
+
if (summary) { fs.writeFileSync(SessionPaths.summaryFile(legDir), summary, { mode: 0o600 }); }
|
|
143
147
|
finalMeta = writeLegPatch(legDir, legPatch);
|
|
144
148
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
149
|
+
if (waveDir) {
|
|
150
|
+
try {
|
|
151
|
+
emitLegTerminal(waveDir, waveId, legId, {
|
|
152
|
+
model: leg.model, status: finalMeta.status,
|
|
153
|
+
durationMs: durationBetween(finalMeta.createdAt, finalMeta.completedAt),
|
|
154
|
+
usage: usage || null,
|
|
155
|
+
}, follow);
|
|
156
|
+
} catch { /* best-effort: a missing duration/emit never fails the leg */ }
|
|
157
|
+
}
|
|
158
|
+
const effectiveResult = finalMeta.status === 'aborted' ? { ...result, aborted: true } : result;
|
|
148
159
|
if (!quiet) {
|
|
149
160
|
process.stderr.write(`[fanout] leg ${legId} (${leg.modelInput}): ${finalMeta.status}\n`);
|
|
150
161
|
}
|
|
151
|
-
|
|
162
|
+
const doc = buildRunResult({
|
|
152
163
|
taskId: legId, metadata: finalMeta, result: effectiveResult, summary,
|
|
153
164
|
modelInput: leg.modelInput, sessionDir: legDir, waveId, usage,
|
|
154
165
|
});
|
|
166
|
+
doc.reason = doc.error || null; // classifier alias (buildRunResult stores it as .error)
|
|
167
|
+
doc.legId = legId;
|
|
168
|
+
return doc;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Run one leg end-to-end. Thin wrapper: fallback OFF (the default) is a
|
|
173
|
+
* single `runSingleAttempt` + exactly one spend row (`attempt:0`, byte-
|
|
174
|
+
* identical to the pre-fallback appendSpend row); fallback ON delegates to
|
|
175
|
+
* `runLegWithFallback` (./fanout-leg-fallback). Never throws.
|
|
176
|
+
*/
|
|
177
|
+
async function runLeg(args) {
|
|
178
|
+
const { leg, legId, waveId, project, fallback } = args;
|
|
179
|
+
const { runLegWithFallback, recordAttemptSpend } = require('./fanout-leg-fallback');
|
|
180
|
+
if (fallback && fallback.enabled) { return runLegWithFallback(args); }
|
|
181
|
+
const doc = await runSingleAttempt(args);
|
|
182
|
+
recordAttemptSpend({ doc, leg, currentModel: leg.model, legId, waveId, project, attempt: 0, originalModel: leg.model }, {});
|
|
183
|
+
return doc;
|
|
155
184
|
}
|
|
156
185
|
|
|
157
|
-
module.exports = {
|
|
186
|
+
module.exports = {
|
|
187
|
+
legStatusFromResult, writeLegPatch, runLeg, buildRoutingFailureLeg, runSingleAttempt,
|
|
188
|
+
...require('./fanout-leg-fallback'),
|
|
189
|
+
};
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
// src/sidecar/fanout-retry.js
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @module fanout-retry
|
|
6
|
+
* `fanout --retry-failed <waveId>` — relaunch ONLY the dead legs of a prior
|
|
7
|
+
* wave as a NEW linked wave, using each failed leg's own saved initial
|
|
8
|
+
* context for a byte-identical retry (spec 6.1). Split out of fanout.js
|
|
9
|
+
* (⚠️ DE-ROT B1): fanout.js is hard-gated at 300 lines and near the cap —
|
|
10
|
+
* inlining this ~120-line surface would blow it past the limit. Mirrors the
|
|
11
|
+
* fanout-leg-fallback.js extraction precedent (Task 18).
|
|
12
|
+
* `wave.json` is NEVER touched by this module — all linkage (retryOf /
|
|
13
|
+
* retriedBy / retry-started event / effective block) is additive, living in
|
|
14
|
+
* metadata.json (mutable via writeWaveMetadata) and the in-memory wave doc.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
|
|
20
|
+
/** Terminal, non-complete leg statuses eligible for --retry-failed. */
|
|
21
|
+
const ELIGIBLE_RETRY = new Set(['error', 'timeout', 'crashed', 'aborted', 'idle-timeout']);
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Parse an initial_context.md written by saveInitialContext (session-utils.js:69):
|
|
25
|
+
* `# System Prompt\n\n<sys>\n\n# User Message (Task)\n\n<user>`
|
|
26
|
+
* Returns null for any file that does not match that exact framing (legacy legs
|
|
27
|
+
* or a hand-edited file) so the caller falls back to briefing.md at launch time.
|
|
28
|
+
* @returns {{systemPrompt: string, userMessage: string}|null}
|
|
29
|
+
*/
|
|
30
|
+
function parseInitialContext(ctx) {
|
|
31
|
+
const head = '# System Prompt\n\n';
|
|
32
|
+
const marker = '\n\n# User Message (Task)\n\n';
|
|
33
|
+
if (!ctx.startsWith(head)) { return null; }
|
|
34
|
+
const mi = ctx.indexOf(marker, head.length);
|
|
35
|
+
if (mi === -1) { return null; }
|
|
36
|
+
return { systemPrompt: ctx.slice(head.length, mi), userMessage: ctx.slice(mi + marker.length) };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Plan a --retry-failed relaunch (spec 6.1). Pure over disk: reads the original
|
|
41
|
+
* wave + leg metadata, selects terminal non-complete legs (optionally filtered
|
|
42
|
+
* by --models), and loads each leg's saved initial context for a byte-identical
|
|
43
|
+
* retry. Refuses while the original wave is still running.
|
|
44
|
+
* @returns {{eligible: Array<{legId, model, systemPrompt, userMessage, hadSavedContext}>, error?: string}}
|
|
45
|
+
*/
|
|
46
|
+
function buildRetryPlan(origWaveId, project, { models } = {}) {
|
|
47
|
+
const { getSessionDir } = require('../session-manager');
|
|
48
|
+
const { SessionPaths } = require('./session-utils');
|
|
49
|
+
const waveDir = getSessionDir(project, origWaveId);
|
|
50
|
+
let waveMeta;
|
|
51
|
+
try { waveMeta = JSON.parse(fs.readFileSync(path.join(waveDir, 'metadata.json'), 'utf-8')); }
|
|
52
|
+
catch { return { eligible: [], error: `wave ${origWaveId} not found` }; }
|
|
53
|
+
if (waveMeta.type !== 'wave') { return { eligible: [], error: `${origWaveId} is not a fan-out wave` }; }
|
|
54
|
+
if (waveMeta.status === 'running') { return { eligible: [], error: `wave ${origWaveId} is still running — wait for it to finish before retrying` }; }
|
|
55
|
+
|
|
56
|
+
const wanted = models && models.length ? new Set(models) : null;
|
|
57
|
+
const eligible = [];
|
|
58
|
+
for (const legId of (waveMeta.legs || [])) {
|
|
59
|
+
let m;
|
|
60
|
+
try { m = JSON.parse(fs.readFileSync(path.join(getSessionDir(project, legId), 'metadata.json'), 'utf-8')); }
|
|
61
|
+
catch { continue; }
|
|
62
|
+
if (!ELIGIBLE_RETRY.has(m.status)) { continue; }
|
|
63
|
+
const model = m.modelInput || m.model;
|
|
64
|
+
if (wanted && !wanted.has(model) && !wanted.has(m.model)) { continue; }
|
|
65
|
+
// load saved initial context (system + user) for a byte-identical retry
|
|
66
|
+
const legDir = getSessionDir(project, legId);
|
|
67
|
+
let systemPrompt = null;
|
|
68
|
+
let userMessage = null;
|
|
69
|
+
let hadSavedContext = false;
|
|
70
|
+
try {
|
|
71
|
+
const parsed = parseInitialContext(fs.readFileSync(SessionPaths.contextFile(legDir), 'utf-8'));
|
|
72
|
+
if (parsed) { ({ systemPrompt, userMessage } = parsed); hadSavedContext = true; }
|
|
73
|
+
} catch { /* legacy leg — fall back to briefing.md at launch time */ }
|
|
74
|
+
eligible.push({ legId, model, systemPrompt, userMessage, hadSavedContext });
|
|
75
|
+
}
|
|
76
|
+
return { eligible };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Launch a --retry-failed wave (spec 6.1). New linked wave, byte-identical
|
|
81
|
+
* relaunch of the original wave's failed legs. Additive linkage only:
|
|
82
|
+
* - new wave metadata + doc gain `retryOf:<origWaveId>`
|
|
83
|
+
* - each new leg metadata gains `retryOf:<origLegId>`
|
|
84
|
+
* - the ORIGINAL wave metadata gains `retriedBy:[<newWaveId>,...]` (mutable,
|
|
85
|
+
* abort-wins via writeWaveMetadata) — wave.json is NOT touched
|
|
86
|
+
* - a `retry-started` event lands in the new wave dir
|
|
87
|
+
* - the doc gains an `effective` block (per original failed slot: the
|
|
88
|
+
* retry leg's latest status + usage)
|
|
89
|
+
* Linkage is gated on `runFanout` actually producing a wave: a pre-flight
|
|
90
|
+
* failure inside runFanout (wave:null, e.g. budget gate) must leave the
|
|
91
|
+
* original wave's metadata untouched — no false retriedBy record for a
|
|
92
|
+
* retry that never launched. runFanout suppresses its OWN stdout doc-print
|
|
93
|
+
* for a retry launch (its `options.retryOfWaveId` gate — see fanout.js),
|
|
94
|
+
* so this function owns printing the enriched doc (retryOf + effective),
|
|
95
|
+
* exactly once, respecting the caller's --json/--quiet.
|
|
96
|
+
* @param {object} [opts.runFanout] - injected for tests (defaults to runFanout)
|
|
97
|
+
* @returns {Promise<{wave: object|null, exitCode: number, errorDoc?: object}>}
|
|
98
|
+
*/
|
|
99
|
+
async function retryFailedWave(origWaveId, project, opts = {}) {
|
|
100
|
+
const { runFanout, deriveLegIds, writeWaveMetadata } = require('./fanout');
|
|
101
|
+
const { generateTaskId } = require('./start');
|
|
102
|
+
const { getSessionDir } = require('../session-manager');
|
|
103
|
+
const { appendEvent } = require('../observe/events');
|
|
104
|
+
const { writeFileAtomic } = require('../utils/atomic-write');
|
|
105
|
+
const { buildWaveResult } = require('../utils/result-schema');
|
|
106
|
+
const runFanoutImpl = opts.runFanout || runFanout;
|
|
107
|
+
|
|
108
|
+
const plan = buildRetryPlan(origWaveId, project, { models: opts.models });
|
|
109
|
+
if (plan.error) {
|
|
110
|
+
// no-console (repo lint gate): this module isn't on the CLI-output
|
|
111
|
+
// allowlist (unlike fanout.js) — write directly, matching failJson's
|
|
112
|
+
// established process.stdout/stderr.write pattern for JSON/human output.
|
|
113
|
+
if (!opts.quiet) { process.stderr.write(plan.error + '\n'); }
|
|
114
|
+
const { buildErrorDoc } = require('../utils/error-doc');
|
|
115
|
+
return { wave: null, errorDoc: buildErrorDoc({ code: 'BAD_ARGS', message: plan.error }), exitCode: 1 };
|
|
116
|
+
}
|
|
117
|
+
if (plan.eligible.length === 0) {
|
|
118
|
+
if (!opts.quiet) {
|
|
119
|
+
if (opts.json) {
|
|
120
|
+
// valid-JSON no-op doc (Finding 5) — reuses the wave schema so a
|
|
121
|
+
// --json caller's stdout stays machine-parseable either way.
|
|
122
|
+
const noopDoc = {
|
|
123
|
+
...buildWaveResult({ waveId: origWaveId, legs: [], status: 'complete' }),
|
|
124
|
+
retryOf: origWaveId, effective: [], note: 'no failed legs',
|
|
125
|
+
};
|
|
126
|
+
process.stdout.write(JSON.stringify(noopDoc, null, 2) + '\n');
|
|
127
|
+
} else {
|
|
128
|
+
process.stdout.write(`No failed legs to retry in ${origWaveId} — nothing to do.\n`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return { wave: null, exitCode: 0 };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const origWaveDir = getSessionDir(project, origWaveId);
|
|
135
|
+
let briefing = '';
|
|
136
|
+
try { briefing = fs.readFileSync(path.join(origWaveDir, 'briefing.md'), 'utf-8'); } catch { /* legacy — empty */ }
|
|
137
|
+
|
|
138
|
+
const newWaveId = generateTaskId();
|
|
139
|
+
const models = plan.eligible.map(e => e.model);
|
|
140
|
+
const retryContexts = plan.eligible.map(e => ({
|
|
141
|
+
origLegId: e.legId, systemPrompt: e.systemPrompt, userMessage: e.userMessage, hadSavedContext: e.hadSavedContext,
|
|
142
|
+
}));
|
|
143
|
+
|
|
144
|
+
// Launch the new wave. `retryContexts`/`retryOfWaveId` are additive options;
|
|
145
|
+
// runFanout uses each slot's saved system/user verbatim when present (Step 4b)
|
|
146
|
+
// and threads retryOfWaveId onto each leg so Task 1's append tags the rows.
|
|
147
|
+
// `models` MUST be the comma-separated STRING runFanout's own validator
|
|
148
|
+
// expects (parseModelsList/validateFanoutModels) — an array silently fails
|
|
149
|
+
// every leg pre-flight (BAD_ARGS), matching run-launch.js:41's precedent.
|
|
150
|
+
// Strip our own injection key so it is never forwarded.
|
|
151
|
+
const fanoutOpts = { ...opts, models: models.join(','), prompt: briefing, project, waveId: newWaveId, retryContexts, retryOfWaveId: origWaveId };
|
|
152
|
+
delete fanoutOpts.runFanout;
|
|
153
|
+
const { wave, exitCode } = await runFanoutImpl(fanoutOpts);
|
|
154
|
+
|
|
155
|
+
// No wave produced (pre-flight failure inside runFanout) — nothing
|
|
156
|
+
// launched, so no linkage may be recorded on the original wave.
|
|
157
|
+
if (!wave) { return { wave: null, exitCode }; }
|
|
158
|
+
|
|
159
|
+
// --- additive linkage (best-effort; a missing dir never throws) ---
|
|
160
|
+
const newWaveDir = getSessionDir(project, newWaveId);
|
|
161
|
+
writeWaveMetadata(newWaveDir, { retryOf: origWaveId });
|
|
162
|
+
wave.retryOf = origWaveId;
|
|
163
|
+
|
|
164
|
+
const newLegIds = deriveLegIds(newWaveId, plan.eligible.length);
|
|
165
|
+
newLegIds.forEach((newLegId, i) => {
|
|
166
|
+
const mp = path.join(getSessionDir(project, newLegId), 'metadata.json');
|
|
167
|
+
try {
|
|
168
|
+
const m = JSON.parse(fs.readFileSync(mp, 'utf-8'));
|
|
169
|
+
m.retryOf = retryContexts[i].origLegId;
|
|
170
|
+
writeFileAtomic(mp, JSON.stringify(m, null, 2), { mode: 0o600 });
|
|
171
|
+
} catch { /* leg dir absent (short-circuited wave) — best-effort */ }
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
// original wave gains retriedBy:[...] (dedup; abort-wins merge)
|
|
175
|
+
let origMeta = {};
|
|
176
|
+
try { origMeta = JSON.parse(fs.readFileSync(path.join(origWaveDir, 'metadata.json'), 'utf-8')); } catch { /* corrupt */ }
|
|
177
|
+
const retriedBy = Array.isArray(origMeta.retriedBy) ? origMeta.retriedBy.slice() : [];
|
|
178
|
+
if (!retriedBy.includes(newWaveId)) { retriedBy.push(newWaveId); }
|
|
179
|
+
writeWaveMetadata(origWaveDir, { retriedBy });
|
|
180
|
+
|
|
181
|
+
// milestone: retry-started into the NEW wave dir (never-throws appendEvent)
|
|
182
|
+
appendEvent(newWaveDir, { event: 'retry-started', id: newWaveId, retryOf: origWaveId, legIds: newLegIds });
|
|
183
|
+
|
|
184
|
+
// effective block: original failed slot -> the retry leg's latest status/usage
|
|
185
|
+
const legs = Array.isArray(wave.legs) ? wave.legs : [];
|
|
186
|
+
const effective = plan.eligible.map((e, i) => ({
|
|
187
|
+
origLegId: e.legId, model: e.model,
|
|
188
|
+
status: legs[i] ? legs[i].status : 'unknown',
|
|
189
|
+
usage: legs[i] ? (legs[i].usage || null) : null,
|
|
190
|
+
}));
|
|
191
|
+
wave.effective = effective;
|
|
192
|
+
|
|
193
|
+
// runFanout suppressed its own stdout print for this retry launch (its
|
|
194
|
+
// options.retryOfWaveId gate) — print the enriched doc (retryOf +
|
|
195
|
+
// effective now attached) ONCE, respecting --json/--quiet (Finding 3).
|
|
196
|
+
if (!opts.quiet) {
|
|
197
|
+
if (opts.json) {
|
|
198
|
+
process.stdout.write(JSON.stringify(wave, null, 2) + '\n');
|
|
199
|
+
} else {
|
|
200
|
+
const { formatWaveHuman } = require('./fanout-output');
|
|
201
|
+
process.stdout.write(formatWaveHuman(wave) + '\n');
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
return { wave, exitCode };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
module.exports = { ELIGIBLE_RETRY, parseInitialContext, buildRetryPlan, retryFailedWave };
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
// src/sidecar/fanout-validate.js
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
|
+
const { logger } = require('../utils/logger');
|
|
5
|
+
|
|
4
6
|
/**
|
|
5
7
|
* @module fanout-validate
|
|
6
8
|
* Fan-out --models parsing + per-leg gateway routing. Split out of fanout.js
|
|
@@ -32,10 +34,18 @@ function parseModelsList(modelsArg) {
|
|
|
32
34
|
* exceeding the leg-count cap — remain wave-level fatal and are returned as
|
|
33
35
|
* a top-level `{error, code}` (nothing to route yet at that point).
|
|
34
36
|
* @param {string} modelsArg - Raw --models value
|
|
35
|
-
* @param {{noValidateModel?: boolean, gatewayMode?: string
|
|
37
|
+
* @param {{noValidateModel?: boolean, gatewayMode?: string, fallback?: object,
|
|
38
|
+
* catalog?: Array}} [opts] `fallback`/`catalog` (v4.3 Task 18, spec §6.2) are
|
|
39
|
+
* additive: when `fallback.enabled`, every resolved primary's chain
|
|
40
|
+
* candidates are pre-resolved so the shared server can register them (see
|
|
41
|
+
* `serverModels` below); omitted/disabled callers are unaffected.
|
|
36
42
|
* @returns {Promise<{legs: Array<{modelInput: string, ok: boolean, model?: string,
|
|
37
|
-
* pricing?: object, gateway?: string, provenance?: object, routeResult?: object}
|
|
38
|
-
*
|
|
43
|
+
* pricing?: object, gateway?: string, provenance?: object, routeResult?: object}>,
|
|
44
|
+
* serverModels?: string[]}
|
|
45
|
+
* | {error: string, code: string}>} `serverModels` (additive) is the UNION of
|
|
46
|
+
* every resolved primary + its fallback-chain candidates' executable ids —
|
|
47
|
+
* present only when `opts.fallback.enabled`; the caller falls back to
|
|
48
|
+
* `okLegs.map(l => l.model)` when absent (unchanged today).
|
|
39
49
|
*/
|
|
40
50
|
async function validateFanoutModels(modelsArg, opts = {}) {
|
|
41
51
|
const raw = parseModelsList(modelsArg);
|
|
@@ -75,7 +85,35 @@ async function validateFanoutModels(modelsArg, opts = {}) {
|
|
|
75
85
|
legs.push({ modelInput, ok: false, routeResult });
|
|
76
86
|
}
|
|
77
87
|
}
|
|
78
|
-
|
|
88
|
+
|
|
89
|
+
// v4.3 Task 18 (spec §6.2 sole-input invariant): when fallback substitution
|
|
90
|
+
// is enabled, pre-resolve every resolved primary's chain candidates and
|
|
91
|
+
// register the UNION of primary + candidate executable ids on the shared
|
|
92
|
+
// server — a substitute that never runs must still be an allowed model if
|
|
93
|
+
// one IS selected mid-wave. An unresolvable candidate is DROPPED (logged),
|
|
94
|
+
// never a wave failure: registration is config, not spend.
|
|
95
|
+
let serverModels;
|
|
96
|
+
if (opts.fallback && opts.fallback.enabled) {
|
|
97
|
+
const { deriveChain } = require('./fallback-chains');
|
|
98
|
+
const ids = new Set(legs.filter(l => l.ok).map(l => l.model));
|
|
99
|
+
for (const leg of legs) {
|
|
100
|
+
if (!leg.ok) { continue; }
|
|
101
|
+
const chain = deriveChain(leg.model, { config: { chains: opts.fallback.chains }, catalog: opts.catalog });
|
|
102
|
+
for (const candidate of chain) {
|
|
103
|
+
let route;
|
|
104
|
+
try {
|
|
105
|
+
route = await resolveRouteForLaunch({ model: candidate, gatewayMode, source: 'fallback', allowSelection: false, validateModel });
|
|
106
|
+
} catch { route = { kind: 'error' }; }
|
|
107
|
+
if (route.kind === 'resolved') {
|
|
108
|
+
ids.add(route.executableId);
|
|
109
|
+
} else {
|
|
110
|
+
logger.warn('Fallback chain candidate failed to route — dropped from server registration', { primary: leg.model, candidate });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
serverModels = [...ids];
|
|
115
|
+
}
|
|
116
|
+
return { legs, serverModels };
|
|
79
117
|
}
|
|
80
118
|
|
|
81
119
|
module.exports = { parseModelsList, DEFAULT_MAX_LEGS, validateFanoutModels };
|
package/src/sidecar/fanout.js
CHANGED
|
@@ -57,7 +57,9 @@ function writeWaveMetadata(waveDir, patch) {
|
|
|
57
57
|
* contextTurns?, contextSince?, contextMaxTokens?, mcp?, mcpConfig?, noMcp?,
|
|
58
58
|
* excludeMcp?, noValidateModel?, gatewayMode? (#61 Task 7.3: --gateway merged
|
|
59
59
|
* with routing.prefer, applied per leg), json?, client?, quiet? (suppress
|
|
60
|
-
* stdout — tests)
|
|
60
|
+
* stdout — tests), councilRunId? / councilName? (v4.3 §7.2: stamped onto legs),
|
|
61
|
+
* fallback? / catalog? (v4.3 Task 18 §6.2: opt-in substitution; off/absent unchanged),
|
|
62
|
+
* retryContexts? / retryOfWaveId? (v4.3 Task 19: --retry-failed relaunch seam; absent -> byte-identical)
|
|
61
63
|
* @returns {Promise<{wave: object, exitCode: number}>} Never rejects for leg errors.
|
|
62
64
|
*/
|
|
63
65
|
async function runFanout(options) {
|
|
@@ -70,11 +72,15 @@ async function runFanout(options) {
|
|
|
70
72
|
const { generateFoldNonce } = require('../utils/fold-marker');
|
|
71
73
|
const { installSignalAbort, markAborted } = require('../utils/session-abort');
|
|
72
74
|
const { getSessionDir } = require('../session-manager');
|
|
75
|
+
const { emitWaveStarted, emitWaveTerminal } = require('../observe/events');
|
|
73
76
|
|
|
74
77
|
const project = options.project || process.cwd();
|
|
75
78
|
const createdAt = new Date().toISOString();
|
|
79
|
+
// v4.3 Task 13: live stderr mirror of this wave's own events, in-process
|
|
80
|
+
// (no tail). Off by default; every emit* call below threads it through.
|
|
81
|
+
const follow = options.follow ? require('../observe/follow').createFollowPrinter({ json: options.json }) : null;
|
|
76
82
|
const emit = (doc) => {
|
|
77
|
-
if (options.quiet) { return; }
|
|
83
|
+
if (options.quiet || options.retryOfWaveId) { return; } // v4.3 T19 FW1#3: retry launches print via fanout-retry.js instead
|
|
78
84
|
if (options.json) {
|
|
79
85
|
console.log(JSON.stringify(doc, null, 2));
|
|
80
86
|
} else {
|
|
@@ -108,15 +114,20 @@ async function runFanout(options) {
|
|
|
108
114
|
const validated = await validateFanoutModels(options.models, {
|
|
109
115
|
noValidateModel: options.noValidateModel,
|
|
110
116
|
gatewayMode: options.gatewayMode,
|
|
117
|
+
fallback: options.fallback, // v4.3 Task 18 §6.2: serverModels union only when enabled
|
|
118
|
+
catalog: options.catalog,
|
|
111
119
|
});
|
|
112
120
|
if (validated.error) { return failPre(validated.code || 'BAD_ARGS', validated.error); }
|
|
113
121
|
const legs = validated.legs;
|
|
122
|
+
// v4.3 §7.2: stamp council attribution onto every leg (fanout-leg.js's
|
|
123
|
+
// existing appendSpend reads it); no-op for every non-council caller.
|
|
124
|
+
if (options.councilRunId || options.councilName) {
|
|
125
|
+
legs.forEach(l => { l.councilRunId = options.councilRunId; l.councilName = options.councilName; });
|
|
126
|
+
}
|
|
114
127
|
const okLegs = legs.filter(l => l.ok);
|
|
115
128
|
// FIX 2 (#61 whole-branch review): a leg's migration notice has no CLI
|
|
116
|
-
// stderr to land on
|
|
117
|
-
//
|
|
118
|
-
// for the same vendor happen to both migrate (only the first ever fires
|
|
119
|
-
// since markMigrationNotified is one-shot per vendor, but dedupe defensively).
|
|
129
|
+
// stderr to land on — surface it on the wave doc instead, deduped in case
|
|
130
|
+
// two legs for the same vendor happen to both migrate.
|
|
120
131
|
const notices = [...new Set(legs.map(l => l.notice).filter(Boolean))];
|
|
121
132
|
|
|
122
133
|
// 1b. Budget gate (pre-creation; refuse before spending). Only legs that
|
|
@@ -146,14 +157,12 @@ async function runFanout(options) {
|
|
|
146
157
|
promptMeta: options.promptMeta || null,
|
|
147
158
|
pid: process.pid, project, createdAt,
|
|
148
159
|
});
|
|
160
|
+
emitWaveStarted(waveDir, waveId, legs.map(l => (l.ok ? l.model : l.modelInput)), legIds, follow);
|
|
149
161
|
|
|
150
162
|
// 2b. All legs failed to route (#61 perf): no leg will ever touch the
|
|
151
|
-
// shared server, so starting one (and
|
|
152
|
-
//
|
|
153
|
-
//
|
|
154
|
-
// (buildRoutingFailureLeg), same aggregation (buildWaveResult /
|
|
155
|
-
// waveStatusFromLegs via the default status param), same exit-code mapping
|
|
156
|
-
// (waveExitCode) — just without the server round-trip.
|
|
163
|
+
// shared server, so starting one (and tearing it down) is pure waste.
|
|
164
|
+
// Short-circuit to the same routing-failure wave the normal path would
|
|
165
|
+
// eventually produce — same per-leg docs, aggregation, and exit mapping.
|
|
157
166
|
if (okLegs.length === 0) {
|
|
158
167
|
const legDocs = legs.map((leg, i) => buildRoutingFailureLeg({ leg, legId: legIds[i], waveId, quiet: options.quiet }));
|
|
159
168
|
const completedAt = new Date().toISOString();
|
|
@@ -163,8 +172,11 @@ async function runFanout(options) {
|
|
|
163
172
|
const wavePath = path.join(waveDir, 'wave.json');
|
|
164
173
|
writeFileAtomic(wavePath, JSON.stringify(wave, null, 2), { mode: 0o600 });
|
|
165
174
|
writeWaveMetadata(waveDir, { status: wave.status, completedAt });
|
|
175
|
+
const routingExitCode = waveExitCode(wave.status);
|
|
176
|
+
emitWaveTerminal(waveDir, waveId, { status: wave.status, counts: wave.counts, usage: wave.usage, exitCode: routingExitCode }, follow);
|
|
177
|
+
await require('../observe/on-complete').fireWaveOnComplete(options.onComplete, wave, { waveId, waveDir, wavePath, exitCode: routingExitCode, project }, options.onCompleteDeps);
|
|
166
178
|
emit(wave);
|
|
167
|
-
return { wave, exitCode:
|
|
179
|
+
return { wave, exitCode: routingExitCode };
|
|
168
180
|
}
|
|
169
181
|
|
|
170
182
|
// 3. Context + prompts built ONCE (model-independent)
|
|
@@ -195,7 +207,7 @@ async function runFanout(options) {
|
|
|
195
207
|
});
|
|
196
208
|
let client, server;
|
|
197
209
|
try {
|
|
198
|
-
({ client, server } = await startOpenCodeServer(mcpServers, { models: okLegs.map(l => l.model) }));
|
|
210
|
+
({ client, server } = await startOpenCodeServer(mcpServers, { models: validated.serverModels || okLegs.map(l => l.model) }));
|
|
199
211
|
} catch (err) {
|
|
200
212
|
writeWaveMetadata(waveDir, { status: 'error', reason: err.message, completedAt: new Date().toISOString() });
|
|
201
213
|
return errorWave(waveId, `Failed to start server: ${err.message}`);
|
|
@@ -225,12 +237,10 @@ async function runFanout(options) {
|
|
|
225
237
|
},
|
|
226
238
|
});
|
|
227
239
|
|
|
228
|
-
// 6. Launch all ROUTABLE legs concurrently (runLeg never rejects). A leg
|
|
229
|
-
//
|
|
230
|
-
//
|
|
231
|
-
|
|
232
|
-
// whole wave (#61 Task 7.3).
|
|
233
|
-
const heartbeat = options.quiet
|
|
240
|
+
// 6. Launch all ROUTABLE legs concurrently (runLeg never rejects). A leg that
|
|
241
|
+
// failed to route (leg.ok === false) resolves to an error run doc in its own
|
|
242
|
+
// slot (buildRoutingFailureLeg), failing only itself, never the wave (#61).
|
|
243
|
+
const heartbeat = (options.quiet || options.follow)
|
|
234
244
|
? { stop() {} }
|
|
235
245
|
: createWaveHeartbeat(
|
|
236
246
|
legs.map((leg, i) => ({ label: leg.modelInput || leg.model, dir: legDirs[i] })),
|
|
@@ -240,15 +250,23 @@ async function runFanout(options) {
|
|
|
240
250
|
const reasoning = options.thinking ? { effort: options.thinking } : undefined;
|
|
241
251
|
let legDocs;
|
|
242
252
|
try {
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
253
|
+
// retryContexts/retryOfWaveId (v4.3 Task 19): absent on a normal wave, so
|
|
254
|
+
// every leg below falls back to the wave-wide prompt — byte-identical.
|
|
255
|
+
legDocs = await Promise.all(legs.map((leg, i) => {
|
|
256
|
+
if (!leg.ok) { return Promise.resolve(buildRoutingFailureLeg({ leg, legId: legIds[i], waveId, quiet: options.quiet })); }
|
|
257
|
+
const rc = options.retryContexts && options.retryContexts[i];
|
|
258
|
+
const saved = rc && rc.hadSavedContext;
|
|
259
|
+
return runLeg({
|
|
260
|
+
leg: options.retryOfWaveId ? { ...leg, retryOfWaveId: options.retryOfWaveId } : leg,
|
|
261
|
+
legId: legIds[i], waveId, project,
|
|
262
|
+
systemPrompt: saved ? rc.systemPrompt : systemPrompt,
|
|
263
|
+
userMessage: saved ? rc.userMessage : userMessage,
|
|
264
|
+
timeoutMs, agent: options.agent, client, server,
|
|
265
|
+
summaryLength: options.summaryLength, reasoning, quiet: options.quiet,
|
|
266
|
+
foldNonce, directory: options.directory, follow,
|
|
267
|
+
fallback: options.fallback, catalog: options.catalog,
|
|
268
|
+
});
|
|
269
|
+
}));
|
|
252
270
|
} finally {
|
|
253
271
|
heartbeat.stop();
|
|
254
272
|
uninstallSignals();
|
|
@@ -264,10 +282,12 @@ async function runFanout(options) {
|
|
|
264
282
|
const wavePath = path.join(waveDir, 'wave.json');
|
|
265
283
|
writeFileAtomic(wavePath, JSON.stringify(wave, null, 2), { mode: 0o600 });
|
|
266
284
|
writeWaveMetadata(waveDir, { status: wave.status, completedAt });
|
|
267
|
-
emit(wave);
|
|
268
285
|
const exitCode = signalled
|
|
269
286
|
? (signalled === 'SIGINT' ? 130 : 143)
|
|
270
287
|
: waveExitCode(wave.status);
|
|
288
|
+
emitWaveTerminal(waveDir, waveId, { status: wave.status, counts: wave.counts, usage: wave.usage, exitCode }, follow);
|
|
289
|
+
await require('../observe/on-complete').fireWaveOnComplete(options.onComplete, wave, { waveId, waveDir, wavePath, exitCode, project }, options.onCompleteDeps);
|
|
290
|
+
emit(wave);
|
|
271
291
|
return { wave, exitCode };
|
|
272
292
|
}
|
|
273
293
|
|
package/src/sidecar/progress.js
CHANGED
|
@@ -113,6 +113,8 @@ function isStalled(lastActivityMs) {
|
|
|
113
113
|
function writeProgress(sessionDir, stage, extra = {}) {
|
|
114
114
|
const progressPath = path.join(sessionDir, 'progress.json');
|
|
115
115
|
const data = {
|
|
116
|
+
schemaVersion: 1,
|
|
117
|
+
type: 'progress',
|
|
116
118
|
stage,
|
|
117
119
|
stageLabel: STAGE_LABELS[stage] || stage,
|
|
118
120
|
updatedAt: new Date().toISOString(),
|
|
@@ -224,6 +226,9 @@ function readProgress(sessionDir) {
|
|
|
224
226
|
if (stage !== undefined) {
|
|
225
227
|
result.stage = stage;
|
|
226
228
|
}
|
|
229
|
+
if (progress && progress.usage) {
|
|
230
|
+
result.usage = progress.usage;
|
|
231
|
+
}
|
|
227
232
|
return result;
|
|
228
233
|
}
|
|
229
234
|
|
package/src/sidecar/resume.js
CHANGED
|
@@ -248,6 +248,18 @@ async function resumeSidecar(options) {
|
|
|
248
248
|
} else {
|
|
249
249
|
finalizeSession(sessionDir, summary, project, updatedMetadata, { quietStdout: json, status: terminal.status });
|
|
250
250
|
}
|
|
251
|
+
// v4.3: attribute resume spend (C9/E4). Reload metadata, write usage + append
|
|
252
|
+
// a ledger row (status: statusFromResult, matching start.js — not terminal.status).
|
|
253
|
+
{
|
|
254
|
+
const { finalizeSpendForReopen } = require('./continue');
|
|
255
|
+
const { statusFromResult } = require('../utils/result-schema');
|
|
256
|
+
const reloaded = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
|
|
257
|
+
const { usage } = finalizeSpendForReopen({
|
|
258
|
+
taskId, model: metadata.model, mode: headless ? 'headless' : 'interactive',
|
|
259
|
+
op: 'resume', result, status: statusFromResult(result), project, metadata: reloaded,
|
|
260
|
+
});
|
|
261
|
+
if (usage) { writeFileAtomic(metaPath, JSON.stringify(reloaded, null, 2), { mode: 0o600 }); }
|
|
262
|
+
}
|
|
251
263
|
|
|
252
264
|
if (json) {
|
|
253
265
|
const { buildRunResult } = require('../utils/result-schema');
|
package/src/sidecar/start.js
CHANGED
|
@@ -257,7 +257,19 @@ async function startSidecar(options) {
|
|
|
257
257
|
// this run's own success must never hinge on ledger bookkeeping either way.
|
|
258
258
|
try {
|
|
259
259
|
const { appendSpend } = require('../utils/spend-ledger');
|
|
260
|
-
|
|
260
|
+
const { statusFromResult } = require('../utils/result-schema');
|
|
261
|
+
appendSpend({
|
|
262
|
+
taskId, model, mode: effectiveHeadless ? 'headless' : 'interactive', usage: runUsage,
|
|
263
|
+
op: 'start', status: statusFromResult(result), project: effectiveProject,
|
|
264
|
+
// ⚠️ DE-ROT: `metadata` is NOT in scope at startSidecar's finalize site — the objects
|
|
265
|
+
// there are `meta` (createSessionMetadata result) and `m`; `metadata` is a local only
|
|
266
|
+
// inside createSessionMetadata. Reading `metadata.gateway` throws a ReferenceError the
|
|
267
|
+
// best-effort catch swallows → EVERY start-mode spend row silently dropped + start-json.test.js
|
|
268
|
+
// goes red. Use an in-scope value (spec-complete for direct/openrouter):
|
|
269
|
+
gateway: String(model).startsWith('openrouter/') ? 'openrouter' : 'direct',
|
|
270
|
+
// (To also attribute v4.2 'local': thread the resolved route gateway — dropped today at
|
|
271
|
+
// cli-handlers-run.js:47 — into createSessionMetadata and read `meta.gateway`, as continue.js:111 does.)
|
|
272
|
+
});
|
|
261
273
|
} catch { /* best-effort */ }
|
|
262
274
|
}
|
|
263
275
|
|