@crewhaus/model-service 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +29 -0
- package/README.md +246 -0
- package/dist/index.d.ts +545 -0
- package/dist/index.js +522 -0
- package/dist/side-calls.d.ts +103 -0
- package/dist/side-calls.js +592 -0
- package/package.json +48 -0
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 0.6.0 PR 9d (plan §7.4, §7.6, §7.8, §10.2, §16 Q6) — the side-call
|
|
3
|
+
* closures the composition root builds from `model_pool.strategy`: the GUIDE
|
|
4
|
+
* (plan-execute), the SHADOW (audition lane) and the COMMITTEE (single-turn
|
|
5
|
+
* hosts only). runtime-core owns each one's lifecycle — when it runs, what
|
|
6
|
+
* its text does, the `model_stage` events, the scoreboard fold — and consumes
|
|
7
|
+
* them through `RunChatLoopOptions.sideCalls`; this module owns the MODEL
|
|
8
|
+
* CALLS, all of which go through ONE nested-single-turn runner:
|
|
9
|
+
*
|
|
10
|
+
* {@link runNestedSingleTurn} — a tool-less `runChatLoop({ singleTurn:
|
|
11
|
+
* true })` on the target model, minted with its OWN child `RunContext` and
|
|
12
|
+
* child `TraceEventBus` (`inheritTraceId`, so OTel stitches the trace),
|
|
13
|
+
* whose `model_request` / `model_response` events are re-published on the
|
|
14
|
+
* PARENT bus under the parent's envelope with the side call's `role` and
|
|
15
|
+
* `stage`, so the parent's cost-tracker prices them, its `budgetMeter`
|
|
16
|
+
* counts them under `judge_share`, and its session mirror persists them.
|
|
17
|
+
* The child runs with `persistSession: false` (§16 Q6): no child session
|
|
18
|
+
* file, no child event log — its spend and stages live in the parent's.
|
|
19
|
+
* Never `adapter.stream` directly, never a parallel `runOneTurn` (§7.6):
|
|
20
|
+
* the members of a committee run one after another, each on its own loop
|
|
21
|
+
* state.
|
|
22
|
+
*
|
|
23
|
+
* The Consult runner (`buildConsultRunner` in index.ts) rides the same
|
|
24
|
+
* runner since this PR, which is what closes the wave-2 carry-over (a consult
|
|
25
|
+
* used to persist one child session per question like a Task child).
|
|
26
|
+
*
|
|
27
|
+
* Classification (Pillar 3): a guide's text enters the PARENT's system region
|
|
28
|
+
* and is classified + lineage-tagged by runtime-core at the injection point,
|
|
29
|
+
* exactly once, at TrustOrigin "consult". Committee members' and the shadow's
|
|
30
|
+
* outputs are the harness's own model output judged blind — not a boundary —
|
|
31
|
+
* and the shadow's text is discarded here, never returned.
|
|
32
|
+
*/
|
|
33
|
+
import { randomBytes, randomUUID } from "node:crypto";
|
|
34
|
+
import { DEFAULT_PRICING, computeCostMicros, resolvePricing } from "@crewhaus/cost-tracker";
|
|
35
|
+
import { judgePairwise, judgeSelect } from "@crewhaus/eval-judge";
|
|
36
|
+
import { createRunContext } from "@crewhaus/run-context";
|
|
37
|
+
import { runChatLoop, } from "@crewhaus/runtime-core";
|
|
38
|
+
import { resolveRosterTarget, rosterFromPool, strongestOf, } from "@crewhaus/tool-consult";
|
|
39
|
+
import { TraceEventBus, } from "@crewhaus/trace-event-bus";
|
|
40
|
+
/**
|
|
41
|
+
* THE nested single-turn side call (plan §7.5 / §7.6). One tool-less turn on
|
|
42
|
+
* the target through `runChatLoop` — never `adapter.stream` — on a child run
|
|
43
|
+
* context whose bus inherits the parent's trace and whose model events are
|
|
44
|
+
* re-published on the parent bus under the parent's envelope with `role` /
|
|
45
|
+
* `stage`. Persists nothing of its own (`persistSession: false`, §16 Q6).
|
|
46
|
+
*/
|
|
47
|
+
export async function runNestedSingleTurn(args) {
|
|
48
|
+
const { parent, deps } = args;
|
|
49
|
+
const childRunId = `run_${randomUUID().slice(0, 8)}`;
|
|
50
|
+
const childSessionId = `sess_${randomBytes(8).toString("hex")}`;
|
|
51
|
+
const childBus = new TraceEventBus({
|
|
52
|
+
runId: childRunId,
|
|
53
|
+
sessionId: childSessionId,
|
|
54
|
+
...(parent !== undefined
|
|
55
|
+
? {
|
|
56
|
+
inheritTraceId: parent.eventBus.traceId,
|
|
57
|
+
inheritParentSpanId: parent.eventBus.currentSpanId,
|
|
58
|
+
logger: parent.logger,
|
|
59
|
+
}
|
|
60
|
+
: {}),
|
|
61
|
+
});
|
|
62
|
+
const child = createRunContext({
|
|
63
|
+
runId: childRunId,
|
|
64
|
+
sessionId: childSessionId,
|
|
65
|
+
...(args.signal !== undefined
|
|
66
|
+
? { abortSignal: args.signal }
|
|
67
|
+
: parent !== undefined
|
|
68
|
+
? { abortSignal: parent.abortSignal }
|
|
69
|
+
: {}),
|
|
70
|
+
...(parent !== undefined ? { logger: parent.logger } : {}),
|
|
71
|
+
eventBus: childBus,
|
|
72
|
+
originStack: [...(parent?.originStack ?? []), "consult"],
|
|
73
|
+
});
|
|
74
|
+
const attribution = {
|
|
75
|
+
role: args.role,
|
|
76
|
+
...(args.stage !== undefined ? { stage: args.stage } : {}),
|
|
77
|
+
};
|
|
78
|
+
const usage = { input: 0, output: 0, cacheRead: 0, cacheCreate: 0 };
|
|
79
|
+
let served;
|
|
80
|
+
let costUsdMicros;
|
|
81
|
+
let durationMs = 0;
|
|
82
|
+
const unsubscribe = childBus.subscribe((event) => {
|
|
83
|
+
if (event.kind === "model_response") {
|
|
84
|
+
const r = event;
|
|
85
|
+
served = { model: r.model, provider: r.provider ?? served?.provider ?? "anthropic" };
|
|
86
|
+
usage.input += r.usage.input;
|
|
87
|
+
usage.output += r.usage.output;
|
|
88
|
+
usage.cacheRead += r.usage.cacheRead ?? 0;
|
|
89
|
+
usage.cacheCreate += r.usage.cacheCreate ?? 0;
|
|
90
|
+
durationMs += r.durationMs;
|
|
91
|
+
const row = r.provider !== undefined ? resolvePricing(DEFAULT_PRICING, r.provider, r.model) : undefined;
|
|
92
|
+
if (row !== undefined) {
|
|
93
|
+
costUsdMicros =
|
|
94
|
+
(costUsdMicros ?? 0) +
|
|
95
|
+
computeCostMicros(row, r.usage.input, r.usage.output, r.usage.cacheRead ?? 0, r.usage.cacheCreate ?? 0);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (event.kind !== "model_request" && event.kind !== "model_response")
|
|
99
|
+
return;
|
|
100
|
+
// Re-publish on the parent so its meter, cost-tracker and session mirror
|
|
101
|
+
// see the call (the child stamped the role; re-stamped defensively).
|
|
102
|
+
if (parent !== undefined) {
|
|
103
|
+
parent.eventBus.publish({ ...event, ...parent.eventBus.envelope(), ...attribution });
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
const injected = deps._consultAdapters?.get(args.target.modelString);
|
|
107
|
+
const t0 = performance.now();
|
|
108
|
+
try {
|
|
109
|
+
const text = await runChatLoop({
|
|
110
|
+
model: args.target.modelString,
|
|
111
|
+
instructions: args.instructions,
|
|
112
|
+
tools: [],
|
|
113
|
+
singleTurn: true,
|
|
114
|
+
seedMessages: args.seedMessages,
|
|
115
|
+
runContext: child,
|
|
116
|
+
sessionName: `${deps.sessionName ?? args.sessionTarget}:${args.sessionTarget}`,
|
|
117
|
+
sessionTarget: args.sessionTarget,
|
|
118
|
+
modelRole: args.role,
|
|
119
|
+
...(args.stage !== undefined ? { modelStage: args.stage } : {}),
|
|
120
|
+
...(args.maxTokens !== undefined ? { maxTokens: args.maxTokens } : {}),
|
|
121
|
+
installSigintHandler: false,
|
|
122
|
+
spinner: false,
|
|
123
|
+
stdout: () => { },
|
|
124
|
+
// A child's rules are the parent's business, not the disk's: the side
|
|
125
|
+
// call runs no tools, so there is nothing to merge.
|
|
126
|
+
settingsDir: null,
|
|
127
|
+
// §16 Q6 — no child session file per side call.
|
|
128
|
+
persistSession: false,
|
|
129
|
+
...(injected !== undefined ? { _adapter: injected } : {}),
|
|
130
|
+
});
|
|
131
|
+
return {
|
|
132
|
+
text,
|
|
133
|
+
model: served?.model ?? args.target.modelString,
|
|
134
|
+
provider: served?.provider ?? "anthropic",
|
|
135
|
+
usage,
|
|
136
|
+
latencyMs: durationMs > 0 ? durationMs : performance.now() - t0,
|
|
137
|
+
...(costUsdMicros !== undefined ? { costUsd: costUsdMicros / 1_000_000 } : {}),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
unsubscribe();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
// ---------------------------------------------------------------------------
|
|
145
|
+
// Transcript projection shared by the side calls
|
|
146
|
+
// ---------------------------------------------------------------------------
|
|
147
|
+
/**
|
|
148
|
+
* The transcript a tool-less side call can be handed: text blocks only
|
|
149
|
+
* (a `tool_use` / `tool_result` pair means nothing to a model that has no
|
|
150
|
+
* tools and would orphan under a different toolset), consecutive same-role
|
|
151
|
+
* messages merged, a leading assistant message dropped, ending on a user
|
|
152
|
+
* message. Returns `[]` when no user text survives.
|
|
153
|
+
*/
|
|
154
|
+
export function textOnlyTranscript(messages) {
|
|
155
|
+
const out = [];
|
|
156
|
+
for (const m of messages) {
|
|
157
|
+
const text = typeof m.content === "string"
|
|
158
|
+
? m.content
|
|
159
|
+
: m.content
|
|
160
|
+
.filter((b) => b.type === "text")
|
|
161
|
+
.map((b) => b.text)
|
|
162
|
+
.join("\n");
|
|
163
|
+
if (text.trim().length === 0)
|
|
164
|
+
continue;
|
|
165
|
+
const last = out[out.length - 1];
|
|
166
|
+
if (last !== undefined && last.role === m.role) {
|
|
167
|
+
last.content = `${last.content}\n\n${text}`;
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
out.push({ role: m.role, content: text });
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
while (out.length > 0 && out[0]?.role !== "user")
|
|
174
|
+
out.shift();
|
|
175
|
+
while (out.length > 0 && out[out.length - 1]?.role !== "user")
|
|
176
|
+
out.pop();
|
|
177
|
+
return out.map((m) => ({ role: m.role, content: m.content }));
|
|
178
|
+
}
|
|
179
|
+
/** The latest user text in a transcript (the turn's input), if any. */
|
|
180
|
+
export function latestUserText(messages) {
|
|
181
|
+
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
|
182
|
+
const m = messages[i];
|
|
183
|
+
if (m === undefined || m.role !== "user")
|
|
184
|
+
continue;
|
|
185
|
+
const text = typeof m.content === "string"
|
|
186
|
+
? m.content
|
|
187
|
+
: m.content
|
|
188
|
+
.filter((b) => b.type === "text")
|
|
189
|
+
.map((b) => b.text)
|
|
190
|
+
.join("\n");
|
|
191
|
+
if (text.trim().length > 0)
|
|
192
|
+
return text;
|
|
193
|
+
}
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
196
|
+
function renderTranscript(messages, maxChars) {
|
|
197
|
+
const lines = textOnlyTranscript(messages).map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content}`);
|
|
198
|
+
const joined = lines.join("\n\n");
|
|
199
|
+
return joined.length > maxChars ? `…${joined.slice(joined.length - maxChars)}` : joined;
|
|
200
|
+
}
|
|
201
|
+
function targetFor(roster, modelString, profile) {
|
|
202
|
+
const member = roster.find((c) => c.modelString === modelString);
|
|
203
|
+
const p = profile ?? member?.profile;
|
|
204
|
+
return { modelString, ...(p !== undefined ? { profile: p } : {}) };
|
|
205
|
+
}
|
|
206
|
+
function sumUsage(parts) {
|
|
207
|
+
const total = { input: 0, output: 0, cacheRead: 0, cacheCreate: 0 };
|
|
208
|
+
for (const u of parts) {
|
|
209
|
+
if (u === undefined)
|
|
210
|
+
continue;
|
|
211
|
+
total.input += u.input;
|
|
212
|
+
total.output += u.output;
|
|
213
|
+
total.cacheRead += u.cacheRead ?? 0;
|
|
214
|
+
total.cacheCreate += u.cacheCreate ?? 0;
|
|
215
|
+
}
|
|
216
|
+
return total;
|
|
217
|
+
}
|
|
218
|
+
// ---------------------------------------------------------------------------
|
|
219
|
+
// Guide (§7.4)
|
|
220
|
+
// ---------------------------------------------------------------------------
|
|
221
|
+
/** The guide model's system prompt — it plans; it never answers the user. */
|
|
222
|
+
export const GUIDE_INSTRUCTIONS = "You are the guide for another model that will actually answer the user. Read the executor's instructions and the conversation so far, then write concise guidance for the executor's next answer: the key steps or plan, the pitfalls to avoid, and what a good answer must contain. Write in the second person to the executor. Do not answer the user yourself, do not address the user, and do not include anything that is not guidance.";
|
|
223
|
+
const GUIDE_TRANSCRIPT_MAX_CHARS = 12_000;
|
|
224
|
+
export const DEFAULT_GUIDE_MAX_TOKENS = 400;
|
|
225
|
+
export function buildGuideSideCall(pool, deps) {
|
|
226
|
+
const guide = pool.strategy?.guide;
|
|
227
|
+
if (guide === undefined)
|
|
228
|
+
return undefined;
|
|
229
|
+
const roster = rosterFromPool(pool);
|
|
230
|
+
const target = targetFor(roster, guide.model, guide.modelProfile);
|
|
231
|
+
const maxTokens = guide.maxTokens ?? DEFAULT_GUIDE_MAX_TOKENS;
|
|
232
|
+
const budgetUsd = guide.budgetUsd;
|
|
233
|
+
let spentUsd = 0;
|
|
234
|
+
return {
|
|
235
|
+
every: guide.every ?? "turn",
|
|
236
|
+
model: guide.model,
|
|
237
|
+
...(target.profile !== undefined ? { profile: target.profile } : {}),
|
|
238
|
+
run: async (ctx) => {
|
|
239
|
+
// `guide.budget_usd` — the guide's own spend cap inside the run: past
|
|
240
|
+
// it the guide is skipped (the stage line says so), never the turn.
|
|
241
|
+
if (budgetUsd !== undefined && spentUsd >= budgetUsd) {
|
|
242
|
+
return { text: null, cause: "guide_budget_exhausted" };
|
|
243
|
+
}
|
|
244
|
+
const prompt = [
|
|
245
|
+
"Executor instructions:",
|
|
246
|
+
ctx.instructions,
|
|
247
|
+
"",
|
|
248
|
+
"Conversation so far:",
|
|
249
|
+
renderTranscript(ctx.messages, GUIDE_TRANSCRIPT_MAX_CHARS) || "(no messages yet)",
|
|
250
|
+
"",
|
|
251
|
+
"Write the guidance for the executor's next answer.",
|
|
252
|
+
].join("\n");
|
|
253
|
+
const result = await runNestedSingleTurn({
|
|
254
|
+
target,
|
|
255
|
+
instructions: GUIDE_INSTRUCTIONS,
|
|
256
|
+
seedMessages: [{ role: "user", content: prompt }],
|
|
257
|
+
role: "guide",
|
|
258
|
+
stage: "guide",
|
|
259
|
+
parent: ctx.runContext,
|
|
260
|
+
signal: ctx.signal,
|
|
261
|
+
maxTokens,
|
|
262
|
+
sessionTarget: "guide",
|
|
263
|
+
deps,
|
|
264
|
+
});
|
|
265
|
+
if (result.costUsd !== undefined)
|
|
266
|
+
spentUsd += result.costUsd;
|
|
267
|
+
const text = result.text.trim();
|
|
268
|
+
return text.length > 0 ? { text } : { text: null, cause: "empty" };
|
|
269
|
+
},
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
// ---------------------------------------------------------------------------
|
|
273
|
+
// Shadow (§7.8)
|
|
274
|
+
// ---------------------------------------------------------------------------
|
|
275
|
+
export const DEFAULT_SHADOW_SAMPLE_RATE = 0.1;
|
|
276
|
+
export function buildShadowSideCall(pool, deps) {
|
|
277
|
+
const shadow = pool.strategy?.shadow;
|
|
278
|
+
if (shadow === undefined)
|
|
279
|
+
return undefined;
|
|
280
|
+
const roster = rosterFromPool(pool);
|
|
281
|
+
const target = targetFor(roster, shadow.candidate, shadow.candidateProfile);
|
|
282
|
+
const strongTag = pool.routing?.strongTag;
|
|
283
|
+
const judgeModel = shadow.gradeWith ??
|
|
284
|
+
(roster.length > 0 ? strongestOf(roster, strongTag).modelString : undefined);
|
|
285
|
+
return {
|
|
286
|
+
candidate: shadow.candidate,
|
|
287
|
+
...(target.profile !== undefined ? { profile: target.profile } : {}),
|
|
288
|
+
sampleRate: shadow.sampleRate ?? DEFAULT_SHADOW_SAMPLE_RATE,
|
|
289
|
+
run: async (ctx) => {
|
|
290
|
+
const seed = textOnlyTranscript(ctx.messages);
|
|
291
|
+
if (seed.length === 0)
|
|
292
|
+
throw new Error("shadow: the transcript carries no user text to replay");
|
|
293
|
+
// The same request on the shadow candidate — the parent's instructions,
|
|
294
|
+
// the text-only transcript, no tools. Its text is discarded below.
|
|
295
|
+
const result = await runNestedSingleTurn({
|
|
296
|
+
target,
|
|
297
|
+
instructions: ctx.instructions,
|
|
298
|
+
seedMessages: seed,
|
|
299
|
+
role: "shadow",
|
|
300
|
+
stage: "shadow",
|
|
301
|
+
parent: ctx.runContext,
|
|
302
|
+
signal: ctx.signal,
|
|
303
|
+
sessionTarget: "shadow",
|
|
304
|
+
deps,
|
|
305
|
+
});
|
|
306
|
+
// Blind, order-swapped pairwise grading against the primary (prev = the
|
|
307
|
+
// served answer, new = the shadow's), metered on the parent bus as judge
|
|
308
|
+
// spend under `judge_share`.
|
|
309
|
+
const comparison = await judgePairwise({
|
|
310
|
+
input: latestUserText(ctx.messages) ?? "",
|
|
311
|
+
prevOutput: ctx.primaryText,
|
|
312
|
+
newOutput: result.text,
|
|
313
|
+
...(judgeModel !== undefined ? { model: judgeModel } : {}),
|
|
314
|
+
...(deps._judgeAdapter !== undefined ? { adapter: deps._judgeAdapter } : {}),
|
|
315
|
+
bus: ctx.bus,
|
|
316
|
+
role: "judge",
|
|
317
|
+
stage: "shadow",
|
|
318
|
+
});
|
|
319
|
+
const verdict = comparison.verdict === "new" ? "shadow" : comparison.verdict === "prev" ? "primary" : "tie";
|
|
320
|
+
return {
|
|
321
|
+
verdict,
|
|
322
|
+
agreed: comparison.agreed,
|
|
323
|
+
model: result.model,
|
|
324
|
+
provider: result.provider,
|
|
325
|
+
usage: result.usage,
|
|
326
|
+
latencyMs: result.latencyMs,
|
|
327
|
+
...(judgeModel !== undefined ? { judgeModel } : {}),
|
|
328
|
+
};
|
|
329
|
+
},
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
// ---------------------------------------------------------------------------
|
|
333
|
+
// Committee (§7.6)
|
|
334
|
+
// ---------------------------------------------------------------------------
|
|
335
|
+
function memberTargets(roster, slots, strongTag) {
|
|
336
|
+
const targets = [];
|
|
337
|
+
for (const slot of slots) {
|
|
338
|
+
const c = resolveRosterTarget(roster, slot, strongTag);
|
|
339
|
+
if (c === undefined) {
|
|
340
|
+
throw new Error(`wireSideCalls: committee member "${slot}" names no enabled model_pool candidate (a tag, a profile or a candidate model string)`);
|
|
341
|
+
}
|
|
342
|
+
if (!targets.includes(c))
|
|
343
|
+
targets.push(c);
|
|
344
|
+
}
|
|
345
|
+
return targets;
|
|
346
|
+
}
|
|
347
|
+
export function buildCommitteeSideCall(pool, deps) {
|
|
348
|
+
const committee = pool.strategy?.committee;
|
|
349
|
+
if (committee === undefined)
|
|
350
|
+
return undefined;
|
|
351
|
+
const roster = rosterFromPool(pool);
|
|
352
|
+
if (roster.length === 0)
|
|
353
|
+
return undefined;
|
|
354
|
+
const strongTag = pool.routing?.strongTag;
|
|
355
|
+
const members = memberTargets(roster, committee.members, strongTag);
|
|
356
|
+
if (members.length < 2) {
|
|
357
|
+
throw new Error("wireSideCalls: a committee needs at least two distinct members");
|
|
358
|
+
}
|
|
359
|
+
const judgeModel = committee.judge ?? strongestOf(roster, strongTag).modelString;
|
|
360
|
+
const tieBreaker = committee.escalateOnDisagreement !== undefined
|
|
361
|
+
? resolveRosterTarget(roster, committee.escalateOnDisagreement, strongTag)
|
|
362
|
+
: undefined;
|
|
363
|
+
if (committee.escalateOnDisagreement !== undefined && tieBreaker === undefined) {
|
|
364
|
+
throw new Error(`wireSideCalls: committee.escalate_on_disagreement "${committee.escalateOnDisagreement}" names no enabled model_pool candidate`);
|
|
365
|
+
}
|
|
366
|
+
const asTarget = (c) => ({
|
|
367
|
+
modelString: c.modelString,
|
|
368
|
+
...(c.profile !== undefined ? { profile: c.profile } : {}),
|
|
369
|
+
});
|
|
370
|
+
const publishStage = (ctx, fields) => {
|
|
371
|
+
ctx.bus.publish({
|
|
372
|
+
...ctx.bus.envelope(),
|
|
373
|
+
kind: "model_stage",
|
|
374
|
+
strategy: "committee",
|
|
375
|
+
...fields,
|
|
376
|
+
});
|
|
377
|
+
};
|
|
378
|
+
return {
|
|
379
|
+
members: members.map(asTarget),
|
|
380
|
+
judge: judgeModel,
|
|
381
|
+
...(tieBreaker !== undefined ? { escalateOnDisagreement: asTarget(tieBreaker) } : {}),
|
|
382
|
+
run: async (ctx) => {
|
|
383
|
+
const seed = textOnlyTranscript(ctx.messages);
|
|
384
|
+
if (seed.length === 0)
|
|
385
|
+
throw new Error("committee: the transcript carries no user text");
|
|
386
|
+
// Members run ONE AFTER ANOTHER — each a nested loop with its own state;
|
|
387
|
+
// never a parallel `runOneTurn` (§7.6).
|
|
388
|
+
const outcomes = [];
|
|
389
|
+
const answers = [];
|
|
390
|
+
// §7.12 — the run's total cap, re-read before every nested call the
|
|
391
|
+
// committee makes: once reached, every call not yet made is excluded
|
|
392
|
+
// (cause `budget`), never opened. Latched so one "stop" is final.
|
|
393
|
+
let capReached = false;
|
|
394
|
+
const budgetStops = async () => {
|
|
395
|
+
if (capReached)
|
|
396
|
+
return true;
|
|
397
|
+
if (ctx.budgetGate === undefined)
|
|
398
|
+
return false;
|
|
399
|
+
capReached = (await ctx.budgetGate()) === "stop";
|
|
400
|
+
return capReached;
|
|
401
|
+
};
|
|
402
|
+
for (const member of members) {
|
|
403
|
+
const t = asTarget(member);
|
|
404
|
+
const stage = {
|
|
405
|
+
stage: "member",
|
|
406
|
+
role: "committee",
|
|
407
|
+
model: t.modelString,
|
|
408
|
+
...(t.profile !== undefined ? { profile: t.profile } : {}),
|
|
409
|
+
};
|
|
410
|
+
if (await budgetStops()) {
|
|
411
|
+
outcomes.push({ ...t, outcome: "skipped", latencyMs: 0, cause: "budget" });
|
|
412
|
+
publishStage(ctx, { ...stage, outcome: "skipped", cause: "budget" });
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
publishStage(ctx, { ...stage, outcome: "started" });
|
|
416
|
+
try {
|
|
417
|
+
const r = await runNestedSingleTurn({
|
|
418
|
+
target: t,
|
|
419
|
+
instructions: ctx.instructions,
|
|
420
|
+
seedMessages: seed,
|
|
421
|
+
role: "committee",
|
|
422
|
+
stage: "member",
|
|
423
|
+
parent: ctx.runContext,
|
|
424
|
+
signal: ctx.signal,
|
|
425
|
+
sessionTarget: "committee",
|
|
426
|
+
deps,
|
|
427
|
+
});
|
|
428
|
+
if (r.text.trim().length === 0)
|
|
429
|
+
throw new Error("member returned no text");
|
|
430
|
+
answers.push({ target: member, text: r.text, usage: r.usage });
|
|
431
|
+
outcomes.push({
|
|
432
|
+
...t,
|
|
433
|
+
model: r.model,
|
|
434
|
+
outcome: "done",
|
|
435
|
+
latencyMs: r.latencyMs,
|
|
436
|
+
usage: r.usage,
|
|
437
|
+
});
|
|
438
|
+
publishStage(ctx, {
|
|
439
|
+
...stage,
|
|
440
|
+
model: r.model,
|
|
441
|
+
outcome: "done",
|
|
442
|
+
...(r.costUsd !== undefined ? { costUsdMicros: Math.round(r.costUsd * 1e6) } : {}),
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
catch (err) {
|
|
446
|
+
// §7.13 — a failed member is excluded.
|
|
447
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
448
|
+
outcomes.push({ ...t, outcome: "failed", latencyMs: 0, cause: message });
|
|
449
|
+
publishStage(ctx, { ...stage, outcome: "failed", cause: message });
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
const served = (pick, agreed, escalated, extraUsage, cause, quality) => ({
|
|
453
|
+
text: pick.text,
|
|
454
|
+
winner: {
|
|
455
|
+
modelString: pick.target.modelString,
|
|
456
|
+
model: outcomes.find((o) => o.modelString === pick.target.modelString)?.model ??
|
|
457
|
+
pick.target.modelString,
|
|
458
|
+
...(pick.target.profile !== undefined ? { profile: pick.target.profile } : {}),
|
|
459
|
+
},
|
|
460
|
+
agreed,
|
|
461
|
+
escalated,
|
|
462
|
+
cause,
|
|
463
|
+
members: outcomes.map((o) => {
|
|
464
|
+
const q = quality?.get(o.modelString);
|
|
465
|
+
return q !== undefined ? { ...o, quality: q } : o;
|
|
466
|
+
}),
|
|
467
|
+
usage: sumUsage([...answers.map((a) => a.usage), extraUsage]),
|
|
468
|
+
});
|
|
469
|
+
if (answers.length === 0) {
|
|
470
|
+
throw new Error("committee: every member failed");
|
|
471
|
+
}
|
|
472
|
+
if (answers.length === 1) {
|
|
473
|
+
// Fewer than two survivors: the step falls back to the one that did.
|
|
474
|
+
const only = answers[0];
|
|
475
|
+
return served(only, false, false, undefined, "single-member");
|
|
476
|
+
}
|
|
477
|
+
// `decide: judge` — the order-controlled N-way pick, metered as judge
|
|
478
|
+
// spend on the parent bus.
|
|
479
|
+
let selection;
|
|
480
|
+
try {
|
|
481
|
+
selection = await judgeSelect({
|
|
482
|
+
input: latestUserText(ctx.messages) ?? "",
|
|
483
|
+
candidates: answers.map((a) => ({ id: a.target.modelString, output: a.text })),
|
|
484
|
+
model: judgeModel,
|
|
485
|
+
...(deps._judgeAdapter !== undefined ? { adapter: deps._judgeAdapter } : {}),
|
|
486
|
+
bus: ctx.bus,
|
|
487
|
+
role: "judge",
|
|
488
|
+
stage: "committee",
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
catch (err) {
|
|
492
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
493
|
+
ctx.runContext.logger.warn("committee judge failed — serving the strongest member", {
|
|
494
|
+
message,
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
if (selection?.winner !== undefined) {
|
|
498
|
+
const winnerId = selection.winner;
|
|
499
|
+
const pick = answers.find((a) => a.target.modelString === winnerId);
|
|
500
|
+
const quality = new Map(answers.map((a) => [a.target.modelString, a.target.modelString === winnerId ? 1 : 0]));
|
|
501
|
+
return served(pick, true, false, undefined, "agreed", quality);
|
|
502
|
+
}
|
|
503
|
+
// Disagreement (or a failed judge): agreement was the acceptor; the
|
|
504
|
+
// strong tie-breaker answers only now, else the strongest survivor's
|
|
505
|
+
// answer stands with the disagreement recorded.
|
|
506
|
+
const unresolved = new Map(answers.map((a) => [a.target.modelString, 0.5]));
|
|
507
|
+
if (selection !== undefined && tieBreaker !== undefined) {
|
|
508
|
+
const t = asTarget(tieBreaker);
|
|
509
|
+
const stage = {
|
|
510
|
+
stage: "tie-break",
|
|
511
|
+
role: "escalation",
|
|
512
|
+
model: t.modelString,
|
|
513
|
+
...(t.profile !== undefined ? { profile: t.profile } : {}),
|
|
514
|
+
};
|
|
515
|
+
if (await budgetStops()) {
|
|
516
|
+
// The strong tie-breaker is the committee's most expensive call:
|
|
517
|
+
// past the cap the strongest survivor stands instead.
|
|
518
|
+
publishStage(ctx, { ...stage, outcome: "skipped", cause: "budget" });
|
|
519
|
+
}
|
|
520
|
+
else {
|
|
521
|
+
publishStage(ctx, { ...stage, outcome: "started", cause: "disagreement" });
|
|
522
|
+
try {
|
|
523
|
+
const r = await runNestedSingleTurn({
|
|
524
|
+
target: t,
|
|
525
|
+
instructions: ctx.instructions,
|
|
526
|
+
seedMessages: seed,
|
|
527
|
+
role: "escalation",
|
|
528
|
+
stage: "tie-break",
|
|
529
|
+
parent: ctx.runContext,
|
|
530
|
+
signal: ctx.signal,
|
|
531
|
+
sessionTarget: "committee",
|
|
532
|
+
deps,
|
|
533
|
+
});
|
|
534
|
+
publishStage(ctx, {
|
|
535
|
+
...stage,
|
|
536
|
+
model: r.model,
|
|
537
|
+
outcome: "done",
|
|
538
|
+
...(r.costUsd !== undefined ? { costUsdMicros: Math.round(r.costUsd * 1e6) } : {}),
|
|
539
|
+
});
|
|
540
|
+
const existing = outcomes.find((o) => o.modelString === t.modelString);
|
|
541
|
+
if (existing === undefined) {
|
|
542
|
+
outcomes.push({
|
|
543
|
+
...t,
|
|
544
|
+
model: r.model,
|
|
545
|
+
outcome: "done",
|
|
546
|
+
latencyMs: r.latencyMs,
|
|
547
|
+
usage: r.usage,
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
return served({ target: tieBreaker, text: r.text }, false, true, r.usage, "escalated", unresolved);
|
|
551
|
+
}
|
|
552
|
+
catch (err) {
|
|
553
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
554
|
+
publishStage(ctx, { ...stage, outcome: "failed", cause: message });
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
const survivors = answers.map((a) => a.target);
|
|
559
|
+
const strongest = strongestOf(survivors, strongTag);
|
|
560
|
+
const pick = answers.find((a) => a.target === strongest);
|
|
561
|
+
return served(pick, false, false, undefined, selection === undefined ? "judge-failed" : "disagreement", unresolved);
|
|
562
|
+
},
|
|
563
|
+
};
|
|
564
|
+
}
|
|
565
|
+
// ---------------------------------------------------------------------------
|
|
566
|
+
// wireSideCalls — the root's entry, and its codegen twin
|
|
567
|
+
// ---------------------------------------------------------------------------
|
|
568
|
+
/** True when the pool's strategy declares a guide, a shadow or a committee. */
|
|
569
|
+
export function hasSideCallStrategy(pool) {
|
|
570
|
+
const st = pool?.strategy;
|
|
571
|
+
return (st !== undefined &&
|
|
572
|
+
(st.guide !== undefined || st.shadow !== undefined || st.committee !== undefined));
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* Build the `sideCalls` option for a pool. Spread-return-`{}`: a pool whose
|
|
576
|
+
* strategy declares none of the three yields `{}`, so every pre-0.6.0 loop
|
|
577
|
+
* call stays byte-identical. Called once per RUN (the guide's budget cap is
|
|
578
|
+
* per run), never cached across runs.
|
|
579
|
+
*/
|
|
580
|
+
export function wireSideCalls(pool, deps = {}) {
|
|
581
|
+
if (!hasSideCallStrategy(pool))
|
|
582
|
+
return {};
|
|
583
|
+
const guide = buildGuideSideCall(pool, deps);
|
|
584
|
+
const shadow = buildShadowSideCall(pool, deps);
|
|
585
|
+
const committee = buildCommitteeSideCall(pool, deps);
|
|
586
|
+
const sideCalls = {
|
|
587
|
+
...(guide !== undefined ? { guide } : {}),
|
|
588
|
+
...(shadow !== undefined ? { shadow } : {}),
|
|
589
|
+
...(committee !== undefined ? { committee } : {}),
|
|
590
|
+
};
|
|
591
|
+
return Object.keys(sideCalls).length > 0 ? { sideCalls } : {};
|
|
592
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@crewhaus/model-service",
|
|
3
|
+
"version": "0.6.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "0.6.0 composition root (plan §2 stance 4, brief 308): wireModels(fragment, deps) turns the lowered model-routing IR into the runChatLoop options fragment — one stable call replacing the per-emitter routing codegen",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"scripts": {
|
|
15
|
+
"test": "bun test src"
|
|
16
|
+
},
|
|
17
|
+
"dependencies": {
|
|
18
|
+
"@crewhaus/adapter-anthropic": "0.6.0",
|
|
19
|
+
"@crewhaus/cost-tracker": "0.6.0",
|
|
20
|
+
"@crewhaus/eval-judge": "0.6.0",
|
|
21
|
+
"@crewhaus/infra-utils": "0.6.0",
|
|
22
|
+
"@crewhaus/ir": "0.6.0",
|
|
23
|
+
"@crewhaus/run-context": "0.6.0",
|
|
24
|
+
"@crewhaus/runtime-core": "0.6.0",
|
|
25
|
+
"@crewhaus/tool-catalog": "0.6.0",
|
|
26
|
+
"@crewhaus/tool-consult": "0.6.0",
|
|
27
|
+
"@crewhaus/trace-event-bus": "0.6.0"
|
|
28
|
+
},
|
|
29
|
+
"license": "Apache-2.0",
|
|
30
|
+
"author": {
|
|
31
|
+
"name": "Max Meier",
|
|
32
|
+
"email": "max@crewhaus.ai",
|
|
33
|
+
"url": "https://crewhaus.ai"
|
|
34
|
+
},
|
|
35
|
+
"repository": {
|
|
36
|
+
"type": "git",
|
|
37
|
+
"url": "git+https://github.com/crewhaus/factory.git",
|
|
38
|
+
"directory": "packages/model-service"
|
|
39
|
+
},
|
|
40
|
+
"homepage": "https://github.com/crewhaus/factory/tree/main/packages/model-service#readme",
|
|
41
|
+
"bugs": {
|
|
42
|
+
"url": "https://github.com/crewhaus/factory/issues"
|
|
43
|
+
},
|
|
44
|
+
"publishConfig": {
|
|
45
|
+
"access": "public"
|
|
46
|
+
},
|
|
47
|
+
"files": ["dist", "README.md", "LICENSE", "NOTICE"]
|
|
48
|
+
}
|