@mupt-ai/dari-router 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +100 -0
- package/dist/anonymous_actions.d.ts +55 -0
- package/dist/anonymous_actions.js +277 -0
- package/dist/anonymous_actions.js.map +1 -0
- package/dist/auto_router.d.ts +10 -0
- package/dist/auto_router.js +56 -0
- package/dist/auto_router.js.map +1 -0
- package/dist/cache_behavior.d.ts +14 -0
- package/dist/cache_behavior.js +112 -0
- package/dist/cache_behavior.js.map +1 -0
- package/dist/cache_scope.d.ts +5 -0
- package/dist/cache_scope.js +49 -0
- package/dist/cache_scope.js.map +1 -0
- package/dist/cache_switch_decision.d.ts +24 -0
- package/dist/cache_switch_decision.js +21 -0
- package/dist/cache_switch_decision.js.map +1 -0
- package/dist/candidate_pruning.d.ts +45 -0
- package/dist/candidate_pruning.js +154 -0
- package/dist/candidate_pruning.js.map +1 -0
- package/dist/candidate_resolution.d.ts +41 -0
- package/dist/candidate_resolution.js +111 -0
- package/dist/candidate_resolution.js.map +1 -0
- package/dist/compatibility.d.ts +22 -0
- package/dist/compatibility.js +76 -0
- package/dist/compatibility.js.map +1 -0
- package/dist/continuation_state.d.ts +36 -0
- package/dist/continuation_state.js +249 -0
- package/dist/continuation_state.js.map +1 -0
- package/dist/cost.d.ts +31 -0
- package/dist/cost.js +314 -0
- package/dist/cost.js.map +1 -0
- package/dist/dari_policy.d.ts +34 -0
- package/dist/dari_policy.js +267 -0
- package/dist/dari_policy.js.map +1 -0
- package/dist/errors.d.ts +7 -0
- package/dist/errors.js +14 -0
- package/dist/errors.js.map +1 -0
- package/dist/fingerprint.d.ts +13 -0
- package/dist/fingerprint.js +130 -0
- package/dist/fingerprint.js.map +1 -0
- package/dist/fixed_turn_cost_config.d.ts +9 -0
- package/dist/fixed_turn_cost_config.js +22 -0
- package/dist/fixed_turn_cost_config.js.map +1 -0
- package/dist/framework_error.d.ts +8 -0
- package/dist/framework_error.js +24 -0
- package/dist/framework_error.js.map +1 -0
- package/dist/framework_lifecycle.d.ts +11 -0
- package/dist/framework_lifecycle.js +57 -0
- package/dist/framework_lifecycle.js.map +1 -0
- package/dist/framework_models.d.ts +3 -0
- package/dist/framework_models.js +120 -0
- package/dist/framework_models.js.map +1 -0
- package/dist/framework_router.d.ts +2 -0
- package/dist/framework_router.js +384 -0
- package/dist/framework_router.js.map +1 -0
- package/dist/framework_streaming.d.ts +26 -0
- package/dist/framework_streaming.js +486 -0
- package/dist/framework_streaming.js.map +1 -0
- package/dist/framework_types.d.ts +272 -0
- package/dist/framework_types.js +5 -0
- package/dist/framework_types.js.map +1 -0
- package/dist/framework_validation.d.ts +7 -0
- package/dist/framework_validation.js +169 -0
- package/dist/framework_validation.js.map +1 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +38 -0
- package/dist/index.js.map +1 -0
- package/dist/json.d.ts +10 -0
- package/dist/json.js +29 -0
- package/dist/json.js.map +1 -0
- package/dist/model_ids.d.ts +4 -0
- package/dist/model_ids.js +26 -0
- package/dist/model_ids.js.map +1 -0
- package/dist/pi_context.d.ts +4 -0
- package/dist/pi_context.js +206 -0
- package/dist/pi_context.js.map +1 -0
- package/dist/pi_options.d.ts +3 -0
- package/dist/pi_options.js +240 -0
- package/dist/pi_options.js.map +1 -0
- package/dist/pi_output.d.ts +4 -0
- package/dist/pi_output.js +328 -0
- package/dist/pi_output.js.map +1 -0
- package/dist/pi_reasoning_signature.d.ts +10 -0
- package/dist/pi_reasoning_signature.js +33 -0
- package/dist/pi_reasoning_signature.js.map +1 -0
- package/dist/pi_runtime.d.ts +3 -0
- package/dist/pi_runtime.js +155 -0
- package/dist/pi_runtime.js.map +1 -0
- package/dist/pi_types.d.ts +35 -0
- package/dist/pi_types.js +2 -0
- package/dist/pi_types.js.map +1 -0
- package/dist/portable_reasoning_state.d.ts +7 -0
- package/dist/portable_reasoning_state.js +81 -0
- package/dist/portable_reasoning_state.js.map +1 -0
- package/dist/previous_decision.d.ts +13 -0
- package/dist/previous_decision.js +82 -0
- package/dist/previous_decision.js.map +1 -0
- package/dist/prompts.d.ts +3 -0
- package/dist/prompts.js +65 -0
- package/dist/prompts.js.map +1 -0
- package/dist/protocol_anthropic.d.ts +18 -0
- package/dist/protocol_anthropic.js +443 -0
- package/dist/protocol_anthropic.js.map +1 -0
- package/dist/protocol_openai_chat.d.ts +25 -0
- package/dist/protocol_openai_chat.js +503 -0
- package/dist/protocol_openai_chat.js.map +1 -0
- package/dist/protocol_validation.d.ts +23 -0
- package/dist/protocol_validation.js +134 -0
- package/dist/protocol_validation.js.map +1 -0
- package/dist/route.d.ts +105 -0
- package/dist/route.js +226 -0
- package/dist/route.js.map +1 -0
- package/dist/routing_history.d.ts +7 -0
- package/dist/routing_history.js +132 -0
- package/dist/routing_history.js.map +1 -0
- package/dist/selector_input.d.ts +42 -0
- package/dist/selector_input.js +153 -0
- package/dist/selector_input.js.map +1 -0
- package/dist/selector_parse.d.ts +13 -0
- package/dist/selector_parse.js +95 -0
- package/dist/selector_parse.js.map +1 -0
- package/dist/selector_request.d.ts +26 -0
- package/dist/selector_request.js +85 -0
- package/dist/selector_request.js.map +1 -0
- package/dist/selector_truncation.d.ts +2 -0
- package/dist/selector_truncation.js +48 -0
- package/dist/selector_truncation.js.map +1 -0
- package/dist/selector_wire.d.ts +10 -0
- package/dist/selector_wire.js +113 -0
- package/dist/selector_wire.js.map +1 -0
- package/dist/types.d.ts +152 -0
- package/dist/types.js +14 -0
- package/dist/types.js.map +1 -0
- package/package.json +64 -0
- package/src/anonymous_actions.ts +434 -0
- package/src/auto_router.ts +95 -0
- package/src/cache_behavior.ts +130 -0
- package/src/cache_scope.ts +58 -0
- package/src/cache_switch_decision.ts +53 -0
- package/src/candidate_pruning.ts +257 -0
- package/src/candidate_resolution.ts +208 -0
- package/src/compatibility.ts +162 -0
- package/src/continuation_state.ts +368 -0
- package/src/cost.ts +455 -0
- package/src/dari_policy.ts +360 -0
- package/src/errors.ts +27 -0
- package/src/fingerprint.ts +148 -0
- package/src/fixed_turn_cost_config.ts +24 -0
- package/src/framework_error.ts +38 -0
- package/src/framework_lifecycle.ts +74 -0
- package/src/framework_models.ts +186 -0
- package/src/framework_router.ts +490 -0
- package/src/framework_streaming.ts +613 -0
- package/src/framework_types.ts +372 -0
- package/src/framework_validation.ts +235 -0
- package/src/index.ts +37 -0
- package/src/json.ts +43 -0
- package/src/model_ids.ts +29 -0
- package/src/pi_context.ts +251 -0
- package/src/pi_options.ts +295 -0
- package/src/pi_output.ts +387 -0
- package/src/pi_reasoning_signature.ts +41 -0
- package/src/pi_runtime.ts +238 -0
- package/src/pi_types.ts +71 -0
- package/src/portable_reasoning_state.ts +129 -0
- package/src/previous_decision.ts +109 -0
- package/src/prompts.ts +69 -0
- package/src/protocol_anthropic.ts +571 -0
- package/src/protocol_openai_chat.ts +645 -0
- package/src/protocol_validation.ts +202 -0
- package/src/route.ts +490 -0
- package/src/routing_history.ts +157 -0
- package/src/selector_input.ts +246 -0
- package/src/selector_parse.ts +138 -0
- package/src/selector_request.ts +130 -0
- package/src/selector_truncation.ts +48 -0
- package/src/selector_wire.ts +167 -0
- package/src/types.ts +231 -0
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
import type { CandidateModelMetadata } from "./compatibility.js";
|
|
2
|
+
import type { PricingLookup } from "./cost.js";
|
|
3
|
+
import { RouterCoreError } from "./errors.js";
|
|
4
|
+
import { RouterFrameworkError } from "./framework_error.js";
|
|
5
|
+
import type {
|
|
6
|
+
RouterCandidate,
|
|
7
|
+
RouterContent,
|
|
8
|
+
RouterRequest,
|
|
9
|
+
RoutingPolicy,
|
|
10
|
+
RoutingPolicyInput,
|
|
11
|
+
} from "./framework_types.js";
|
|
12
|
+
import { optionsFingerprints } from "./fingerprint.js";
|
|
13
|
+
import {
|
|
14
|
+
finalizeRoute,
|
|
15
|
+
prepareRoute,
|
|
16
|
+
type PreparedRoute,
|
|
17
|
+
type RouteInput,
|
|
18
|
+
type RouteResult,
|
|
19
|
+
type Selector,
|
|
20
|
+
} from "./route.js";
|
|
21
|
+
import type {
|
|
22
|
+
ChatCompletionRequest,
|
|
23
|
+
ChatMessage,
|
|
24
|
+
CustomRouterConfig,
|
|
25
|
+
RouterEval,
|
|
26
|
+
RouterModelPrice,
|
|
27
|
+
RouterPrefixHit,
|
|
28
|
+
} from "./types.js";
|
|
29
|
+
|
|
30
|
+
export type DariPolicyState = {
|
|
31
|
+
chainsByModel?: ReadonlyMap<string, string[]>;
|
|
32
|
+
prefixHits?: RouterPrefixHit[];
|
|
33
|
+
looseChain?: readonly string[];
|
|
34
|
+
nowMs?: number;
|
|
35
|
+
toolChoiceFingerprint?: string;
|
|
36
|
+
responseFormatFingerprint?: string;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export type DariRoutingPolicyOptions<Metadata = unknown> = {
|
|
40
|
+
selector?: Selector | ((request: ChatCompletionRequest, signal: AbortSignal) => Promise<string>);
|
|
41
|
+
runtime?: {
|
|
42
|
+
select(request: ChatCompletionRequest, signal?: AbortSignal): Promise<string>;
|
|
43
|
+
};
|
|
44
|
+
selectorModel: string;
|
|
45
|
+
selectorContextWindowChars: number;
|
|
46
|
+
metadata?: (model: RouterCandidate<Metadata>) => CandidateModelMetadata;
|
|
47
|
+
pricing?: PricingLookup;
|
|
48
|
+
evals?: RouterEval[] | ((input: RoutingPolicyInput<Metadata>) => RouterEval[] | Promise<RouterEval[]>);
|
|
49
|
+
state?: (
|
|
50
|
+
input: RoutingPolicyInput<Metadata>,
|
|
51
|
+
) => DariPolicyState | Promise<DariPolicyState>;
|
|
52
|
+
strategy?: "slm" | "custom";
|
|
53
|
+
customConfig?: CustomRouterConfig | null;
|
|
54
|
+
modelPrices?: Record<string, RouterModelPrice>;
|
|
55
|
+
averageOutputTokensByModel?: RouteInput["averageOutputTokensByModel"];
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export type DariPolicyDetails = {
|
|
59
|
+
prepared: PreparedRoute;
|
|
60
|
+
selectorOutput: string;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export function createDariRoutingPolicy<Metadata = unknown>(
|
|
64
|
+
options: DariRoutingPolicyOptions<Metadata>,
|
|
65
|
+
): RoutingPolicy<Metadata> {
|
|
66
|
+
const selector = options.selector ?? options.runtime;
|
|
67
|
+
if (selector === undefined) {
|
|
68
|
+
throw new RouterFrameworkError(
|
|
69
|
+
"configuration",
|
|
70
|
+
"Dari routing requires a selector or Pi runtime.",
|
|
71
|
+
"selector_missing",
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
return async (input) => {
|
|
75
|
+
const chatRequest = routerRequestToChat(input.request);
|
|
76
|
+
const state = await options.state?.(input) ?? {};
|
|
77
|
+
const evals = typeof options.evals === "function"
|
|
78
|
+
? await options.evals(input)
|
|
79
|
+
: options.evals ?? [];
|
|
80
|
+
const fingerprints = optionsFingerprints(chatRequest);
|
|
81
|
+
const candidateModels = input.candidates.map((candidate) => candidate.id);
|
|
82
|
+
const metadataByModel = new Map(
|
|
83
|
+
input.candidates.map((candidate) => [
|
|
84
|
+
candidate.id,
|
|
85
|
+
options.metadata?.(candidate) ?? defaultDariMetadata(candidate),
|
|
86
|
+
]),
|
|
87
|
+
);
|
|
88
|
+
let prepared: PreparedRoute;
|
|
89
|
+
try {
|
|
90
|
+
prepared = prepareRoute({
|
|
91
|
+
candidateModels,
|
|
92
|
+
metadataLookup: (model) => metadataByModel.get(model)!,
|
|
93
|
+
requiredCapabilities: [],
|
|
94
|
+
...(input.request.reasoning?.effort === undefined
|
|
95
|
+
? {}
|
|
96
|
+
: { requestedReasoningEffort: input.request.reasoning.effort }),
|
|
97
|
+
...(input.request.reasoning?.enabled === undefined
|
|
98
|
+
? {}
|
|
99
|
+
: { thinkingEnabled: input.request.reasoning.enabled }),
|
|
100
|
+
...(chatRequest.tool_choice === undefined ? {} : { toolChoice: chatRequest.tool_choice }),
|
|
101
|
+
modelThinkingLevels: Object.fromEntries(
|
|
102
|
+
input.candidates.map((candidate) => [candidate.id, [...candidate.reasoningEfforts]]),
|
|
103
|
+
),
|
|
104
|
+
strategy: options.strategy ?? "slm",
|
|
105
|
+
...(options.customConfig === undefined ? {} : { customConfig: options.customConfig }),
|
|
106
|
+
modelPrices: options.modelPrices ?? {},
|
|
107
|
+
pricing: options.pricing ?? (() => null),
|
|
108
|
+
...(options.averageOutputTokensByModel === undefined
|
|
109
|
+
? {}
|
|
110
|
+
: { averageOutputTokensByModel: options.averageOutputTokensByModel }),
|
|
111
|
+
messages: chatRequest.messages ?? [],
|
|
112
|
+
chainsByModel: state.chainsByModel ?? new Map(
|
|
113
|
+
candidateModels.map((model) => [model, []]),
|
|
114
|
+
),
|
|
115
|
+
prefixHits: state.prefixHits ?? [],
|
|
116
|
+
nowMs: state.nowMs ?? Date.now(),
|
|
117
|
+
...(state.looseChain === undefined ? {} : { looseChain: state.looseChain }),
|
|
118
|
+
toolChoiceFp: state.toolChoiceFingerprint ?? fingerprints.tool_choice_fp,
|
|
119
|
+
responseFormatFp: state.responseFormatFingerprint ?? fingerprints.response_format_fp,
|
|
120
|
+
evals,
|
|
121
|
+
selectorModel: options.selectorModel,
|
|
122
|
+
selectorContextWindowChars: options.selectorContextWindowChars,
|
|
123
|
+
// Selection fallback is deliberately not provider retry. The framework
|
|
124
|
+
// dispatches one chosen candidate and leaves retries to its executor.
|
|
125
|
+
modelFallbackEnabled: false,
|
|
126
|
+
});
|
|
127
|
+
} catch (error) {
|
|
128
|
+
throw mapDariPolicyError(error);
|
|
129
|
+
}
|
|
130
|
+
if (prepared.selectorPreparation === null) {
|
|
131
|
+
throw new RouterFrameworkError(
|
|
132
|
+
"configuration",
|
|
133
|
+
"Dari custom routing requires a non-empty customConfig.",
|
|
134
|
+
"custom_config_missing",
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const selectorRequest = prepared.selectorPreparation.selectorRequest;
|
|
139
|
+
let selectorOutput: string;
|
|
140
|
+
try {
|
|
141
|
+
selectorOutput = typeof selector === "function"
|
|
142
|
+
? await selector(selectorRequest, input.signal)
|
|
143
|
+
: await selector.select(selectorRequest, input.signal);
|
|
144
|
+
} catch (error) {
|
|
145
|
+
throw mapSelectorError(error);
|
|
146
|
+
}
|
|
147
|
+
let result: RouteResult;
|
|
148
|
+
try {
|
|
149
|
+
result = finalizeRoute(prepared, selectorOutput);
|
|
150
|
+
} catch (error) {
|
|
151
|
+
throw mapDariPolicyError(error);
|
|
152
|
+
}
|
|
153
|
+
return {
|
|
154
|
+
model: result.decision.selectedModel,
|
|
155
|
+
reasoningEffort: result.decision.reasoningEffort,
|
|
156
|
+
reason: result.decision.reason,
|
|
157
|
+
...(result.decision.leaseTurnsRemaining === undefined
|
|
158
|
+
? {}
|
|
159
|
+
: { leaseTurnsRemaining: result.decision.leaseTurnsRemaining }),
|
|
160
|
+
details: { prepared, selectorOutput } satisfies DariPolicyDetails,
|
|
161
|
+
};
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function mapSelectorError(error: unknown): unknown {
|
|
166
|
+
if (!(error instanceof RouterFrameworkError) || error.kind !== "executor") return error;
|
|
167
|
+
return new RouterFrameworkError(
|
|
168
|
+
"policy",
|
|
169
|
+
error.message,
|
|
170
|
+
error.code,
|
|
171
|
+
error.param,
|
|
172
|
+
{ cause: error },
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function mapDariPolicyError(error: unknown): unknown {
|
|
177
|
+
if (!(error instanceof RouterCoreError)) return error;
|
|
178
|
+
const kind = error.kind === "invalid_request"
|
|
179
|
+
? "invalid_request"
|
|
180
|
+
: error.kind === "configuration"
|
|
181
|
+
? "configuration"
|
|
182
|
+
: "policy";
|
|
183
|
+
return new RouterFrameworkError(
|
|
184
|
+
kind,
|
|
185
|
+
error.message,
|
|
186
|
+
error.code,
|
|
187
|
+
error.param,
|
|
188
|
+
{ cause: error },
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function defaultDariMetadata<Metadata>(
|
|
193
|
+
candidate: RouterCandidate<Metadata>,
|
|
194
|
+
): CandidateModelMetadata {
|
|
195
|
+
return {
|
|
196
|
+
provider: candidate.provider,
|
|
197
|
+
api: candidate.api,
|
|
198
|
+
supportsImageInput: candidate.capabilities.imageInput,
|
|
199
|
+
supportsHostedWebSearch: false,
|
|
200
|
+
supportsStructuredOutput: candidate.capabilities.structuredOutput,
|
|
201
|
+
supportedThinkingLevels: [...candidate.reasoningEfforts],
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function routerRequestToChat(request: RouterRequest): ChatCompletionRequest {
|
|
206
|
+
const messages: ChatMessage[] = [];
|
|
207
|
+
for (const item of request.items) {
|
|
208
|
+
if (item.type === "message") {
|
|
209
|
+
const content = chatContent(item.content);
|
|
210
|
+
const previous = messages.at(-1);
|
|
211
|
+
if (item.role === "assistant" && previous?.role === "assistant") {
|
|
212
|
+
const previousContent = previous.content;
|
|
213
|
+
previous.content = Array.isArray(previousContent)
|
|
214
|
+
? [...previousContent, ...content]
|
|
215
|
+
: content;
|
|
216
|
+
} else {
|
|
217
|
+
messages.push({ role: item.role, content });
|
|
218
|
+
}
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
if (item.type === "reasoning") {
|
|
222
|
+
const readable = [...(item.summary ?? []), ...item.content].join("\n");
|
|
223
|
+
let assistant = messages.at(-1);
|
|
224
|
+
if (!assistant || assistant.role !== "assistant" || hasAssistantContent(assistant)) {
|
|
225
|
+
assistant = { role: "assistant", content: null };
|
|
226
|
+
messages.push(assistant);
|
|
227
|
+
}
|
|
228
|
+
if (readable) {
|
|
229
|
+
assistant.reasoning_content = appendReasoningText(assistant.reasoning_content, readable);
|
|
230
|
+
}
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
if (item.type === "tool_result") {
|
|
234
|
+
messages.push({
|
|
235
|
+
role: "tool",
|
|
236
|
+
tool_call_id: item.toolCallId,
|
|
237
|
+
content: chatContent(item.content),
|
|
238
|
+
});
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
let assistant = messages.at(-1);
|
|
243
|
+
if (assistant?.role !== "assistant") {
|
|
244
|
+
assistant = { role: "assistant", content: null, tool_calls: [] };
|
|
245
|
+
messages.push(assistant);
|
|
246
|
+
}
|
|
247
|
+
assistant.tool_calls ??= [];
|
|
248
|
+
if (item.type === "hosted_tool_call") {
|
|
249
|
+
const payload: Record<string, unknown> = {
|
|
250
|
+
...item.payload,
|
|
251
|
+
type: item.providerType,
|
|
252
|
+
id: item.id,
|
|
253
|
+
};
|
|
254
|
+
if (item.status) payload.status = item.status;
|
|
255
|
+
else delete payload.status;
|
|
256
|
+
assistant.tool_calls.push({
|
|
257
|
+
id: item.id,
|
|
258
|
+
type: "function",
|
|
259
|
+
function: { name: item.tool, arguments: JSON.stringify(payload) },
|
|
260
|
+
});
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
assistant.tool_calls.push({
|
|
264
|
+
id: item.id,
|
|
265
|
+
type: "function",
|
|
266
|
+
function: {
|
|
267
|
+
name: item.name,
|
|
268
|
+
arguments: typeof item.arguments === "string"
|
|
269
|
+
? item.arguments
|
|
270
|
+
: JSON.stringify(item.arguments),
|
|
271
|
+
},
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
return {
|
|
276
|
+
model: request.requestedModel,
|
|
277
|
+
messages,
|
|
278
|
+
stream: request.stream,
|
|
279
|
+
source_protocol: request.protocol === "anthropic_messages"
|
|
280
|
+
? "anthropic_messages"
|
|
281
|
+
: "openai_chat",
|
|
282
|
+
...(request.cacheKey === undefined ? {} : { prompt_cache_key: request.cacheKey }),
|
|
283
|
+
...(request.generation.temperature === undefined
|
|
284
|
+
? {}
|
|
285
|
+
: { temperature: request.generation.temperature }),
|
|
286
|
+
...(request.generation.topP === undefined ? {} : { top_p: request.generation.topP }),
|
|
287
|
+
...(request.generation.maxOutputTokens === undefined
|
|
288
|
+
? {}
|
|
289
|
+
: { max_completion_tokens: request.generation.maxOutputTokens }),
|
|
290
|
+
...(request.generation.stop === undefined ? {} : { stop: request.generation.stop }),
|
|
291
|
+
...(request.tools.length === 0 ? {} : { tools: request.tools.map(chatTool) }),
|
|
292
|
+
...(request.toolChoice === undefined ? {} : { tool_choice: chatToolChoice(request.toolChoice) }),
|
|
293
|
+
...(request.parallelToolCalls === undefined
|
|
294
|
+
? {}
|
|
295
|
+
: { parallel_tool_calls: request.parallelToolCalls }),
|
|
296
|
+
...(request.responseFormat === undefined
|
|
297
|
+
? {}
|
|
298
|
+
: { response_format: chatResponseFormat(request.responseFormat) }),
|
|
299
|
+
...(request.reasoning?.effort === undefined
|
|
300
|
+
? {}
|
|
301
|
+
: { reasoning_effort: request.reasoning.effort }),
|
|
302
|
+
...(request.metadata === undefined ? {} : { metadata: request.metadata }),
|
|
303
|
+
...(request.user === undefined ? {} : { user: request.user }),
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function chatContent(content: readonly RouterContent[]): Array<Record<string, unknown>> {
|
|
308
|
+
return content.map((part) =>
|
|
309
|
+
part.type === "text"
|
|
310
|
+
? { type: "text", text: part.text }
|
|
311
|
+
: {
|
|
312
|
+
type: "image_url",
|
|
313
|
+
image_url: {
|
|
314
|
+
url: part.url,
|
|
315
|
+
...(part.detail === undefined ? {} : { detail: part.detail }),
|
|
316
|
+
},
|
|
317
|
+
}
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function hasAssistantContent(message: ChatMessage): boolean {
|
|
322
|
+
if (message.content !== undefined && message.content !== null) return true;
|
|
323
|
+
return (message.tool_calls?.length ?? 0) > 0;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function appendReasoningText(existing: string | undefined, text: string): string {
|
|
327
|
+
return existing ? `${existing}\n${text}` : text;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function chatTool(tool: RouterRequest["tools"][number]): Record<string, unknown> {
|
|
331
|
+
return {
|
|
332
|
+
type: "function",
|
|
333
|
+
function: {
|
|
334
|
+
name: tool.name,
|
|
335
|
+
...(tool.description === undefined ? {} : { description: tool.description }),
|
|
336
|
+
parameters: tool.inputSchema,
|
|
337
|
+
...(tool.strict === undefined ? {} : { strict: tool.strict }),
|
|
338
|
+
},
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function chatToolChoice(choice: NonNullable<RouterRequest["toolChoice"]>): string | Record<string, unknown> {
|
|
343
|
+
if (typeof choice === "string") return choice;
|
|
344
|
+
return { type: "function", function: { name: choice.name } };
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function chatResponseFormat(
|
|
348
|
+
format: NonNullable<RouterRequest["responseFormat"]>,
|
|
349
|
+
): NonNullable<ChatCompletionRequest["response_format"]> {
|
|
350
|
+
if (format.type !== "json_schema") return format;
|
|
351
|
+
return {
|
|
352
|
+
type: "json_schema",
|
|
353
|
+
json_schema: {
|
|
354
|
+
name: format.name,
|
|
355
|
+
schema: format.schema,
|
|
356
|
+
...(format.description === undefined ? {} : { description: format.description }),
|
|
357
|
+
...(format.strict === undefined ? {} : { strict: format.strict }),
|
|
358
|
+
},
|
|
359
|
+
};
|
|
360
|
+
}
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// Typed errors for the deterministic routing core. The core never speaks
|
|
2
|
+
// HTTP; hosts map `kind` to their own transport semantics (Dari's hosted
|
|
3
|
+
// runtime maps invalid_request -> 400, selector_output -> 502, and
|
|
4
|
+
// configuration -> 500 while preserving `code`, `param`, and the message).
|
|
5
|
+
export type RouterCoreErrorKind =
|
|
6
|
+
| "invalid_request"
|
|
7
|
+
| "selector_output"
|
|
8
|
+
| "configuration";
|
|
9
|
+
|
|
10
|
+
export class RouterCoreError extends Error {
|
|
11
|
+
readonly kind: RouterCoreErrorKind;
|
|
12
|
+
readonly code: string;
|
|
13
|
+
readonly param?: string;
|
|
14
|
+
|
|
15
|
+
constructor(
|
|
16
|
+
kind: RouterCoreErrorKind,
|
|
17
|
+
message: string,
|
|
18
|
+
code: string,
|
|
19
|
+
param?: string,
|
|
20
|
+
) {
|
|
21
|
+
super(message);
|
|
22
|
+
this.name = "RouterCoreError";
|
|
23
|
+
this.kind = kind;
|
|
24
|
+
this.code = code;
|
|
25
|
+
if (param !== undefined) this.param = param;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import type { ChatCompletionRequest, ChatMessage } from "./types.js";
|
|
4
|
+
|
|
5
|
+
const FP_VERSION = "v1";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Conversation identity is an exact-prefix match over what providers actually
|
|
9
|
+
* cache: system prompt, tools, and the message sequence. Sampling parameters
|
|
10
|
+
* (temperature, top_p, max_tokens, ...) never enter provider cache keys and
|
|
11
|
+
* are deliberately excluded. tool_choice and response_format are also kept
|
|
12
|
+
* out of the chain — folding them in would break conversation identity over
|
|
13
|
+
* a one-turn knob flip — and are tracked via optionsFingerprints instead.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
function sha256(text: string): string {
|
|
17
|
+
return createHash("sha256").update(text, "utf8").digest("hex");
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// JSON with recursively sorted object keys so semantically identical payloads
|
|
21
|
+
// hash identically regardless of client key ordering. Provider caches match
|
|
22
|
+
// tokenized content, not raw JSON byte order.
|
|
23
|
+
export function stableStringify(value: unknown): string {
|
|
24
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
|
|
25
|
+
if (Array.isArray(value)) return `[${value.map((item) => stableStringify(item)).join(",")}]`;
|
|
26
|
+
const record = value as Record<string, unknown>;
|
|
27
|
+
const keys = Object.keys(record).sort();
|
|
28
|
+
return `{${keys.map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function canonicalToolCalls(toolCalls: Array<Record<string, unknown>> | undefined): unknown {
|
|
32
|
+
if (!toolCalls?.length) return null;
|
|
33
|
+
return toolCalls.map((call) => {
|
|
34
|
+
const fn = isRecord(call.function) ? call.function : {};
|
|
35
|
+
return {
|
|
36
|
+
id: typeof call.id === "string" ? call.id : "",
|
|
37
|
+
name: typeof fn.name === "string" ? fn.name : "",
|
|
38
|
+
// Keep string arguments byte-exact; never parse-and-restringify, the
|
|
39
|
+
// provider saw (and cached) exactly these bytes inside its prompt.
|
|
40
|
+
arguments: typeof fn.arguments === "string" ? fn.arguments : stableStringify(fn.arguments ?? null),
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function canonicalMessage(message: ChatMessage): string {
|
|
46
|
+
return stableStringify({
|
|
47
|
+
role: message.role,
|
|
48
|
+
content: message.content ?? null,
|
|
49
|
+
tool_calls: canonicalToolCalls(message.tool_calls),
|
|
50
|
+
tool_call_id: message.tool_call_id ?? null,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function isSystemRole(role: string): boolean {
|
|
55
|
+
return role === "system" || role === "developer";
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function conversationMessages(request: ChatCompletionRequest): ChatMessage[] {
|
|
59
|
+
return (request.messages ?? []).filter((message) => !isSystemRole(message.role));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Approximates how many Anthropic content blocks a message renders to: one
|
|
63
|
+
// per content part (or one for string content), plus one per tool call; tool
|
|
64
|
+
// results are a single tool_result block. Used only for the Anthropic
|
|
65
|
+
// breakpoint-lookback gate, where blocks — not messages — are the unit.
|
|
66
|
+
export function messageBlockCount(message: ChatMessage): number {
|
|
67
|
+
if (message.role === "tool") return 1;
|
|
68
|
+
let blocks = 0;
|
|
69
|
+
if (typeof message.content === "string") {
|
|
70
|
+
if (message.content.length > 0) blocks += 1;
|
|
71
|
+
} else if (Array.isArray(message.content)) {
|
|
72
|
+
blocks += message.content.length;
|
|
73
|
+
}
|
|
74
|
+
blocks += message.tool_calls?.length ?? 0;
|
|
75
|
+
return Math.max(1, blocks);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Content-block position of this request's cache breakpoint (its last user
|
|
79
|
+
// message): total blocks across the non-system messages.
|
|
80
|
+
export function conversationBlockCount(request: ChatCompletionRequest): number {
|
|
81
|
+
return conversationMessages(request).reduce(
|
|
82
|
+
(total, message) => total + messageBlockCount(message),
|
|
83
|
+
0
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Anchors and scopes the chain. routerId is a pure namespace so two routers
|
|
88
|
+
// receiving identical conversations never share entries; system/tools changes
|
|
89
|
+
// bust provider caches, so they bust ours too.
|
|
90
|
+
// System/developer messages are folded in here regardless of their position in
|
|
91
|
+
// the message list, so a request that moves one mid-conversation hashes the
|
|
92
|
+
// same as system-first. Clients virtually always send system first; if they
|
|
93
|
+
// don't, the warmth estimate is merely optimistic — estimates never bill.
|
|
94
|
+
// NUL is the component delimiter throughout: JSON.stringify always escapes
|
|
95
|
+
// control characters, so canonical output can never contain a raw NUL and
|
|
96
|
+
// component boundaries are unambiguous.
|
|
97
|
+
export function headHash(routerId: string, request: ChatCompletionRequest): string {
|
|
98
|
+
const systemMessages = (request.messages ?? [])
|
|
99
|
+
.filter((message) => isSystemRole(message.role))
|
|
100
|
+
.map((message) => canonicalMessage(message));
|
|
101
|
+
return sha256(
|
|
102
|
+
[FP_VERSION, routerId, `[${systemMessages.join(",")}]`, stableStringify(request.tools ?? [])].join("\u0000")
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// h[i] = sha256(h[i-1] ‖ canonical(message_i)) over non-system messages.
|
|
107
|
+
// Returns [h1 ... hn]; chain[i] identifies the conversation prefix ending at
|
|
108
|
+
// non-system message i+1. System messages live in the head hash only.
|
|
109
|
+
export function prefixChain(routerId: string, request: ChatCompletionRequest): string[] {
|
|
110
|
+
let hash = headHash(routerId, request);
|
|
111
|
+
const chain: string[] = [];
|
|
112
|
+
for (const message of conversationMessages(request)) {
|
|
113
|
+
hash = sha256(`${hash}\u0000${canonicalMessage(message)}`);
|
|
114
|
+
chain.push(hash);
|
|
115
|
+
}
|
|
116
|
+
return chain;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Chain extended with the assistant reply we are about to return, in the
|
|
120
|
+
// exact OpenAI shape the client will echo back next turn.
|
|
121
|
+
export function extendChain(lastHash: string, assistantMessage: Record<string, unknown>): string {
|
|
122
|
+
const canonical = canonicalMessage({
|
|
123
|
+
role: "assistant",
|
|
124
|
+
content: (assistantMessage.content as ChatMessage["content"]) ?? null,
|
|
125
|
+
tool_calls: Array.isArray(assistantMessage.tool_calls)
|
|
126
|
+
? (assistantMessage.tool_calls as Array<Record<string, unknown>>)
|
|
127
|
+
: undefined,
|
|
128
|
+
});
|
|
129
|
+
return sha256(`${lastHash}\u0000${canonical}`);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// NOT part of the chain (see the module comment): compared against the
|
|
133
|
+
// matched entry's stored values to downgrade cache-warmth estimates when
|
|
134
|
+
// these knobs flip mid-conversation. Do not "fix" this by folding them into
|
|
135
|
+
// headHash — that would break conversation identity over a one-turn change.
|
|
136
|
+
export function optionsFingerprints(request: ChatCompletionRequest): {
|
|
137
|
+
tool_choice_fp: string;
|
|
138
|
+
response_format_fp: string;
|
|
139
|
+
} {
|
|
140
|
+
return {
|
|
141
|
+
tool_choice_fp: sha256(stableStringify(request.tool_choice ?? null)),
|
|
142
|
+
response_format_fp: sha256(stableStringify(request.response_format ?? null)),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
147
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
148
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Loop horizons every candidate is projected at, ascending. These match the
|
|
2
|
+
// lease lengths the selector can commit to, so every option is priced at
|
|
3
|
+
// exactly the horizon its action would hold for — the longer horizons are
|
|
4
|
+
// where cache behavior and output pricing separate models that look alike on
|
|
5
|
+
// a single turn.
|
|
6
|
+
export const FIXED_TURN_COST_PROJECTED_TURNS = [5, 10, 30] as const;
|
|
7
|
+
|
|
8
|
+
// Turn commitments the selector may choose between: how many turns the routed
|
|
9
|
+
// model keeps serving before the router reconsiders. Committing amortizes the
|
|
10
|
+
// cold start it pays on the first turn across the warm turns that follow.
|
|
11
|
+
export const SELECTOR_LEASE_TURNS = [5, 10, 30] as const;
|
|
12
|
+
|
|
13
|
+
// The horizon used wherever a single scalar cost is needed rather than the
|
|
14
|
+
// whole curve, notably warm-incumbent pruning.
|
|
15
|
+
export const FIXED_TURN_COST_COMPARISON_TURNS = 10;
|
|
16
|
+
|
|
17
|
+
export const FIXED_TURN_CACHE_HIT_PROBABILITY = {
|
|
18
|
+
openai: 0.95,
|
|
19
|
+
anthropic: 0.99,
|
|
20
|
+
fireworks: 0.88,
|
|
21
|
+
// 48/50 follow-up requests hit in the July 2026 25-agent Pi benchmark.
|
|
22
|
+
// First turns are excluded because fixed-turn turn one uses live warmth.
|
|
23
|
+
meta: 0.96,
|
|
24
|
+
} as const;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export type RouterFrameworkErrorKind =
|
|
2
|
+
| "invalid_request"
|
|
3
|
+
| "not_found"
|
|
4
|
+
| "configuration"
|
|
5
|
+
| "cancelled"
|
|
6
|
+
| "policy"
|
|
7
|
+
| "executor";
|
|
8
|
+
|
|
9
|
+
const STATUS_BY_KIND: Record<RouterFrameworkErrorKind, number> = {
|
|
10
|
+
invalid_request: 400,
|
|
11
|
+
not_found: 404,
|
|
12
|
+
configuration: 500,
|
|
13
|
+
cancelled: 499,
|
|
14
|
+
policy: 502,
|
|
15
|
+
executor: 502,
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export class RouterFrameworkError extends Error {
|
|
19
|
+
readonly kind: RouterFrameworkErrorKind;
|
|
20
|
+
readonly code: string;
|
|
21
|
+
readonly param?: string;
|
|
22
|
+
readonly status: number;
|
|
23
|
+
|
|
24
|
+
constructor(
|
|
25
|
+
kind: RouterFrameworkErrorKind,
|
|
26
|
+
message: string,
|
|
27
|
+
code: string,
|
|
28
|
+
param?: string,
|
|
29
|
+
options?: ErrorOptions,
|
|
30
|
+
) {
|
|
31
|
+
super(message, options);
|
|
32
|
+
this.name = "RouterFrameworkError";
|
|
33
|
+
this.kind = kind;
|
|
34
|
+
this.code = code;
|
|
35
|
+
this.status = STATUS_BY_KIND[kind];
|
|
36
|
+
if (param !== undefined) this.param = param;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { RouterFrameworkError } from "./framework_error.js";
|
|
2
|
+
|
|
3
|
+
export function executorStartError(error: unknown, executor: string): RouterFrameworkError {
|
|
4
|
+
if (error instanceof RouterFrameworkError) return error;
|
|
5
|
+
return new RouterFrameworkError(
|
|
6
|
+
"executor",
|
|
7
|
+
errorMessage(error, `Executor '${executor}' failed to start.`),
|
|
8
|
+
"executor_setup_failed",
|
|
9
|
+
undefined,
|
|
10
|
+
{ cause: error },
|
|
11
|
+
);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function safeHook(run: () => void): void {
|
|
15
|
+
try {
|
|
16
|
+
run();
|
|
17
|
+
} catch {
|
|
18
|
+
// Hooks are fire-and-forget; a throwing hook must never affect the
|
|
19
|
+
// request or stream.
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function normalizeError(error: unknown): RouterFrameworkError {
|
|
24
|
+
if (error instanceof RouterFrameworkError) return error;
|
|
25
|
+
return new RouterFrameworkError(
|
|
26
|
+
"configuration",
|
|
27
|
+
errorMessage(error, "Internal router error."),
|
|
28
|
+
"internal_error",
|
|
29
|
+
undefined,
|
|
30
|
+
{ cause: error },
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function completionContract(message: string): RouterFrameworkError {
|
|
35
|
+
return new RouterFrameworkError("executor", message, "executor_output_invalid");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function streamContract(message: string): RouterFrameworkError {
|
|
39
|
+
return new RouterFrameworkError("executor", message, "stream_invalid");
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function cancelled(reason: unknown): RouterFrameworkError {
|
|
43
|
+
return new RouterFrameworkError(
|
|
44
|
+
"cancelled",
|
|
45
|
+
errorMessage(reason, "Request cancelled."),
|
|
46
|
+
"request_cancelled",
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function throwIfAborted(signal: AbortSignal): void {
|
|
51
|
+
if (signal.aborted) throw cancelled(signal.reason);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export async function closeAsyncIterable(iterable: AsyncIterable<unknown>): Promise<void> {
|
|
55
|
+
try {
|
|
56
|
+
await closeAsyncIterator(iterable[Symbol.asyncIterator]());
|
|
57
|
+
} catch {
|
|
58
|
+
// Cleanup errors must not replace the mode mismatch.
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export async function closeAsyncIterator(iterator: AsyncIterator<unknown>): Promise<void> {
|
|
63
|
+
try {
|
|
64
|
+
await iterator.return?.();
|
|
65
|
+
} catch {
|
|
66
|
+
// Cleanup errors must not replace the request or provider failure.
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function errorMessage(error: unknown, fallback: string): string {
|
|
71
|
+
if (error instanceof Error && error.message.trim()) return error.message;
|
|
72
|
+
if (typeof error === "string" && error.trim()) return error;
|
|
73
|
+
return fallback;
|
|
74
|
+
}
|