@mupt-ai/dari-router 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -60
- package/dist/anonymous_actions.js +32 -10
- package/dist/anonymous_actions.js.map +1 -1
- package/dist/auto_router.d.ts +4 -1
- package/dist/auto_router.js +3 -3
- package/dist/auto_router.js.map +1 -1
- package/dist/cache_behavior.d.ts +8 -6
- package/dist/cache_behavior.js +60 -30
- package/dist/cache_behavior.js.map +1 -1
- package/dist/cache_scope.d.ts +2 -2
- package/dist/cache_scope.js +23 -18
- package/dist/cache_scope.js.map +1 -1
- package/dist/cost.d.ts +3 -2
- package/dist/cost.js +32 -23
- package/dist/cost.js.map +1 -1
- package/dist/dari_policy.d.ts +8 -2
- package/dist/dari_policy.js +39 -1
- package/dist/dari_policy.js.map +1 -1
- package/dist/eval_score_imputation.d.ts +26 -0
- package/dist/eval_score_imputation.js +121 -0
- package/dist/eval_score_imputation.js.map +1 -0
- package/dist/framework_lifecycle.d.ts +1 -1
- package/dist/framework_lifecycle.js +4 -1
- package/dist/framework_lifecycle.js.map +1 -1
- package/dist/framework_models.js +6 -0
- package/dist/framework_models.js.map +1 -1
- package/dist/framework_router.js +22 -11
- package/dist/framework_router.js.map +1 -1
- package/dist/framework_types.d.ts +9 -5
- package/dist/index.d.ts +9 -35
- package/dist/index.js +6 -37
- package/dist/index.js.map +1 -1
- package/dist/model_ids.js +4 -0
- package/dist/model_ids.js.map +1 -1
- package/dist/pi_runtime.js +19 -11
- package/dist/pi_runtime.js.map +1 -1
- package/dist/pi_types.d.ts +2 -2
- package/{src/index.ts → dist/policy-engine.d.ts} +2 -14
- package/dist/policy-engine.js +25 -0
- package/dist/policy-engine.js.map +1 -0
- package/dist/prompts.d.ts +1 -1
- package/dist/prompts.js +3 -3
- package/dist/prompts.js.map +1 -1
- package/dist/protocols.d.ts +5 -0
- package/dist/protocols.js +6 -0
- package/dist/protocols.js.map +1 -0
- package/dist/route.d.ts +3 -4
- package/dist/route.js +17 -10
- package/dist/route.js.map +1 -1
- package/dist/selector_input.d.ts +1 -0
- package/dist/selector_input.js +66 -78
- package/dist/selector_input.js.map +1 -1
- package/dist/selector_parse.d.ts +1 -0
- package/dist/selector_parse.js +2 -1
- package/dist/selector_parse.js.map +1 -1
- package/dist/selector_request.d.ts +1 -0
- package/dist/selector_request.js +1 -0
- package/dist/selector_request.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/examples/auto_router.ts +40 -0
- package/examples/basic_route.ts +130 -0
- package/examples/create_router.ts +114 -0
- package/examples/dari_policy.ts +86 -0
- package/examples/lease_stores.ts +159 -0
- package/examples/managed/README.md +50 -0
- package/examples/managed/byok-router/README.md +13 -0
- package/examples/managed/byok-router/router.yml +13 -0
- package/examples/managed/custom-rules-router/README.md +12 -0
- package/examples/managed/custom-rules-router/router.yml +24 -0
- package/examples/managed/eval-router/README.md +18 -0
- package/examples/managed/eval-router/router.yml +11 -0
- package/examples/managed/slm-router/README.md +12 -0
- package/examples/managed/slm-router/router.yml +8 -0
- package/examples/pi_runtime.ts +55 -0
- package/package.json +26 -7
- package/src/anonymous_actions.ts +0 -434
- package/src/auto_router.ts +0 -95
- package/src/cache_behavior.ts +0 -130
- package/src/cache_scope.ts +0 -58
- package/src/cache_switch_decision.ts +0 -53
- package/src/candidate_pruning.ts +0 -257
- package/src/candidate_resolution.ts +0 -208
- package/src/compatibility.ts +0 -162
- package/src/continuation_state.ts +0 -368
- package/src/cost.ts +0 -455
- package/src/dari_policy.ts +0 -360
- package/src/errors.ts +0 -27
- package/src/fingerprint.ts +0 -148
- package/src/fixed_turn_cost_config.ts +0 -24
- package/src/framework_error.ts +0 -38
- package/src/framework_lifecycle.ts +0 -74
- package/src/framework_models.ts +0 -186
- package/src/framework_router.ts +0 -490
- package/src/framework_streaming.ts +0 -613
- package/src/framework_types.ts +0 -372
- package/src/framework_validation.ts +0 -235
- package/src/json.ts +0 -43
- package/src/model_ids.ts +0 -29
- package/src/pi_context.ts +0 -251
- package/src/pi_options.ts +0 -295
- package/src/pi_output.ts +0 -387
- package/src/pi_reasoning_signature.ts +0 -41
- package/src/pi_runtime.ts +0 -238
- package/src/pi_types.ts +0 -71
- package/src/portable_reasoning_state.ts +0 -129
- package/src/previous_decision.ts +0 -109
- package/src/prompts.ts +0 -69
- package/src/protocol_anthropic.ts +0 -571
- package/src/protocol_openai_chat.ts +0 -645
- package/src/protocol_validation.ts +0 -202
- package/src/route.ts +0 -490
- package/src/routing_history.ts +0 -157
- package/src/selector_input.ts +0 -246
- package/src/selector_parse.ts +0 -138
- package/src/selector_request.ts +0 -130
- package/src/selector_truncation.ts +0 -48
- package/src/selector_wire.ts +0 -167
- package/src/types.ts +0 -231
|
@@ -1,202 +0,0 @@
|
|
|
1
|
-
import { RouterFrameworkError } from "./framework_error.js";
|
|
2
|
-
import type {
|
|
3
|
-
RouterContent,
|
|
4
|
-
RouterSelection,
|
|
5
|
-
RouterTool,
|
|
6
|
-
RouterToolChoice,
|
|
7
|
-
} from "./framework_types.js";
|
|
8
|
-
import { isRecord } from "./types.js";
|
|
9
|
-
|
|
10
|
-
export function invalidProtocolRequest(
|
|
11
|
-
message: string,
|
|
12
|
-
param?: string,
|
|
13
|
-
): RouterFrameworkError {
|
|
14
|
-
return new RouterFrameworkError(
|
|
15
|
-
"invalid_request",
|
|
16
|
-
message,
|
|
17
|
-
"invalid_request",
|
|
18
|
-
param,
|
|
19
|
-
);
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export function rejectUnknownFields(
|
|
23
|
-
value: Record<string, unknown>,
|
|
24
|
-
allowed: readonly string[],
|
|
25
|
-
prefix?: string,
|
|
26
|
-
): void {
|
|
27
|
-
const field = Object.keys(value).find((name) => !allowed.includes(name));
|
|
28
|
-
if (field === undefined) return;
|
|
29
|
-
const param = prefix ? `${prefix}.${field}` : field;
|
|
30
|
-
throw new RouterFrameworkError(
|
|
31
|
-
"invalid_request",
|
|
32
|
-
`${param} is not supported by the portable router contract.`,
|
|
33
|
-
"unsupported_field",
|
|
34
|
-
param,
|
|
35
|
-
);
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export function validateToolChoice(
|
|
39
|
-
choice: RouterToolChoice | undefined,
|
|
40
|
-
tools: readonly RouterTool[],
|
|
41
|
-
): void {
|
|
42
|
-
if (choice === undefined || choice === "none" || choice === "auto") return;
|
|
43
|
-
if (tools.length === 0) {
|
|
44
|
-
throw invalidProtocolRequest("tool_choice requires at least one tool.", "tool_choice");
|
|
45
|
-
}
|
|
46
|
-
if (typeof choice !== "string" && !tools.some((tool) => tool.name === choice.name)) {
|
|
47
|
-
throw invalidProtocolRequest(
|
|
48
|
-
`tool_choice references unknown tool '${choice.name}'.`,
|
|
49
|
-
"tool_choice",
|
|
50
|
-
);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
// `dari_routing` response metadata, matching the managed Dari platform's wire shape.
|
|
55
|
-
export function dariRoutingPayload<Metadata>(
|
|
56
|
-
requestedModel: string,
|
|
57
|
-
selection: RouterSelection<Metadata>,
|
|
58
|
-
): Record<string, unknown> {
|
|
59
|
-
return {
|
|
60
|
-
requested_model: requestedModel,
|
|
61
|
-
selected_model: selection.decision.selectedModel,
|
|
62
|
-
reasoning_effort: selection.decision.reasoningEffort,
|
|
63
|
-
reason: selection.decision.reason,
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export function requiredString(value: unknown, param: string): string {
|
|
68
|
-
if (typeof value !== "string" || !value.trim()) {
|
|
69
|
-
throw invalidProtocolRequest(`${param} must be a non-empty string.`, param);
|
|
70
|
-
}
|
|
71
|
-
return value;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
export function optionalString(value: unknown, param: string): string | undefined {
|
|
75
|
-
if (value === undefined || value === null) return undefined;
|
|
76
|
-
if (typeof value !== "string") {
|
|
77
|
-
throw invalidProtocolRequest(`${param} must be a string.`, param);
|
|
78
|
-
}
|
|
79
|
-
return value;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
// Empty and whitespace-only cache keys mean "no cache key": preserving them
|
|
83
|
-
// would make every such request share one lease bucket.
|
|
84
|
-
export function optionalCacheKey(value: unknown, param: string): string | undefined {
|
|
85
|
-
const cacheKey = optionalString(value, param);
|
|
86
|
-
return cacheKey !== undefined && cacheKey.trim() ? cacheKey : undefined;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
export function optionalNumberInRange(
|
|
90
|
-
value: unknown,
|
|
91
|
-
param: string,
|
|
92
|
-
minimum: number,
|
|
93
|
-
maximum: number,
|
|
94
|
-
): number | undefined {
|
|
95
|
-
if (value === undefined || value === null) return undefined;
|
|
96
|
-
if (
|
|
97
|
-
typeof value !== "number" ||
|
|
98
|
-
!Number.isFinite(value) ||
|
|
99
|
-
value < minimum ||
|
|
100
|
-
value > maximum
|
|
101
|
-
) {
|
|
102
|
-
throw invalidProtocolRequest(
|
|
103
|
-
`${param} must be between ${minimum} and ${maximum}.`,
|
|
104
|
-
param,
|
|
105
|
-
);
|
|
106
|
-
}
|
|
107
|
-
return value;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
export function positiveInteger(value: unknown, param: string): number {
|
|
111
|
-
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
|
|
112
|
-
throw invalidProtocolRequest(`${param} must be a positive integer.`, param);
|
|
113
|
-
}
|
|
114
|
-
return value;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
export function optionalPositiveInteger(
|
|
118
|
-
value: unknown,
|
|
119
|
-
param: string,
|
|
120
|
-
): number | undefined {
|
|
121
|
-
if (value === undefined || value === null) return undefined;
|
|
122
|
-
return positiveInteger(value, param);
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
export function optionalBoolean(value: unknown, param: string): boolean | undefined {
|
|
126
|
-
if (value === undefined || value === null) return undefined;
|
|
127
|
-
if (typeof value !== "boolean") {
|
|
128
|
-
throw invalidProtocolRequest(`${param} must be a boolean.`, param);
|
|
129
|
-
}
|
|
130
|
-
return value;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
export function optionalRecord(
|
|
134
|
-
value: unknown,
|
|
135
|
-
param: string,
|
|
136
|
-
): Record<string, unknown> | undefined {
|
|
137
|
-
if (value === undefined || value === null) return undefined;
|
|
138
|
-
if (!isRecord(value)) {
|
|
139
|
-
throw invalidProtocolRequest(`${param} must be an object.`, param);
|
|
140
|
-
}
|
|
141
|
-
return value;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
export function stopSequenceValues(
|
|
145
|
-
value: unknown,
|
|
146
|
-
param: string,
|
|
147
|
-
allowBareString: boolean,
|
|
148
|
-
): string | string[] | undefined {
|
|
149
|
-
if (value === undefined || value === null) return undefined;
|
|
150
|
-
if (typeof value === "string" && allowBareString) return value;
|
|
151
|
-
if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) {
|
|
152
|
-
throw invalidProtocolRequest(
|
|
153
|
-
allowBareString
|
|
154
|
-
? `${param} must be a string or array of strings.`
|
|
155
|
-
: `${param} must be an array of strings.`,
|
|
156
|
-
param,
|
|
157
|
-
);
|
|
158
|
-
}
|
|
159
|
-
return value;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
export function isAllEmptyText(parts: readonly RouterContent[]): boolean {
|
|
163
|
-
return parts.every((part) => part.type === "text" && part.text.length === 0);
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
export function prefixedWireId(value: string, prefix: string): string {
|
|
167
|
-
return (value.startsWith(prefix) ? value : `${prefix}${value.replace(/[^A-Za-z0-9_-]/g, "_")}`)
|
|
168
|
-
.slice(0, 64);
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
export function portableTool(
|
|
172
|
-
fields: Record<string, unknown>,
|
|
173
|
-
options: {
|
|
174
|
-
names: Set<string>;
|
|
175
|
-
param: string;
|
|
176
|
-
schemaField: string;
|
|
177
|
-
schemaRequired: boolean;
|
|
178
|
-
},
|
|
179
|
-
): RouterTool {
|
|
180
|
-
const name = requiredString(fields.name, `${options.param}.name`);
|
|
181
|
-
if (options.names.has(name)) {
|
|
182
|
-
throw invalidProtocolRequest(`Duplicate tool '${name}'.`, `${options.param}.name`);
|
|
183
|
-
}
|
|
184
|
-
options.names.add(name);
|
|
185
|
-
const schemaParam = `${options.param}.${options.schemaField}`;
|
|
186
|
-
const schema = fields[options.schemaField];
|
|
187
|
-
if (schema === undefined) {
|
|
188
|
-
if (options.schemaRequired) {
|
|
189
|
-
throw invalidProtocolRequest(`${schemaParam} must be an object.`, schemaParam);
|
|
190
|
-
}
|
|
191
|
-
} else if (!isRecord(schema)) {
|
|
192
|
-
throw invalidProtocolRequest(`${schemaParam} must be an object.`, schemaParam);
|
|
193
|
-
}
|
|
194
|
-
const description = optionalString(fields.description, `${options.param}.description`);
|
|
195
|
-
const strict = optionalBoolean(fields.strict, `${options.param}.strict`);
|
|
196
|
-
return {
|
|
197
|
-
name,
|
|
198
|
-
...(description === undefined ? {} : { description }),
|
|
199
|
-
inputSchema: schema ?? { type: "object" },
|
|
200
|
-
...(strict === undefined ? {} : { strict }),
|
|
201
|
-
};
|
|
202
|
-
}
|
package/src/route.ts
DELETED
|
@@ -1,490 +0,0 @@
|
|
|
1
|
-
// Deterministic route preparation and finalization. These compositions call
|
|
2
|
-
// the same phase functions Dari's hosted runtime executes; hosts with
|
|
3
|
-
// interleaved effects (speculative execution, telemetry) may call the phases
|
|
4
|
-
// directly instead.
|
|
5
|
-
|
|
6
|
-
import { resolveCompatibleCandidates, type CandidateMetadataLookup } from "./compatibility.js";
|
|
7
|
-
import { conversationBlockCount, conversationMessages } from "./fingerprint.js";
|
|
8
|
-
import {
|
|
9
|
-
deepestIdentityHit,
|
|
10
|
-
deriveModelChainHits,
|
|
11
|
-
estimateCandidateCosts,
|
|
12
|
-
selectPromptAnchor,
|
|
13
|
-
type CandidatePromptEstimate,
|
|
14
|
-
type PricingLookup,
|
|
15
|
-
} from "./cost.js";
|
|
16
|
-
import { warmPrefixHits } from "./cache_behavior.js";
|
|
17
|
-
import type { ReasoningCacheScopeLookup } from "./cache_scope.js";
|
|
18
|
-
import { RouterCoreError } from "./errors.js";
|
|
19
|
-
import {
|
|
20
|
-
activeLeaseFromHit,
|
|
21
|
-
compatiblePreviousDecision,
|
|
22
|
-
pendingLeaseFromHit,
|
|
23
|
-
previousDecisionFromHit,
|
|
24
|
-
type ActiveLease,
|
|
25
|
-
type PendingLease,
|
|
26
|
-
} from "./previous_decision.js";
|
|
27
|
-
import {
|
|
28
|
-
resolveStrategyCandidates,
|
|
29
|
-
type StrategyCandidateResolution,
|
|
30
|
-
} from "./candidate_resolution.js";
|
|
31
|
-
import { selectorSafeMessages } from "./selector_input.js";
|
|
32
|
-
import { buildSizedSelectorRequest, type BuiltSelectorRequest } from "./selector_request.js";
|
|
33
|
-
import { parseSelectorDecision, validateSelectorDecisions } from "./selector_parse.js";
|
|
34
|
-
import { routingCandidateKey } from "./types.js";
|
|
35
|
-
import type {
|
|
36
|
-
CandidateCostEstimate,
|
|
37
|
-
ChainHits,
|
|
38
|
-
ChatCompletionRequest,
|
|
39
|
-
ChatMessage,
|
|
40
|
-
CustomRouterConfig,
|
|
41
|
-
HardCapability,
|
|
42
|
-
PrefixHit,
|
|
43
|
-
PreviousDecision,
|
|
44
|
-
ReasoningEffort,
|
|
45
|
-
RouterEval,
|
|
46
|
-
RouterModelPrice,
|
|
47
|
-
RouterPrefixHit,
|
|
48
|
-
RoutingDecision,
|
|
49
|
-
RoutingStrategy,
|
|
50
|
-
} from "./types.js";
|
|
51
|
-
|
|
52
|
-
export type RoutePreparationWarning = {
|
|
53
|
-
phase: "cost_estimation";
|
|
54
|
-
error: unknown;
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
export type CandidatePromptAnchor = {
|
|
58
|
-
hash: string;
|
|
59
|
-
depth: number;
|
|
60
|
-
};
|
|
61
|
-
|
|
62
|
-
// Candidate preparation consumes explicit prompt-accounting facts instead of
|
|
63
|
-
// inferring them from the selector's lossy message projection. Hosts may use
|
|
64
|
-
// provider-visible IR semantics; prepareRoute supplies the legacy Chat
|
|
65
|
-
// accounting behavior for public Chat-shaped callers.
|
|
66
|
-
export type CandidatePromptAccounting = {
|
|
67
|
-
anchorSemantics: "matched_prefix" | "stored_input";
|
|
68
|
-
defaultBlockDepth: number;
|
|
69
|
-
providerBlockDepthFor: (model: string) => number;
|
|
70
|
-
// Return the full candidate-visible prompt when no anchor is supplied, or
|
|
71
|
-
// the suffix after a same-model anchor when one is supplied. Foreign-model
|
|
72
|
-
// token counts are never a valid baseline for this candidate's prompt.
|
|
73
|
-
promptEstimateFor: (
|
|
74
|
-
model: string,
|
|
75
|
-
anchor?: CandidatePromptAnchor,
|
|
76
|
-
) => CandidatePromptEstimate;
|
|
77
|
-
};
|
|
78
|
-
|
|
79
|
-
export type LegacyTailCharsFor = (
|
|
80
|
-
model: string,
|
|
81
|
-
hash: string,
|
|
82
|
-
) => number | undefined;
|
|
83
|
-
|
|
84
|
-
type CandidatePreparationSharedInput = {
|
|
85
|
-
candidateModels: string[];
|
|
86
|
-
metadataLookup: CandidateMetadataLookup;
|
|
87
|
-
requiredCapabilities: readonly HardCapability[];
|
|
88
|
-
requestedReasoningEffort?: ReasoningEffort;
|
|
89
|
-
thinkingEnabled?: boolean;
|
|
90
|
-
toolChoice?: unknown;
|
|
91
|
-
modelThinkingLevels?: Readonly<Record<string, ReasoningEffort[]>>;
|
|
92
|
-
strategy: RoutingStrategy;
|
|
93
|
-
customConfig?: CustomRouterConfig | null;
|
|
94
|
-
modelPrices?: Record<string, RouterModelPrice>;
|
|
95
|
-
pricing: PricingLookup;
|
|
96
|
-
averageOutputTokensByModel?: Readonly<
|
|
97
|
-
Record<string, Partial<Record<ReasoningEffort, number>> | null>
|
|
98
|
-
>;
|
|
99
|
-
// Per-candidate-model prefix-hash chains for the incoming conversation, as
|
|
100
|
-
// computed by the host's fingerprinting (see fingerprint.ts / prefixChain).
|
|
101
|
-
chainsByModel: ReadonlyMap<string, string[]>;
|
|
102
|
-
// Prefix entries resolved by the host's prefix-hit storage: every entry
|
|
103
|
-
// inside the host's conversation lookback, warm or not. Which of them are
|
|
104
|
-
// still warm is decided here against nowMs, so a host never has to encode
|
|
105
|
-
// the provider cache-warmth window itself.
|
|
106
|
-
prefixHits?: RouterPrefixHit[];
|
|
107
|
-
// The instant this request arrived, in epoch milliseconds. This package
|
|
108
|
-
// reads no clock, and prefix warmth is measured from here.
|
|
109
|
-
nowMs: number;
|
|
110
|
-
// The request's loose chain (client-durable canonicalization), as computed
|
|
111
|
-
// by the host's fingerprinting. Identity-only; never used for warmth.
|
|
112
|
-
looseChain?: readonly string[];
|
|
113
|
-
toolChoiceFp: string;
|
|
114
|
-
responseFormatFp: string;
|
|
115
|
-
previousDecision?: PreviousDecision;
|
|
116
|
-
cacheScope?: ReasoningCacheScopeLookup;
|
|
117
|
-
};
|
|
118
|
-
|
|
119
|
-
// Rich hosts provide provider-visible accounting. Existing Chat-shaped hosts
|
|
120
|
-
// provide messages and may override same-model suffix sizing. These modes are
|
|
121
|
-
// mutually exclusive so preparation never silently ignores one representation.
|
|
122
|
-
export type CandidatePreparationInput = CandidatePreparationSharedInput & (
|
|
123
|
-
| {
|
|
124
|
-
promptAccounting: CandidatePromptAccounting;
|
|
125
|
-
messages?: never;
|
|
126
|
-
tailCharsFor?: never;
|
|
127
|
-
}
|
|
128
|
-
| {
|
|
129
|
-
promptAccounting?: undefined;
|
|
130
|
-
messages: ChatMessage[];
|
|
131
|
-
tailCharsFor?: LegacyTailCharsFor;
|
|
132
|
-
}
|
|
133
|
-
);
|
|
134
|
-
|
|
135
|
-
export type CandidatePreparation = {
|
|
136
|
-
blockDepth: number;
|
|
137
|
-
hits: ChainHits;
|
|
138
|
-
// Conversation id recovered from the deepest identity hit (strict or
|
|
139
|
-
// loose tier, warm or merely recent); null for a fresh conversation
|
|
140
|
-
// (hosts typically substitute a new random id).
|
|
141
|
-
conversationId: string | null;
|
|
142
|
-
// Depth of that identity hit, for host reporting. Null when no hit.
|
|
143
|
-
identityMatchDepth: number | null;
|
|
144
|
-
// The same matched entry, exposed so hosts can carry synchronized
|
|
145
|
-
// conversation-level state forward without repeating identity matching.
|
|
146
|
-
identityHit?: PrefixHit;
|
|
147
|
-
previousDecision?: PreviousDecision;
|
|
148
|
-
// Unexpired lease recovered from the identity hit, when its target is still
|
|
149
|
-
// a compatible candidate. The host may serve it without consulting the
|
|
150
|
-
// selector.
|
|
151
|
-
activeLease?: ActiveLease;
|
|
152
|
-
// Next lease prefetched before expiry; adopted when activeLease is absent.
|
|
153
|
-
pendingLease?: PendingLease;
|
|
154
|
-
// Pre-strategy estimates for every compatible candidate, kept for
|
|
155
|
-
// reporting/audit even when pruning later narrows the candidate set.
|
|
156
|
-
costEstimates: CandidateCostEstimate[];
|
|
157
|
-
candidateResolution: StrategyCandidateResolution;
|
|
158
|
-
warnings: RoutePreparationWarning[];
|
|
159
|
-
};
|
|
160
|
-
|
|
161
|
-
// Deterministic candidate preparation: eligibility, cache-aware cost
|
|
162
|
-
// estimates, previous-decision resolution, and strategy resolution/pruning.
|
|
163
|
-
// Cost estimation is best-effort: a failure there surfaces as a warning and
|
|
164
|
-
// leaves estimates empty rather than failing the route.
|
|
165
|
-
export function prepareCandidates(input: CandidatePreparationInput): CandidatePreparation {
|
|
166
|
-
return prepareCandidatesWithAccounting(
|
|
167
|
-
input,
|
|
168
|
-
resolvePromptAccounting(input),
|
|
169
|
-
);
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
function prepareCandidatesWithAccounting(
|
|
173
|
-
input: CandidatePreparationSharedInput,
|
|
174
|
-
promptAccounting: CandidatePromptAccounting,
|
|
175
|
-
): CandidatePreparation {
|
|
176
|
-
const compatible = resolveCompatibleCandidates({
|
|
177
|
-
requiredCapabilities: input.requiredCapabilities,
|
|
178
|
-
requestedReasoningEffort: input.requestedReasoningEffort,
|
|
179
|
-
thinkingEnabled: input.thinkingEnabled,
|
|
180
|
-
toolChoice: input.toolChoice,
|
|
181
|
-
candidateModels: input.candidateModels,
|
|
182
|
-
metadataLookup: input.metadataLookup,
|
|
183
|
-
modelThinkingLevels: input.modelThinkingLevels,
|
|
184
|
-
});
|
|
185
|
-
const blockDepth = promptAccounting.defaultBlockDepth;
|
|
186
|
-
const warnings: RoutePreparationWarning[] = [];
|
|
187
|
-
|
|
188
|
-
let hits: ChainHits = { perModel: new Map(), perModelBuckets: new Map() };
|
|
189
|
-
let costEstimates: CandidateCostEstimate[] = [];
|
|
190
|
-
let previousDecision = compatiblePreviousDecision(
|
|
191
|
-
input.previousDecision,
|
|
192
|
-
compatible.candidates,
|
|
193
|
-
);
|
|
194
|
-
let conversationId: string | null = null;
|
|
195
|
-
let identityMatchDepth: number | null = null;
|
|
196
|
-
let activeLease: ActiveLease | undefined;
|
|
197
|
-
let pendingLease: PendingLease | undefined;
|
|
198
|
-
let identityHit: PrefixHit | undefined;
|
|
199
|
-
|
|
200
|
-
try {
|
|
201
|
-
// Warmth: strict chains over hits still inside the warmth window — cost
|
|
202
|
-
// estimation and the freshness touch must reflect what the provider cache
|
|
203
|
-
// can actually re-read. Identity: either tier over every recent hit — a
|
|
204
|
-
// stale or loose-only match still names the same conversation.
|
|
205
|
-
hits = deriveModelChainHits(
|
|
206
|
-
input.chainsByModel,
|
|
207
|
-
warmPrefixHits(input.prefixHits, input.nowMs),
|
|
208
|
-
);
|
|
209
|
-
identityHit = deepestIdentityHit(
|
|
210
|
-
input.chainsByModel,
|
|
211
|
-
input.looseChain,
|
|
212
|
-
input.prefixHits,
|
|
213
|
-
);
|
|
214
|
-
if (identityHit) {
|
|
215
|
-
conversationId = identityHit.entry.conversation_id;
|
|
216
|
-
identityMatchDepth = identityHit.depth;
|
|
217
|
-
previousDecision ??= previousDecisionFromHit(
|
|
218
|
-
identityHit.entry,
|
|
219
|
-
compatible.candidates,
|
|
220
|
-
);
|
|
221
|
-
activeLease = activeLeaseFromHit(identityHit.entry, compatible.candidates);
|
|
222
|
-
// The prefetch write lands on whatever rows existed when the policy
|
|
223
|
-
// finished, and later turns append rows without it, so the pending
|
|
224
|
-
// lease is recovered from any recent row of this conversation — but
|
|
225
|
-
// only when the previous lease ran to completion. The final hold turn
|
|
226
|
-
// writes a zero countdown; a fallback-served turn writes none, and its
|
|
227
|
-
// broken commitment must not be resurrected by a stale prefetch.
|
|
228
|
-
if (identityHit.entry.lease_turns_remaining === 0) {
|
|
229
|
-
const matchConversationId = identityHit.entry.conversation_id;
|
|
230
|
-
const conversationHits = (input.prefixHits ?? [])
|
|
231
|
-
.filter((hit) => hit.conversation_id === matchConversationId)
|
|
232
|
-
.sort((left, right) => right.updated_at.localeCompare(left.updated_at));
|
|
233
|
-
for (const hit of conversationHits) {
|
|
234
|
-
pendingLease = pendingLeaseFromHit(hit, compatible.candidates);
|
|
235
|
-
if (pendingLease !== undefined) break;
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
const promptEstimatesByCandidate = new Map(
|
|
241
|
-
compatible.candidates.map((candidate) => {
|
|
242
|
-
const anchor = selectPromptAnchor(
|
|
243
|
-
hits,
|
|
244
|
-
candidate.model,
|
|
245
|
-
candidate.reasoningEffort,
|
|
246
|
-
input.cacheScope,
|
|
247
|
-
);
|
|
248
|
-
const candidateAnchor = promptAnchorFor({
|
|
249
|
-
accounting: promptAccounting,
|
|
250
|
-
// Stored prompt tokens include the previous response-format schema.
|
|
251
|
-
anchor:
|
|
252
|
-
anchor?.entry.response_format_fp === input.responseFormatFp
|
|
253
|
-
? anchor
|
|
254
|
-
: undefined,
|
|
255
|
-
chain: input.chainsByModel.get(candidate.model),
|
|
256
|
-
});
|
|
257
|
-
return [
|
|
258
|
-
routingCandidateKey(candidate),
|
|
259
|
-
promptAccounting.promptEstimateFor(
|
|
260
|
-
candidate.model,
|
|
261
|
-
candidateAnchor,
|
|
262
|
-
),
|
|
263
|
-
];
|
|
264
|
-
}),
|
|
265
|
-
);
|
|
266
|
-
costEstimates = estimateCandidateCosts({
|
|
267
|
-
candidates: compatible.candidates,
|
|
268
|
-
hits,
|
|
269
|
-
incomingBlockCount: blockDepth,
|
|
270
|
-
incomingProviderBlockCountFor:
|
|
271
|
-
promptAccounting.providerBlockDepthFor,
|
|
272
|
-
promptEstimatesByCandidate,
|
|
273
|
-
toolChoiceFp: input.toolChoiceFp,
|
|
274
|
-
responseFormatFp: input.responseFormatFp,
|
|
275
|
-
pricing: input.pricing,
|
|
276
|
-
averageOutputTokensByModel: input.averageOutputTokensByModel,
|
|
277
|
-
cacheScope: input.cacheScope,
|
|
278
|
-
});
|
|
279
|
-
} catch (error) {
|
|
280
|
-
warnings.push({ phase: "cost_estimation", error });
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
const candidateResolution = resolveStrategyCandidates({
|
|
284
|
-
strategy: input.strategy,
|
|
285
|
-
candidates: compatible.candidates,
|
|
286
|
-
costEstimates,
|
|
287
|
-
previousDecision,
|
|
288
|
-
customConfig: input.customConfig,
|
|
289
|
-
modelPrices: input.modelPrices ?? {},
|
|
290
|
-
});
|
|
291
|
-
|
|
292
|
-
return {
|
|
293
|
-
blockDepth,
|
|
294
|
-
hits,
|
|
295
|
-
conversationId,
|
|
296
|
-
identityMatchDepth,
|
|
297
|
-
...(identityHit !== undefined ? { identityHit } : {}),
|
|
298
|
-
...(previousDecision !== undefined ? { previousDecision } : {}),
|
|
299
|
-
...(activeLease !== undefined ? { activeLease } : {}),
|
|
300
|
-
...(pendingLease !== undefined ? { pendingLease } : {}),
|
|
301
|
-
costEstimates,
|
|
302
|
-
candidateResolution,
|
|
303
|
-
warnings,
|
|
304
|
-
};
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
// Resolved fallback posture for the phased prepareRoute API. Distinct from
|
|
308
|
-
// createRouter's RouterFallbackConfig: here both flags default ON.
|
|
309
|
-
export type RouteFallbackConfig = {
|
|
310
|
-
enabled: boolean;
|
|
311
|
-
requiresDifferentProvider: boolean;
|
|
312
|
-
};
|
|
313
|
-
|
|
314
|
-
type RouteAccountingInput =
|
|
315
|
-
| {
|
|
316
|
-
promptAccounting: CandidatePromptAccounting;
|
|
317
|
-
tailCharsFor?: never;
|
|
318
|
-
}
|
|
319
|
-
| {
|
|
320
|
-
promptAccounting?: undefined;
|
|
321
|
-
tailCharsFor?: LegacyTailCharsFor;
|
|
322
|
-
};
|
|
323
|
-
|
|
324
|
-
export type RouteInput = CandidatePreparationSharedInput & RouteAccountingInput & {
|
|
325
|
-
// Full request message list, system messages included.
|
|
326
|
-
messages: ChatMessage[];
|
|
327
|
-
evals?: RouterEval[];
|
|
328
|
-
selectorModel: string;
|
|
329
|
-
// Selector context budget in characters (typically the selector model's
|
|
330
|
-
// context window in tokens times CHARS_PER_TOKEN).
|
|
331
|
-
selectorContextWindowChars: number;
|
|
332
|
-
modelFallbackEnabled?: boolean;
|
|
333
|
-
fallbackRequiresDifferentProvider?: boolean;
|
|
334
|
-
};
|
|
335
|
-
|
|
336
|
-
export type PreparedRoute = CandidatePreparation & {
|
|
337
|
-
fallbackConfig: RouteFallbackConfig;
|
|
338
|
-
// Null when the custom strategy is selected but no custom rule
|
|
339
|
-
// configuration resolved; hosts surface their own error for that state.
|
|
340
|
-
selectorPreparation: BuiltSelectorRequest | null;
|
|
341
|
-
};
|
|
342
|
-
|
|
343
|
-
export function prepareRoute(input: RouteInput): PreparedRoute {
|
|
344
|
-
const preparation = prepareCandidatesWithAccounting(
|
|
345
|
-
input,
|
|
346
|
-
input.promptAccounting
|
|
347
|
-
?? chatPromptAccounting(input.messages, input.tailCharsFor),
|
|
348
|
-
);
|
|
349
|
-
const resolution = preparation.candidateResolution;
|
|
350
|
-
const fallbackConfig: RouteFallbackConfig = {
|
|
351
|
-
enabled: input.modelFallbackEnabled !== false,
|
|
352
|
-
requiresDifferentProvider: input.fallbackRequiresDifferentProvider !== false,
|
|
353
|
-
};
|
|
354
|
-
const custom = resolution.strategy === "custom" ? resolution.custom : undefined;
|
|
355
|
-
if (custom === null) {
|
|
356
|
-
return { ...preparation, fallbackConfig, selectorPreparation: null };
|
|
357
|
-
}
|
|
358
|
-
const selectorPreparation = buildSizedSelectorRequest({
|
|
359
|
-
candidates: resolution.candidates,
|
|
360
|
-
evals: input.evals ?? [],
|
|
361
|
-
...(preparation.previousDecision !== undefined
|
|
362
|
-
? { previousDecision: preparation.previousDecision }
|
|
363
|
-
: {}),
|
|
364
|
-
costEstimates: resolution.costEstimates,
|
|
365
|
-
...(custom ? { customRules: custom.rules, defaultTarget: custom.defaultTarget } : {}),
|
|
366
|
-
selectorModel: input.selectorModel,
|
|
367
|
-
contextWindowChars: input.selectorContextWindowChars,
|
|
368
|
-
messages: selectorSafeMessages(input.messages),
|
|
369
|
-
modelFallbackEnabled: fallbackConfig.enabled,
|
|
370
|
-
fallbackRequiresDifferentProvider: fallbackConfig.requiresDifferentProvider,
|
|
371
|
-
});
|
|
372
|
-
return { ...preparation, fallbackConfig, selectorPreparation };
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
export type RouteResult = {
|
|
376
|
-
decision: RoutingDecision;
|
|
377
|
-
fallbackDecision?: RoutingDecision;
|
|
378
|
-
outputText: string;
|
|
379
|
-
};
|
|
380
|
-
|
|
381
|
-
// Parses and validates the selector's raw output against the prepared
|
|
382
|
-
// candidate set and fallback policy.
|
|
383
|
-
export function finalizeRoute(
|
|
384
|
-
prepared: Pick<PreparedRoute, "candidateResolution" | "fallbackConfig">,
|
|
385
|
-
selectorOutputText: string,
|
|
386
|
-
): RouteResult {
|
|
387
|
-
const parsed = parseSelectorDecision(selectorOutputText);
|
|
388
|
-
validateSelectorDecisions({
|
|
389
|
-
decision: parsed.decision,
|
|
390
|
-
...(parsed.fallbackDecision !== undefined
|
|
391
|
-
? { fallbackDecision: parsed.fallbackDecision }
|
|
392
|
-
: {}),
|
|
393
|
-
candidates: prepared.candidateResolution.candidates,
|
|
394
|
-
modelFallbackEnabled: prepared.fallbackConfig.enabled,
|
|
395
|
-
fallbackRequiresDifferentProvider: prepared.fallbackConfig.requiresDifferentProvider,
|
|
396
|
-
});
|
|
397
|
-
return {
|
|
398
|
-
decision: parsed.decision,
|
|
399
|
-
...(parsed.fallbackDecision !== undefined
|
|
400
|
-
? { fallbackDecision: parsed.fallbackDecision }
|
|
401
|
-
: {}),
|
|
402
|
-
outputText: selectorOutputText,
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
// The only effect in the pipeline: executing the selector request against
|
|
407
|
-
// some model. Implementations receive a complete chat-completions-shaped
|
|
408
|
-
// request and return the model's raw output text. Credentials, transport,
|
|
409
|
-
// and retries are entirely the implementation's concern.
|
|
410
|
-
export type Selector = {
|
|
411
|
-
select(request: ChatCompletionRequest, signal?: AbortSignal): Promise<string>;
|
|
412
|
-
};
|
|
413
|
-
|
|
414
|
-
export async function route(
|
|
415
|
-
input: RouteInput,
|
|
416
|
-
selector: Selector,
|
|
417
|
-
): Promise<RouteResult & { prepared: PreparedRoute }> {
|
|
418
|
-
const prepared = prepareRoute(input);
|
|
419
|
-
if (prepared.selectorPreparation === null) {
|
|
420
|
-
throw new RouterCoreError(
|
|
421
|
-
"configuration",
|
|
422
|
-
"Router custom rule configuration is missing.",
|
|
423
|
-
"custom_config_missing",
|
|
424
|
-
);
|
|
425
|
-
}
|
|
426
|
-
const outputText = await selector.select(prepared.selectorPreparation.selectorRequest);
|
|
427
|
-
return { ...finalizeRoute(prepared, outputText), prepared };
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
export function chatPromptAccounting(
|
|
431
|
-
messages: ChatMessage[],
|
|
432
|
-
tailCharsFor?: LegacyTailCharsFor,
|
|
433
|
-
): CandidatePromptAccounting {
|
|
434
|
-
const conversation = conversationMessages({ messages });
|
|
435
|
-
const blockDepth = conversationBlockCount({ messages });
|
|
436
|
-
return {
|
|
437
|
-
anchorSemantics: "matched_prefix",
|
|
438
|
-
defaultBlockDepth: blockDepth,
|
|
439
|
-
providerBlockDepthFor: () => blockDepth,
|
|
440
|
-
promptEstimateFor: (model, anchor) => {
|
|
441
|
-
if (anchor) {
|
|
442
|
-
const exact = tailCharsFor?.(model, anchor.hash);
|
|
443
|
-
if (exact !== undefined) {
|
|
444
|
-
return { chars: exact, reusesStoredPromptTokens: true };
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
return {
|
|
448
|
-
chars: JSON.stringify(
|
|
449
|
-
anchor ? conversation.slice(anchor.depth) : messages,
|
|
450
|
-
).length,
|
|
451
|
-
reusesStoredPromptTokens: anchor !== undefined,
|
|
452
|
-
};
|
|
453
|
-
},
|
|
454
|
-
};
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
function promptAnchorFor(args: {
|
|
458
|
-
accounting: CandidatePromptAccounting;
|
|
459
|
-
anchor: ReturnType<typeof selectPromptAnchor>;
|
|
460
|
-
chain: readonly string[] | undefined;
|
|
461
|
-
}): CandidatePromptAnchor | undefined {
|
|
462
|
-
if (!args.anchor) return undefined;
|
|
463
|
-
const depth =
|
|
464
|
-
args.accounting.anchorSemantics === "stored_input"
|
|
465
|
-
? args.anchor.entry.prompt_anchor_depth
|
|
466
|
-
: args.anchor.depth;
|
|
467
|
-
if (
|
|
468
|
-
depth === undefined
|
|
469
|
-
|| depth === null
|
|
470
|
-
|| depth <= 0
|
|
471
|
-
) {
|
|
472
|
-
return undefined;
|
|
473
|
-
}
|
|
474
|
-
const hash = args.chain?.[depth - 1];
|
|
475
|
-
return hash === undefined ? undefined : { hash, depth };
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
function resolvePromptAccounting(
|
|
479
|
-
input: CandidatePreparationInput,
|
|
480
|
-
): CandidatePromptAccounting {
|
|
481
|
-
if (input.promptAccounting) return input.promptAccounting;
|
|
482
|
-
if (input.messages) {
|
|
483
|
-
return chatPromptAccounting(input.messages, input.tailCharsFor);
|
|
484
|
-
}
|
|
485
|
-
throw new RouterCoreError(
|
|
486
|
-
"invalid_request",
|
|
487
|
-
"Candidate preparation requires promptAccounting or messages",
|
|
488
|
-
"prompt_accounting_required",
|
|
489
|
-
);
|
|
490
|
-
}
|