@mupt-ai/dari-gateway 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +135 -2
- package/dari-gateway.example.yaml +28 -0
- package/package.json +42 -4
- package/src/bun.ts +4 -0
- package/src/context/eval_catalog.ts +11 -0
- package/src/context/interaction_fingerprint.ts +691 -0
- package/src/context/interaction_fingerprint_format.ts +189 -0
- package/src/context/interaction_lease_depth.ts +150 -0
- package/src/context/lease_history.ts +200 -0
- package/src/context/pricing.ts +30 -0
- package/src/context/provider_prompt_accounting.ts +241 -0
- package/src/context/routing_context.ts +337 -0
- package/src/http/auth.ts +43 -0
- package/src/http/errors.ts +114 -0
- package/src/http/handler.ts +110 -0
- package/src/http/http2_server.ts +122 -0
- package/src/http/serve.ts +119 -0
- package/src/index.ts +22 -0
- package/src/ingress/anthropic_messages.ts +162 -0
- package/src/ingress/anthropic_models.ts +28 -0
- package/src/ingress/chat_completions.ts +49 -0
- package/src/ingress/claude_code/request_context.ts +36 -0
- package/src/ingress/codex.ts +81 -0
- package/src/ingress/codex_models.ts +113 -0
- package/src/ingress/json.ts +113 -0
- package/src/ingress/models.ts +55 -0
- package/src/interaction/capabilities.ts +30 -0
- package/src/interaction/image.ts +12 -0
- package/src/interaction/portable_reasoning_state.ts +25 -0
- package/src/interaction/provider_continuation_state.ts +36 -0
- package/src/interaction/tool_identity.ts +65 -0
- package/src/interaction/types.ts +282 -0
- package/src/models/context_limits.ts +8 -0
- package/src/models/execution_provider.ts +22 -0
- package/src/models/gateways.ts +73 -0
- package/src/models/ids.ts +26 -0
- package/src/models/provider_defaults.ts +47 -0
- package/src/models/runtime_overrides.ts +221 -0
- package/src/observability/errors.ts +25 -0
- package/src/platform/platform.ts +204 -0
- package/src/platform/routing_contract.ts +6 -0
- package/src/protocol/input/anthropic_messages.ts +531 -0
- package/src/protocol/input/chat_completions.ts +255 -0
- package/src/protocol/input/chat_completions_validation.ts +293 -0
- package/src/protocol/input/openai_responses.ts +749 -0
- package/src/protocol/input/router_model.ts +53 -0
- package/src/protocol/output/anthropic_messages.ts +32 -0
- package/src/protocol/output/anthropic_messages_output.ts +129 -0
- package/src/protocol/output/anthropic_messages_stream.ts +281 -0
- package/src/protocol/output/chat_completions.ts +451 -0
- package/src/protocol/output/chat_reasoning_detail.ts +19 -0
- package/src/protocol/output/delivered_stream_snapshot.ts +31 -0
- package/src/protocol/output/openai_responses.ts +30 -0
- package/src/protocol/output/openai_responses_output.ts +159 -0
- package/src/protocol/output/openai_responses_stream.ts +631 -0
- package/src/protocol/output/sse_stream.ts +102 -0
- package/src/providers/anthropic_safeguards.ts +115 -0
- package/src/providers/aws_assume_role.ts +147 -0
- package/src/providers/cancel_estimate.ts +140 -0
- package/src/providers/execution.ts +800 -0
- package/src/providers/model_capabilities.ts +50 -0
- package/src/providers/pi.ts +499 -0
- package/src/providers/pi_context.ts +466 -0
- package/src/providers/pi_options.ts +416 -0
- package/src/providers/pi_output.ts +500 -0
- package/src/providers/provider_attempt_runner.ts +153 -0
- package/src/providers/provider_call.ts +433 -0
- package/src/providers/provider_reported_cost.ts +138 -0
- package/src/providers/reasoning.ts +59 -0
- package/src/providers/response_capture.ts +65 -0
- package/src/providers/stream_finalization.ts +264 -0
- package/src/providers/subscription_limits.ts +208 -0
- package/src/providers/subscription_usage.ts +99 -0
- package/src/runtime/claude_code/auxiliary_routing.ts +80 -0
- package/src/runtime/customer_disconnect.ts +66 -0
- package/src/runtime/explicit_model.ts +86 -0
- package/src/runtime/harness_identity/conversation.ts +87 -0
- package/src/runtime/harness_identity/extract.ts +323 -0
- package/src/runtime/harness_identity/index.ts +14 -0
- package/src/runtime/prepare_routed_completion.ts +224 -0
- package/src/runtime/provider_stream_lifecycle.ts +105 -0
- package/src/runtime/provider_stream_outcome.ts +83 -0
- package/src/runtime/request_tags.ts +34 -0
- package/src/runtime/routed_completion.ts +102 -0
- package/src/runtime/routed_protocol.ts +33 -0
- package/src/runtime/routed_runtime.ts +722 -0
- package/src/runtime/routing_lease.ts +273 -0
- package/src/runtime/runtime_reporter.ts +115 -0
- package/src/self_host/config.ts +173 -0
- package/src/self_host/main.ts +104 -0
- package/src/self_host/platform.ts +63 -0
- package/src/self_host/prefix_store.ts +315 -0
- package/src/self_host/resolve.ts +240 -0
- package/src/self_host/subscriptions/access.ts +43 -0
- package/src/self_host/subscriptions/http.ts +204 -0
- package/src/self_host/subscriptions/index.ts +43 -0
- package/src/self_host/subscriptions/oauth.ts +395 -0
- package/src/self_host/subscriptions/store.ts +339 -0
- package/src/strategies/auto_routing_selector.ts +72 -0
- package/src/strategies/eval_relevance.ts +57 -0
- package/src/strategies/fallback_decision.ts +30 -0
- package/src/strategies/priority_selector.ts +40 -0
- package/src/strategies/route_decision.ts +106 -0
- package/src/strategies/routing.ts +455 -0
- package/src/strategies/selector_projection.ts +207 -0
- package/src/telemetry/background.ts +40 -0
- package/src/telemetry/benchmark_replay.ts +160 -0
- package/src/telemetry/billing_finalization.ts +167 -0
- package/src/telemetry/ingest_values.ts +15 -0
- package/src/telemetry/model_cost.ts +14 -0
- package/src/telemetry/prefix_state.ts +183 -0
- package/src/telemetry/report.ts +832 -0
- package/src/telemetry/requests/interaction_observability.ts +258 -0
- package/src/telemetry/requests/retained_interaction_request.ts +20 -0
- package/src/telemetry/scrub.ts +38 -0
- package/src/telemetry/skills/claude/catalog.ts +119 -0
- package/src/telemetry/skills/claude/file_reads.ts +153 -0
- package/src/telemetry/skills/claude/observations.ts +450 -0
- package/src/telemetry/skills/codex/catalog.ts +171 -0
- package/src/telemetry/skills/codex/exec_file_reads.ts +55 -0
- package/src/telemetry/skills/codex/explicit.ts +116 -0
- package/src/telemetry/skills/codex/file_reads.ts +51 -0
- package/src/telemetry/skills/codex/identity.ts +21 -0
- package/src/telemetry/skills/observations.ts +94 -0
- package/src/telemetry/skills/paths.ts +82 -0
- package/src/telemetry/skills/pi.ts +276 -0
- package/src/telemetry/skills/shell_file_reads.ts +267 -0
- package/src/telemetry/skills/shell_words.ts +202 -0
- package/src/telemetry/tools/input.ts +16 -0
- package/src/telemetry/tools/observations.ts +349 -0
- package/src/telemetry/usage.ts +173 -0
- package/src/types.ts +150 -0
|
@@ -0,0 +1,455 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CHARS_PER_TOKEN,
|
|
3
|
+
RouterCoreError,
|
|
4
|
+
buildSizedSelectorRequest,
|
|
5
|
+
isRecord,
|
|
6
|
+
parseSelectorDecision,
|
|
7
|
+
sameRoutingCandidate,
|
|
8
|
+
selectorSafeMessages,
|
|
9
|
+
type CandidateCostEstimate,
|
|
10
|
+
type ChatCompletionRequest,
|
|
11
|
+
type ChatMessage,
|
|
12
|
+
type CustomDefaultTarget,
|
|
13
|
+
type CustomRouterRule,
|
|
14
|
+
type CustomSelectorInput,
|
|
15
|
+
type PreviousDecision,
|
|
16
|
+
type ReasoningEffort,
|
|
17
|
+
type ResolvedJevConfig,
|
|
18
|
+
type RouterEval,
|
|
19
|
+
type RoutingCandidate,
|
|
20
|
+
type RoutingDecision,
|
|
21
|
+
type SelectorInput,
|
|
22
|
+
type SelectorLeaseHistoryEntry,
|
|
23
|
+
} from "@mupt-ai/dari-router/policy-engine";
|
|
24
|
+
import { HttpError } from "../http/errors";
|
|
25
|
+
import { executionProviderForModel } from "../models/execution_provider";
|
|
26
|
+
import { providerLabel } from "../models/provider_defaults";
|
|
27
|
+
import { internalChatCompletionsToInteraction } from "../protocol/input/chat_completions";
|
|
28
|
+
import { gatewayPlatform, type ResolveRouterRequest } from "../platform/platform";
|
|
29
|
+
import { isBedrockApi } from "../providers/model_capabilities";
|
|
30
|
+
import { completeInteractionModel, modelContextWindow } from "../providers/pi";
|
|
31
|
+
import type {
|
|
32
|
+
AwsAssumeRoleProviderAuth,
|
|
33
|
+
AwsSigV4ProviderAuth,
|
|
34
|
+
ProviderAuth,
|
|
35
|
+
ResolvedRouter,
|
|
36
|
+
} from "../types";
|
|
37
|
+
import { deriveFallbackDecision } from "./fallback_decision";
|
|
38
|
+
import type { SelectorImage } from "./selector_projection";
|
|
39
|
+
|
|
40
|
+
export function resolveRouter(args: ResolveRouterRequest): Promise<ResolvedRouter> {
|
|
41
|
+
return gatewayPlatform().resolveRouter(args);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type SelectorTrace = {
|
|
45
|
+
selectorModel: string;
|
|
46
|
+
candidates: RoutingCandidate[];
|
|
47
|
+
selectorInput: SelectorInput | CustomSelectorInput;
|
|
48
|
+
selectorRequest: ChatCompletionRequest;
|
|
49
|
+
outputText: string;
|
|
50
|
+
rejectedOutputTexts?: string[];
|
|
51
|
+
usage: Record<string, unknown> | null;
|
|
52
|
+
latencyMs: number;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
export type SelectorRun = SelectorTrace & {
|
|
56
|
+
decision: RoutingDecision;
|
|
57
|
+
fallbackDecision?: RoutingDecision;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
export type SelectModelArgs = {
|
|
61
|
+
// The configured roster before pruning; anonymous-action selectors assign
|
|
62
|
+
// conversation-stable letters over it. Optional: absent, the active set is
|
|
63
|
+
// treated as the roster.
|
|
64
|
+
allCandidates?: RoutingCandidate[];
|
|
65
|
+
messages: ChatMessage[];
|
|
66
|
+
candidates: RoutingCandidate[];
|
|
67
|
+
evals?: RouterEval[];
|
|
68
|
+
previousDecision?: PreviousDecision;
|
|
69
|
+
costEstimates?: CandidateCostEstimate[];
|
|
70
|
+
customRules?: CustomRouterRule[];
|
|
71
|
+
defaultTarget?: CustomDefaultTarget | null;
|
|
72
|
+
sessionId?: string;
|
|
73
|
+
modelFallbackEnabled?: boolean;
|
|
74
|
+
fallbackRequiresDifferentProvider?: boolean;
|
|
75
|
+
executionProviderForModel?: (model: string) => string;
|
|
76
|
+
imputeEvalScores?: boolean;
|
|
77
|
+
imputationReferenceEvals?: RouterEval[];
|
|
78
|
+
leaseHistory?: SelectorLeaseHistoryEntry[];
|
|
79
|
+
// Absent means ENABLED — deliberate: training and experiment callers
|
|
80
|
+
// predate the flag and must keep lease-context behavior. When false, the
|
|
81
|
+
// selector input carries no lease_history key (byte-identical to the
|
|
82
|
+
// pre-lease-context shape).
|
|
83
|
+
leaseContextEnabled?: boolean;
|
|
84
|
+
// Images behind the `<image N>` placeholders in `messages`. Only a backend
|
|
85
|
+
// that advertises image input attaches them; every other selector ignores
|
|
86
|
+
// them and routes on the placeholders alone.
|
|
87
|
+
images?: SelectorImage[];
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
export type SelectorAdapterArgs = SelectModelArgs & {
|
|
91
|
+
selectorModel: string;
|
|
92
|
+
selectorInput: SelectorInput | CustomSelectorInput;
|
|
93
|
+
selectorRequest: ChatCompletionRequest;
|
|
94
|
+
startedAt: number;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
export type SelectorAdapter = (args: SelectorAdapterArgs) => Promise<SelectorRun>;
|
|
98
|
+
|
|
99
|
+
export type SelectorBackend = {
|
|
100
|
+
selectorModel: string;
|
|
101
|
+
contextWindowChars: () => Promise<number>;
|
|
102
|
+
prepareMessages?: (messages: ChatMessage[]) => ChatMessage[];
|
|
103
|
+
select: SelectorAdapter;
|
|
104
|
+
// Turn commitments this backend's deployed policy advertises; empty or
|
|
105
|
+
// absent means per-turn decisions (the runtime then applies its stateless
|
|
106
|
+
// hold). Async because the menu is discovered from the policy endpoint.
|
|
107
|
+
negotiatedLeaseTurns?: () => Promise<readonly number[]>;
|
|
108
|
+
// Whether the runtime may serve the incumbent while this backend selects.
|
|
109
|
+
// Defaults to true; unavailable/configuration-error backends must disable it.
|
|
110
|
+
supportsSpeculativeRouting?: boolean;
|
|
111
|
+
// How many request images the deployed policy accepts per decision; 0 or
|
|
112
|
+
// absent means the backend routes on text alone.
|
|
113
|
+
maxImages?: () => Promise<number>;
|
|
114
|
+
// Jev decides which eval cards (the organization's plus the public
|
|
115
|
+
// catalog) are relevant to each request before selection. Production
|
|
116
|
+
// wiring sets it on the selector chain; test backends may leave it out to see
|
|
117
|
+
// the organization's cards as given.
|
|
118
|
+
evalRelevance?: ResolvedJevConfig;
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
export function createLlmSelectorBackend(config: {
|
|
122
|
+
model: string;
|
|
123
|
+
apiKey: string;
|
|
124
|
+
reasoningEffort?: ReasoningEffort;
|
|
125
|
+
}): SelectorBackend {
|
|
126
|
+
const model = requiredSelectorConfig(config.model, "LLM selector model");
|
|
127
|
+
const apiKey = requiredSelectorConfig(config.apiKey, "LLM selector API key");
|
|
128
|
+
return {
|
|
129
|
+
selectorModel: model,
|
|
130
|
+
contextWindowChars: () => selectorContextWindowChars(model),
|
|
131
|
+
select: (args) => callLlmSelector(args, apiKey, config.reasoningEffort),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export const SELECTOR_OUTPUT_ATTEMPTS = 2;
|
|
136
|
+
|
|
137
|
+
// Exported so other selector backends can attach the raw policy completion
|
|
138
|
+
// to failures the same way the LLM selector does. Without this wrapping,
|
|
139
|
+
// selectorTraceFromError() returns null for their failures and the malformed
|
|
140
|
+
// completion is never reported.
|
|
141
|
+
export class SelectorOutputError extends HttpError {
|
|
142
|
+
readonly selectorTrace: SelectorTrace;
|
|
143
|
+
|
|
144
|
+
constructor(error: HttpError, selectorTrace: SelectorTrace) {
|
|
145
|
+
super(error.status, error.message, error.code, error.param);
|
|
146
|
+
this.selectorTrace = selectorTrace;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function selectorTraceFromError(error: unknown): SelectorTrace | null {
|
|
151
|
+
return error instanceof SelectorOutputError ? error.selectorTrace : null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export async function selectModel(
|
|
155
|
+
args: SelectModelArgs & { selectorBackend: SelectorBackend },
|
|
156
|
+
): Promise<SelectorRun> {
|
|
157
|
+
const { selectorBackend } = args;
|
|
158
|
+
const selectorModel = selectorBackend.selectorModel;
|
|
159
|
+
const startedAt = Date.now();
|
|
160
|
+
const contextWindowChars = await selectorBackend.contextWindowChars();
|
|
161
|
+
const safeMessages = selectorSafeMessages(args.messages);
|
|
162
|
+
const leaseContextEnabled = args.leaseContextEnabled !== false;
|
|
163
|
+
const { selectorInput, selectorRequest } = buildSizedSelectorRequest({
|
|
164
|
+
candidates: args.candidates,
|
|
165
|
+
evals: args.evals,
|
|
166
|
+
previousDecision: args.previousDecision,
|
|
167
|
+
costEstimates: args.costEstimates,
|
|
168
|
+
customRules: args.customRules,
|
|
169
|
+
defaultTarget: args.defaultTarget,
|
|
170
|
+
selectorModel,
|
|
171
|
+
contextWindowChars,
|
|
172
|
+
messages: selectorBackend.prepareMessages?.(safeMessages) ?? safeMessages,
|
|
173
|
+
...(leaseContextEnabled ? { leaseHistory: args.leaseHistory ?? [] } : {}),
|
|
174
|
+
modelFallbackEnabled: args.modelFallbackEnabled,
|
|
175
|
+
fallbackRequiresDifferentProvider: args.fallbackRequiresDifferentProvider,
|
|
176
|
+
imputeEvalScores: args.imputeEvalScores,
|
|
177
|
+
imputationReferenceEvals: args.imputationReferenceEvals,
|
|
178
|
+
});
|
|
179
|
+
const runArgs: SelectorAdapterArgs = {
|
|
180
|
+
...args,
|
|
181
|
+
selectorModel,
|
|
182
|
+
selectorInput,
|
|
183
|
+
selectorRequest,
|
|
184
|
+
startedAt,
|
|
185
|
+
};
|
|
186
|
+
const run = await selectorBackend.select(runArgs);
|
|
187
|
+
return validateSelectorRun(run, args);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async function selectorContextWindowChars(selectorModel: string): Promise<number> {
|
|
191
|
+
const contextWindow = modelContextWindow(selectorModel);
|
|
192
|
+
if (contextWindow === null) {
|
|
193
|
+
throw new HttpError(
|
|
194
|
+
500,
|
|
195
|
+
`No context window is configured for selector model ${selectorModel}.`,
|
|
196
|
+
"configuration_error"
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
return contextWindow * CHARS_PER_TOKEN;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async function callLlmSelector(
|
|
203
|
+
args: SelectorAdapterArgs,
|
|
204
|
+
selectorApiKey: string,
|
|
205
|
+
reasoningEffort?: ReasoningEffort,
|
|
206
|
+
): Promise<SelectorRun> {
|
|
207
|
+
const rejectedOutputTexts: string[] = [];
|
|
208
|
+
const usages: Array<Record<string, unknown> | null> = [];
|
|
209
|
+
|
|
210
|
+
for (let attempt = 1; attempt <= SELECTOR_OUTPUT_ATTEMPTS; attempt += 1) {
|
|
211
|
+
let message: Record<string, unknown>;
|
|
212
|
+
try {
|
|
213
|
+
const completion = await completeInteractionModel({
|
|
214
|
+
model: args.selectorModel,
|
|
215
|
+
request: internalChatCompletionsToInteraction(args.selectorRequest),
|
|
216
|
+
apiKey: selectorApiKey,
|
|
217
|
+
sessionId: args.sessionId,
|
|
218
|
+
...(reasoningEffort ? { selectedReasoningEffort: reasoningEffort } : {}),
|
|
219
|
+
});
|
|
220
|
+
message = completion.message;
|
|
221
|
+
} catch {
|
|
222
|
+
throw selectorExecutionError(args, rejectedOutputTexts, usages);
|
|
223
|
+
}
|
|
224
|
+
if (message.stopReason === "error" || message.stopReason === "aborted") {
|
|
225
|
+
throw selectorExecutionError(args, rejectedOutputTexts, usages);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const text = assistantText(message);
|
|
229
|
+
usages.push(isRecord(message.usage) ? message.usage : null);
|
|
230
|
+
try {
|
|
231
|
+
// The managed LLM selector never negotiates leases, so drop any
|
|
232
|
+
// lease_turns_remaining a non-strict provider stamps on the output.
|
|
233
|
+
const { leaseTurnsRemaining: _leaseTurnsRemaining, ...decision } =
|
|
234
|
+
parseSelectorDecision(text).decision;
|
|
235
|
+
return {
|
|
236
|
+
decision,
|
|
237
|
+
selectorModel: args.selectorModel,
|
|
238
|
+
candidates: args.candidates,
|
|
239
|
+
selectorInput: args.selectorInput,
|
|
240
|
+
selectorRequest: args.selectorRequest,
|
|
241
|
+
outputText: text,
|
|
242
|
+
...(rejectedOutputTexts.length > 0
|
|
243
|
+
? { rejectedOutputTexts: [...rejectedOutputTexts] }
|
|
244
|
+
: {}),
|
|
245
|
+
usage: aggregateSelectorUsage(usages),
|
|
246
|
+
latencyMs: Date.now() - args.startedAt,
|
|
247
|
+
};
|
|
248
|
+
} catch (error) {
|
|
249
|
+
const mapped = selectorOutputHttpError(error);
|
|
250
|
+
if (!mapped || !isMalformedSelectorOutput(mapped)) throw error;
|
|
251
|
+
if (attempt < SELECTOR_OUTPUT_ATTEMPTS) {
|
|
252
|
+
rejectedOutputTexts.push(text);
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
throw new SelectorOutputError(mapped, {
|
|
256
|
+
selectorModel: args.selectorModel,
|
|
257
|
+
candidates: args.candidates,
|
|
258
|
+
selectorInput: args.selectorInput,
|
|
259
|
+
selectorRequest: args.selectorRequest,
|
|
260
|
+
outputText: text,
|
|
261
|
+
...(rejectedOutputTexts.length > 0
|
|
262
|
+
? { rejectedOutputTexts: [...rejectedOutputTexts] }
|
|
263
|
+
: {}),
|
|
264
|
+
usage: aggregateSelectorUsage(usages),
|
|
265
|
+
latencyMs: Date.now() - args.startedAt,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
throw new HttpError(502, "Selector request failed.", "selector_execution_failed");
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function validateSelectorRun(run: SelectorRun, args: SelectModelArgs): SelectorRun {
|
|
274
|
+
const selected = {
|
|
275
|
+
model: run.decision.selectedModel,
|
|
276
|
+
reasoningEffort: run.decision.reasoningEffort,
|
|
277
|
+
};
|
|
278
|
+
if (!args.candidates.some((candidate) => sameRoutingCandidate(candidate, selected))) {
|
|
279
|
+
throw new SelectorOutputError(
|
|
280
|
+
new HttpError(
|
|
281
|
+
502,
|
|
282
|
+
"Selector returned a model/thinking-level pair outside this router's candidates.",
|
|
283
|
+
"selector_invalid_candidate",
|
|
284
|
+
),
|
|
285
|
+
run,
|
|
286
|
+
);
|
|
287
|
+
}
|
|
288
|
+
const { fallbackDecision: _selectorFallback, ...primaryRun } = run;
|
|
289
|
+
const fallbackDecision = deriveFallbackDecision({
|
|
290
|
+
candidates: args.candidates,
|
|
291
|
+
selected,
|
|
292
|
+
enabled: args.modelFallbackEnabled,
|
|
293
|
+
requiresDifferentProvider: args.fallbackRequiresDifferentProvider,
|
|
294
|
+
providerForModel: args.executionProviderForModel,
|
|
295
|
+
});
|
|
296
|
+
return fallbackDecision
|
|
297
|
+
? { ...primaryRun, fallbackDecision }
|
|
298
|
+
: primaryRun;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function requiredSelectorConfig(value: string, label: string): string {
|
|
302
|
+
const normalized = value.trim();
|
|
303
|
+
if (!normalized) {
|
|
304
|
+
throw new HttpError(500, `${label} is not configured.`, "configuration_error");
|
|
305
|
+
}
|
|
306
|
+
return normalized;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export function providerAuthForSelectedModel(args: {
|
|
310
|
+
selectedModel: string;
|
|
311
|
+
credentials: ResolvedRouter;
|
|
312
|
+
}): ProviderAuth {
|
|
313
|
+
const provider = executionProviderForModel(args.credentials, args.selectedModel);
|
|
314
|
+
const config = args.credentials.model_configs?.[args.selectedModel];
|
|
315
|
+
const structured = args.credentials.provider_auth?.[provider];
|
|
316
|
+
const key = args.credentials.provider_keys[provider]?.trim();
|
|
317
|
+
if (structured !== undefined) {
|
|
318
|
+
if (
|
|
319
|
+
key
|
|
320
|
+
|| (provider !== "amazon-bedrock" && !isBedrockApi(config?.api))
|
|
321
|
+
|| !(isAwsSigV4ProviderAuth(structured) || isAwsAssumeRoleProviderAuth(structured))
|
|
322
|
+
) {
|
|
323
|
+
throw new HttpError(
|
|
324
|
+
500,
|
|
325
|
+
`Invalid ${providerLabel(provider)} provider authentication is configured for this router.`,
|
|
326
|
+
"provider_configuration_error",
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
return structured;
|
|
330
|
+
}
|
|
331
|
+
if (key) return { type: "bearer", apiKey: key };
|
|
332
|
+
const source = args.credentials.provider_key_sources?.[provider] ?? "user";
|
|
333
|
+
const prefix = source === "managed" ? "Dari-managed" : "stored";
|
|
334
|
+
throw new HttpError(
|
|
335
|
+
401,
|
|
336
|
+
`No ${prefix} ${providerLabel(provider)} provider credential is configured for this router.`,
|
|
337
|
+
"missing_provider_key",
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function isAwsAssumeRoleProviderAuth(
|
|
342
|
+
value: unknown,
|
|
343
|
+
): value is AwsAssumeRoleProviderAuth {
|
|
344
|
+
return (
|
|
345
|
+
isRecord(value)
|
|
346
|
+
&& value.type === "aws_assume_role"
|
|
347
|
+
&& typeof value.role_arn === "string"
|
|
348
|
+
&& Boolean(value.role_arn.trim())
|
|
349
|
+
&& typeof value.external_id === "string"
|
|
350
|
+
&& Boolean(value.external_id.trim())
|
|
351
|
+
&& typeof value.region === "string"
|
|
352
|
+
&& Boolean(value.region.trim())
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function isAwsSigV4ProviderAuth(
|
|
357
|
+
value: unknown,
|
|
358
|
+
): value is AwsSigV4ProviderAuth {
|
|
359
|
+
return (
|
|
360
|
+
isRecord(value)
|
|
361
|
+
&& value.type === "aws_sigv4"
|
|
362
|
+
&& typeof value.access_key_id === "string"
|
|
363
|
+
&& Boolean(value.access_key_id.trim())
|
|
364
|
+
&& typeof value.secret_access_key === "string"
|
|
365
|
+
&& Boolean(value.secret_access_key.trim())
|
|
366
|
+
&& typeof value.region === "string"
|
|
367
|
+
&& Boolean(value.region.trim())
|
|
368
|
+
&& (
|
|
369
|
+
value.session_token == null
|
|
370
|
+
|| (
|
|
371
|
+
typeof value.session_token === "string"
|
|
372
|
+
&& Boolean(value.session_token.trim())
|
|
373
|
+
)
|
|
374
|
+
)
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function assistantText(message: Record<string, unknown>): string {
|
|
379
|
+
const content = message.content;
|
|
380
|
+
if (!Array.isArray(content)) return "";
|
|
381
|
+
return content
|
|
382
|
+
.filter(isRecord)
|
|
383
|
+
.filter((item) => item.type === "text" && typeof item.text === "string")
|
|
384
|
+
.map((item) => String(item.text))
|
|
385
|
+
.join("");
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function selectorOutputHttpError(error: unknown): HttpError | null {
|
|
389
|
+
if (error instanceof HttpError) return error;
|
|
390
|
+
if (!(error instanceof RouterCoreError) || error.kind !== "selector_output") return null;
|
|
391
|
+
return new HttpError(502, error.message, error.code, error.param);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function isMalformedSelectorOutput(error: HttpError): boolean {
|
|
395
|
+
return error.code === "selector_invalid_json" || error.code === "selector_invalid_response";
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
function selectorExecutionError(
|
|
399
|
+
args: SelectorAdapterArgs,
|
|
400
|
+
rejectedOutputTexts: string[],
|
|
401
|
+
usages: Array<Record<string, unknown> | null>,
|
|
402
|
+
): HttpError {
|
|
403
|
+
const error = new HttpError(502, "Selector request failed.", "selector_execution_failed");
|
|
404
|
+
const lastOutput = rejectedOutputTexts.at(-1);
|
|
405
|
+
if (lastOutput === undefined) return error;
|
|
406
|
+
return new SelectorOutputError(error, {
|
|
407
|
+
selectorModel: args.selectorModel,
|
|
408
|
+
candidates: args.candidates,
|
|
409
|
+
selectorInput: args.selectorInput,
|
|
410
|
+
selectorRequest: args.selectorRequest,
|
|
411
|
+
outputText: lastOutput,
|
|
412
|
+
...(rejectedOutputTexts.length > 1
|
|
413
|
+
? { rejectedOutputTexts: rejectedOutputTexts.slice(0, -1) }
|
|
414
|
+
: {}),
|
|
415
|
+
usage: aggregateSelectorUsage(usages),
|
|
416
|
+
latencyMs: Date.now() - args.startedAt,
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
function aggregateSelectorUsage(
|
|
421
|
+
usages: Array<Record<string, unknown> | null>,
|
|
422
|
+
): Record<string, unknown> | null {
|
|
423
|
+
const records = usages.filter((usage): usage is Record<string, unknown> => usage !== null);
|
|
424
|
+
if (records.length === 0) return null;
|
|
425
|
+
if (records.length === 1) return records[0];
|
|
426
|
+
const aggregate: Record<string, unknown> = { ...records.at(-1) };
|
|
427
|
+
for (const key of ["input", "cacheRead", "cacheWrite", "output", "totalTokens"]) {
|
|
428
|
+
const total = sumFiniteField(records, key);
|
|
429
|
+
if (total !== null) aggregate[key] = total;
|
|
430
|
+
}
|
|
431
|
+
const costs = records.flatMap((usage) => isRecord(usage.cost) ? [usage.cost] : []);
|
|
432
|
+
if (costs.length > 0) {
|
|
433
|
+
const aggregateCost: Record<string, unknown> = {
|
|
434
|
+
...(isRecord(aggregate.cost) ? aggregate.cost : {}),
|
|
435
|
+
};
|
|
436
|
+
for (const key of ["input", "cacheRead", "cacheWrite", "output", "total"]) {
|
|
437
|
+
const total = sumFiniteField(costs, key);
|
|
438
|
+
if (total !== null) aggregateCost[key] = total;
|
|
439
|
+
}
|
|
440
|
+
aggregate.cost = aggregateCost;
|
|
441
|
+
}
|
|
442
|
+
return aggregate;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function sumFiniteField(
|
|
446
|
+
records: Array<Record<string, unknown>>,
|
|
447
|
+
key: string,
|
|
448
|
+
): number | null {
|
|
449
|
+
const values = records.flatMap((record) =>
|
|
450
|
+
typeof record[key] === "number" && Number.isFinite(record[key])
|
|
451
|
+
? [record[key] as number]
|
|
452
|
+
: [],
|
|
453
|
+
);
|
|
454
|
+
return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) : null;
|
|
455
|
+
}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import type { ChatMessage } from "@mupt-ai/dari-router/policy-engine";
|
|
2
|
+
|
|
3
|
+
import { toolWireName } from "../interaction/tool_identity";
|
|
4
|
+
import type {
|
|
5
|
+
InteractionContent,
|
|
6
|
+
InteractionMessage,
|
|
7
|
+
InteractionRequest,
|
|
8
|
+
InteractionToolCall,
|
|
9
|
+
} from "../interaction/types";
|
|
10
|
+
|
|
11
|
+
export const IMAGE_OMITTED = "<image omitted>";
|
|
12
|
+
// Matches every numbered placeholder in a rendered selector prompt.
|
|
13
|
+
export const SELECTOR_IMAGE_PLACEHOLDERS = /<image (\d+)>/g;
|
|
14
|
+
const OPAQUE_REASONING_OMITTED = "<opaque reasoning omitted>";
|
|
15
|
+
|
|
16
|
+
// An image the selector may receive out of band. `index` is the 1-based
|
|
17
|
+
// position of its `<image N>` placeholder in the projected messages.
|
|
18
|
+
export type SelectorImage = {
|
|
19
|
+
index: number;
|
|
20
|
+
url: string;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type SelectorProjection = {
|
|
24
|
+
messages: ChatMessage[];
|
|
25
|
+
// Empty unless image input was requested; then every image part in
|
|
26
|
+
// `messages` is a numbered placeholder with its bytes listed here.
|
|
27
|
+
images: SelectorImage[];
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
// The selector needs readable task history, not provider replay state. This
|
|
31
|
+
// projection intentionally keeps text and tool exchanges while omitting
|
|
32
|
+
// opaque continuation payloads, source item ids, phases, and image bytes.
|
|
33
|
+
// With `imageInput`, image parts become numbered `<image N>` placeholders and
|
|
34
|
+
// the data URLs are returned separately so the policy adapter can attach them
|
|
35
|
+
// as ordered image parts. Without it the projection stays byte-identical to
|
|
36
|
+
// the pre-image-input shape, which the deployed policy was trained on.
|
|
37
|
+
export function selectorMessagesForInteraction(
|
|
38
|
+
interaction: InteractionRequest,
|
|
39
|
+
options: { imageInput?: boolean } = {},
|
|
40
|
+
): SelectorProjection {
|
|
41
|
+
const messages: ChatMessage[] = [];
|
|
42
|
+
const images: SelectorImage[] = [];
|
|
43
|
+
const imagePart = (url: string, detail: string | undefined): Record<string, unknown> => {
|
|
44
|
+
let placeholder = IMAGE_OMITTED;
|
|
45
|
+
if (options.imageInput === true) {
|
|
46
|
+
images.push({ index: images.length + 1, url });
|
|
47
|
+
placeholder = `<image ${images.length}>`;
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
type: "image_url",
|
|
51
|
+
image_url: {
|
|
52
|
+
url: placeholder,
|
|
53
|
+
...(detail !== undefined ? { detail } : {}),
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
};
|
|
57
|
+
// Hosted calls are complete provider output groups. Harness calls can be
|
|
58
|
+
// persisted before reasoning that belongs to the same assistant turn.
|
|
59
|
+
const hostedToolCarriers = new WeakSet<ChatMessage>();
|
|
60
|
+
if (interaction.instructions) {
|
|
61
|
+
messages.push({ role: "system", content: interaction.instructions });
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
for (const item of interaction.items) {
|
|
65
|
+
if (item.type === "reasoning") {
|
|
66
|
+
const readable = [...item.summary, ...item.content].join("\n");
|
|
67
|
+
let assistant = messages.at(-1);
|
|
68
|
+
if (
|
|
69
|
+
!assistant
|
|
70
|
+
|| assistant.role !== "assistant"
|
|
71
|
+
|| hasAssistantContent(assistant)
|
|
72
|
+
|| hostedToolCarriers.has(assistant)
|
|
73
|
+
) {
|
|
74
|
+
assistant = { role: "assistant", content: null };
|
|
75
|
+
messages.push(assistant);
|
|
76
|
+
}
|
|
77
|
+
const selectorReasoning =
|
|
78
|
+
readable || OPAQUE_REASONING_OMITTED;
|
|
79
|
+
assistant.reasoning_content = assistant.reasoning_content
|
|
80
|
+
? `${assistant.reasoning_content}\n${selectorReasoning}`
|
|
81
|
+
: selectorReasoning;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (item.type === "reasoning_configuration") {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (item.type === "tool_result") {
|
|
90
|
+
messages.push({
|
|
91
|
+
role: "tool",
|
|
92
|
+
tool_call_id: item.callId,
|
|
93
|
+
...selectorContentField(item.output, item.outputShape, imagePart),
|
|
94
|
+
});
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (item.type === "tool_call") {
|
|
99
|
+
appendToolCall(messages, item);
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (item.type === "hosted_tool_call") {
|
|
104
|
+
const assistant = appendToolCall(messages, {
|
|
105
|
+
type: "tool_call",
|
|
106
|
+
id: item.id,
|
|
107
|
+
callId: item.id,
|
|
108
|
+
kind: "function",
|
|
109
|
+
name: item.tool,
|
|
110
|
+
input: item.payload,
|
|
111
|
+
});
|
|
112
|
+
hostedToolCarriers.add(assistant);
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const content = selectorContentField(item.content, item.contentShape, imagePart);
|
|
117
|
+
const previous = messages.at(-1);
|
|
118
|
+
if (
|
|
119
|
+
item.role === "assistant"
|
|
120
|
+
&& previous?.role === "assistant"
|
|
121
|
+
&& isReasoningOnly(previous)
|
|
122
|
+
) {
|
|
123
|
+
if ("content" in content) previous.content = content.content;
|
|
124
|
+
else delete previous.content;
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
messages.push({ role: item.role, ...content });
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return { messages, images };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function appendToolCall(
|
|
134
|
+
messages: ChatMessage[],
|
|
135
|
+
item: InteractionToolCall,
|
|
136
|
+
): ChatMessage {
|
|
137
|
+
let assistant = messages.at(-1);
|
|
138
|
+
if (!assistant || assistant.role !== "assistant") {
|
|
139
|
+
assistant = { role: "assistant", content: null };
|
|
140
|
+
messages.push(assistant);
|
|
141
|
+
}
|
|
142
|
+
assistant.tool_calls = [
|
|
143
|
+
...(assistant.tool_calls ?? []),
|
|
144
|
+
{
|
|
145
|
+
id: item.callId,
|
|
146
|
+
type: "function",
|
|
147
|
+
function: {
|
|
148
|
+
name: toolWireName(item.namespace, item.name),
|
|
149
|
+
...(item.input !== undefined
|
|
150
|
+
? {
|
|
151
|
+
arguments:
|
|
152
|
+
item.kind === "custom"
|
|
153
|
+
? JSON.stringify({ input: String(item.input) })
|
|
154
|
+
: item.input,
|
|
155
|
+
}
|
|
156
|
+
: {}),
|
|
157
|
+
},
|
|
158
|
+
},
|
|
159
|
+
];
|
|
160
|
+
return assistant;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function selectorContentField(
|
|
164
|
+
content: InteractionContent[] | string | null,
|
|
165
|
+
shape: InteractionMessage["contentShape"],
|
|
166
|
+
imagePart: (url: string, detail: string | undefined) => Record<string, unknown>,
|
|
167
|
+
): Pick<ChatMessage, "content"> {
|
|
168
|
+
if (shape === "absent") return {};
|
|
169
|
+
if (shape === "null") return { content: null };
|
|
170
|
+
if (shape === "string") {
|
|
171
|
+
const text = Array.isArray(content)
|
|
172
|
+
? content
|
|
173
|
+
.filter((part) => part.type === "text")
|
|
174
|
+
.map((part) => part.text)
|
|
175
|
+
.join("")
|
|
176
|
+
: content;
|
|
177
|
+
return { content: text ?? "" };
|
|
178
|
+
}
|
|
179
|
+
if (!Array.isArray(content)) {
|
|
180
|
+
return {
|
|
181
|
+
content:
|
|
182
|
+
content === null
|
|
183
|
+
? []
|
|
184
|
+
: [{ type: "text", text: content }],
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
return {
|
|
188
|
+
content: content.map((part) => {
|
|
189
|
+
if (part.type === "text") return { type: "text", text: part.text };
|
|
190
|
+
if (part.type === "tool_reference") {
|
|
191
|
+
return { type: "text", text: `Referenced tool: ${part.toolName}` };
|
|
192
|
+
}
|
|
193
|
+
return imagePart(part.url, part.detail);
|
|
194
|
+
}),
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function hasAssistantContent(message: ChatMessage): boolean {
|
|
199
|
+
if (typeof message.content === "string") return message.content.length > 0;
|
|
200
|
+
return Array.isArray(message.content) && message.content.length > 0;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function isReasoningOnly(message: ChatMessage): boolean {
|
|
204
|
+
return !message.tool_calls?.length
|
|
205
|
+
&& !hasAssistantContent(message)
|
|
206
|
+
&& message.reasoning_content !== undefined;
|
|
207
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const pendingTelemetry = new Set<Promise<unknown>>();
|
|
2
|
+
|
|
3
|
+
export function trackBackgroundTelemetry(task: Promise<unknown>): void {
|
|
4
|
+
pendingTelemetry.add(task);
|
|
5
|
+
void task.then(
|
|
6
|
+
() => pendingTelemetry.delete(task),
|
|
7
|
+
() => pendingTelemetry.delete(task),
|
|
8
|
+
);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export async function drainBackgroundTelemetry(): Promise<void> {
|
|
12
|
+
while (pendingTelemetry.size > 0) {
|
|
13
|
+
await Promise.allSettled([...pendingTelemetry]);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Drain until every tracked task settles or the timeout passes. Tasks tracked
|
|
19
|
+
* while draining are included. Returns how many tasks were still pending.
|
|
20
|
+
*/
|
|
21
|
+
export async function drainBackgroundTelemetryFor(
|
|
22
|
+
timeoutMs: number,
|
|
23
|
+
): Promise<number> {
|
|
24
|
+
let expired = false;
|
|
25
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
26
|
+
const deadline = new Promise<void>((resolve) => {
|
|
27
|
+
timer = setTimeout(() => {
|
|
28
|
+
expired = true;
|
|
29
|
+
resolve();
|
|
30
|
+
}, Math.max(0, timeoutMs));
|
|
31
|
+
});
|
|
32
|
+
try {
|
|
33
|
+
while (pendingTelemetry.size > 0 && !expired) {
|
|
34
|
+
await Promise.race([Promise.allSettled([...pendingTelemetry]), deadline]);
|
|
35
|
+
}
|
|
36
|
+
} finally {
|
|
37
|
+
clearTimeout(timer);
|
|
38
|
+
}
|
|
39
|
+
return pendingTelemetry.size;
|
|
40
|
+
}
|