@mupt-ai/dari-gateway 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -6
- package/dari-gateway.example.yaml +3 -2
- package/dist/http/auth.js +2 -2
- package/dist/http/http2_server.d.ts +1 -1
- package/dist/http/http2_server.js +3 -3
- package/dist/http/serve.d.ts +1 -0
- package/dist/http/serve.js +5 -4
- package/dist/ingress/chat_completions.js +2 -2
- package/dist/ingress/codex.js +3 -1
- package/dist/ingress/codex_models.d.ts +1 -1
- package/dist/ingress/codex_models.js +22 -13
- package/dist/platform/platform.d.ts +13 -3
- package/dist/protocol/input/openai_responses.js +2 -4
- package/dist/protocol/output/anthropic_messages_output.d.ts +2 -2
- package/dist/protocol/output/anthropic_messages_output.js +1 -0
- package/dist/protocol/output/openai_responses_stream.js +19 -1
- package/dist/providers/execution.d.ts +0 -5
- package/dist/providers/execution.js +5 -8
- package/dist/providers/pi_output.d.ts +4 -0
- package/dist/providers/pi_output.js +1 -1
- package/dist/providers/provider_attempt_runner.d.ts +2 -3
- package/dist/providers/provider_attempt_runner.js +14 -24
- package/dist/providers/provider_call.d.ts +3 -3
- package/dist/providers/provider_call.js +1 -0
- package/dist/providers/provider_reported_cost.js +17 -2
- package/dist/providers/stream_finalization.d.ts +0 -1
- package/dist/providers/stream_finalization.js +0 -1
- package/dist/runtime/claude_code/auxiliary_routing.js +1 -9
- package/dist/runtime/explicit_model.js +1 -1
- package/dist/runtime/harness_identity/conversation.d.ts +5 -0
- package/dist/runtime/harness_identity/conversation.js +6 -0
- package/dist/runtime/harness_identity/extract.d.ts +1 -0
- package/dist/runtime/harness_identity/extract.js +13 -5
- package/dist/runtime/harness_identity/index.d.ts +2 -2
- package/dist/runtime/harness_identity/index.js +2 -2
- package/dist/runtime/prepare_routed_completion.d.ts +5 -1
- package/dist/runtime/prepare_routed_completion.js +26 -2
- package/dist/runtime/routed_runtime.js +33 -53
- package/dist/runtime/routing_lease.d.ts +4 -6
- package/dist/runtime/routing_lease.js +2 -47
- package/dist/runtime/runtime_reporter.d.ts +0 -2
- package/dist/runtime/runtime_reporter.js +0 -1
- package/dist/runtime/session_pin.d.ts +7 -0
- package/dist/runtime/session_pin.js +51 -0
- package/dist/self_host/config.d.ts +2 -3
- package/dist/self_host/config.js +1 -2
- package/dist/self_host/main.d.ts +2 -1
- package/dist/self_host/main.js +20 -4
- package/dist/self_host/platform.d.ts +4 -2
- package/dist/self_host/platform.js +6 -2
- package/dist/self_host/resolve.d.ts +8 -1
- package/dist/self_host/resolve.js +31 -8
- package/dist/self_host/session_pin_store.d.ts +14 -0
- package/dist/self_host/session_pin_store.js +44 -0
- package/dist/strategies/priority_selector.d.ts +1 -2
- package/dist/strategies/priority_selector.js +1 -2
- package/dist/strategies/route_decision.d.ts +0 -1
- package/dist/strategies/route_decision.js +0 -3
- package/dist/strategies/routing.d.ts +0 -3
- package/dist/strategies/routing.js +1 -14
- package/dist/telemetry/billing_finalization.d.ts +1 -2
- package/dist/telemetry/billing_finalization.js +1 -1
- package/dist/telemetry/prefix_state.d.ts +0 -1
- package/dist/telemetry/prefix_state.js +1 -3
- package/dist/telemetry/report.d.ts +0 -2
- package/dist/telemetry/report.js +5 -41
- package/dist/types.d.ts +5 -3
- package/package.json +2 -2
- package/src/http/auth.ts +2 -2
- package/src/http/http2_server.ts +7 -3
- package/src/http/serve.ts +6 -5
- package/src/ingress/chat_completions.ts +2 -1
- package/src/ingress/codex.ts +4 -1
- package/src/ingress/codex_models.ts +18 -7
- package/src/platform/platform.ts +16 -2
- package/src/protocol/input/openai_responses.ts +2 -9
- package/src/protocol/output/anthropic_messages_output.ts +3 -2
- package/src/protocol/output/openai_responses_stream.ts +17 -1
- package/src/providers/execution.ts +5 -15
- package/src/providers/pi_output.ts +1 -1
- package/src/providers/provider_attempt_runner.ts +10 -21
- package/src/providers/provider_call.ts +3 -1
- package/src/providers/provider_reported_cost.ts +16 -2
- package/src/providers/stream_finalization.ts +0 -2
- package/src/providers/subscription_limits.ts +1 -1
- package/src/runtime/claude_code/auxiliary_routing.ts +1 -8
- package/src/runtime/explicit_model.ts +1 -1
- package/src/runtime/harness_identity/conversation.ts +11 -0
- package/src/runtime/harness_identity/extract.ts +14 -5
- package/src/runtime/harness_identity/index.ts +2 -0
- package/src/runtime/prepare_routed_completion.ts +32 -2
- package/src/runtime/routed_runtime.ts +39 -59
- package/src/runtime/routing_lease.ts +2 -56
- package/src/runtime/runtime_reporter.ts +0 -3
- package/src/runtime/session_pin.ts +61 -0
- package/src/self_host/config.ts +3 -6
- package/src/self_host/main.ts +23 -4
- package/src/self_host/platform.ts +8 -2
- package/src/self_host/resolve.ts +37 -7
- package/src/self_host/session_pin_store.ts +58 -0
- package/src/strategies/priority_selector.ts +1 -2
- package/src/strategies/route_decision.ts +0 -5
- package/src/strategies/routing.ts +1 -17
- package/src/telemetry/billing_finalization.ts +4 -3
- package/src/telemetry/prefix_state.ts +1 -6
- package/src/telemetry/report.ts +4 -48
- package/src/types.ts +12 -6
- package/dist/strategies/fallback_decision.d.ts +0 -8
- package/dist/strategies/fallback_decision.js +0 -17
- package/src/strategies/fallback_decision.ts +0 -30
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { type CandidateCostEstimate, type ModelPricing, type PricingLookup, type Provider, type ReasoningEffort
|
|
1
|
+
import { type CandidateCostEstimate, type ModelPricing, type PricingLookup, type Provider, type ReasoningEffort } from "@mupt-ai/dari-router/policy-engine";
|
|
2
2
|
import type { RequestTags } from "../platform/platform.js";
|
|
3
3
|
import type { InteractionRequest, ProviderIdentity } from "../interaction/types.js";
|
|
4
4
|
import { type ProviderAttemptAbort } from "../runtime/customer_disconnect.js";
|
|
5
5
|
import { type RoutingUsageContext } from "../telemetry/usage.js";
|
|
6
|
-
import type { ProviderAuth, ProviderKeySource, ResolvedRouter, ResolvedRouterModelConfig } from "../types.js";
|
|
6
|
+
import type { ProviderAuth, ProviderKeySource, ResolvedRouter, ResolvedRouterModelConfig, ServedDecision } from "../types.js";
|
|
7
7
|
import { type ProviderResponseCapture } from "./response_capture.js";
|
|
8
8
|
import { type ProviderEndpoint, type ProviderEventStream } from "./pi.js";
|
|
9
9
|
import { type PromptTokenEstimate } from "./cancel_estimate.js";
|
|
@@ -45,7 +45,7 @@ export type StartedStream = {
|
|
|
45
45
|
responseCapture?: ProviderResponseCapture;
|
|
46
46
|
};
|
|
47
47
|
export declare function prepareProviderExecution(args: {
|
|
48
|
-
decision:
|
|
48
|
+
decision: ServedDecision;
|
|
49
49
|
credentials: ResolvedRouter;
|
|
50
50
|
billable: boolean;
|
|
51
51
|
routerId: string;
|
|
@@ -73,6 +73,7 @@ export function prepareProviderExecution(args) {
|
|
|
73
73
|
...(args.decision.leaseTurnsRemaining === undefined
|
|
74
74
|
? {}
|
|
75
75
|
: { "X-Dari-Lease-Turns-Remaining": String(args.decision.leaseTurnsRemaining) }),
|
|
76
|
+
...(args.decision.sessionPinned === true ? { "X-Dari-Session-Pinned": "true" } : {}),
|
|
76
77
|
},
|
|
77
78
|
};
|
|
78
79
|
}
|
|
@@ -2,7 +2,8 @@ import { isRecord } from "@mupt-ai/dari-router/policy-engine";
|
|
|
2
2
|
import { gatewayFor } from "../models/gateways.js";
|
|
3
3
|
import { providerFailureFromCaught } from "../runtime/provider_stream_outcome.js";
|
|
4
4
|
// Gateways bill per call and say what they billed: OpenRouter as `usage.cost`
|
|
5
|
-
// on the final chunk, Vercel AI
|
|
5
|
+
// on the final chunk (plus the provider's charge on a BYOK call), Vercel AI
|
|
6
|
+
// Gateway as `providerMetadata.gateway.cost`.
|
|
6
7
|
// Pi prices usage from model rates and drops both, so the amount is read off
|
|
7
8
|
// the raw response bytes and replaces the priced total.
|
|
8
9
|
export function reportsProviderCost(model) {
|
|
@@ -63,7 +64,21 @@ export function reportedCost(body) {
|
|
|
63
64
|
const gateway = isRecord(body.providerMetadata) && isRecord(body.providerMetadata.gateway)
|
|
64
65
|
? body.providerMetadata.gateway
|
|
65
66
|
: undefined;
|
|
66
|
-
return
|
|
67
|
+
return openRouterCost(body.usage) ?? usd(gateway?.cost);
|
|
68
|
+
}
|
|
69
|
+
// On a BYOK call OpenRouter bills the provider's charge to the org's own key:
|
|
70
|
+
// `usage.cost` is only its fee (often 0), and the charge is in
|
|
71
|
+
// `cost_details.upstream_inference_cost`.
|
|
72
|
+
function openRouterCost(usage) {
|
|
73
|
+
if (!isRecord(usage))
|
|
74
|
+
return undefined;
|
|
75
|
+
const cost = usd(usage.cost);
|
|
76
|
+
if (cost === undefined || usage.is_byok !== true)
|
|
77
|
+
return cost;
|
|
78
|
+
const upstream = isRecord(usage.cost_details)
|
|
79
|
+
? usd(usage.cost_details.upstream_inference_cost)
|
|
80
|
+
: undefined;
|
|
81
|
+
return cost + (upstream ?? 0);
|
|
67
82
|
}
|
|
68
83
|
function usd(value) {
|
|
69
84
|
const amount = typeof value === "string" && value.trim() !== "" ? Number(value) : value;
|
|
@@ -23,7 +23,6 @@ type CreateStreamFinalizerArgs = {
|
|
|
23
23
|
reporter: RoutedRuntimeReporter;
|
|
24
24
|
protocol: RoutedProtocol;
|
|
25
25
|
providerLatencyMs: () => number;
|
|
26
|
-
role: "primary" | "fallback";
|
|
27
26
|
attempt: number;
|
|
28
27
|
reportBarrier?: Promise<unknown>;
|
|
29
28
|
delivered: () => DeliveredOutput;
|
|
@@ -2,7 +2,6 @@ import { routingCandidateKey, } from "@mupt-ai/dari-router/policy-engine";
|
|
|
2
2
|
import { isBackgroundRequest } from "../../interaction/types.js";
|
|
3
3
|
import { HttpError } from "../../http/errors.js";
|
|
4
4
|
import { assertCreditAvailableBeforeRouting } from "../../telemetry/usage.js";
|
|
5
|
-
import { leaseFallbackDecision } from "../routing_lease.js";
|
|
6
5
|
// Explicit background classes bypass the selector and every lease path.
|
|
7
6
|
// Keep the existing deterministic cheapest-input-cost serving policy.
|
|
8
7
|
export function auxiliaryDecision(prepared) {
|
|
@@ -19,14 +18,7 @@ export function auxiliaryDecision(prepared) {
|
|
|
19
18
|
reasoningEffort: cheapest.reasoning_effort,
|
|
20
19
|
reason: `${requestClass} request served by the cheapest routable model without selection.`,
|
|
21
20
|
};
|
|
22
|
-
|
|
23
|
-
candidates: prepared.routing.candidateResolution.candidates,
|
|
24
|
-
decision,
|
|
25
|
-
credentials: prepared.credentials,
|
|
26
|
-
});
|
|
27
|
-
if (fallbackDecision)
|
|
28
|
-
prepared.reporter.attachFallbackDecision(fallbackDecision);
|
|
29
|
-
return fallbackDecision ? { decision, fallbackDecision } : { decision };
|
|
21
|
+
return { decision };
|
|
30
22
|
}
|
|
31
23
|
/* Ranked on estimated input cost: these requests carry a whole transcript in
|
|
32
24
|
and a sentence out, so the prompt dominates. Estimates cover the configured
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { HttpError } from "../http/errors.js";
|
|
2
2
|
import { explicitModelId } from "../protocol/input/router_model.js";
|
|
3
3
|
/* A request for `dari/model/<id>` is served by exactly that enabled model,
|
|
4
|
-
with no selector
|
|
4
|
+
with no selector or lease: the user chose it, like picking a
|
|
5
5
|
model in their agent's /model menu. The client's thinking level is kept
|
|
6
6
|
when the model supports it; otherwise the model's default level is used. */
|
|
7
7
|
export function explicitModelDecision(prepared) {
|
|
@@ -25,6 +25,11 @@ export declare function scopedHarnessSessionId(args: {
|
|
|
25
25
|
sourceProtocol: SourceProtocol;
|
|
26
26
|
scope: HarnessConversationScope;
|
|
27
27
|
}): string;
|
|
28
|
+
export declare function scopedHarnessAgentId(args: {
|
|
29
|
+
identity: HarnessConversationIdentity;
|
|
30
|
+
sourceProtocol: SourceProtocol;
|
|
31
|
+
scope: HarnessConversationScope;
|
|
32
|
+
}): string;
|
|
28
33
|
export declare function scopedHarnessConversationId(args: {
|
|
29
34
|
identity: HarnessConversationIdentity;
|
|
30
35
|
sourceProtocol: SourceProtocol;
|
|
@@ -11,6 +11,12 @@ export function scopedHarnessSessionId(args) {
|
|
|
11
11
|
: args.identity;
|
|
12
12
|
return scopedHarnessId({ ...args, identity });
|
|
13
13
|
}
|
|
14
|
+
// One agent of a session: the main agent on the bare session, each subagent
|
|
15
|
+
// on its own qualifier or thread. Unlike the conversation id, it survives
|
|
16
|
+
// compaction.
|
|
17
|
+
export function scopedHarnessAgentId(args) {
|
|
18
|
+
return scopedHarnessId(args);
|
|
19
|
+
}
|
|
14
20
|
export function scopedHarnessConversationId(args) {
|
|
15
21
|
return scopedHarnessId(args, args.conversationPrefix);
|
|
16
22
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AgentRequestContext, InteractionRequestPurpose } from "../../interaction/types.js";
|
|
2
2
|
import type { HarnessConversationIdentity } from "./conversation.js";
|
|
3
3
|
export declare function chatCompletionsConversationIdentity(headers: Headers, promptCacheKey?: string): HarnessConversationIdentity | undefined;
|
|
4
|
+
export declare function piRequestContext(headers: Headers): AgentRequestContext | undefined;
|
|
4
5
|
export declare function openCodeRequestContext(headers: Headers): AgentRequestContext | undefined;
|
|
5
6
|
export declare function codexConversationIdentity(headers: Headers, payload: Record<string, unknown>): HarnessConversationIdentity | undefined;
|
|
6
7
|
export declare function codexRequestPurpose(headers: Headers, payload: Record<string, unknown>): InteractionRequestPurpose | undefined;
|
|
@@ -2,11 +2,10 @@ import { isRecord } from "@mupt-ai/dari-router/policy-engine";
|
|
|
2
2
|
import { HttpError } from "../../http/errors.js";
|
|
3
3
|
const MAX_ID_CHARS = 200;
|
|
4
4
|
export function chatCompletionsConversationIdentity(headers, promptCacheKey) {
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
|
|
9
|
-
const sessionId = boundedIdentity(headers.get("x-session-affinity"), "x-session-affinity") ?? boundedIdentity(promptCacheKey, "prompt_cache_key");
|
|
5
|
+
// Pi summaries use fresh cache/affinity IDs. Its explicit session header
|
|
6
|
+
// groups them with the main turns without changing provider cache affinity.
|
|
7
|
+
// Other callers use x-session-affinity, falling back to prompt_cache_key.
|
|
8
|
+
const sessionId = boundedIdentity(headers.get("x-pi-session-id"), "x-pi-session-id") ?? boundedIdentity(headers.get("x-session-affinity"), "x-session-affinity") ?? boundedIdentity(promptCacheKey, "prompt_cache_key");
|
|
10
9
|
if (sessionId === undefined)
|
|
11
10
|
return undefined;
|
|
12
11
|
// OpenCode sends a subagent's parent session as its affinity, so subagents
|
|
@@ -19,6 +18,15 @@ export function chatCompletionsConversationIdentity(headers, promptCacheKey) {
|
|
|
19
18
|
...(subagentId === undefined ? {} : { qualifiers: [subagentId] }),
|
|
20
19
|
};
|
|
21
20
|
}
|
|
21
|
+
export function piRequestContext(headers) {
|
|
22
|
+
if (boundedIdentity(headers.get("x-pi-session-id"), "x-pi-session-id") === undefined) {
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
const kind = headers.get("x-pi-request-kind")?.trim();
|
|
26
|
+
return {
|
|
27
|
+
requestClass: kind === "turn" ? "main" : kind === "compaction" ? "compaction" : "unknown",
|
|
28
|
+
};
|
|
29
|
+
}
|
|
22
30
|
// OpenCode names the session behind every request, and a subagent's parent
|
|
23
31
|
// session too. The `dari opencode` plugin adds the request kind and agent,
|
|
24
32
|
// which OpenCode does not send itself. Like Claude Code's headers, these are
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { scopedHarnessConversationId, scopedHarnessSessionId, type HarnessConversationIdentity, type HarnessConversationScope, } from "./conversation.js";
|
|
2
|
-
export { chatCompletionsConversationIdentity, claudeCodeConversationIdentity, codexConversationIdentity, codexRequestPurpose, codexRequestContext, openCodeRequestContext, } from "./extract.js";
|
|
1
|
+
export { scopedHarnessAgentId, scopedHarnessConversationId, scopedHarnessSessionId, type HarnessConversationIdentity, type HarnessConversationScope, } from "./conversation.js";
|
|
2
|
+
export { chatCompletionsConversationIdentity, claudeCodeConversationIdentity, codexConversationIdentity, codexRequestPurpose, codexRequestContext, openCodeRequestContext, piRequestContext, } from "./extract.js";
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { scopedHarnessConversationId, scopedHarnessSessionId, } from "./conversation.js";
|
|
2
|
-
export { chatCompletionsConversationIdentity, claudeCodeConversationIdentity, codexConversationIdentity, codexRequestPurpose, codexRequestContext, openCodeRequestContext, } from "./extract.js";
|
|
1
|
+
export { scopedHarnessAgentId, scopedHarnessConversationId, scopedHarnessSessionId, } from "./conversation.js";
|
|
2
|
+
export { chatCompletionsConversationIdentity, claudeCodeConversationIdentity, codexConversationIdentity, codexRequestPurpose, codexRequestContext, openCodeRequestContext, piRequestContext, } from "./extract.js";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type RoutingContext } from "../context/routing_context.js";
|
|
2
2
|
import { type InteractionRequest } from "../interaction/types.js";
|
|
3
|
-
import { type RequestTags } from "../platform/platform.js";
|
|
3
|
+
import { type RequestTags, type SessionPin } from "../platform/platform.js";
|
|
4
4
|
import type { ClientHarness, ResolvedRouter } from "../types.js";
|
|
5
5
|
import type { RoutedRuntimeReporter } from "./runtime_reporter.js";
|
|
6
6
|
import type { RoutedProtocol } from "./routed_protocol.js";
|
|
@@ -16,6 +16,10 @@ export type PreparedRoutedCompletion = {
|
|
|
16
16
|
tags: RequestTags;
|
|
17
17
|
harnessConversationIdentity?: HarnessConversationIdentity;
|
|
18
18
|
routing: RoutingContext;
|
|
19
|
+
sessionPin?: {
|
|
20
|
+
key: string;
|
|
21
|
+
pin: SessionPin | null;
|
|
22
|
+
};
|
|
19
23
|
reporter: RoutedRuntimeReporter;
|
|
20
24
|
};
|
|
21
25
|
export declare function prepareRoutedCompletion(args: {
|
|
@@ -3,13 +3,14 @@ import { buildRoutingContext, derivePrefixLookupHashes, } from "../context/routi
|
|
|
3
3
|
import { isBackgroundRequest } from "../interaction/types.js";
|
|
4
4
|
import { executionProviderForModel } from "../models/execution_provider.js";
|
|
5
5
|
import { codexModelConfig, subscriptionServes } from "../models/subscriptions.js";
|
|
6
|
+
import { explicitModelId } from "../protocol/input/router_model.js";
|
|
6
7
|
import { gatewayPlatform } from "../platform/platform.js";
|
|
7
8
|
import { resolveRouter } from "../strategies/routing.js";
|
|
8
9
|
import { RequestReporter } from "../telemetry/report.js";
|
|
9
10
|
import { RoutingBillingFinalization } from "../telemetry/billing_finalization.js";
|
|
10
11
|
import { requestTags } from "./request_tags.js";
|
|
11
12
|
import { strictProviderAccess } from "./strict_provider_access.js";
|
|
12
|
-
import { scopedHarnessConversationId, scopedHarnessSessionId, } from "./harness_identity/index.js";
|
|
13
|
+
import { scopedHarnessAgentId, scopedHarnessConversationId, scopedHarnessSessionId, } from "./harness_identity/index.js";
|
|
13
14
|
export async function prepareRoutedCompletion(args) {
|
|
14
15
|
const tags = requestTags(args.tagsHeader);
|
|
15
16
|
// Eval relevance draws from the public catalog, which resolve only returns
|
|
@@ -94,6 +95,28 @@ export async function prepareRoutedCompletion(args) {
|
|
|
94
95
|
receivedAt: args.receivedAt,
|
|
95
96
|
});
|
|
96
97
|
const candidateResolution = routing.candidateResolution;
|
|
98
|
+
let sessionPin;
|
|
99
|
+
if (credentials.session_pinning === true &&
|
|
100
|
+
credentials.strict !== true &&
|
|
101
|
+
!background &&
|
|
102
|
+
explicitModelId(args.interaction.model) === null) {
|
|
103
|
+
// Harness identity keys the main agent and each subagent separately and
|
|
104
|
+
// survives compaction. Claude Code names its subagents only with gateway
|
|
105
|
+
// hint headers; without them, and for callers with no harness identity,
|
|
106
|
+
// each conversation is pinned on its own.
|
|
107
|
+
const identity = args.harnessConversationIdentity;
|
|
108
|
+
const unclassifiedClaudeCode = identity?.kind === "claude_code_session"
|
|
109
|
+
&& args.interaction.agentContext?.requestClass === "unknown";
|
|
110
|
+
const key = identity === undefined || unclassifiedClaudeCode
|
|
111
|
+
? routing.conversationId
|
|
112
|
+
: scopedHarnessAgentId({
|
|
113
|
+
identity,
|
|
114
|
+
sourceProtocol: args.interaction.sourceProtocol,
|
|
115
|
+
scope: harnessScope,
|
|
116
|
+
});
|
|
117
|
+
const pin = await gatewayPlatform().readSessionPin({ routerId: canonicalRouterId, pinKey: key });
|
|
118
|
+
sessionPin = { key, pin };
|
|
119
|
+
}
|
|
97
120
|
const reporter = new RequestReporter({
|
|
98
121
|
requestId: args.requestId,
|
|
99
122
|
clientRequestId: args.clientRequestId,
|
|
@@ -134,11 +157,12 @@ export async function prepareRoutedCompletion(args) {
|
|
|
134
157
|
tags,
|
|
135
158
|
harnessConversationIdentity: args.harnessConversationIdentity,
|
|
136
159
|
routing,
|
|
160
|
+
...(sessionPin === undefined ? {} : { sessionPin }),
|
|
137
161
|
reporter,
|
|
138
162
|
};
|
|
139
163
|
}
|
|
140
164
|
// Ask the platform for each enabled model's credential up front, so provider
|
|
141
|
-
// execution
|
|
165
|
+
// execution stays synchronous.
|
|
142
166
|
async function withProviderCredentials(credentials, apiKey, clientHarness) {
|
|
143
167
|
const platform = gatewayPlatform();
|
|
144
168
|
if (platform.resolveProviderCredential === undefined)
|
|
@@ -11,7 +11,8 @@ import { buildCompletedPrefixHit } from "../telemetry/prefix_state.js";
|
|
|
11
11
|
import { scrubSecrets } from "../telemetry/scrub.js";
|
|
12
12
|
import { auxiliaryDecision } from "./claude_code/auxiliary_routing.js";
|
|
13
13
|
import { explicitModelDecision, strictModelDecision } from "./explicit_model.js";
|
|
14
|
-
import { clearPendingLease,
|
|
14
|
+
import { clearPendingLease, leaseHoldDecision, leasePrefetchDue, leasedDecision, pendingLeaseDecision, prefetchNextLease, selectorNegotiatesLeases, } from "./routing_lease.js";
|
|
15
|
+
import { pinSession, sessionPinHoldDecision, sessionPinnedSelection } from "./session_pin.js";
|
|
15
16
|
import { createExperimentRoutedRuntimeReporter, } from "./runtime_reporter.js";
|
|
16
17
|
import { HttpError } from "../http/errors.js";
|
|
17
18
|
export class RoutedRuntimeProviderSettledError extends Error {
|
|
@@ -38,7 +39,6 @@ export async function runPreparedRoutedCompletion(args) {
|
|
|
38
39
|
interaction: prepared.interaction,
|
|
39
40
|
primary: plan.primary,
|
|
40
41
|
primaryRetries: plan.primaryRetries,
|
|
41
|
-
resolveFallback: plan.resolveFallback,
|
|
42
42
|
reporter: prepared.reporter,
|
|
43
43
|
protocol: prepared.protocol,
|
|
44
44
|
startedCompletion: plan.startedCompletion,
|
|
@@ -51,7 +51,6 @@ export async function runPreparedRoutedCompletion(args) {
|
|
|
51
51
|
interaction: prepared.interaction,
|
|
52
52
|
primary: plan.primary,
|
|
53
53
|
primaryRetries: plan.primaryRetries,
|
|
54
|
-
resolveFallback: plan.resolveFallback,
|
|
55
54
|
reporter: prepared.reporter,
|
|
56
55
|
startedStream: plan.startedStream,
|
|
57
56
|
initialError: plan.initialStreamError,
|
|
@@ -84,7 +83,6 @@ export async function runPreparedRoutedInteraction(args) {
|
|
|
84
83
|
interaction: args.prepared.interaction,
|
|
85
84
|
primary: plan.primary,
|
|
86
85
|
primaryRetries: plan.primaryRetries,
|
|
87
|
-
resolveFallback: plan.resolveFallback,
|
|
88
86
|
reporter: args.prepared.reporter,
|
|
89
87
|
protocol: args.prepared.protocol,
|
|
90
88
|
startedCompletion: plan.startedCompletion,
|
|
@@ -129,8 +127,7 @@ async function prepareExplicitRoutedCompletion(args) {
|
|
|
129
127
|
}
|
|
130
128
|
async function prepareRuntimeExecution(args) {
|
|
131
129
|
const { prepared } = args;
|
|
132
|
-
// A strict router never selects
|
|
133
|
-
// an error.
|
|
130
|
+
// A strict router never selects or leases: the pinned pair or an error.
|
|
134
131
|
if (prepared.credentials.strict === true) {
|
|
135
132
|
const pinned = strictModelDecision(prepared);
|
|
136
133
|
prepared.reporter.attachServingDecision(pinned.decision);
|
|
@@ -171,14 +168,16 @@ async function prepareRuntimeExecution(args) {
|
|
|
171
168
|
});
|
|
172
169
|
return selectedExecutionPlan(prepared, auxiliary);
|
|
173
170
|
}
|
|
171
|
+
// A session-pinning router serves the session's pin, or selects once and
|
|
172
|
+
// pins the result. Leases and speculation never apply to a pinned session.
|
|
173
|
+
if (prepared.sessionPin !== undefined && !args.requireSelectorDecision) {
|
|
174
|
+
return selectedExecutionPlan(prepared, await sessionPinnedDecision(prepared, args.selectorBackend));
|
|
175
|
+
}
|
|
174
176
|
// Persisted policy leases (negotiated {action, turns}) take precedence.
|
|
175
177
|
// Explicit in-process runs are selector experiments: they must observe the
|
|
176
178
|
// selector and must not consume or prefetch live conversations' leases.
|
|
177
179
|
const held = args.requireSelectorDecision ? null : leaseHoldDecision(prepared);
|
|
178
180
|
if (held) {
|
|
179
|
-
// Serving-only attach: nextDecision stays unset so if a fallback serves
|
|
180
|
-
// this turn, execution's attachServingDecision moves both the prefix's
|
|
181
|
-
// next_model and inherited countdown onto the healthy serving pair.
|
|
182
181
|
prepared.reporter.attachServingDecision(held);
|
|
183
182
|
prepared.reporter.attachDecisionDetails({
|
|
184
183
|
decision_source: "lease_hold",
|
|
@@ -197,7 +196,7 @@ async function prepareRuntimeExecution(args) {
|
|
|
197
196
|
if (leasePrefetchDue(prepared)) {
|
|
198
197
|
trackBackgroundTelemetry(prefetchNextLease(prepared, args.selectorBackend));
|
|
199
198
|
}
|
|
200
|
-
return selectedExecutionPlan(prepared,
|
|
199
|
+
return selectedExecutionPlan(prepared, { decision: held });
|
|
201
200
|
}
|
|
202
201
|
const adopted = args.requireSelectorDecision ? null : pendingLeaseDecision(prepared);
|
|
203
202
|
if (adopted) {
|
|
@@ -222,7 +221,7 @@ async function prepareRuntimeExecution(args) {
|
|
|
222
221
|
// wait elapses, the next lease's own prefetch overwrites the stale rows
|
|
223
222
|
// before its expiry (the menu's shortest lease exceeds the lag).
|
|
224
223
|
await clearPendingLease(prepared);
|
|
225
|
-
return selectedExecutionPlan(prepared,
|
|
224
|
+
return selectedExecutionPlan(prepared, { decision: adopted });
|
|
226
225
|
}
|
|
227
226
|
// The stateless ten-turn hold covers per-turn policies. A lease-negotiating
|
|
228
227
|
// backend must reach the selector at expiry — the modulo hold would mask it
|
|
@@ -250,27 +249,15 @@ async function prepareRuntimeExecution(args) {
|
|
|
250
249
|
const primary = {
|
|
251
250
|
decision: speculative.decision,
|
|
252
251
|
execution: speculative.execution,
|
|
253
|
-
role: "primary",
|
|
254
252
|
};
|
|
255
253
|
const selectionPromise = selectDecision(prepared, speculative.decision, args.selectorBackend, args.requireSelectorDecision === true);
|
|
256
254
|
// The execution layer awaits selection as its report barrier only after the
|
|
257
255
|
// provider settles, which can be long after a rejection. Mark the promise
|
|
258
|
-
// handled now;
|
|
256
|
+
// handled now; the report barrier still observes the original rejection.
|
|
259
257
|
selectionPromise.catch(() => undefined);
|
|
260
|
-
let fallbackPromise = null;
|
|
261
|
-
const resolveFallback = () => {
|
|
262
|
-
fallbackPromise ??= selectionPromise.then((selected) => {
|
|
263
|
-
const decision = fallbackForPrimary(prepared, primary.decision, selected);
|
|
264
|
-
if (decision)
|
|
265
|
-
prepared.reporter.attachFallbackDecision(decision);
|
|
266
|
-
return decision ? providerChoice(prepared, decision, "fallback") : null;
|
|
267
|
-
});
|
|
268
|
-
return fallbackPromise;
|
|
269
|
-
};
|
|
270
258
|
return {
|
|
271
259
|
primary,
|
|
272
260
|
primaryRetries: configuredPrimaryRetries(prepared.credentials.primary_retries),
|
|
273
|
-
resolveFallback,
|
|
274
261
|
reportBarrier: selectionPromise,
|
|
275
262
|
startedCompletion: speculative.transport === "completion" ? speculative.started : undefined,
|
|
276
263
|
startedStream: speculative.transport === "stream" ? speculative.started : undefined,
|
|
@@ -281,19 +268,8 @@ async function prepareRuntimeExecution(args) {
|
|
|
281
268
|
}
|
|
282
269
|
function selectedExecutionPlan(prepared, selected) {
|
|
283
270
|
return {
|
|
284
|
-
primary: providerChoice(prepared, selected.decision
|
|
271
|
+
primary: providerChoice(prepared, selected.decision),
|
|
285
272
|
primaryRetries: configuredPrimaryRetries(prepared.credentials.primary_retries),
|
|
286
|
-
resolveFallback: async () => {
|
|
287
|
-
if (!selected.fallbackDecision)
|
|
288
|
-
return null;
|
|
289
|
-
const fallbackDecision = selected.decision.leaseTurnsRemaining === undefined
|
|
290
|
-
? selected.fallbackDecision
|
|
291
|
-
: {
|
|
292
|
-
...selected.fallbackDecision,
|
|
293
|
-
leaseTurnsRemaining: selected.decision.leaseTurnsRemaining,
|
|
294
|
-
};
|
|
295
|
-
return providerChoice(prepared, fallbackDecision, "fallback");
|
|
296
|
-
},
|
|
297
273
|
};
|
|
298
274
|
}
|
|
299
275
|
async function startSpeculativeExecution(prepared, selectorBackend, clientSignal) {
|
|
@@ -349,10 +325,28 @@ async function startSpeculativeExecution(prepared, selectorBackend, clientSignal
|
|
|
349
325
|
return null;
|
|
350
326
|
}
|
|
351
327
|
}
|
|
328
|
+
async function sessionPinnedDecision(prepared, selectorBackend) {
|
|
329
|
+
const held = sessionPinHoldDecision(prepared);
|
|
330
|
+
if (held) {
|
|
331
|
+
prepared.reporter.attachDecision(held);
|
|
332
|
+
prepared.reporter.attachDecisionDetails({
|
|
333
|
+
decision_source: "session_pin",
|
|
334
|
+
session_pin: { model: held.selectedModel, reasoning_effort: held.reasoningEffort },
|
|
335
|
+
});
|
|
336
|
+
return { decision: held };
|
|
337
|
+
}
|
|
338
|
+
const selected = await selectDecision(prepared, null, selectorBackend);
|
|
339
|
+
prepared.reporter.attachDecisionDetails({
|
|
340
|
+
decision_source: "selector",
|
|
341
|
+
session_pin_established: true,
|
|
342
|
+
});
|
|
343
|
+
await pinSession(prepared, selected.decision);
|
|
344
|
+
return selected;
|
|
345
|
+
}
|
|
352
346
|
async function selectDecision(prepared, servingDecision, selectorBackend, requireSelectorDecision = false) {
|
|
353
347
|
const { credentials, reporter, routing } = prepared;
|
|
354
348
|
try {
|
|
355
|
-
const
|
|
349
|
+
const selected = await routeDecision({
|
|
356
350
|
selectorMessages: routing.selectorMessages,
|
|
357
351
|
selectorImages: routing.selectorImages,
|
|
358
352
|
resolution: routing.candidateResolution,
|
|
@@ -362,6 +356,7 @@ async function selectDecision(prepared, servingDecision, selectorBackend, requir
|
|
|
362
356
|
selectorBackend,
|
|
363
357
|
leaseHistory: routing.leaseHistory,
|
|
364
358
|
});
|
|
359
|
+
const routed = prepared.sessionPin === undefined ? selected : sessionPinnedSelection(selected);
|
|
365
360
|
if (routed.details)
|
|
366
361
|
reporter.attachDecisionDetails(routed.details);
|
|
367
362
|
if (servingDecision) {
|
|
@@ -387,7 +382,6 @@ async function selectDecision(prepared, servingDecision, selectorBackend, requir
|
|
|
387
382
|
}
|
|
388
383
|
else {
|
|
389
384
|
reporter.attachDecision(routed.decision);
|
|
390
|
-
reporter.attachFallbackDecision(routed.fallbackDecision);
|
|
391
385
|
}
|
|
392
386
|
return routed;
|
|
393
387
|
}
|
|
@@ -430,24 +424,10 @@ function configuredPrimaryRetries(value) {
|
|
|
430
424
|
const retries = value ?? 1;
|
|
431
425
|
return Number.isInteger(retries) && retries >= 0 && retries <= 5 ? retries : 1;
|
|
432
426
|
}
|
|
433
|
-
function
|
|
434
|
-
if (prepared.credentials.model_fallback_enabled === false)
|
|
435
|
-
return undefined;
|
|
436
|
-
const fallback = [selected.fallbackDecision, selected.decision].find((candidate) => candidate !== undefined &&
|
|
437
|
-
candidate.selectedModel !== primary.selectedModel);
|
|
438
|
-
if (!fallback || selected.decision.leaseTurnsRemaining === undefined) {
|
|
439
|
-
return fallback;
|
|
440
|
-
}
|
|
441
|
-
return {
|
|
442
|
-
...fallback,
|
|
443
|
-
leaseTurnsRemaining: selected.decision.leaseTurnsRemaining,
|
|
444
|
-
};
|
|
445
|
-
}
|
|
446
|
-
function providerChoice(prepared, decision, role) {
|
|
427
|
+
function providerChoice(prepared, decision) {
|
|
447
428
|
return {
|
|
448
429
|
decision,
|
|
449
430
|
execution: providerExecution(prepared, decision),
|
|
450
|
-
role,
|
|
451
431
|
};
|
|
452
432
|
}
|
|
453
433
|
function providerExecution(prepared, decision) {
|
|
@@ -12,15 +12,13 @@ export declare function leasedRoutingDecision(args: {
|
|
|
12
12
|
previousDecision?: PreviousDecision;
|
|
13
13
|
}): RoutingLeaseDecision | null;
|
|
14
14
|
export declare function leasedDecision(prepared: PreparedRoutedCompletion): RoutedDecision | null;
|
|
15
|
-
export declare function committedLeaseDecision(prepared: PreparedRoutedCompletion, decision: RoutingDecision): RoutedDecision;
|
|
16
|
-
export declare function leaseFallbackDecision(args: {
|
|
17
|
-
candidates: PreparedRoutedCompletion["routing"]["candidateResolution"]["candidates"];
|
|
18
|
-
decision: RoutingDecision;
|
|
19
|
-
credentials: PreparedRoutedCompletion["credentials"];
|
|
20
|
-
}): RoutingDecision | undefined;
|
|
21
15
|
export declare function selectorNegotiatesLeases(backend: SelectorBackend): Promise<boolean>;
|
|
22
16
|
export declare function prefetchNextLease(prepared: PreparedRoutedCompletion, selectorBackend: SelectorBackend): Promise<void>;
|
|
23
17
|
export declare function leasePrefetchDue(prepared: PreparedRoutedCompletion): boolean;
|
|
24
18
|
export declare function clearPendingLease(prepared: PreparedRoutedCompletion): Promise<void>;
|
|
19
|
+
export declare function canServeCommittedPair(prepared: PreparedRoutedCompletion, pair: {
|
|
20
|
+
model: string;
|
|
21
|
+
reasoningEffort: string;
|
|
22
|
+
}): boolean;
|
|
25
23
|
export declare function pendingLeaseDecision(prepared: PreparedRoutedCompletion): RoutingDecision | null;
|
|
26
24
|
export declare function leaseHoldDecision(prepared: PreparedRoutedCompletion): RoutingDecision | null;
|
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import { FIXED_TURN_COST_COMPARISON_TURNS, LEASE_DECISION_TURN_LAG, } from "@mupt-ai/dari-router/policy-engine";
|
|
2
|
-
import { deriveFallbackDecision } from "../strategies/fallback_decision.js";
|
|
3
|
-
import { executionProviderForModel } from "../models/execution_provider.js";
|
|
4
2
|
import { routeDecision, } from "../strategies/route_decision.js";
|
|
5
3
|
import { reportPendingLease } from "../telemetry/prefix_state.js";
|
|
6
4
|
import { scrubSecrets } from "../telemetry/scrub.js";
|
|
@@ -42,17 +40,7 @@ export function leasedDecision(prepared) {
|
|
|
42
40
|
// incumbent cannot hide an available user-owned key behind the lease.
|
|
43
41
|
if (prepared.billable && prepared.credentials.credit_exhausted)
|
|
44
42
|
return null;
|
|
45
|
-
const fallbackDecision = leaseFallbackDecision({
|
|
46
|
-
candidates: prepared.routing.candidateResolution.candidates,
|
|
47
|
-
decision: lease.decision,
|
|
48
|
-
credentials: prepared.credentials,
|
|
49
|
-
});
|
|
50
|
-
// Use attachServingDecision rather than attachDecision so next_model
|
|
51
|
-
// follows the model that actually serves. When fallback handles a primary
|
|
52
|
-
// failure, it inherits this countdown and subsequent held turns continue
|
|
53
|
-
// on the healthy serving pair instead of retrying the failed primary.
|
|
54
43
|
prepared.reporter.attachServingDecision(lease.decision);
|
|
55
|
-
prepared.reporter.attachFallbackDecision(fallbackDecision);
|
|
56
44
|
prepared.reporter.attachDecisionDetails({
|
|
57
45
|
decision_source: "lease_hold",
|
|
58
46
|
routing_lease: {
|
|
@@ -61,40 +49,7 @@ export function leasedDecision(prepared) {
|
|
|
61
49
|
turn_in_lease: lease.turnInLease,
|
|
62
50
|
},
|
|
63
51
|
});
|
|
64
|
-
return { decision: lease.decision
|
|
65
|
-
}
|
|
66
|
-
// Wraps a lease-held (or adopted) decision with a runtime-derived fallback so
|
|
67
|
-
// a mid-lease provider failure degrades like any other turn instead of
|
|
68
|
-
// erroring the request. The fallback inherits the remaining commitment so the
|
|
69
|
-
// completed prefix follows the healthy serving pair for the rest of the lease.
|
|
70
|
-
export function committedLeaseDecision(prepared, decision) {
|
|
71
|
-
const fallbackDecision = leaseFallbackDecision({
|
|
72
|
-
candidates: prepared.routing.candidateResolution.candidates,
|
|
73
|
-
decision,
|
|
74
|
-
credentials: prepared.credentials,
|
|
75
|
-
});
|
|
76
|
-
if (fallbackDecision)
|
|
77
|
-
prepared.reporter.attachFallbackDecision(fallbackDecision);
|
|
78
|
-
return fallbackDecision ? { decision, fallbackDecision } : { decision };
|
|
79
|
-
}
|
|
80
|
-
export function leaseFallbackDecision(args) {
|
|
81
|
-
const fallback = deriveFallbackDecision({
|
|
82
|
-
candidates: args.candidates,
|
|
83
|
-
selected: {
|
|
84
|
-
model: args.decision.selectedModel,
|
|
85
|
-
reasoningEffort: args.decision.reasoningEffort,
|
|
86
|
-
},
|
|
87
|
-
enabled: args.credentials.model_fallback_enabled !== false,
|
|
88
|
-
requiresDifferentProvider: args.credentials.fallback_requires_different_provider !== false,
|
|
89
|
-
providerForModel: (model) => executionProviderForModel(args.credentials, model),
|
|
90
|
-
});
|
|
91
|
-
if (!fallback || args.decision.leaseTurnsRemaining === undefined) {
|
|
92
|
-
return fallback;
|
|
93
|
-
}
|
|
94
|
-
return {
|
|
95
|
-
...fallback,
|
|
96
|
-
leaseTurnsRemaining: args.decision.leaseTurnsRemaining,
|
|
97
|
-
};
|
|
52
|
+
return { decision: lease.decision };
|
|
98
53
|
}
|
|
99
54
|
export async function selectorNegotiatesLeases(backend) {
|
|
100
55
|
if (backend.negotiatedLeaseTurns === undefined)
|
|
@@ -164,7 +119,7 @@ export async function clearPendingLease(prepared) {
|
|
|
164
119
|
lease: null,
|
|
165
120
|
}), PENDING_LEASE_CLEAR_WAIT_MS);
|
|
166
121
|
}
|
|
167
|
-
function canServeCommittedPair(prepared, pair) {
|
|
122
|
+
export function canServeCommittedPair(prepared, pair) {
|
|
168
123
|
if (prepared.billable && prepared.credentials.credit_exhausted)
|
|
169
124
|
return false;
|
|
170
125
|
return prepared.routing.candidateResolution.candidates.some((candidate) => candidate.model === pair.model &&
|
|
@@ -31,7 +31,6 @@ export type ReportableProviderOutcome = {
|
|
|
31
31
|
export type ProviderAttemptReport = {
|
|
32
32
|
model: string;
|
|
33
33
|
reasoning_effort: string;
|
|
34
|
-
role: "primary" | "fallback";
|
|
35
34
|
attempt: number;
|
|
36
35
|
outcome: "completed" | "error" | "terminal_error" | "canceled";
|
|
37
36
|
};
|
|
@@ -47,7 +46,6 @@ export type RoutedRuntimeReporter = {
|
|
|
47
46
|
attachSelectorRun(run: SelectorRun): void;
|
|
48
47
|
attachSpeculativeSelectorRun(run: SelectorRun, servingDecision: RoutingDecision): void;
|
|
49
48
|
attachDecision(decision: RoutingDecision): void;
|
|
50
|
-
attachFallbackDecision(decision: RoutingDecision | undefined): void;
|
|
51
49
|
attachServingDecision(decision: RoutingDecision): void;
|
|
52
50
|
recordProviderAttempt(attempt: ProviderAttemptRecord): void;
|
|
53
51
|
modelCost(attempt?: ProviderAttemptRecord): number | null;
|
|
@@ -14,7 +14,6 @@ export function createExperimentRoutedRuntimeReporter() {
|
|
|
14
14
|
attachSelectorRun() { },
|
|
15
15
|
attachSpeculativeSelectorRun() { },
|
|
16
16
|
attachDecision() { },
|
|
17
|
-
attachFallbackDecision() { },
|
|
18
17
|
attachServingDecision() { },
|
|
19
18
|
recordProviderAttempt() { },
|
|
20
19
|
modelCost() { return null; },
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { RoutingDecision } from "@mupt-ai/dari-router/policy-engine";
|
|
2
|
+
import type { RoutedDecision } from "../strategies/route_decision.js";
|
|
3
|
+
import type { ServedDecision } from "../types.js";
|
|
4
|
+
import type { PreparedRoutedCompletion } from "./prepare_routed_completion.js";
|
|
5
|
+
export declare function sessionPinHoldDecision(prepared: PreparedRoutedCompletion): ServedDecision | null;
|
|
6
|
+
export declare function sessionPinnedSelection(routed: RoutedDecision): RoutedDecision;
|
|
7
|
+
export declare function pinSession(prepared: PreparedRoutedCompletion, decision: RoutingDecision): Promise<void>;
|