@mupt-ai/dari-gateway 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -11
- package/dari-gateway.example.yaml +3 -2
- package/dist/context/interaction_fingerprint.js +10 -1
- package/dist/http/auth.js +2 -2
- package/dist/http/http2_server.d.ts +1 -1
- package/dist/http/http2_server.js +3 -3
- package/dist/http/serve.d.ts +1 -0
- package/dist/http/serve.js +5 -4
- package/dist/ingress/chat_completions.js +2 -2
- package/dist/ingress/codex.js +7 -3
- package/dist/ingress/codex_models.d.ts +1 -1
- package/dist/ingress/codex_models.js +22 -13
- package/dist/platform/platform.d.ts +13 -3
- package/dist/protocol/input/openai_responses.js +2 -4
- package/dist/protocol/output/anthropic_messages_output.d.ts +2 -2
- package/dist/protocol/output/anthropic_messages_output.js +1 -0
- package/dist/protocol/output/openai_responses_stream.js +19 -1
- package/dist/providers/execution.d.ts +0 -5
- package/dist/providers/execution.js +5 -8
- package/dist/providers/pi_output.d.ts +4 -0
- package/dist/providers/pi_output.js +1 -1
- package/dist/providers/provider_attempt_runner.d.ts +2 -3
- package/dist/providers/provider_attempt_runner.js +14 -24
- package/dist/providers/provider_call.d.ts +3 -3
- package/dist/providers/provider_call.js +1 -0
- package/dist/providers/stream_finalization.d.ts +0 -1
- package/dist/providers/stream_finalization.js +0 -1
- package/dist/runtime/claude_code/auxiliary_routing.js +1 -9
- package/dist/runtime/explicit_model.js +1 -1
- package/dist/runtime/harness_identity/conversation.d.ts +9 -0
- package/dist/runtime/harness_identity/conversation.js +6 -0
- package/dist/runtime/harness_identity/extract.d.ts +2 -0
- package/dist/runtime/harness_identity/extract.js +20 -5
- package/dist/runtime/harness_identity/index.d.ts +2 -2
- package/dist/runtime/harness_identity/index.js +2 -2
- package/dist/runtime/prepare_routed_completion.d.ts +5 -1
- package/dist/runtime/prepare_routed_completion.js +26 -2
- package/dist/runtime/routed_runtime.js +33 -53
- package/dist/runtime/routing_lease.d.ts +4 -6
- package/dist/runtime/routing_lease.js +2 -47
- package/dist/runtime/runtime_reporter.d.ts +0 -2
- package/dist/runtime/runtime_reporter.js +0 -1
- package/dist/runtime/session_pin.d.ts +7 -0
- package/dist/runtime/session_pin.js +51 -0
- package/dist/self_host/config.d.ts +2 -3
- package/dist/self_host/config.js +3 -3
- package/dist/self_host/main.d.ts +2 -1
- package/dist/self_host/main.js +26 -6
- package/dist/self_host/platform.d.ts +4 -2
- package/dist/self_host/platform.js +6 -2
- package/dist/self_host/resolve.d.ts +8 -1
- package/dist/self_host/resolve.js +31 -8
- package/dist/self_host/session_pin_store.d.ts +14 -0
- package/dist/self_host/session_pin_store.js +44 -0
- package/dist/strategies/priority_selector.d.ts +1 -2
- package/dist/strategies/priority_selector.js +1 -2
- package/dist/strategies/route_decision.d.ts +0 -1
- package/dist/strategies/route_decision.js +0 -3
- package/dist/strategies/routing.d.ts +0 -3
- package/dist/strategies/routing.js +1 -14
- package/dist/telemetry/billing_finalization.d.ts +1 -2
- package/dist/telemetry/billing_finalization.js +1 -1
- package/dist/telemetry/prefix_state.d.ts +0 -1
- package/dist/telemetry/prefix_state.js +1 -3
- package/dist/telemetry/report.d.ts +0 -2
- package/dist/telemetry/report.js +5 -41
- package/dist/types.d.ts +5 -3
- package/package.json +2 -2
- package/src/context/interaction_fingerprint.ts +15 -1
- package/src/http/auth.ts +2 -2
- package/src/http/http2_server.ts +7 -3
- package/src/http/serve.ts +6 -5
- package/src/ingress/chat_completions.ts +2 -1
- package/src/ingress/codex.ts +8 -5
- package/src/ingress/codex_models.ts +18 -7
- package/src/platform/platform.ts +16 -2
- package/src/protocol/input/openai_responses.ts +2 -9
- package/src/protocol/output/anthropic_messages_output.ts +3 -2
- package/src/protocol/output/openai_responses_stream.ts +17 -1
- package/src/providers/execution.ts +5 -15
- package/src/providers/pi_output.ts +1 -1
- package/src/providers/provider_attempt_runner.ts +10 -21
- package/src/providers/provider_call.ts +3 -1
- package/src/providers/stream_finalization.ts +0 -2
- package/src/providers/subscription_limits.ts +1 -1
- package/src/runtime/claude_code/auxiliary_routing.ts +1 -8
- package/src/runtime/explicit_model.ts +1 -1
- package/src/runtime/harness_identity/conversation.ts +16 -0
- package/src/runtime/harness_identity/extract.ts +24 -5
- package/src/runtime/harness_identity/index.ts +3 -0
- package/src/runtime/prepare_routed_completion.ts +32 -2
- package/src/runtime/routed_runtime.ts +39 -59
- package/src/runtime/routing_lease.ts +2 -56
- package/src/runtime/runtime_reporter.ts +0 -3
- package/src/runtime/session_pin.ts +61 -0
- package/src/self_host/config.ts +7 -7
- package/src/self_host/main.ts +29 -6
- package/src/self_host/platform.ts +8 -2
- package/src/self_host/resolve.ts +37 -7
- package/src/self_host/session_pin_store.ts +58 -0
- package/src/strategies/priority_selector.ts +1 -2
- package/src/strategies/route_decision.ts +0 -5
- package/src/strategies/routing.ts +1 -17
- package/src/telemetry/billing_finalization.ts +4 -3
- package/src/telemetry/prefix_state.ts +1 -6
- package/src/telemetry/report.ts +4 -48
- package/src/types.ts +12 -6
- package/dist/strategies/fallback_decision.d.ts +0 -8
- package/dist/strategies/fallback_decision.js +0 -17
- package/src/strategies/fallback_decision.ts +0 -30
|
@@ -14,7 +14,6 @@ export function createExperimentRoutedRuntimeReporter() {
|
|
|
14
14
|
attachSelectorRun() { },
|
|
15
15
|
attachSpeculativeSelectorRun() { },
|
|
16
16
|
attachDecision() { },
|
|
17
|
-
attachFallbackDecision() { },
|
|
18
17
|
attachServingDecision() { },
|
|
19
18
|
recordProviderAttempt() { },
|
|
20
19
|
modelCost() { return null; },
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { RoutingDecision } from "@mupt-ai/dari-router/policy-engine";
|
|
2
|
+
import type { RoutedDecision } from "../strategies/route_decision.js";
|
|
3
|
+
import type { ServedDecision } from "../types.js";
|
|
4
|
+
import type { PreparedRoutedCompletion } from "./prepare_routed_completion.js";
|
|
5
|
+
export declare function sessionPinHoldDecision(prepared: PreparedRoutedCompletion): ServedDecision | null;
|
|
6
|
+
export declare function sessionPinnedSelection(routed: RoutedDecision): RoutedDecision;
|
|
7
|
+
export declare function pinSession(prepared: PreparedRoutedCompletion, decision: RoutingDecision): Promise<void>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { gatewayPlatform } from "../platform/platform.js";
|
|
2
|
+
import { canServeCommittedPair } from "./routing_lease.js";
|
|
3
|
+
// The session's pinned pair, while this request may still serve it. A pin
|
|
4
|
+
// that can no longer serve (model disabled, credits exhausted, image input on
|
|
5
|
+
// a text-only model) falls through to selection, which moves the pin.
|
|
6
|
+
export function sessionPinHoldDecision(prepared) {
|
|
7
|
+
const pin = prepared.sessionPin?.pin;
|
|
8
|
+
if (pin === null || pin === undefined)
|
|
9
|
+
return null;
|
|
10
|
+
if (!canServeCommittedPair(prepared, pin))
|
|
11
|
+
return null;
|
|
12
|
+
return {
|
|
13
|
+
selectedModel: pin.model,
|
|
14
|
+
reasoningEffort: pin.reasoningEffort,
|
|
15
|
+
reason: `Session pin: ${pin.model}/${pin.reasoningEffort} serves this whole session.`,
|
|
16
|
+
sessionPinned: true,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
// A pinned session never holds leases: the pin outlives any countdown.
|
|
20
|
+
export function sessionPinnedSelection(routed) {
|
|
21
|
+
const pinned = (decision) => ({ ...unleased(decision), sessionPinned: true });
|
|
22
|
+
return {
|
|
23
|
+
...routed,
|
|
24
|
+
decision: pinned(routed.decision),
|
|
25
|
+
...(routed.run === undefined ? {} : { run: { ...routed.run, decision: pinned(routed.run.decision) } }),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function unleased(decision) {
|
|
29
|
+
const { leaseTurnsRemaining: _, ...rest } = decision;
|
|
30
|
+
return rest;
|
|
31
|
+
}
|
|
32
|
+
// A failed write only costs the pin: the next turn selects again.
|
|
33
|
+
export async function pinSession(prepared, decision) {
|
|
34
|
+
if (prepared.sessionPin === undefined)
|
|
35
|
+
return;
|
|
36
|
+
const pin = { model: decision.selectedModel, reasoningEffort: decision.reasoningEffort };
|
|
37
|
+
try {
|
|
38
|
+
await gatewayPlatform().writeSessionPin({
|
|
39
|
+
routerId: prepared.routerId,
|
|
40
|
+
pinKey: prepared.sessionPin.key,
|
|
41
|
+
...pin,
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
gatewayPlatform().captureOperationalError(error, {
|
|
46
|
+
code: "session_pin_write_failed",
|
|
47
|
+
requestId: prepared.requestId,
|
|
48
|
+
routerId: prepared.routerId,
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -9,8 +9,8 @@ import { type ReasoningEffort } from "@mupt-ai/dari-router/policy-engine";
|
|
|
9
9
|
* - anthropic/claude-sonnet-5-5
|
|
10
10
|
* - id: openai/gpt-5.5
|
|
11
11
|
* thinking: [medium, high]
|
|
12
|
-
* fallback: true # retry on the next model when a provider fails
|
|
13
12
|
* subscriptions: true # serve with connected Claude Code / ChatGPT plans
|
|
13
|
+
* sessionPinning: false # route each session and subagent once, then keep its model
|
|
14
14
|
* providers: # optional: OpenAI-compatible or self-run endpoints
|
|
15
15
|
* local:
|
|
16
16
|
* baseUrl: http://localhost:8000/v1
|
|
@@ -32,10 +32,9 @@ export type ProviderEndpointConfig = {
|
|
|
32
32
|
};
|
|
33
33
|
export type RouterConfig = {
|
|
34
34
|
models: RouterModelConfig[];
|
|
35
|
-
fallback: boolean;
|
|
36
|
-
fallbackRequiresDifferentProvider: boolean;
|
|
37
35
|
retries: number;
|
|
38
36
|
subscriptions: boolean;
|
|
37
|
+
sessionPinning: boolean;
|
|
39
38
|
zdr: boolean;
|
|
40
39
|
};
|
|
41
40
|
export type RouterModelConfig = {
|
package/dist/self_host/config.js
CHANGED
|
@@ -5,7 +5,8 @@ const MODEL_ID_PATTERN = /^[a-z0-9][a-z0-9.-]*\/\S+$/;
|
|
|
5
5
|
export async function loadGatewayConfig(path) {
|
|
6
6
|
const file = Bun.file(path);
|
|
7
7
|
if (!(await file.exists())) {
|
|
8
|
-
throw new Error(`Gateway config not found at ${path}
|
|
8
|
+
throw new Error(`Gateway config not found at ${path}. Write one from dari-gateway.example.yaml, `
|
|
9
|
+
+ "or set DARI_GATEWAY_CONFIG (in the container image, mount it at /config/dari-gateway.yaml).");
|
|
9
10
|
}
|
|
10
11
|
const text = await file.text();
|
|
11
12
|
const raw = path.endsWith(".json") ? JSON.parse(text) : Bun.YAML.parse(text);
|
|
@@ -67,10 +68,9 @@ function parseRouter(id, raw) {
|
|
|
67
68
|
}
|
|
68
69
|
return {
|
|
69
70
|
models,
|
|
70
|
-
fallback: optionalBoolean(raw.fallback, `routers.${id}.fallback`) ?? true,
|
|
71
|
-
fallbackRequiresDifferentProvider: optionalBoolean(raw.fallbackRequiresDifferentProvider, `routers.${id}.fallbackRequiresDifferentProvider`) ?? false,
|
|
72
71
|
retries: optionalRetries(raw.retries, `routers.${id}.retries`) ?? 1,
|
|
73
72
|
subscriptions: optionalBoolean(raw.subscriptions, `routers.${id}.subscriptions`) ?? true,
|
|
73
|
+
sessionPinning: optionalBoolean(raw.sessionPinning, `routers.${id}.sessionPinning`) ?? false,
|
|
74
74
|
zdr: optionalBoolean(raw.zdr, `routers.${id}.zdr`) ?? false,
|
|
75
75
|
};
|
|
76
76
|
}
|
package/dist/self_host/main.d.ts
CHANGED
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
* state (default `dari-gateway.db`). It holds subscription tokens.
|
|
9
9
|
* - `DARI_AUTO_ROUTING_API_KEY`: choose among a router's models with Dari
|
|
10
10
|
* Auto Routing; without it, `dari/routing` serves the first listed model.
|
|
11
|
-
* - `PORT` (default 8080),
|
|
11
|
+
* - `PORT` (default 8080), `DARI_GATEWAY_HOST` (default 127.0.0.1; the container
|
|
12
|
+
* image sets 0.0.0.0), and each provider's standard API key variable.
|
|
12
13
|
*/
|
|
13
14
|
export declare function startSelfHostedGateway(env?: Record<string, string>): Promise<{
|
|
14
15
|
server: import("../http/http2_server.js").Http2RoutingServer | Bun.Server<undefined>;
|
package/dist/self_host/main.js
CHANGED
|
@@ -10,8 +10,9 @@ import { createAutoRoutingSelectorBackend } from "../strategies/auto_routing_sel
|
|
|
10
10
|
import { createPrioritySelectorBackend } from "../strategies/priority_selector.js";
|
|
11
11
|
import { loadGatewayConfig } from "./config.js";
|
|
12
12
|
import { createSelfHostedPlatform } from "./platform.js";
|
|
13
|
-
import { gatewayKeyIndex } from "./resolve.js";
|
|
13
|
+
import { gatewayKeyIndex, providersWithoutCredentials } from "./resolve.js";
|
|
14
14
|
import { PrefixStore } from "./prefix_store.js";
|
|
15
|
+
import { SessionPinStore } from "./session_pin_store.js";
|
|
15
16
|
import { handleSubscriptionRequest, SubscriptionStore } from "./subscriptions/index.js";
|
|
16
17
|
const SUBSCRIPTION_ROUTES = "/v1/organizations/current/credentials";
|
|
17
18
|
/**
|
|
@@ -23,7 +24,8 @@ const SUBSCRIPTION_ROUTES = "/v1/organizations/current/credentials";
|
|
|
23
24
|
* state (default `dari-gateway.db`). It holds subscription tokens.
|
|
24
25
|
* - `DARI_AUTO_ROUTING_API_KEY`: choose among a router's models with Dari
|
|
25
26
|
* Auto Routing; without it, `dari/routing` serves the first listed model.
|
|
26
|
-
* - `PORT` (default 8080),
|
|
27
|
+
* - `PORT` (default 8080), `DARI_GATEWAY_HOST` (default 127.0.0.1; the container
|
|
28
|
+
* image sets 0.0.0.0), and each provider's standard API key variable.
|
|
27
29
|
*/
|
|
28
30
|
export async function startSelfHostedGateway(env = process.env) {
|
|
29
31
|
const gatewayKeys = (env.DARI_GATEWAY_KEYS ?? "")
|
|
@@ -42,20 +44,31 @@ export async function startSelfHostedGateway(env = process.env) {
|
|
|
42
44
|
db.exec("PRAGMA journal_mode = WAL");
|
|
43
45
|
const subscriptions = new SubscriptionStore(db);
|
|
44
46
|
const prefixes = new PrefixStore(db);
|
|
47
|
+
const sessionPins = new SessionPinStore(db);
|
|
45
48
|
// Pi's provider auth reads each provider's standard key variable, such as
|
|
46
49
|
// OPENAI_API_KEY, from this environment only.
|
|
47
50
|
const piModels = builtinModels({
|
|
48
51
|
authContext: { env: async (name) => env[name], fileExists: async () => false },
|
|
49
52
|
});
|
|
53
|
+
const providerApiKey = async (provider) => (await piModels.getAuth(provider))?.auth.apiKey;
|
|
50
54
|
const platform = createSelfHostedPlatform({
|
|
51
55
|
config,
|
|
52
56
|
gatewayKeys,
|
|
53
57
|
subscriptions,
|
|
54
58
|
prefixes,
|
|
55
|
-
|
|
59
|
+
sessionPins,
|
|
60
|
+
providerApiKey,
|
|
56
61
|
env,
|
|
57
62
|
fetch,
|
|
58
63
|
});
|
|
64
|
+
// A model without a credential is dropped at request time, so a missing key
|
|
65
|
+
// would otherwise surface only as a 424 on the first request.
|
|
66
|
+
for (const [routerId, router] of Object.entries(config.routers)) {
|
|
67
|
+
const missing = await providersWithoutCredentials({ config, subscriptions, providerApiKey, env }, router);
|
|
68
|
+
if (missing.length > 0) {
|
|
69
|
+
console.warn("dari-gateway router has providers without credentials; their models are not served", JSON.stringify({ router: routerId, providers: missing }));
|
|
70
|
+
}
|
|
71
|
+
}
|
|
59
72
|
const autoRoutingKey = env.DARI_AUTO_ROUTING_API_KEY?.trim();
|
|
60
73
|
const selectorBackend = autoRoutingKey
|
|
61
74
|
? createAutoRoutingSelectorBackend({
|
|
@@ -81,8 +94,11 @@ export async function startSelfHostedGateway(env = process.env) {
|
|
|
81
94
|
?? errorResponse(new HttpError(404, "Not found", "not_found"));
|
|
82
95
|
};
|
|
83
96
|
const port = Number(env.PORT ?? "8080");
|
|
84
|
-
|
|
97
|
+
// Loopback unless DARI_GATEWAY_HOST says otherwise (see the doc comment).
|
|
98
|
+
const hostname = (env.DARI_GATEWAY_HOST ?? "").trim() || "127.0.0.1";
|
|
99
|
+
const server = startServer(port, handler, { hostname });
|
|
85
100
|
console.info("dari-gateway listening", JSON.stringify({
|
|
101
|
+
hostname,
|
|
86
102
|
port,
|
|
87
103
|
routers: Object.keys(config.routers),
|
|
88
104
|
auto_routing: autoRoutingKey ? "dari" : "priority",
|
|
@@ -90,6 +106,10 @@ export async function startSelfHostedGateway(env = process.env) {
|
|
|
90
106
|
return { server, close: () => db.close() };
|
|
91
107
|
}
|
|
92
108
|
if (import.meta.main) {
|
|
93
|
-
|
|
94
|
-
|
|
109
|
+
// Startup errors are configuration mistakes: print the message, not a stack.
|
|
110
|
+
const started = await startSelfHostedGateway().catch((error) => {
|
|
111
|
+
console.error(`dari-gateway: ${error instanceof Error ? error.message : String(error)}`);
|
|
112
|
+
process.exit(1);
|
|
113
|
+
});
|
|
114
|
+
installShutdownHandlers(started.server, async () => started.close());
|
|
95
115
|
}
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import type { GatewayPlatform } from "../platform/platform.js";
|
|
2
2
|
import type { PrefixStore } from "./prefix_store.js";
|
|
3
|
+
import type { SessionPinStore } from "./session_pin_store.js";
|
|
3
4
|
import { type SelfHostedResolverDeps } from "./resolve.js";
|
|
4
5
|
import type { SubscriptionStore } from "./subscriptions/index.js";
|
|
5
6
|
/**
|
|
6
|
-
* The self-hosted platform: routers from the config file, subscriptions
|
|
7
|
-
* prefix-cache state in the gateway's SQLite database, and one content-free
|
|
7
|
+
* The self-hosted platform: routers from the config file, subscriptions,
|
|
8
|
+
* prefix-cache state, and session pins in the gateway's SQLite database, and one content-free
|
|
8
9
|
* log line per request.
|
|
9
10
|
*/
|
|
10
11
|
export declare function createSelfHostedPlatform(deps: Omit<SelfHostedResolverDeps, "prefixHits"> & {
|
|
11
12
|
prefixes: PrefixStore;
|
|
13
|
+
sessionPins: SessionPinStore;
|
|
12
14
|
subscriptions: SubscriptionStore;
|
|
13
15
|
}): GatewayPlatform;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { errorCode } from "../http/errors.js";
|
|
2
2
|
import { resolveSelfHostedRouter } from "./resolve.js";
|
|
3
3
|
/**
|
|
4
|
-
* The self-hosted platform: routers from the config file, subscriptions
|
|
5
|
-
* prefix-cache state in the gateway's SQLite database, and one content-free
|
|
4
|
+
* The self-hosted platform: routers from the config file, subscriptions,
|
|
5
|
+
* prefix-cache state, and session pins in the gateway's SQLite database, and one content-free
|
|
6
6
|
* log line per request.
|
|
7
7
|
*/
|
|
8
8
|
export function createSelfHostedPlatform(deps) {
|
|
@@ -18,6 +18,10 @@ export function createSelfHostedPlatform(deps) {
|
|
|
18
18
|
writePendingLease: async (write) => {
|
|
19
19
|
deps.prefixes.writePendingLease(write, deps.config.routers[write.router_id]?.zdr ?? false);
|
|
20
20
|
},
|
|
21
|
+
readSessionPin: async (key) => deps.sessionPins.read(key),
|
|
22
|
+
writeSessionPin: async (write) => {
|
|
23
|
+
deps.sessionPins.write(write);
|
|
24
|
+
},
|
|
21
25
|
markSubscriptionUnavailable: async (hold) => hold.credentialId !== undefined &&
|
|
22
26
|
deps.subscriptions.markUnavailable(hold.credentialId, hold.limit.resetsAt, hold.limit.reason),
|
|
23
27
|
async recordUsage() {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type RouterPrefixHit } from "@mupt-ai/dari-router/policy-engine";
|
|
2
2
|
import type { ResolveRouterRequest } from "../platform/platform.js";
|
|
3
3
|
import type { ResolvedRouter } from "../types.js";
|
|
4
|
-
import type { GatewayConfig } from "./config.js";
|
|
4
|
+
import type { GatewayConfig, RouterConfig } from "./config.js";
|
|
5
5
|
import { type OAuthFetch, type SubscriptionStore } from "./subscriptions/index.js";
|
|
6
6
|
export declare const SELF_HOSTED_ORGANIZATION_ID = "self-hosted";
|
|
7
7
|
export declare const SELF_HOSTED_USER_ID = "self-hosted";
|
|
@@ -26,3 +26,10 @@ export type SelfHostedResolverDeps = {
|
|
|
26
26
|
*/
|
|
27
27
|
export declare function resolveSelfHostedRouter(deps: SelfHostedResolverDeps, request: ResolveRouterRequest): Promise<ResolvedRouter>;
|
|
28
28
|
export declare function gatewayKeyIndex(keys: readonly string[], presented: string): number | null;
|
|
29
|
+
/**
|
|
30
|
+
* Providers a router lists that no credential can serve: no API key and no
|
|
31
|
+
* connected subscription the router allows. Startup uses this, so it counts a
|
|
32
|
+
* held subscription as connected (the request-time 424 reports holds) and
|
|
33
|
+
* checks both harnesses, since startup does not know the client.
|
|
34
|
+
*/
|
|
35
|
+
export declare function providersWithoutCredentials(deps: Pick<SelfHostedResolverDeps, "subscriptions" | "providerApiKey" | "config" | "env">, router: RouterConfig): Promise<string[]>;
|
|
@@ -17,7 +17,7 @@ export const SELF_HOSTED_USER_ID = "self-hosted";
|
|
|
17
17
|
export async function resolveSelfHostedRouter(deps, request) {
|
|
18
18
|
const keyIndex = gatewayKeyIndex(deps.gatewayKeys, request.apiKey);
|
|
19
19
|
if (keyIndex === null) {
|
|
20
|
-
throw new HttpError(401, "Missing or invalid
|
|
20
|
+
throw new HttpError(401, "Missing or invalid gateway key.", "authentication_error");
|
|
21
21
|
}
|
|
22
22
|
const router = deps.config.routers[request.routerId];
|
|
23
23
|
if (router === undefined) {
|
|
@@ -48,8 +48,10 @@ export async function resolveSelfHostedRouter(deps, request) {
|
|
|
48
48
|
credentials.set(provider, apiKey ? { key: apiKey, source: "user" } : null);
|
|
49
49
|
}
|
|
50
50
|
const models = router.models.filter((model) => credentials.get(providerForModel(model.id)) != null);
|
|
51
|
-
if (models.length === 0)
|
|
52
|
-
|
|
51
|
+
if (models.length === 0) {
|
|
52
|
+
const missing = [...credentials].filter(([, credential]) => credential == null).map(([provider]) => provider);
|
|
53
|
+
throw unavailableRouterError(request.routerId, exhausted, missing);
|
|
54
|
+
}
|
|
53
55
|
const resolved = {
|
|
54
56
|
router_id: request.routerId,
|
|
55
57
|
organization_id: SELF_HOSTED_ORGANIZATION_ID,
|
|
@@ -61,9 +63,8 @@ export async function resolveSelfHostedRouter(deps, request) {
|
|
|
61
63
|
subscription_credential_ids: {},
|
|
62
64
|
zdr: router.zdr,
|
|
63
65
|
speculative_routing: false,
|
|
66
|
+
session_pinning: router.sessionPinning,
|
|
64
67
|
primary_retries: router.retries,
|
|
65
|
-
model_fallback_enabled: router.fallback,
|
|
66
|
-
fallback_requires_different_provider: router.fallbackRequiresDifferentProvider,
|
|
67
68
|
lease_context_enabled: false,
|
|
68
69
|
routing_strategy: "dari",
|
|
69
70
|
custom_config: null,
|
|
@@ -156,12 +157,34 @@ const SUBSCRIPTION_PLAN_NAMES = {
|
|
|
156
157
|
openai_codex: "ChatGPT",
|
|
157
158
|
meta_muse: "Muse Code",
|
|
158
159
|
};
|
|
159
|
-
|
|
160
|
+
/**
|
|
161
|
+
* Providers a router lists that no credential can serve: no API key and no
|
|
162
|
+
* connected subscription the router allows. Startup uses this, so it counts a
|
|
163
|
+
* held subscription as connected (the request-time 424 reports holds) and
|
|
164
|
+
* checks both harnesses, since startup does not know the client.
|
|
165
|
+
*/
|
|
166
|
+
export async function providersWithoutCredentials(deps, router) {
|
|
167
|
+
const missing = [];
|
|
168
|
+
for (const provider of new Set(router.models.map((model) => providerForModel(model.id)))) {
|
|
169
|
+
const plans = router.subscriptions
|
|
170
|
+
? [subscriptionProviderFor(provider, "claude_code"), subscriptionProviderFor(provider, "muse_code")]
|
|
171
|
+
: [];
|
|
172
|
+
const connected = plans.some((plan) => plan !== null && deps.subscriptions.list(plan).length > 0);
|
|
173
|
+
if (connected)
|
|
174
|
+
continue;
|
|
175
|
+
const apiKey = await providerApiKey(deps, provider);
|
|
176
|
+
if (apiKey === undefined)
|
|
177
|
+
missing.push(provider);
|
|
178
|
+
}
|
|
179
|
+
return missing;
|
|
180
|
+
}
|
|
181
|
+
function unavailableRouterError(routerId, exhausted, missing) {
|
|
182
|
+
const missingNote = missing.length > 0 ? ` Missing providers: ${missing.join(", ")}.` : "";
|
|
160
183
|
if (exhausted.size > 0) {
|
|
161
184
|
const plans = [...exhausted]
|
|
162
185
|
.map((provider) => SUBSCRIPTION_PLAN_NAMES[provider])
|
|
163
186
|
.join(" and ");
|
|
164
|
-
return new HttpError(424, `Every connected ${plans} subscription is at its usage limit, and no API key can serve router ${routerId} instead
|
|
187
|
+
return new HttpError(424, `Every connected ${plans} subscription is at its usage limit, and no API key can serve router ${routerId} instead.${missingNote}`, "personal_subscription_unavailable");
|
|
165
188
|
}
|
|
166
|
-
return new HttpError(424, `No credentials can serve router ${routerId}
|
|
189
|
+
return new HttpError(424, `No credentials can serve router ${routerId}.${missingNote} Set each provider's API key or connect a subscription that can serve it.`, "provider_credentials_missing");
|
|
167
190
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite storage for session pins: the model and thinking level each session
|
|
3
|
+
* (and each subagent) of a session-pinning router was routed to on its first
|
|
4
|
+
* turn. Serving a pin refreshes it at most daily; pins unused for longer than
|
|
5
|
+
* the retention window are no longer served and are deleted.
|
|
6
|
+
*/
|
|
7
|
+
import type { Database } from "bun:sqlite";
|
|
8
|
+
import type { SessionPin, SessionPinKey } from "../platform/platform.js";
|
|
9
|
+
export declare class SessionPinStore {
|
|
10
|
+
private readonly db;
|
|
11
|
+
constructor(db: Database);
|
|
12
|
+
read(key: SessionPinKey, now?: Date): SessionPin | null;
|
|
13
|
+
write(write: SessionPinKey & SessionPin, now?: Date): void;
|
|
14
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
const RETENTION_MS = 30 * 24 * 60 * 60_000;
|
|
2
|
+
const REFRESH_MS = 24 * 60 * 60_000;
|
|
3
|
+
export class SessionPinStore {
|
|
4
|
+
db;
|
|
5
|
+
constructor(db) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
db.run(`CREATE TABLE IF NOT EXISTS routing_session_pins (
|
|
8
|
+
router_id TEXT NOT NULL,
|
|
9
|
+
pin_key TEXT NOT NULL,
|
|
10
|
+
model TEXT NOT NULL,
|
|
11
|
+
reasoning_effort TEXT NOT NULL,
|
|
12
|
+
updated_at INTEGER NOT NULL,
|
|
13
|
+
PRIMARY KEY (router_id, pin_key)
|
|
14
|
+
)`);
|
|
15
|
+
db.run("CREATE INDEX IF NOT EXISTS ix_routing_session_pins_updated_at ON routing_session_pins (updated_at)");
|
|
16
|
+
}
|
|
17
|
+
read(key, now = new Date()) {
|
|
18
|
+
const row = this.db
|
|
19
|
+
.query(`SELECT model, reasoning_effort, updated_at FROM routing_session_pins
|
|
20
|
+
WHERE router_id = ? AND pin_key = ? AND updated_at >= ?`)
|
|
21
|
+
.get(key.routerId, key.pinKey, now.getTime() - RETENTION_MS);
|
|
22
|
+
if (row === null)
|
|
23
|
+
return null;
|
|
24
|
+
if (row.updated_at < now.getTime() - REFRESH_MS) {
|
|
25
|
+
this.db
|
|
26
|
+
.query("UPDATE routing_session_pins SET updated_at = ? WHERE router_id = ? AND pin_key = ?")
|
|
27
|
+
.run(now.getTime(), key.routerId, key.pinKey);
|
|
28
|
+
}
|
|
29
|
+
return { model: row.model, reasoningEffort: row.reasoning_effort };
|
|
30
|
+
}
|
|
31
|
+
write(write, now = new Date()) {
|
|
32
|
+
this.db.transaction(() => {
|
|
33
|
+
this.db
|
|
34
|
+
.query(`INSERT INTO routing_session_pins (router_id, pin_key, model, reasoning_effort, updated_at)
|
|
35
|
+
VALUES (?, ?, ?, ?, ?)
|
|
36
|
+
ON CONFLICT (router_id, pin_key) DO UPDATE SET
|
|
37
|
+
model = excluded.model,
|
|
38
|
+
reasoning_effort = excluded.reasoning_effort,
|
|
39
|
+
updated_at = excluded.updated_at`)
|
|
40
|
+
.run(write.routerId, write.pinKey, write.model, write.reasoningEffort, now.getTime());
|
|
41
|
+
this.db.query("DELETE FROM routing_session_pins WHERE updated_at < ?").run(now.getTime() - RETENTION_MS);
|
|
42
|
+
})();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { SelectorBackend } from "./routing.js";
|
|
2
2
|
/**
|
|
3
3
|
* Serve the router's first eligible model/thinking-level pair, in the order
|
|
4
|
-
* the router lists them.
|
|
5
|
-
* when the router enables model fallback.
|
|
4
|
+
* the router lists them.
|
|
6
5
|
*/
|
|
7
6
|
export declare function createPrioritySelectorBackend(): SelectorBackend;
|
|
@@ -4,8 +4,7 @@ const PRIORITY_SELECTOR_MODEL = "dari-gateway/priority";
|
|
|
4
4
|
const PRIORITY_CONTEXT_WINDOW_CHARS = 1_000_000;
|
|
5
5
|
/**
|
|
6
6
|
* Serve the router's first eligible model/thinking-level pair, in the order
|
|
7
|
-
* the router lists them.
|
|
8
|
-
* when the router enables model fallback.
|
|
7
|
+
* the router lists them.
|
|
9
8
|
*/
|
|
10
9
|
export function createPrioritySelectorBackend() {
|
|
11
10
|
return {
|
|
@@ -4,7 +4,6 @@ import type { SelectorImage } from "./selector_projection.js";
|
|
|
4
4
|
import type { ResolvedRouter } from "../types.js";
|
|
5
5
|
export type RoutedDecision = {
|
|
6
6
|
decision: RoutingDecision;
|
|
7
|
-
fallbackDecision?: RoutingDecision;
|
|
8
7
|
run?: SelectorRun;
|
|
9
8
|
details?: Record<string, unknown>;
|
|
10
9
|
};
|
|
@@ -42,8 +42,6 @@ export async function routeDecision(args) {
|
|
|
42
42
|
? { customRules: custom.rules, defaultTarget: custom.defaultTarget }
|
|
43
43
|
: {}),
|
|
44
44
|
selectorBackend: args.selectorBackend,
|
|
45
|
-
modelFallbackEnabled: credentials.model_fallback_enabled !== false,
|
|
46
|
-
fallbackRequiresDifferentProvider: credentials.fallback_requires_different_provider !== false,
|
|
47
45
|
executionProviderForModel: (model) => executionProviderForModel(credentials, model),
|
|
48
46
|
imputeEvalScores: credentials.eval_score_imputation === true,
|
|
49
47
|
leaseHistory: args.leaseHistory,
|
|
@@ -58,7 +56,6 @@ export async function routeDecision(args) {
|
|
|
58
56
|
};
|
|
59
57
|
return {
|
|
60
58
|
decision: run.decision,
|
|
61
|
-
fallbackDecision: run.fallbackDecision,
|
|
62
59
|
run,
|
|
63
60
|
...(Object.keys(details).length > 0 ? { details } : {}),
|
|
64
61
|
};
|
|
@@ -16,7 +16,6 @@ export type SelectorTrace = {
|
|
|
16
16
|
};
|
|
17
17
|
export type SelectorRun = SelectorTrace & {
|
|
18
18
|
decision: RoutingDecision;
|
|
19
|
-
fallbackDecision?: RoutingDecision;
|
|
20
19
|
};
|
|
21
20
|
export type SelectModelArgs = {
|
|
22
21
|
allCandidates?: RoutingCandidate[];
|
|
@@ -28,8 +27,6 @@ export type SelectModelArgs = {
|
|
|
28
27
|
customRules?: CustomRouterRule[];
|
|
29
28
|
defaultTarget?: CustomDefaultTarget | null;
|
|
30
29
|
sessionId?: string;
|
|
31
|
-
modelFallbackEnabled?: boolean;
|
|
32
|
-
fallbackRequiresDifferentProvider?: boolean;
|
|
33
30
|
executionProviderForModel?: (model: string) => string;
|
|
34
31
|
imputeEvalScores?: boolean;
|
|
35
32
|
imputationReferenceEvals?: RouterEval[];
|
|
@@ -6,7 +6,6 @@ import { internalChatCompletionsToInteraction } from "../protocol/input/chat_com
|
|
|
6
6
|
import { gatewayPlatform } from "../platform/platform.js";
|
|
7
7
|
import { isBedrockApi } from "../providers/model_capabilities.js";
|
|
8
8
|
import { completeInteractionModel, modelContextWindow } from "../providers/pi.js";
|
|
9
|
-
import { deriveFallbackDecision } from "./fallback_decision.js";
|
|
10
9
|
export function resolveRouter(args) {
|
|
11
10
|
return gatewayPlatform().resolveRouter(args);
|
|
12
11
|
}
|
|
@@ -52,8 +51,6 @@ export async function selectModel(args) {
|
|
|
52
51
|
contextWindowChars,
|
|
53
52
|
messages: selectorBackend.prepareMessages?.(safeMessages) ?? safeMessages,
|
|
54
53
|
...(leaseContextEnabled ? { leaseHistory: args.leaseHistory ?? [] } : {}),
|
|
55
|
-
modelFallbackEnabled: args.modelFallbackEnabled,
|
|
56
|
-
fallbackRequiresDifferentProvider: args.fallbackRequiresDifferentProvider,
|
|
57
54
|
imputeEvalScores: args.imputeEvalScores,
|
|
58
55
|
imputationReferenceEvals: args.imputationReferenceEvals,
|
|
59
56
|
});
|
|
@@ -147,17 +144,7 @@ function validateSelectorRun(run, args) {
|
|
|
147
144
|
if (!args.candidates.some((candidate) => sameRoutingCandidate(candidate, selected))) {
|
|
148
145
|
throw new SelectorOutputError(new HttpError(502, "Selector returned a model/thinking-level pair outside this router's candidates.", "selector_invalid_candidate"), run);
|
|
149
146
|
}
|
|
150
|
-
|
|
151
|
-
const fallbackDecision = deriveFallbackDecision({
|
|
152
|
-
candidates: args.candidates,
|
|
153
|
-
selected,
|
|
154
|
-
enabled: args.modelFallbackEnabled,
|
|
155
|
-
requiresDifferentProvider: args.fallbackRequiresDifferentProvider,
|
|
156
|
-
providerForModel: args.executionProviderForModel,
|
|
157
|
-
});
|
|
158
|
-
return fallbackDecision
|
|
159
|
-
? { ...primaryRun, fallbackDecision }
|
|
160
|
-
: primaryRun;
|
|
147
|
+
return run;
|
|
161
148
|
}
|
|
162
149
|
function requiredSelectorConfig(value, label) {
|
|
163
150
|
const normalized = value.trim();
|
|
@@ -5,7 +5,7 @@ export type RoutingBillingProviderAttemptFacts = {
|
|
|
5
5
|
provider: string;
|
|
6
6
|
model: string;
|
|
7
7
|
provider_key_source: ProviderKeySource;
|
|
8
|
-
role: "primary"
|
|
8
|
+
role: "primary";
|
|
9
9
|
attempt: number;
|
|
10
10
|
outcome: "completed" | "provider_failed" | "customer_canceled";
|
|
11
11
|
provider_request_id: string | null;
|
|
@@ -54,7 +54,6 @@ export declare function routingBillingProviderAttemptFacts(args: {
|
|
|
54
54
|
provider: string;
|
|
55
55
|
model: string;
|
|
56
56
|
providerKeySource: ProviderKeySource;
|
|
57
|
-
role: "primary" | "fallback";
|
|
58
57
|
attempt: number;
|
|
59
58
|
outcome: "completed" | "provider_failed" | "customer_canceled";
|
|
60
59
|
providerMessage: Record<string, unknown> | null;
|
|
@@ -62,7 +62,7 @@ export function routingBillingProviderAttemptFacts(args) {
|
|
|
62
62
|
provider: args.provider,
|
|
63
63
|
model: args.model,
|
|
64
64
|
provider_key_source: args.providerKeySource,
|
|
65
|
-
role:
|
|
65
|
+
role: "primary",
|
|
66
66
|
attempt: args.attempt,
|
|
67
67
|
outcome: args.outcome,
|
|
68
68
|
provider_request_id: providerRequestId(args.providerMessage),
|
|
@@ -29,7 +29,6 @@ export declare function buildCompletedPrefixHit(args: {
|
|
|
29
29
|
export declare function leaseCountdownForPrefix(args: {
|
|
30
30
|
served: RoutingDecision | null;
|
|
31
31
|
next: RoutingDecision | null;
|
|
32
|
-
fallbackServed?: boolean;
|
|
33
32
|
}): number | null;
|
|
34
33
|
export declare function reportPendingLease(args: {
|
|
35
34
|
routerId: string;
|
|
@@ -41,12 +41,10 @@ export function buildCompletedPrefixHit(args) {
|
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
43
|
export function leaseCountdownForPrefix(args) {
|
|
44
|
-
const granted = args.
|
|
44
|
+
const granted = args.next ?? args.served;
|
|
45
45
|
const remaining = granted?.leaseTurnsRemaining;
|
|
46
46
|
if (remaining === undefined || remaining === null)
|
|
47
47
|
return null;
|
|
48
|
-
if (args.fallbackServed === true)
|
|
49
|
-
return remaining;
|
|
50
48
|
const servedGrantedPair = args.next === null ||
|
|
51
49
|
(args.served !== null &&
|
|
52
50
|
args.served.selectedModel === args.next.selectedModel &&
|
|
@@ -20,7 +20,6 @@ export declare class RequestReporter {
|
|
|
20
20
|
private selectorOutputAccepted;
|
|
21
21
|
private decision;
|
|
22
22
|
private nextDecision;
|
|
23
|
-
private fallbackDecision;
|
|
24
23
|
private providerAttempts;
|
|
25
24
|
private billingProviderAttempts;
|
|
26
25
|
private decisionDetails;
|
|
@@ -62,7 +61,6 @@ export declare class RequestReporter {
|
|
|
62
61
|
attachSelectorRun(run: SelectorRun): void;
|
|
63
62
|
attachSpeculativeSelectorRun(run: SelectorRun, servingDecision: RoutingDecision): void;
|
|
64
63
|
attachDecision(decision: RoutingDecision): void;
|
|
65
|
-
attachFallbackDecision(decision: RoutingDecision | undefined): void;
|
|
66
64
|
attachServingDecision(decision: RoutingDecision): void;
|
|
67
65
|
recordProviderAttempt(attempt: ProviderAttemptRecord): void;
|
|
68
66
|
modelCost(attempt?: ProviderAttemptRecord): number | null;
|