@mupt-ai/dari-gateway 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -11
- package/dari-gateway.example.yaml +3 -2
- package/dist/context/interaction_fingerprint.js +10 -1
- package/dist/http/auth.js +2 -2
- package/dist/http/http2_server.d.ts +1 -1
- package/dist/http/http2_server.js +3 -3
- package/dist/http/serve.d.ts +1 -0
- package/dist/http/serve.js +5 -4
- package/dist/ingress/chat_completions.js +2 -2
- package/dist/ingress/codex.js +7 -3
- package/dist/ingress/codex_models.d.ts +1 -1
- package/dist/ingress/codex_models.js +22 -13
- package/dist/platform/platform.d.ts +13 -3
- package/dist/protocol/input/openai_responses.js +2 -4
- package/dist/protocol/output/anthropic_messages_output.d.ts +2 -2
- package/dist/protocol/output/anthropic_messages_output.js +1 -0
- package/dist/protocol/output/openai_responses_stream.js +19 -1
- package/dist/providers/execution.d.ts +0 -5
- package/dist/providers/execution.js +5 -8
- package/dist/providers/pi_output.d.ts +4 -0
- package/dist/providers/pi_output.js +1 -1
- package/dist/providers/provider_attempt_runner.d.ts +2 -3
- package/dist/providers/provider_attempt_runner.js +14 -24
- package/dist/providers/provider_call.d.ts +3 -3
- package/dist/providers/provider_call.js +1 -0
- package/dist/providers/stream_finalization.d.ts +0 -1
- package/dist/providers/stream_finalization.js +0 -1
- package/dist/runtime/claude_code/auxiliary_routing.js +1 -9
- package/dist/runtime/explicit_model.js +1 -1
- package/dist/runtime/harness_identity/conversation.d.ts +9 -0
- package/dist/runtime/harness_identity/conversation.js +6 -0
- package/dist/runtime/harness_identity/extract.d.ts +2 -0
- package/dist/runtime/harness_identity/extract.js +20 -5
- package/dist/runtime/harness_identity/index.d.ts +2 -2
- package/dist/runtime/harness_identity/index.js +2 -2
- package/dist/runtime/prepare_routed_completion.d.ts +5 -1
- package/dist/runtime/prepare_routed_completion.js +26 -2
- package/dist/runtime/routed_runtime.js +33 -53
- package/dist/runtime/routing_lease.d.ts +4 -6
- package/dist/runtime/routing_lease.js +2 -47
- package/dist/runtime/runtime_reporter.d.ts +0 -2
- package/dist/runtime/runtime_reporter.js +0 -1
- package/dist/runtime/session_pin.d.ts +7 -0
- package/dist/runtime/session_pin.js +51 -0
- package/dist/self_host/config.d.ts +2 -3
- package/dist/self_host/config.js +3 -3
- package/dist/self_host/main.d.ts +2 -1
- package/dist/self_host/main.js +26 -6
- package/dist/self_host/platform.d.ts +4 -2
- package/dist/self_host/platform.js +6 -2
- package/dist/self_host/resolve.d.ts +8 -1
- package/dist/self_host/resolve.js +31 -8
- package/dist/self_host/session_pin_store.d.ts +14 -0
- package/dist/self_host/session_pin_store.js +44 -0
- package/dist/strategies/priority_selector.d.ts +1 -2
- package/dist/strategies/priority_selector.js +1 -2
- package/dist/strategies/route_decision.d.ts +0 -1
- package/dist/strategies/route_decision.js +0 -3
- package/dist/strategies/routing.d.ts +0 -3
- package/dist/strategies/routing.js +1 -14
- package/dist/telemetry/billing_finalization.d.ts +1 -2
- package/dist/telemetry/billing_finalization.js +1 -1
- package/dist/telemetry/prefix_state.d.ts +0 -1
- package/dist/telemetry/prefix_state.js +1 -3
- package/dist/telemetry/report.d.ts +0 -2
- package/dist/telemetry/report.js +5 -41
- package/dist/types.d.ts +5 -3
- package/package.json +2 -2
- package/src/context/interaction_fingerprint.ts +15 -1
- package/src/http/auth.ts +2 -2
- package/src/http/http2_server.ts +7 -3
- package/src/http/serve.ts +6 -5
- package/src/ingress/chat_completions.ts +2 -1
- package/src/ingress/codex.ts +8 -5
- package/src/ingress/codex_models.ts +18 -7
- package/src/platform/platform.ts +16 -2
- package/src/protocol/input/openai_responses.ts +2 -9
- package/src/protocol/output/anthropic_messages_output.ts +3 -2
- package/src/protocol/output/openai_responses_stream.ts +17 -1
- package/src/providers/execution.ts +5 -15
- package/src/providers/pi_output.ts +1 -1
- package/src/providers/provider_attempt_runner.ts +10 -21
- package/src/providers/provider_call.ts +3 -1
- package/src/providers/stream_finalization.ts +0 -2
- package/src/providers/subscription_limits.ts +1 -1
- package/src/runtime/claude_code/auxiliary_routing.ts +1 -8
- package/src/runtime/explicit_model.ts +1 -1
- package/src/runtime/harness_identity/conversation.ts +16 -0
- package/src/runtime/harness_identity/extract.ts +24 -5
- package/src/runtime/harness_identity/index.ts +3 -0
- package/src/runtime/prepare_routed_completion.ts +32 -2
- package/src/runtime/routed_runtime.ts +39 -59
- package/src/runtime/routing_lease.ts +2 -56
- package/src/runtime/runtime_reporter.ts +0 -3
- package/src/runtime/session_pin.ts +61 -0
- package/src/self_host/config.ts +7 -7
- package/src/self_host/main.ts +29 -6
- package/src/self_host/platform.ts +8 -2
- package/src/self_host/resolve.ts +37 -7
- package/src/self_host/session_pin_store.ts +58 -0
- package/src/strategies/priority_selector.ts +1 -2
- package/src/strategies/route_decision.ts +0 -5
- package/src/strategies/routing.ts +1 -17
- package/src/telemetry/billing_finalization.ts +4 -3
- package/src/telemetry/prefix_state.ts +1 -6
- package/src/telemetry/report.ts +4 -48
- package/src/types.ts +12 -6
- package/dist/strategies/fallback_decision.d.ts +0 -8
- package/dist/strategies/fallback_decision.js +0 -17
- package/src/strategies/fallback_decision.ts +0 -30
|
@@ -6,8 +6,6 @@ import {
|
|
|
6
6
|
type RoutingDecision,
|
|
7
7
|
} from "@mupt-ai/dari-router/policy-engine";
|
|
8
8
|
|
|
9
|
-
import { deriveFallbackDecision } from "../strategies/fallback_decision.js";
|
|
10
|
-
import { executionProviderForModel } from "../models/execution_provider.js";
|
|
11
9
|
import type { SelectorBackend } from "../strategies/routing.js";
|
|
12
10
|
import {
|
|
13
11
|
routeDecision,
|
|
@@ -71,17 +69,7 @@ export function leasedDecision(
|
|
|
71
69
|
// incumbent cannot hide an available user-owned key behind the lease.
|
|
72
70
|
if (prepared.billable && prepared.credentials.credit_exhausted) return null;
|
|
73
71
|
|
|
74
|
-
const fallbackDecision = leaseFallbackDecision({
|
|
75
|
-
candidates: prepared.routing.candidateResolution.candidates,
|
|
76
|
-
decision: lease.decision,
|
|
77
|
-
credentials: prepared.credentials,
|
|
78
|
-
});
|
|
79
|
-
// Use attachServingDecision rather than attachDecision so next_model
|
|
80
|
-
// follows the model that actually serves. When fallback handles a primary
|
|
81
|
-
// failure, it inherits this countdown and subsequent held turns continue
|
|
82
|
-
// on the healthy serving pair instead of retrying the failed primary.
|
|
83
72
|
prepared.reporter.attachServingDecision(lease.decision);
|
|
84
|
-
prepared.reporter.attachFallbackDecision(fallbackDecision);
|
|
85
73
|
prepared.reporter.attachDecisionDetails({
|
|
86
74
|
decision_source: "lease_hold",
|
|
87
75
|
routing_lease: {
|
|
@@ -90,49 +78,7 @@ export function leasedDecision(
|
|
|
90
78
|
turn_in_lease: lease.turnInLease,
|
|
91
79
|
},
|
|
92
80
|
});
|
|
93
|
-
return { decision: lease.decision
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// Wraps a lease-held (or adopted) decision with a runtime-derived fallback so
|
|
97
|
-
// a mid-lease provider failure degrades like any other turn instead of
|
|
98
|
-
// erroring the request. The fallback inherits the remaining commitment so the
|
|
99
|
-
// completed prefix follows the healthy serving pair for the rest of the lease.
|
|
100
|
-
export function committedLeaseDecision(
|
|
101
|
-
prepared: PreparedRoutedCompletion,
|
|
102
|
-
decision: RoutingDecision,
|
|
103
|
-
): RoutedDecision {
|
|
104
|
-
const fallbackDecision = leaseFallbackDecision({
|
|
105
|
-
candidates: prepared.routing.candidateResolution.candidates,
|
|
106
|
-
decision,
|
|
107
|
-
credentials: prepared.credentials,
|
|
108
|
-
});
|
|
109
|
-
if (fallbackDecision) prepared.reporter.attachFallbackDecision(fallbackDecision);
|
|
110
|
-
return fallbackDecision ? { decision, fallbackDecision } : { decision };
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
export function leaseFallbackDecision(args: {
|
|
114
|
-
candidates: PreparedRoutedCompletion["routing"]["candidateResolution"]["candidates"];
|
|
115
|
-
decision: RoutingDecision;
|
|
116
|
-
credentials: PreparedRoutedCompletion["credentials"];
|
|
117
|
-
}): RoutingDecision | undefined {
|
|
118
|
-
const fallback = deriveFallbackDecision({
|
|
119
|
-
candidates: args.candidates,
|
|
120
|
-
selected: {
|
|
121
|
-
model: args.decision.selectedModel,
|
|
122
|
-
reasoningEffort: args.decision.reasoningEffort,
|
|
123
|
-
},
|
|
124
|
-
enabled: args.credentials.model_fallback_enabled !== false,
|
|
125
|
-
requiresDifferentProvider:
|
|
126
|
-
args.credentials.fallback_requires_different_provider !== false,
|
|
127
|
-
providerForModel: (model) => executionProviderForModel(args.credentials, model),
|
|
128
|
-
});
|
|
129
|
-
if (!fallback || args.decision.leaseTurnsRemaining === undefined) {
|
|
130
|
-
return fallback;
|
|
131
|
-
}
|
|
132
|
-
return {
|
|
133
|
-
...fallback,
|
|
134
|
-
leaseTurnsRemaining: args.decision.leaseTurnsRemaining,
|
|
135
|
-
};
|
|
81
|
+
return { decision: lease.decision };
|
|
136
82
|
}
|
|
137
83
|
|
|
138
84
|
export async function selectorNegotiatesLeases(
|
|
@@ -220,7 +166,7 @@ export async function clearPendingLease(
|
|
|
220
166
|
);
|
|
221
167
|
}
|
|
222
168
|
|
|
223
|
-
function canServeCommittedPair(
|
|
169
|
+
export function canServeCommittedPair(
|
|
224
170
|
prepared: PreparedRoutedCompletion,
|
|
225
171
|
pair: { model: string; reasoningEffort: string },
|
|
226
172
|
): boolean {
|
|
@@ -46,7 +46,6 @@ export type ReportableProviderOutcome =
|
|
|
46
46
|
export type ProviderAttemptReport = {
|
|
47
47
|
model: string;
|
|
48
48
|
reasoning_effort: string;
|
|
49
|
-
role: "primary" | "fallback";
|
|
50
49
|
attempt: number;
|
|
51
50
|
outcome: "completed" | "error" | "terminal_error" | "canceled";
|
|
52
51
|
};
|
|
@@ -65,7 +64,6 @@ export type RoutedRuntimeReporter = {
|
|
|
65
64
|
attachSelectorRun(run: SelectorRun): void;
|
|
66
65
|
attachSpeculativeSelectorRun(run: SelectorRun, servingDecision: RoutingDecision): void;
|
|
67
66
|
attachDecision(decision: RoutingDecision): void;
|
|
68
|
-
attachFallbackDecision(decision: RoutingDecision | undefined): void;
|
|
69
67
|
attachServingDecision(decision: RoutingDecision): void;
|
|
70
68
|
recordProviderAttempt(attempt: ProviderAttemptRecord): void;
|
|
71
69
|
modelCost(attempt?: ProviderAttemptRecord): number | null;
|
|
@@ -94,7 +92,6 @@ export function createExperimentRoutedRuntimeReporter(): ExperimentRoutedRuntime
|
|
|
94
92
|
attachSelectorRun() {},
|
|
95
93
|
attachSpeculativeSelectorRun() {},
|
|
96
94
|
attachDecision() {},
|
|
97
|
-
attachFallbackDecision() {},
|
|
98
95
|
attachServingDecision() {},
|
|
99
96
|
recordProviderAttempt() {},
|
|
100
97
|
modelCost() { return null; },
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { RoutingDecision } from "@mupt-ai/dari-router/policy-engine";
|
|
2
|
+
|
|
3
|
+
import { gatewayPlatform } from "../platform/platform.js";
|
|
4
|
+
import type { RoutedDecision } from "../strategies/route_decision.js";
|
|
5
|
+
import type { ServedDecision } from "../types.js";
|
|
6
|
+
import type { PreparedRoutedCompletion } from "./prepare_routed_completion.js";
|
|
7
|
+
import { canServeCommittedPair } from "./routing_lease.js";
|
|
8
|
+
|
|
9
|
+
// The session's pinned pair, while this request may still serve it. A pin
|
|
10
|
+
// that can no longer serve (model disabled, credits exhausted, image input on
|
|
11
|
+
// a text-only model) falls through to selection, which moves the pin.
|
|
12
|
+
export function sessionPinHoldDecision(
|
|
13
|
+
prepared: PreparedRoutedCompletion,
|
|
14
|
+
): ServedDecision | null {
|
|
15
|
+
const pin = prepared.sessionPin?.pin;
|
|
16
|
+
if (pin === null || pin === undefined) return null;
|
|
17
|
+
if (!canServeCommittedPair(prepared, pin)) return null;
|
|
18
|
+
return {
|
|
19
|
+
selectedModel: pin.model,
|
|
20
|
+
reasoningEffort: pin.reasoningEffort,
|
|
21
|
+
reason: `Session pin: ${pin.model}/${pin.reasoningEffort} serves this whole session.`,
|
|
22
|
+
sessionPinned: true,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// A pinned session never holds leases: the pin outlives any countdown.
|
|
27
|
+
export function sessionPinnedSelection(routed: RoutedDecision): RoutedDecision {
|
|
28
|
+
const pinned = (decision: RoutingDecision): ServedDecision => ({ ...unleased(decision), sessionPinned: true });
|
|
29
|
+
return {
|
|
30
|
+
...routed,
|
|
31
|
+
decision: pinned(routed.decision),
|
|
32
|
+
...(routed.run === undefined ? {} : { run: { ...routed.run, decision: pinned(routed.run.decision) } }),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function unleased(decision: RoutingDecision): RoutingDecision {
|
|
37
|
+
const { leaseTurnsRemaining: _, ...rest } = decision;
|
|
38
|
+
return rest;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// A failed write only costs the pin: the next turn selects again.
|
|
42
|
+
export async function pinSession(
|
|
43
|
+
prepared: PreparedRoutedCompletion,
|
|
44
|
+
decision: RoutingDecision,
|
|
45
|
+
): Promise<void> {
|
|
46
|
+
if (prepared.sessionPin === undefined) return;
|
|
47
|
+
const pin = { model: decision.selectedModel, reasoningEffort: decision.reasoningEffort };
|
|
48
|
+
try {
|
|
49
|
+
await gatewayPlatform().writeSessionPin({
|
|
50
|
+
routerId: prepared.routerId,
|
|
51
|
+
pinKey: prepared.sessionPin.key,
|
|
52
|
+
...pin,
|
|
53
|
+
});
|
|
54
|
+
} catch (error) {
|
|
55
|
+
gatewayPlatform().captureOperationalError(error, {
|
|
56
|
+
code: "session_pin_write_failed",
|
|
57
|
+
requestId: prepared.requestId,
|
|
58
|
+
routerId: prepared.routerId,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
package/src/self_host/config.ts
CHANGED
|
@@ -14,8 +14,8 @@ import {
|
|
|
14
14
|
* - anthropic/claude-sonnet-5-5
|
|
15
15
|
* - id: openai/gpt-5.5
|
|
16
16
|
* thinking: [medium, high]
|
|
17
|
-
* fallback: true # retry on the next model when a provider fails
|
|
18
17
|
* subscriptions: true # serve with connected Claude Code / ChatGPT plans
|
|
18
|
+
* sessionPinning: false # route each session and subagent once, then keep its model
|
|
19
19
|
* providers: # optional: OpenAI-compatible or self-run endpoints
|
|
20
20
|
* local:
|
|
21
21
|
* baseUrl: http://localhost:8000/v1
|
|
@@ -44,10 +44,9 @@ export type ProviderEndpointConfig = {
|
|
|
44
44
|
|
|
45
45
|
export type RouterConfig = {
|
|
46
46
|
models: RouterModelConfig[];
|
|
47
|
-
fallback: boolean;
|
|
48
|
-
fallbackRequiresDifferentProvider: boolean;
|
|
49
47
|
retries: number;
|
|
50
48
|
subscriptions: boolean;
|
|
49
|
+
sessionPinning: boolean;
|
|
51
50
|
zdr: boolean;
|
|
52
51
|
};
|
|
53
52
|
|
|
@@ -66,7 +65,10 @@ const MODEL_ID_PATTERN = /^[a-z0-9][a-z0-9.-]*\/\S+$/;
|
|
|
66
65
|
export async function loadGatewayConfig(path: string): Promise<GatewayConfig> {
|
|
67
66
|
const file = Bun.file(path);
|
|
68
67
|
if (!(await file.exists())) {
|
|
69
|
-
throw new Error(
|
|
68
|
+
throw new Error(
|
|
69
|
+
`Gateway config not found at ${path}. Write one from dari-gateway.example.yaml, `
|
|
70
|
+
+ "or set DARI_GATEWAY_CONFIG (in the container image, mount it at /config/dari-gateway.yaml).",
|
|
71
|
+
);
|
|
70
72
|
}
|
|
71
73
|
const text = await file.text();
|
|
72
74
|
const raw: unknown = path.endsWith(".json") ? JSON.parse(text) : Bun.YAML.parse(text);
|
|
@@ -129,11 +131,9 @@ function parseRouter(id: string, raw: unknown): RouterConfig {
|
|
|
129
131
|
}
|
|
130
132
|
return {
|
|
131
133
|
models,
|
|
132
|
-
fallback: optionalBoolean(raw.fallback, `routers.${id}.fallback`) ?? true,
|
|
133
|
-
fallbackRequiresDifferentProvider:
|
|
134
|
-
optionalBoolean(raw.fallbackRequiresDifferentProvider, `routers.${id}.fallbackRequiresDifferentProvider`) ?? false,
|
|
135
134
|
retries: optionalRetries(raw.retries, `routers.${id}.retries`) ?? 1,
|
|
136
135
|
subscriptions: optionalBoolean(raw.subscriptions, `routers.${id}.subscriptions`) ?? true,
|
|
136
|
+
sessionPinning: optionalBoolean(raw.sessionPinning, `routers.${id}.sessionPinning`) ?? false,
|
|
137
137
|
zdr: optionalBoolean(raw.zdr, `routers.${id}.zdr`) ?? false,
|
|
138
138
|
};
|
|
139
139
|
}
|
package/src/self_host/main.ts
CHANGED
|
@@ -12,8 +12,9 @@ import { createPrioritySelectorBackend } from "../strategies/priority_selector.j
|
|
|
12
12
|
import type { SelectorBackend } from "../strategies/routing.js";
|
|
13
13
|
import { loadGatewayConfig } from "./config.js";
|
|
14
14
|
import { createSelfHostedPlatform } from "./platform.js";
|
|
15
|
-
import { gatewayKeyIndex } from "./resolve.js";
|
|
15
|
+
import { gatewayKeyIndex, providersWithoutCredentials } from "./resolve.js";
|
|
16
16
|
import { PrefixStore } from "./prefix_store.js";
|
|
17
|
+
import { SessionPinStore } from "./session_pin_store.js";
|
|
17
18
|
import { handleSubscriptionRequest, SubscriptionStore } from "./subscriptions/index.js";
|
|
18
19
|
|
|
19
20
|
const SUBSCRIPTION_ROUTES = "/v1/organizations/current/credentials";
|
|
@@ -27,7 +28,8 @@ const SUBSCRIPTION_ROUTES = "/v1/organizations/current/credentials";
|
|
|
27
28
|
* state (default `dari-gateway.db`). It holds subscription tokens.
|
|
28
29
|
* - `DARI_AUTO_ROUTING_API_KEY`: choose among a router's models with Dari
|
|
29
30
|
* Auto Routing; without it, `dari/routing` serves the first listed model.
|
|
30
|
-
* - `PORT` (default 8080),
|
|
31
|
+
* - `PORT` (default 8080), `DARI_GATEWAY_HOST` (default 127.0.0.1; the container
|
|
32
|
+
* image sets 0.0.0.0), and each provider's standard API key variable.
|
|
31
33
|
*/
|
|
32
34
|
export async function startSelfHostedGateway(env: Record<string, string> = process.env as Record<string, string>) {
|
|
33
35
|
const gatewayKeys = (env.DARI_GATEWAY_KEYS ?? "")
|
|
@@ -45,21 +47,35 @@ export async function startSelfHostedGateway(env: Record<string, string> = proce
|
|
|
45
47
|
db.exec("PRAGMA journal_mode = WAL");
|
|
46
48
|
const subscriptions = new SubscriptionStore(db);
|
|
47
49
|
const prefixes = new PrefixStore(db);
|
|
50
|
+
const sessionPins = new SessionPinStore(db);
|
|
48
51
|
// Pi's provider auth reads each provider's standard key variable, such as
|
|
49
52
|
// OPENAI_API_KEY, from this environment only.
|
|
50
53
|
const piModels = builtinModels({
|
|
51
54
|
authContext: { env: async (name) => env[name], fileExists: async () => false },
|
|
52
55
|
});
|
|
53
56
|
|
|
57
|
+
const providerApiKey = async (provider: string) => (await piModels.getAuth(provider))?.auth.apiKey;
|
|
54
58
|
const platform = createSelfHostedPlatform({
|
|
55
59
|
config,
|
|
56
60
|
gatewayKeys,
|
|
57
61
|
subscriptions,
|
|
58
62
|
prefixes,
|
|
59
|
-
|
|
63
|
+
sessionPins,
|
|
64
|
+
providerApiKey,
|
|
60
65
|
env,
|
|
61
66
|
fetch,
|
|
62
67
|
});
|
|
68
|
+
// A model without a credential is dropped at request time, so a missing key
|
|
69
|
+
// would otherwise surface only as a 424 on the first request.
|
|
70
|
+
for (const [routerId, router] of Object.entries(config.routers)) {
|
|
71
|
+
const missing = await providersWithoutCredentials({ config, subscriptions, providerApiKey, env }, router);
|
|
72
|
+
if (missing.length > 0) {
|
|
73
|
+
console.warn(
|
|
74
|
+
"dari-gateway router has providers without credentials; their models are not served",
|
|
75
|
+
JSON.stringify({ router: routerId, providers: missing }),
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
63
79
|
const autoRoutingKey = env.DARI_AUTO_ROUTING_API_KEY?.trim();
|
|
64
80
|
const selectorBackend: SelectorBackend = autoRoutingKey
|
|
65
81
|
? createAutoRoutingSelectorBackend({
|
|
@@ -86,10 +102,13 @@ export async function startSelfHostedGateway(env: Record<string, string> = proce
|
|
|
86
102
|
};
|
|
87
103
|
|
|
88
104
|
const port = Number(env.PORT ?? "8080");
|
|
89
|
-
|
|
105
|
+
// Loopback unless DARI_GATEWAY_HOST says otherwise (see the doc comment).
|
|
106
|
+
const hostname = (env.DARI_GATEWAY_HOST ?? "").trim() || "127.0.0.1";
|
|
107
|
+
const server = startServer(port, handler, { hostname });
|
|
90
108
|
console.info(
|
|
91
109
|
"dari-gateway listening",
|
|
92
110
|
JSON.stringify({
|
|
111
|
+
hostname,
|
|
93
112
|
port,
|
|
94
113
|
routers: Object.keys(config.routers),
|
|
95
114
|
auto_routing: autoRoutingKey ? "dari" : "priority",
|
|
@@ -99,6 +118,10 @@ export async function startSelfHostedGateway(env: Record<string, string> = proce
|
|
|
99
118
|
}
|
|
100
119
|
|
|
101
120
|
if (import.meta.main) {
|
|
102
|
-
|
|
103
|
-
|
|
121
|
+
// Startup errors are configuration mistakes: print the message, not a stack.
|
|
122
|
+
const started = await startSelfHostedGateway().catch((error: unknown) => {
|
|
123
|
+
console.error(`dari-gateway: ${error instanceof Error ? error.message : String(error)}`);
|
|
124
|
+
process.exit(1);
|
|
125
|
+
});
|
|
126
|
+
installShutdownHandlers(started.server, async () => started.close());
|
|
104
127
|
}
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
import { errorCode } from "../http/errors.js";
|
|
2
2
|
import type { GatewayPlatform } from "../platform/platform.js";
|
|
3
3
|
import type { PrefixStore } from "./prefix_store.js";
|
|
4
|
+
import type { SessionPinStore } from "./session_pin_store.js";
|
|
4
5
|
import { resolveSelfHostedRouter, type SelfHostedResolverDeps } from "./resolve.js";
|
|
5
6
|
import type { SubscriptionStore } from "./subscriptions/index.js";
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
|
-
* The self-hosted platform: routers from the config file, subscriptions
|
|
9
|
-
* prefix-cache state in the gateway's SQLite database, and one content-free
|
|
9
|
+
* The self-hosted platform: routers from the config file, subscriptions,
|
|
10
|
+
* prefix-cache state, and session pins in the gateway's SQLite database, and one content-free
|
|
10
11
|
* log line per request.
|
|
11
12
|
*/
|
|
12
13
|
export function createSelfHostedPlatform(deps: Omit<SelfHostedResolverDeps, "prefixHits"> & {
|
|
13
14
|
prefixes: PrefixStore;
|
|
15
|
+
sessionPins: SessionPinStore;
|
|
14
16
|
subscriptions: SubscriptionStore;
|
|
15
17
|
}): GatewayPlatform {
|
|
16
18
|
const resolver: SelfHostedResolverDeps = {
|
|
@@ -25,6 +27,10 @@ export function createSelfHostedPlatform(deps: Omit<SelfHostedResolverDeps, "pre
|
|
|
25
27
|
writePendingLease: async (write) => {
|
|
26
28
|
deps.prefixes.writePendingLease(write, deps.config.routers[write.router_id]?.zdr ?? false);
|
|
27
29
|
},
|
|
30
|
+
readSessionPin: async (key) => deps.sessionPins.read(key),
|
|
31
|
+
writeSessionPin: async (write) => {
|
|
32
|
+
deps.sessionPins.write(write);
|
|
33
|
+
},
|
|
28
34
|
markSubscriptionUnavailable: async (hold) =>
|
|
29
35
|
hold.credentialId !== undefined &&
|
|
30
36
|
deps.subscriptions.markUnavailable(hold.credentialId, hold.limit.resetsAt, hold.limit.reason),
|
package/src/self_host/resolve.ts
CHANGED
|
@@ -58,7 +58,7 @@ export async function resolveSelfHostedRouter(
|
|
|
58
58
|
): Promise<ResolvedRouter> {
|
|
59
59
|
const keyIndex = gatewayKeyIndex(deps.gatewayKeys, request.apiKey);
|
|
60
60
|
if (keyIndex === null) {
|
|
61
|
-
throw new HttpError(401, "Missing or invalid
|
|
61
|
+
throw new HttpError(401, "Missing or invalid gateway key.", "authentication_error");
|
|
62
62
|
}
|
|
63
63
|
const router = deps.config.routers[request.routerId];
|
|
64
64
|
if (router === undefined) {
|
|
@@ -91,7 +91,10 @@ export async function resolveSelfHostedRouter(
|
|
|
91
91
|
}
|
|
92
92
|
|
|
93
93
|
const models = router.models.filter((model) => credentials.get(providerForModel(model.id)) != null);
|
|
94
|
-
if (models.length === 0)
|
|
94
|
+
if (models.length === 0) {
|
|
95
|
+
const missing = [...credentials].filter(([, credential]) => credential == null).map(([provider]) => provider);
|
|
96
|
+
throw unavailableRouterError(request.routerId, exhausted, missing);
|
|
97
|
+
}
|
|
95
98
|
|
|
96
99
|
const resolved: ResolvedRouter = {
|
|
97
100
|
router_id: request.routerId,
|
|
@@ -104,9 +107,8 @@ export async function resolveSelfHostedRouter(
|
|
|
104
107
|
subscription_credential_ids: {},
|
|
105
108
|
zdr: router.zdr,
|
|
106
109
|
speculative_routing: false,
|
|
110
|
+
session_pinning: router.sessionPinning,
|
|
107
111
|
primary_retries: router.retries,
|
|
108
|
-
model_fallback_enabled: router.fallback,
|
|
109
|
-
fallback_requires_different_provider: router.fallbackRequiresDifferentProvider,
|
|
110
112
|
lease_context_enabled: false,
|
|
111
113
|
routing_strategy: "dari",
|
|
112
114
|
custom_config: null,
|
|
@@ -152,7 +154,10 @@ export async function resolveSelfHostedRouter(
|
|
|
152
154
|
|
|
153
155
|
// Configured endpoints read their own key variable, or send a placeholder
|
|
154
156
|
// when they take none; known providers use their standard variable.
|
|
155
|
-
async function providerApiKey(
|
|
157
|
+
async function providerApiKey(
|
|
158
|
+
deps: Pick<SelfHostedResolverDeps, "config" | "providerApiKey" | "env">,
|
|
159
|
+
provider: string,
|
|
160
|
+
): Promise<string | undefined> {
|
|
156
161
|
const endpoint = deps.config.providers[provider];
|
|
157
162
|
if (endpoint === undefined) return (await deps.providerApiKey(provider))?.trim() || undefined;
|
|
158
163
|
if (endpoint.apiKeyEnv === undefined) return "no-key";
|
|
@@ -204,23 +209,48 @@ const SUBSCRIPTION_PLAN_NAMES: Record<SubscriptionProvider, string> = {
|
|
|
204
209
|
meta_muse: "Muse Code",
|
|
205
210
|
};
|
|
206
211
|
|
|
212
|
+
/**
|
|
213
|
+
* Providers a router lists that no credential can serve: no API key and no
|
|
214
|
+
* connected subscription the router allows. Startup uses this, so it counts a
|
|
215
|
+
* held subscription as connected (the request-time 424 reports holds) and
|
|
216
|
+
* checks both harnesses, since startup does not know the client.
|
|
217
|
+
*/
|
|
218
|
+
export async function providersWithoutCredentials(
|
|
219
|
+
deps: Pick<SelfHostedResolverDeps, "subscriptions" | "providerApiKey" | "config" | "env">,
|
|
220
|
+
router: RouterConfig,
|
|
221
|
+
): Promise<string[]> {
|
|
222
|
+
const missing: string[] = [];
|
|
223
|
+
for (const provider of new Set(router.models.map((model) => providerForModel(model.id)))) {
|
|
224
|
+
const plans = router.subscriptions
|
|
225
|
+
? [subscriptionProviderFor(provider, "claude_code"), subscriptionProviderFor(provider, "muse_code")]
|
|
226
|
+
: [];
|
|
227
|
+
const connected = plans.some((plan) => plan !== null && deps.subscriptions.list(plan).length > 0);
|
|
228
|
+
if (connected) continue;
|
|
229
|
+
const apiKey = await providerApiKey(deps, provider);
|
|
230
|
+
if (apiKey === undefined) missing.push(provider);
|
|
231
|
+
}
|
|
232
|
+
return missing;
|
|
233
|
+
}
|
|
234
|
+
|
|
207
235
|
function unavailableRouterError(
|
|
208
236
|
routerId: string,
|
|
209
237
|
exhausted: ReadonlySet<SubscriptionProvider>,
|
|
238
|
+
missing: readonly string[],
|
|
210
239
|
): HttpError {
|
|
240
|
+
const missingNote = missing.length > 0 ? ` Missing providers: ${missing.join(", ")}.` : "";
|
|
211
241
|
if (exhausted.size > 0) {
|
|
212
242
|
const plans = [...exhausted]
|
|
213
243
|
.map((provider) => SUBSCRIPTION_PLAN_NAMES[provider])
|
|
214
244
|
.join(" and ");
|
|
215
245
|
return new HttpError(
|
|
216
246
|
424,
|
|
217
|
-
`Every connected ${plans} subscription is at its usage limit, and no API key can serve router ${routerId} instead
|
|
247
|
+
`Every connected ${plans} subscription is at its usage limit, and no API key can serve router ${routerId} instead.${missingNote}`,
|
|
218
248
|
"personal_subscription_unavailable",
|
|
219
249
|
);
|
|
220
250
|
}
|
|
221
251
|
return new HttpError(
|
|
222
252
|
424,
|
|
223
|
-
`No credentials can serve router ${routerId}
|
|
253
|
+
`No credentials can serve router ${routerId}.${missingNote} Set each provider's API key or connect a subscription that can serve it.`,
|
|
224
254
|
"provider_credentials_missing",
|
|
225
255
|
);
|
|
226
256
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite storage for session pins: the model and thinking level each session
|
|
3
|
+
* (and each subagent) of a session-pinning router was routed to on its first
|
|
4
|
+
* turn. Serving a pin refreshes it at most daily; pins unused for longer than
|
|
5
|
+
* the retention window are no longer served and are deleted.
|
|
6
|
+
*/
|
|
7
|
+
import type { Database } from "bun:sqlite";
|
|
8
|
+
|
|
9
|
+
import type { SessionPin, SessionPinKey } from "../platform/platform.js";
|
|
10
|
+
|
|
11
|
+
const RETENTION_MS = 30 * 24 * 60 * 60_000;
|
|
12
|
+
const REFRESH_MS = 24 * 60 * 60_000;
|
|
13
|
+
|
|
14
|
+
type PinRow = { model: string; reasoning_effort: SessionPin["reasoningEffort"]; updated_at: number };
|
|
15
|
+
|
|
16
|
+
export class SessionPinStore {
|
|
17
|
+
constructor(private readonly db: Database) {
|
|
18
|
+
db.run(`CREATE TABLE IF NOT EXISTS routing_session_pins (
|
|
19
|
+
router_id TEXT NOT NULL,
|
|
20
|
+
pin_key TEXT NOT NULL,
|
|
21
|
+
model TEXT NOT NULL,
|
|
22
|
+
reasoning_effort TEXT NOT NULL,
|
|
23
|
+
updated_at INTEGER NOT NULL,
|
|
24
|
+
PRIMARY KEY (router_id, pin_key)
|
|
25
|
+
)`);
|
|
26
|
+
db.run("CREATE INDEX IF NOT EXISTS ix_routing_session_pins_updated_at ON routing_session_pins (updated_at)");
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
read(key: SessionPinKey, now: Date = new Date()): SessionPin | null {
|
|
30
|
+
const row = this.db
|
|
31
|
+
.query<PinRow, [string, string, number]>(
|
|
32
|
+
`SELECT model, reasoning_effort, updated_at FROM routing_session_pins
|
|
33
|
+
WHERE router_id = ? AND pin_key = ? AND updated_at >= ?`,
|
|
34
|
+
)
|
|
35
|
+
.get(key.routerId, key.pinKey, now.getTime() - RETENTION_MS);
|
|
36
|
+
if (row === null) return null;
|
|
37
|
+
if (row.updated_at < now.getTime() - REFRESH_MS) {
|
|
38
|
+
this.db
|
|
39
|
+
.query("UPDATE routing_session_pins SET updated_at = ? WHERE router_id = ? AND pin_key = ?")
|
|
40
|
+
.run(now.getTime(), key.routerId, key.pinKey);
|
|
41
|
+
}
|
|
42
|
+
return { model: row.model, reasoningEffort: row.reasoning_effort };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
write(write: SessionPinKey & SessionPin, now: Date = new Date()): void {
|
|
46
|
+
this.db.transaction(() => {
|
|
47
|
+
this.db
|
|
48
|
+
.query(`INSERT INTO routing_session_pins (router_id, pin_key, model, reasoning_effort, updated_at)
|
|
49
|
+
VALUES (?, ?, ?, ?, ?)
|
|
50
|
+
ON CONFLICT (router_id, pin_key) DO UPDATE SET
|
|
51
|
+
model = excluded.model,
|
|
52
|
+
reasoning_effort = excluded.reasoning_effort,
|
|
53
|
+
updated_at = excluded.updated_at`)
|
|
54
|
+
.run(write.routerId, write.pinKey, write.model, write.reasoningEffort, now.getTime());
|
|
55
|
+
this.db.query("DELETE FROM routing_session_pins WHERE updated_at < ?").run(now.getTime() - RETENTION_MS);
|
|
56
|
+
})();
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -7,8 +7,7 @@ const PRIORITY_CONTEXT_WINDOW_CHARS = 1_000_000;
|
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Serve the router's first eligible model/thinking-level pair, in the order
|
|
10
|
-
* the router lists them.
|
|
11
|
-
* when the router enables model fallback.
|
|
10
|
+
* the router lists them.
|
|
12
11
|
*/
|
|
13
12
|
export function createPrioritySelectorBackend(): SelectorBackend {
|
|
14
13
|
return {
|
|
@@ -15,7 +15,6 @@ import type { ResolvedRouter } from "../types.js";
|
|
|
15
15
|
|
|
16
16
|
export type RoutedDecision = {
|
|
17
17
|
decision: RoutingDecision;
|
|
18
|
-
fallbackDecision?: RoutingDecision;
|
|
19
18
|
run?: SelectorRun;
|
|
20
19
|
details?: Record<string, unknown>;
|
|
21
20
|
};
|
|
@@ -81,9 +80,6 @@ export async function routeDecision(
|
|
|
81
80
|
? { customRules: custom.rules, defaultTarget: custom.defaultTarget }
|
|
82
81
|
: {}),
|
|
83
82
|
selectorBackend: args.selectorBackend,
|
|
84
|
-
modelFallbackEnabled: credentials.model_fallback_enabled !== false,
|
|
85
|
-
fallbackRequiresDifferentProvider:
|
|
86
|
-
credentials.fallback_requires_different_provider !== false,
|
|
87
83
|
executionProviderForModel: (model) =>
|
|
88
84
|
executionProviderForModel(credentials, model),
|
|
89
85
|
imputeEvalScores: credentials.eval_score_imputation === true,
|
|
@@ -99,7 +95,6 @@ export async function routeDecision(
|
|
|
99
95
|
};
|
|
100
96
|
return {
|
|
101
97
|
decision: run.decision,
|
|
102
|
-
fallbackDecision: run.fallbackDecision,
|
|
103
98
|
run,
|
|
104
99
|
...(Object.keys(details).length > 0 ? { details } : {}),
|
|
105
100
|
};
|
|
@@ -34,7 +34,6 @@ import type {
|
|
|
34
34
|
ProviderAuth,
|
|
35
35
|
ResolvedRouter,
|
|
36
36
|
} from "../types.js";
|
|
37
|
-
import { deriveFallbackDecision } from "./fallback_decision.js";
|
|
38
37
|
import type { SelectorImage } from "./selector_projection.js";
|
|
39
38
|
|
|
40
39
|
export function resolveRouter(args: ResolveRouterRequest): Promise<ResolvedRouter> {
|
|
@@ -54,7 +53,6 @@ export type SelectorTrace = {
|
|
|
54
53
|
|
|
55
54
|
export type SelectorRun = SelectorTrace & {
|
|
56
55
|
decision: RoutingDecision;
|
|
57
|
-
fallbackDecision?: RoutingDecision;
|
|
58
56
|
};
|
|
59
57
|
|
|
60
58
|
export type SelectModelArgs = {
|
|
@@ -70,8 +68,6 @@ export type SelectModelArgs = {
|
|
|
70
68
|
customRules?: CustomRouterRule[];
|
|
71
69
|
defaultTarget?: CustomDefaultTarget | null;
|
|
72
70
|
sessionId?: string;
|
|
73
|
-
modelFallbackEnabled?: boolean;
|
|
74
|
-
fallbackRequiresDifferentProvider?: boolean;
|
|
75
71
|
executionProviderForModel?: (model: string) => string;
|
|
76
72
|
imputeEvalScores?: boolean;
|
|
77
73
|
imputationReferenceEvals?: RouterEval[];
|
|
@@ -171,8 +167,6 @@ export async function selectModel(
|
|
|
171
167
|
contextWindowChars,
|
|
172
168
|
messages: selectorBackend.prepareMessages?.(safeMessages) ?? safeMessages,
|
|
173
169
|
...(leaseContextEnabled ? { leaseHistory: args.leaseHistory ?? [] } : {}),
|
|
174
|
-
modelFallbackEnabled: args.modelFallbackEnabled,
|
|
175
|
-
fallbackRequiresDifferentProvider: args.fallbackRequiresDifferentProvider,
|
|
176
170
|
imputeEvalScores: args.imputeEvalScores,
|
|
177
171
|
imputationReferenceEvals: args.imputationReferenceEvals,
|
|
178
172
|
});
|
|
@@ -285,17 +279,7 @@ function validateSelectorRun(run: SelectorRun, args: SelectModelArgs): SelectorR
|
|
|
285
279
|
run,
|
|
286
280
|
);
|
|
287
281
|
}
|
|
288
|
-
|
|
289
|
-
const fallbackDecision = deriveFallbackDecision({
|
|
290
|
-
candidates: args.candidates,
|
|
291
|
-
selected,
|
|
292
|
-
enabled: args.modelFallbackEnabled,
|
|
293
|
-
requiresDifferentProvider: args.fallbackRequiresDifferentProvider,
|
|
294
|
-
providerForModel: args.executionProviderForModel,
|
|
295
|
-
});
|
|
296
|
-
return fallbackDecision
|
|
297
|
-
? { ...primaryRun, fallbackDecision }
|
|
298
|
-
: primaryRun;
|
|
282
|
+
return run;
|
|
299
283
|
}
|
|
300
284
|
|
|
301
285
|
function requiredSelectorConfig(value: string, label: string): string {
|
|
@@ -14,7 +14,9 @@ export type RoutingBillingProviderAttemptFacts = {
|
|
|
14
14
|
provider: string;
|
|
15
15
|
model: string;
|
|
16
16
|
provider_key_source: ProviderKeySource;
|
|
17
|
-
|
|
17
|
+
// The control-plane billing contract numbers attempts within a role; every
|
|
18
|
+
// attempt retries the routed choice, so the role is always primary.
|
|
19
|
+
role: "primary";
|
|
18
20
|
attempt: number;
|
|
19
21
|
outcome: "completed" | "provider_failed" | "customer_canceled";
|
|
20
22
|
provider_request_id: string | null;
|
|
@@ -94,7 +96,6 @@ export function routingBillingProviderAttemptFacts(args: {
|
|
|
94
96
|
provider: string;
|
|
95
97
|
model: string;
|
|
96
98
|
providerKeySource: ProviderKeySource;
|
|
97
|
-
role: "primary" | "fallback";
|
|
98
99
|
attempt: number;
|
|
99
100
|
outcome: "completed" | "provider_failed" | "customer_canceled";
|
|
100
101
|
providerMessage: Record<string, unknown> | null;
|
|
@@ -129,7 +130,7 @@ export function routingBillingProviderAttemptFacts(args: {
|
|
|
129
130
|
provider: args.provider,
|
|
130
131
|
model: args.model,
|
|
131
132
|
provider_key_source: args.providerKeySource,
|
|
132
|
-
role:
|
|
133
|
+
role: "primary",
|
|
133
134
|
attempt: args.attempt,
|
|
134
135
|
outcome: args.outcome,
|
|
135
136
|
provider_request_id: providerRequestId(args.providerMessage),
|
|
@@ -80,15 +80,10 @@ export function buildCompletedPrefixHit(args: {
|
|
|
80
80
|
export function leaseCountdownForPrefix(args: {
|
|
81
81
|
served: RoutingDecision | null;
|
|
82
82
|
next: RoutingDecision | null;
|
|
83
|
-
// True when a completed fallback attempt served this turn. A lease-aware
|
|
84
|
-
// fallback carries the remaining countdown on the serving decision; an
|
|
85
|
-
// ordinary fallback has no countdown and ends any primary commitment.
|
|
86
|
-
fallbackServed?: boolean;
|
|
87
83
|
}): number | null {
|
|
88
|
-
const granted = args.
|
|
84
|
+
const granted = args.next ?? args.served;
|
|
89
85
|
const remaining = granted?.leaseTurnsRemaining;
|
|
90
86
|
if (remaining === undefined || remaining === null) return null;
|
|
91
|
-
if (args.fallbackServed === true) return remaining;
|
|
92
87
|
const servedGrantedPair =
|
|
93
88
|
args.next === null ||
|
|
94
89
|
(args.served !== null &&
|