@cloudpeers-jkl/model-router 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/dist/adapters/anthropic.d.ts +7 -0
- package/dist/adapters/anthropic.js +121 -0
- package/dist/adapters/gemini.d.ts +40 -0
- package/dist/adapters/gemini.js +151 -0
- package/dist/adapters/selfhosted.d.ts +14 -0
- package/dist/adapters/selfhosted.js +96 -0
- package/dist/capabilities.d.ts +18 -0
- package/dist/capabilities.js +14 -0
- package/dist/errors.d.ts +23 -0
- package/dist/errors.js +32 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.js +22 -0
- package/dist/policy.d.ts +35 -0
- package/dist/policy.js +48 -0
- package/dist/router.d.ts +36 -0
- package/dist/router.js +328 -0
- package/dist/schema.d.ts +335 -0
- package/dist/schema.js +68 -0
- package/dist/types.d.ts +256 -0
- package/dist/types.js +16 -0
- package/package.json +33 -0
package/README.md
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# @cloudpeers-jkl/model-router
|
|
2
|
+
|
|
3
|
+
cloudpeers inference router — **embedded library** (Amendment A1.1: no
|
|
4
|
+
central data plane). Each service embeds the router behind its existing
|
|
5
|
+
seam; the sovereignty privacy gate runs in-process, where the data lives.
|
|
6
|
+
Control plane (tier config, approved providers, gate stage, metering
|
|
7
|
+
aggregation) distributes from the platform as deploy-time config.
|
|
8
|
+
|
|
9
|
+
Canonical spec: `cloudpeers-github/docs/architecture/INFERENCE_ROUTER_SPEC.md`
|
|
10
|
+
(read Amendment A1 first). Pointer + contract map: `mcp/docs/INFERENCE_ROUTER_SPEC.md`.
|
|
11
|
+
|
|
12
|
+
## Two APIs, one enforcement path
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
import {
|
|
16
|
+
routeModel, routeMessages, defaultBackendAdapters,
|
|
17
|
+
} from '@cloudpeers-jkl/model-router';
|
|
18
|
+
|
|
19
|
+
const deps = {
|
|
20
|
+
adapters: defaultBackendAdapters(), // selfhosted, gemini, anthropic — no OpenAI (directive)
|
|
21
|
+
meter, // the embedding service's token meter
|
|
22
|
+
quota, // and quota check (token-tracker in mcp)
|
|
23
|
+
hooks: {}, // §7 mandate/policy/attribution — fail closed when named but unwired
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
// v1 convenience API — prompt in, text out
|
|
27
|
+
const r = await routeModel({
|
|
28
|
+
taskClass: 'coaching',
|
|
29
|
+
prompt: '…',
|
|
30
|
+
sovereigntyClass: 'local_only', // REQUIRED — the caller declares the data class
|
|
31
|
+
operation: 'lab_coaching',
|
|
32
|
+
userId, serviceId: 'experience-labs',
|
|
33
|
+
}, deps);
|
|
34
|
+
|
|
35
|
+
// Spec API — Anthropic Messages shape + cloudpeers envelope (§4)
|
|
36
|
+
const { response, usage } = await routeMessages({
|
|
37
|
+
messages: [{ role: 'user', content: '…' }],
|
|
38
|
+
max_tokens: 1000,
|
|
39
|
+
cloudpeers: { v: 1, sovereignty_class: 'aggregate_only', task_class: 'reasoning' },
|
|
40
|
+
}, { userId, serviceId, operation: 'my_op' }, deps);
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
In mcp, don't construct deps yourself — import the prewired composition from
|
|
44
|
+
`server/lib/model-router.ts`.
|
|
45
|
+
|
|
46
|
+
## Behavior contracts
|
|
47
|
+
|
|
48
|
+
- **Privacy gate stages** (interim cloud-PHI policy 2026-07-09):
|
|
49
|
+
MONITOR (default, logs `[privacy-gate-monitor]` would_block lines) →
|
|
50
|
+
BAA egress (`MODEL_ROUTER_BAA_PROVIDERS`) → enforce
|
|
51
|
+
(`MODEL_PRIVACY_GATE_ENFORCE=true`, fail closed). The monitor log lines are
|
|
52
|
+
a stable interface — the Stage-3 flip decision is made from them.
|
|
53
|
+
- **Fail closed:** `mandate_id`/`policy_id` named with no wired hook → blocked
|
|
54
|
+
(412). BYOK → 501 (deferred, A1.2). `tool_use_strict` with no capable
|
|
55
|
+
backend → 422. Errors carry `statusCode`/`code` for HTTP mapping.
|
|
56
|
+
- **Static routing** (confidence v1): task-class tier chains in `policy.ts`;
|
|
57
|
+
schema-validation failure escalates a tier; exhausted chain returns
|
|
58
|
+
best-effort marked `degraded`.
|
|
59
|
+
- **Uniform metering:** every call reports through the injected meter with a
|
|
60
|
+
`router` metadata block — provider, tier, taskClass, gate.
|
|
61
|
+
|
|
62
|
+
## Env (deploy-time control plane)
|
|
63
|
+
|
|
64
|
+
`MODEL_ROUTER_TIER0_MODEL` / `MODEL_ROUTER_TIER0_URL` (Tier 0 self-hosted),
|
|
65
|
+
`MODEL_ROUTER_GEMINI_MODEL`, `MODEL_ROUTER_ANTHROPIC_MODEL`,
|
|
66
|
+
`MODEL_ROUTER_BAA_PROVIDERS`, `MODEL_PRIVACY_GATE_ENFORCE`,
|
|
67
|
+
plus the provider keys (`ANTHROPIC_API_KEY`, `GEMINI_SERVER_API_KEY`/`GEMINI_API_KEY`/`GOOGLE_AI_API_KEY`).
|
|
68
|
+
|
|
69
|
+
## Publishing
|
|
70
|
+
|
|
71
|
+
Not yet published. Consumed in-repo via relative imports (see
|
|
72
|
+
`server/lib/model-router.ts`). To publish for external consumers
|
|
73
|
+
(carepeers seams, gemini-proxy in Deno): `npm run build && npm publish`
|
|
74
|
+
from this directory — `dist/` is what ships.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anthropic adapter — the lingua franca IS the Anthropic Messages shape, so
|
|
3
|
+
* translation is identity (§5.3: smoke test only). fetch-based, the
|
|
4
|
+
* carepeers-established pattern (no SDK dependency).
|
|
5
|
+
*/
|
|
6
|
+
import type { BackendAdapter } from '../types.js';
|
|
7
|
+
export declare function anthropicAdapter(): BackendAdapter;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
const API_URL = 'https://api.anthropic.com/v1/messages';
|
|
2
|
+
const API_VERSION = '2023-06-01';
|
|
3
|
+
const TIMEOUT_MS = 30000;
|
|
4
|
+
function apiKey() {
|
|
5
|
+
return process.env.ANTHROPIC_API_KEY ?? '';
|
|
6
|
+
}
|
|
7
|
+
function buildBody(req, stream) {
|
|
8
|
+
return JSON.stringify({
|
|
9
|
+
model: req.model,
|
|
10
|
+
max_tokens: req.max_tokens ?? 1024,
|
|
11
|
+
...(req.temperature !== undefined ? { temperature: req.temperature } : {}),
|
|
12
|
+
...(req.system ? { system: req.system } : {}),
|
|
13
|
+
...(req.tools?.length ? { tools: req.tools } : {}),
|
|
14
|
+
...(req.tool_choice ? { tool_choice: req.tool_choice } : {}),
|
|
15
|
+
messages: req.messages,
|
|
16
|
+
...(stream ? { stream: true } : {}),
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
export function anthropicAdapter() {
|
|
20
|
+
return {
|
|
21
|
+
id: 'anthropic',
|
|
22
|
+
display_name: 'Anthropic Messages API',
|
|
23
|
+
surface: 'cloud',
|
|
24
|
+
supports_streaming: true,
|
|
25
|
+
supports_tool_use: true,
|
|
26
|
+
supports_prompt_caching: true,
|
|
27
|
+
tool_use: {
|
|
28
|
+
supports_tool_use: true,
|
|
29
|
+
supports_parallel_tool_use: true,
|
|
30
|
+
preserves_block_position: true,
|
|
31
|
+
supports_streaming_tool_use: true,
|
|
32
|
+
max_tools_per_request: null,
|
|
33
|
+
},
|
|
34
|
+
available: () => !!apiKey(),
|
|
35
|
+
async isEligible(_req, _ctx) {
|
|
36
|
+
if (!apiKey())
|
|
37
|
+
return { eligible: false, reason: 'ANTHROPIC_API_KEY not configured' };
|
|
38
|
+
return { eligible: true, estimated_latency_ms: 2000, estimated_cost_cents: 0 };
|
|
39
|
+
},
|
|
40
|
+
async invoke(req, ctx) {
|
|
41
|
+
const controller = new AbortController();
|
|
42
|
+
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
|
43
|
+
ctx.signal?.addEventListener('abort', () => controller.abort(), { once: true });
|
|
44
|
+
try {
|
|
45
|
+
const res = await fetch(API_URL, {
|
|
46
|
+
method: 'POST',
|
|
47
|
+
headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey(), 'anthropic-version': API_VERSION },
|
|
48
|
+
body: buildBody(req, false),
|
|
49
|
+
signal: controller.signal,
|
|
50
|
+
});
|
|
51
|
+
if (!res.ok)
|
|
52
|
+
throw new Error(`Anthropic ${res.status}`);
|
|
53
|
+
const data = (await res.json());
|
|
54
|
+
if (!data.content?.length)
|
|
55
|
+
throw new Error('Anthropic returned empty content');
|
|
56
|
+
return data;
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
clearTimeout(timer);
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
async *invokeStream(req, ctx) {
|
|
63
|
+
const controller = new AbortController();
|
|
64
|
+
// Timeout covers time-to-first-byte only; a healthy stream is unbounded.
|
|
65
|
+
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
|
66
|
+
ctx.signal?.addEventListener('abort', () => controller.abort(), { once: true });
|
|
67
|
+
let res;
|
|
68
|
+
try {
|
|
69
|
+
res = await fetch(API_URL, {
|
|
70
|
+
method: 'POST',
|
|
71
|
+
headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey(), 'anthropic-version': API_VERSION },
|
|
72
|
+
body: buildBody(req, true),
|
|
73
|
+
signal: controller.signal,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
finally {
|
|
77
|
+
clearTimeout(timer);
|
|
78
|
+
}
|
|
79
|
+
if (!res.ok || !res.body)
|
|
80
|
+
throw new Error(`Anthropic ${res.status}`);
|
|
81
|
+
const reader = res.body.getReader();
|
|
82
|
+
const decoder = new TextDecoder();
|
|
83
|
+
let buffer = '';
|
|
84
|
+
try {
|
|
85
|
+
for (;;) {
|
|
86
|
+
const { done, value } = await reader.read();
|
|
87
|
+
if (done)
|
|
88
|
+
break;
|
|
89
|
+
buffer += decoder.decode(value, { stream: true });
|
|
90
|
+
const lines = buffer.split('\n');
|
|
91
|
+
buffer = lines.pop() ?? '';
|
|
92
|
+
for (const line of lines) {
|
|
93
|
+
if (!line.startsWith('data: '))
|
|
94
|
+
continue;
|
|
95
|
+
const payload = line.slice(6).trim();
|
|
96
|
+
if (!payload)
|
|
97
|
+
continue;
|
|
98
|
+
const event = JSON.parse(payload);
|
|
99
|
+
yield event;
|
|
100
|
+
if (event.type === 'message_stop')
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
reader.releaseLock();
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
reportUsage(_req, res) {
|
|
110
|
+
return {
|
|
111
|
+
backend_id: 'anthropic',
|
|
112
|
+
tokens_in: res.usage?.input_tokens ?? 0,
|
|
113
|
+
tokens_out: res.usage?.output_tokens ?? 0,
|
|
114
|
+
latency_ms: 0, // overwritten by the router's measured value
|
|
115
|
+
cost_cents: 0, // real cost computed by the platform meter (calculate_token_cost)
|
|
116
|
+
cache_hit: false, // v1 router does not set cache_control
|
|
117
|
+
sovereignty_class: 'externalizable', // overwritten by the router from the envelope
|
|
118
|
+
};
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gemini adapter — owns the Anthropic-shape ⇄ generativelanguage translation
|
|
3
|
+
* (§5.2). Translation helpers are exported pure so the mandatory round-trip
|
|
4
|
+
* tests (§13) run without a network mock.
|
|
5
|
+
*
|
|
6
|
+
* Tool use is declared unsupported in v1: the Anthropic tool_use ⇄ Gemini
|
|
7
|
+
* functionCall mapping is the §8 hard problem and the anthropic backend is
|
|
8
|
+
* the v1 tool path. Requests with tools skip this backend (§8.2).
|
|
9
|
+
*/
|
|
10
|
+
import { GoogleGenerativeAI } from '@google/generative-ai';
|
|
11
|
+
import type { AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicSSEEvent, BackendAdapter } from '../types.js';
|
|
12
|
+
/** Anthropic messages → Gemini `contents` (role user→user, assistant→model). */
|
|
13
|
+
export declare function anthropicToGeminiContents(req: AnthropicMessagesRequest): {
|
|
14
|
+
role: "user" | "model";
|
|
15
|
+
parts: {
|
|
16
|
+
text: string;
|
|
17
|
+
}[];
|
|
18
|
+
}[];
|
|
19
|
+
export declare function geminiFinishToStopReason(finishReason: string | undefined): string;
|
|
20
|
+
/** Gemini response fields → Anthropic Messages response shape. */
|
|
21
|
+
export declare function geminiResponseToAnthropic(fields: {
|
|
22
|
+
text: string;
|
|
23
|
+
model: string;
|
|
24
|
+
finishReason?: string;
|
|
25
|
+
promptTokens?: number;
|
|
26
|
+
completionTokens?: number;
|
|
27
|
+
}): AnthropicMessagesResponse;
|
|
28
|
+
/**
|
|
29
|
+
* Synthesize the Anthropic SSE event sequence from a Gemini chunk stream,
|
|
30
|
+
* preserving content_block_* boundaries (§5.2 streaming shape).
|
|
31
|
+
*/
|
|
32
|
+
export declare function geminiChunksToSSE(chunks: AsyncIterable<{
|
|
33
|
+
text: string;
|
|
34
|
+
promptTokens?: number;
|
|
35
|
+
completionTokens?: number;
|
|
36
|
+
finishReason?: string;
|
|
37
|
+
}>, model: string): AsyncIterable<AnthropicSSEEvent>;
|
|
38
|
+
export declare function geminiAdapter(opts?: {
|
|
39
|
+
client?: GoogleGenerativeAI;
|
|
40
|
+
}): BackendAdapter;
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gemini adapter — owns the Anthropic-shape ⇄ generativelanguage translation
|
|
3
|
+
* (§5.2). Translation helpers are exported pure so the mandatory round-trip
|
|
4
|
+
* tests (§13) run without a network mock.
|
|
5
|
+
*
|
|
6
|
+
* Tool use is declared unsupported in v1: the Anthropic tool_use ⇄ Gemini
|
|
7
|
+
* functionCall mapping is the §8 hard problem and the anthropic backend is
|
|
8
|
+
* the v1 tool path. Requests with tools skip this backend (§8.2).
|
|
9
|
+
*/
|
|
10
|
+
import { GoogleGenerativeAI } from '@google/generative-ai';
|
|
11
|
+
import { randomUUID } from 'node:crypto';
|
|
12
|
+
function blockText(content) {
|
|
13
|
+
if (typeof content === 'string')
|
|
14
|
+
return content;
|
|
15
|
+
return content
|
|
16
|
+
.filter((b) => b.type === 'text')
|
|
17
|
+
.map((b) => b.text)
|
|
18
|
+
.join('\n');
|
|
19
|
+
}
|
|
20
|
+
/** Anthropic messages → Gemini `contents` (role user→user, assistant→model). */
|
|
21
|
+
export function anthropicToGeminiContents(req) {
|
|
22
|
+
return req.messages.map((m) => ({
|
|
23
|
+
role: m.role === 'assistant' ? 'model' : 'user',
|
|
24
|
+
parts: [{ text: blockText(m.content) }],
|
|
25
|
+
}));
|
|
26
|
+
}
|
|
27
|
+
export function geminiFinishToStopReason(finishReason) {
|
|
28
|
+
if (finishReason === 'MAX_TOKENS')
|
|
29
|
+
return 'max_tokens';
|
|
30
|
+
return 'end_turn';
|
|
31
|
+
}
|
|
32
|
+
/** Gemini response fields → Anthropic Messages response shape. */
|
|
33
|
+
export function geminiResponseToAnthropic(fields) {
|
|
34
|
+
return {
|
|
35
|
+
id: `msg_gemini_${randomUUID()}`,
|
|
36
|
+
type: 'message',
|
|
37
|
+
role: 'assistant',
|
|
38
|
+
content: [{ type: 'text', text: fields.text }],
|
|
39
|
+
model: fields.model,
|
|
40
|
+
stop_reason: geminiFinishToStopReason(fields.finishReason),
|
|
41
|
+
usage: {
|
|
42
|
+
input_tokens: fields.promptTokens ?? 0,
|
|
43
|
+
output_tokens: fields.completionTokens ?? 0,
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Synthesize the Anthropic SSE event sequence from a Gemini chunk stream,
|
|
49
|
+
* preserving content_block_* boundaries (§5.2 streaming shape).
|
|
50
|
+
*/
|
|
51
|
+
export async function* geminiChunksToSSE(chunks, model) {
|
|
52
|
+
const id = `msg_gemini_${randomUUID()}`;
|
|
53
|
+
yield {
|
|
54
|
+
type: 'message_start',
|
|
55
|
+
message: { id, type: 'message', role: 'assistant', content: [], model, stop_reason: null, usage: { input_tokens: 0, output_tokens: 0 } },
|
|
56
|
+
};
|
|
57
|
+
yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } };
|
|
58
|
+
let inputTokens = 0;
|
|
59
|
+
let outputTokens = 0;
|
|
60
|
+
let finishReason;
|
|
61
|
+
for await (const chunk of chunks) {
|
|
62
|
+
if (chunk.text)
|
|
63
|
+
yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: chunk.text } };
|
|
64
|
+
if (chunk.promptTokens !== undefined)
|
|
65
|
+
inputTokens = chunk.promptTokens;
|
|
66
|
+
if (chunk.completionTokens !== undefined)
|
|
67
|
+
outputTokens = chunk.completionTokens;
|
|
68
|
+
if (chunk.finishReason)
|
|
69
|
+
finishReason = chunk.finishReason;
|
|
70
|
+
}
|
|
71
|
+
yield { type: 'content_block_stop', index: 0 };
|
|
72
|
+
yield {
|
|
73
|
+
type: 'message_delta',
|
|
74
|
+
delta: { stop_reason: geminiFinishToStopReason(finishReason) },
|
|
75
|
+
usage: { input_tokens: inputTokens, output_tokens: outputTokens },
|
|
76
|
+
};
|
|
77
|
+
yield { type: 'message_stop' };
|
|
78
|
+
}
|
|
79
|
+
export function geminiAdapter(opts = {}) {
|
|
80
|
+
const apiKey = process.env.GEMINI_SERVER_API_KEY || process.env.GEMINI_API_KEY || process.env.GOOGLE_AI_API_KEY || '';
|
|
81
|
+
const client = opts.client ?? (apiKey ? new GoogleGenerativeAI(apiKey) : null);
|
|
82
|
+
function model(req) {
|
|
83
|
+
if (!client)
|
|
84
|
+
throw new Error('Gemini key not configured');
|
|
85
|
+
return client.getGenerativeModel({
|
|
86
|
+
model: req.model ?? '',
|
|
87
|
+
systemInstruction: req.system,
|
|
88
|
+
generationConfig: { temperature: req.temperature, maxOutputTokens: req.max_tokens },
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
return {
|
|
92
|
+
id: 'gemini',
|
|
93
|
+
display_name: 'Google Gemini (generativelanguage)',
|
|
94
|
+
surface: 'cloud',
|
|
95
|
+
supports_streaming: true,
|
|
96
|
+
supports_tool_use: false,
|
|
97
|
+
supports_prompt_caching: false,
|
|
98
|
+
tool_use: {
|
|
99
|
+
supports_tool_use: false,
|
|
100
|
+
supports_parallel_tool_use: false,
|
|
101
|
+
preserves_block_position: false,
|
|
102
|
+
supports_streaming_tool_use: false,
|
|
103
|
+
max_tools_per_request: 0,
|
|
104
|
+
},
|
|
105
|
+
available: () => !!client,
|
|
106
|
+
async isEligible(req, _ctx) {
|
|
107
|
+
if (!client)
|
|
108
|
+
return { eligible: false, reason: 'Gemini key not configured' };
|
|
109
|
+
if (req.tools?.length)
|
|
110
|
+
return { eligible: false, reason: 'tool use not supported by gemini adapter (v1)' };
|
|
111
|
+
return { eligible: true, estimated_latency_ms: 300, estimated_cost_cents: 0 };
|
|
112
|
+
},
|
|
113
|
+
async invoke(req, _ctx) {
|
|
114
|
+
const result = await model(req).generateContent({ contents: anthropicToGeminiContents(req) });
|
|
115
|
+
const usage = result.response.usageMetadata;
|
|
116
|
+
return geminiResponseToAnthropic({
|
|
117
|
+
text: result.response.text(),
|
|
118
|
+
model: req.model ?? 'gemini',
|
|
119
|
+
finishReason: result.response.candidates?.[0]?.finishReason,
|
|
120
|
+
promptTokens: usage?.promptTokenCount,
|
|
121
|
+
completionTokens: usage?.candidatesTokenCount,
|
|
122
|
+
});
|
|
123
|
+
},
|
|
124
|
+
async *invokeStream(req, _ctx) {
|
|
125
|
+
const result = await model(req).generateContentStream({ contents: anthropicToGeminiContents(req) });
|
|
126
|
+
async function* chunks() {
|
|
127
|
+
for await (const chunk of result.stream) {
|
|
128
|
+
yield {
|
|
129
|
+
text: chunk.text(),
|
|
130
|
+
promptTokens: chunk.usageMetadata?.promptTokenCount,
|
|
131
|
+
completionTokens: chunk.usageMetadata?.candidatesTokenCount,
|
|
132
|
+
finishReason: chunk.candidates?.[0]?.finishReason,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
yield* geminiChunksToSSE(chunks(), req.model ?? 'gemini');
|
|
137
|
+
},
|
|
138
|
+
reportUsage(_req, res) {
|
|
139
|
+
return {
|
|
140
|
+
backend_id: 'gemini',
|
|
141
|
+
tokens_in: res.usage?.input_tokens ?? 0,
|
|
142
|
+
tokens_out: res.usage?.output_tokens ?? 0,
|
|
143
|
+
latency_ms: 0, // overwritten by the router's measured value
|
|
144
|
+
cost_cents: 0, // real cost computed by the platform meter (calculate_token_cost)
|
|
145
|
+
cache_hit: false,
|
|
146
|
+
cache_unsupported: true,
|
|
147
|
+
sovereignty_class: 'externalizable', // overwritten by the router from the envelope
|
|
148
|
+
};
|
|
149
|
+
},
|
|
150
|
+
};
|
|
151
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tier 0 — self-hosted in-boundary (surface: self_hosted). Honest stub until
|
|
3
|
+
* the OpenMed/Nemotron serving path lands (Q3 eval tracks): unavailable
|
|
4
|
+
* unless MODEL_ROUTER_TIER0_URL is set, then a minimal
|
|
5
|
+
* POST {model, prompt} → {text} contract. No streaming, no tools.
|
|
6
|
+
*/
|
|
7
|
+
import type { AnthropicMessagesRequest, BackendAdapter } from '../types.js';
|
|
8
|
+
/**
|
|
9
|
+
* Flatten Anthropic messages to a single prompt. A lone user message passes
|
|
10
|
+
* through verbatim (byte-identical to what routeModel v1 sent); multi-turn
|
|
11
|
+
* conversations get role-prefixed lines.
|
|
12
|
+
*/
|
|
13
|
+
export declare function messagesToPrompt(req: AnthropicMessagesRequest): string;
|
|
14
|
+
export declare function selfhostedAdapter(): BackendAdapter;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
const TIMEOUT_MS = 30000;
|
|
2
|
+
/**
|
|
3
|
+
* Flatten Anthropic messages to a single prompt. A lone user message passes
|
|
4
|
+
* through verbatim (byte-identical to what routeModel v1 sent); multi-turn
|
|
5
|
+
* conversations get role-prefixed lines.
|
|
6
|
+
*/
|
|
7
|
+
export function messagesToPrompt(req) {
|
|
8
|
+
const text = (content) => typeof content === 'string'
|
|
9
|
+
? content
|
|
10
|
+
: content
|
|
11
|
+
.filter((b) => b.type === 'text')
|
|
12
|
+
.map((b) => b.text)
|
|
13
|
+
.join('\n');
|
|
14
|
+
if (req.messages.length === 1 && req.messages[0].role === 'user')
|
|
15
|
+
return text(req.messages[0].content);
|
|
16
|
+
return req.messages.map((m) => `${m.role === 'assistant' ? 'Assistant' : 'User'}: ${text(m.content)}`).join('\n\n');
|
|
17
|
+
}
|
|
18
|
+
export function selfhostedAdapter() {
|
|
19
|
+
return {
|
|
20
|
+
id: 'selfhosted',
|
|
21
|
+
display_name: 'cloudpeers self-hosted in-boundary (Tier 0)',
|
|
22
|
+
surface: 'self_hosted',
|
|
23
|
+
supports_streaming: false,
|
|
24
|
+
supports_tool_use: false,
|
|
25
|
+
supports_prompt_caching: false,
|
|
26
|
+
tool_use: {
|
|
27
|
+
supports_tool_use: false,
|
|
28
|
+
supports_parallel_tool_use: false,
|
|
29
|
+
preserves_block_position: false,
|
|
30
|
+
supports_streaming_tool_use: false,
|
|
31
|
+
max_tools_per_request: 0,
|
|
32
|
+
},
|
|
33
|
+
available: () => !!process.env.MODEL_ROUTER_TIER0_URL,
|
|
34
|
+
async isEligible(req, _ctx) {
|
|
35
|
+
if (!process.env.MODEL_ROUTER_TIER0_URL)
|
|
36
|
+
return { eligible: false, reason: 'Tier 0 serving not configured (MODEL_ROUTER_TIER0_URL)' };
|
|
37
|
+
if (req.tools?.length)
|
|
38
|
+
return { eligible: false, reason: 'tool use not supported by Tier 0 serving' };
|
|
39
|
+
return { eligible: true, estimated_latency_ms: 100, estimated_cost_cents: 0 };
|
|
40
|
+
},
|
|
41
|
+
async invoke(req, ctx) {
|
|
42
|
+
const url = process.env.MODEL_ROUTER_TIER0_URL;
|
|
43
|
+
if (!url)
|
|
44
|
+
throw new Error('Tier 0 serving not configured');
|
|
45
|
+
const controller = new AbortController();
|
|
46
|
+
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
|
47
|
+
ctx.signal?.addEventListener('abort', () => controller.abort(), { once: true });
|
|
48
|
+
try {
|
|
49
|
+
const res = await fetch(`${url.replace(/\/$/, '')}/v1/generate`, {
|
|
50
|
+
method: 'POST',
|
|
51
|
+
headers: { 'Content-Type': 'application/json' },
|
|
52
|
+
body: JSON.stringify({
|
|
53
|
+
model: req.model,
|
|
54
|
+
prompt: messagesToPrompt(req),
|
|
55
|
+
system: req.system,
|
|
56
|
+
max_tokens: req.max_tokens,
|
|
57
|
+
}),
|
|
58
|
+
signal: controller.signal,
|
|
59
|
+
});
|
|
60
|
+
if (!res.ok)
|
|
61
|
+
throw new Error(`Tier0 ${res.status}`);
|
|
62
|
+
const data = (await res.json());
|
|
63
|
+
if (!data.text)
|
|
64
|
+
throw new Error('Tier0 returned empty text');
|
|
65
|
+
return {
|
|
66
|
+
id: `msg_tier0_${Math.random().toString(36).slice(2)}`,
|
|
67
|
+
type: 'message',
|
|
68
|
+
role: 'assistant',
|
|
69
|
+
content: [{ type: 'text', text: data.text }],
|
|
70
|
+
model: req.model ?? 'tier0',
|
|
71
|
+
stop_reason: 'end_turn',
|
|
72
|
+
usage: { input_tokens: data.prompt_tokens ?? 0, output_tokens: data.completion_tokens ?? 0 },
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
clearTimeout(timer);
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
// eslint-disable-next-line require-yield
|
|
80
|
+
async *invokeStream(_req, _ctx) {
|
|
81
|
+
throw new Error('Tier 0 serving does not support streaming (supports_streaming: false)');
|
|
82
|
+
},
|
|
83
|
+
reportUsage(_req, res) {
|
|
84
|
+
return {
|
|
85
|
+
backend_id: 'selfhosted',
|
|
86
|
+
tokens_in: res.usage?.input_tokens ?? 0,
|
|
87
|
+
tokens_out: res.usage?.output_tokens ?? 0,
|
|
88
|
+
latency_ms: 0, // overwritten by the router's measured value
|
|
89
|
+
cost_cents: 0,
|
|
90
|
+
cache_hit: false,
|
|
91
|
+
cache_unsupported: true,
|
|
92
|
+
sovereignty_class: 'local_only', // overwritten by the router from the envelope
|
|
93
|
+
};
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §8.1 — capability matrix, built from adapter declarations so it cannot
|
|
3
|
+
* drift from what the adapters actually claim (§13: "capability endpoint
|
|
4
|
+
* reflects truth"). Embedded-library v1 exposes this as a function; the
|
|
5
|
+
* mcp server surfaces it on its discovery endpoints.
|
|
6
|
+
*/
|
|
7
|
+
import type { BackendAdapter, BackendSurface, Provider, ToolUseCapability } from './types.js';
|
|
8
|
+
export interface BackendCapabilities {
|
|
9
|
+
id: Provider;
|
|
10
|
+
display_name: string;
|
|
11
|
+
surface: BackendSurface;
|
|
12
|
+
available: boolean;
|
|
13
|
+
supports_streaming: boolean;
|
|
14
|
+
supports_tool_use: boolean;
|
|
15
|
+
supports_prompt_caching: boolean;
|
|
16
|
+
tool_use: ToolUseCapability;
|
|
17
|
+
}
|
|
18
|
+
export declare function capabilityMatrix(adapters: Partial<Record<Provider, BackendAdapter>>): BackendCapabilities[];
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export function capabilityMatrix(adapters) {
|
|
2
|
+
return Object.values(adapters)
|
|
3
|
+
.filter((a) => !!a)
|
|
4
|
+
.map((a) => ({
|
|
5
|
+
id: a.id,
|
|
6
|
+
display_name: a.display_name,
|
|
7
|
+
surface: a.surface,
|
|
8
|
+
available: a.available(),
|
|
9
|
+
supports_streaming: a.supports_streaming,
|
|
10
|
+
supports_tool_use: a.supports_tool_use,
|
|
11
|
+
supports_prompt_caching: a.supports_prompt_caching,
|
|
12
|
+
tool_use: a.tool_use,
|
|
13
|
+
}));
|
|
14
|
+
}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Router errors carry an HTTP-ish statusCode so embedding servers can map
|
|
3
|
+
* them onto responses without string-matching messages (§7.1 status
|
|
4
|
+
* semantics: 400 shape, 409 revoked, 410 expired, 412 no eligible backend,
|
|
5
|
+
* 422 tool-use strict, 501 deferred capability).
|
|
6
|
+
*/
|
|
7
|
+
export declare class RouterError extends Error {
|
|
8
|
+
readonly statusCode: number;
|
|
9
|
+
readonly code: string;
|
|
10
|
+
constructor(message: string, statusCode: number, code: string);
|
|
11
|
+
}
|
|
12
|
+
/** §7.1 #1 — request failed envelope/shape validation. */
|
|
13
|
+
export declare class RouterValidationError extends RouterError {
|
|
14
|
+
constructor(message: string);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Enforcement blocked the request (privacy gate, mandate, policy, capability).
|
|
18
|
+
* Name and default semantics preserved from ModelRouter v1 — callers catch
|
|
19
|
+
* `RouterBlockedError` and existing behavior is unchanged.
|
|
20
|
+
*/
|
|
21
|
+
export declare class RouterBlockedError extends RouterError {
|
|
22
|
+
constructor(message: string, statusCode?: number, code?: string);
|
|
23
|
+
}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Router errors carry an HTTP-ish statusCode so embedding servers can map
|
|
3
|
+
* them onto responses without string-matching messages (§7.1 status
|
|
4
|
+
* semantics: 400 shape, 409 revoked, 410 expired, 412 no eligible backend,
|
|
5
|
+
* 422 tool-use strict, 501 deferred capability).
|
|
6
|
+
*/
|
|
7
|
+
export class RouterError extends Error {
|
|
8
|
+
constructor(message, statusCode, code) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = 'RouterError';
|
|
11
|
+
this.statusCode = statusCode;
|
|
12
|
+
this.code = code;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
/** §7.1 #1 — request failed envelope/shape validation. */
|
|
16
|
+
export class RouterValidationError extends RouterError {
|
|
17
|
+
constructor(message) {
|
|
18
|
+
super(message, 400, 'invalid_request');
|
|
19
|
+
this.name = 'RouterValidationError';
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Enforcement blocked the request (privacy gate, mandate, policy, capability).
|
|
24
|
+
* Name and default semantics preserved from ModelRouter v1 — callers catch
|
|
25
|
+
* `RouterBlockedError` and existing behavior is unchanged.
|
|
26
|
+
*/
|
|
27
|
+
export class RouterBlockedError extends RouterError {
|
|
28
|
+
constructor(message, statusCode = 412, code = 'routing_blocked') {
|
|
29
|
+
super(message, statusCode, code);
|
|
30
|
+
this.name = 'RouterBlockedError';
|
|
31
|
+
}
|
|
32
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @cloudpeers-jkl/model-router — cloudpeers inference router, embedded
|
|
3
|
+
* library surface (Amendment A1.1: no central data plane; the embedding
|
|
4
|
+
* service injects metering/quota/hooks and the envelope's sovereignty gate
|
|
5
|
+
* runs in-process, where the data lives).
|
|
6
|
+
*/
|
|
7
|
+
export * from './types.js';
|
|
8
|
+
export { RouterError, RouterValidationError, RouterBlockedError } from './errors.js';
|
|
9
|
+
export { parseRouterRequest, routerMessagesRequestSchema, cloudpeersEnvelopeSchema } from './schema.js';
|
|
10
|
+
export { TIERS, TASK_ROUTES, evaluateGate, gateEnforced, resolveChain } from './policy.js';
|
|
11
|
+
export { routeModel, routeMessages, routeMessagesStream } from './router.js';
|
|
12
|
+
export type { RouteContext, MessagesStreamResult } from './router.js';
|
|
13
|
+
export { capabilityMatrix } from './capabilities.js';
|
|
14
|
+
export type { BackendCapabilities } from './capabilities.js';
|
|
15
|
+
export { anthropicAdapter } from './adapters/anthropic.js';
|
|
16
|
+
export { geminiAdapter, anthropicToGeminiContents, geminiResponseToAnthropic, geminiChunksToSSE, geminiFinishToStopReason, } from './adapters/gemini.js';
|
|
17
|
+
export { selfhostedAdapter, messagesToPrompt } from './adapters/selfhosted.js';
|
|
18
|
+
import type { BackendAdapter, Provider } from './types.js';
|
|
19
|
+
/** The approved backend set (A1.2): selfhosted, gemini, anthropic. No OpenAI. */
|
|
20
|
+
export declare function defaultBackendAdapters(): Record<Provider, BackendAdapter>;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @cloudpeers-jkl/model-router — cloudpeers inference router, embedded
|
|
3
|
+
* library surface (Amendment A1.1: no central data plane; the embedding
|
|
4
|
+
* service injects metering/quota/hooks and the envelope's sovereignty gate
|
|
5
|
+
* runs in-process, where the data lives).
|
|
6
|
+
*/
|
|
7
|
+
export * from './types.js';
|
|
8
|
+
export { RouterError, RouterValidationError, RouterBlockedError } from './errors.js';
|
|
9
|
+
export { parseRouterRequest, routerMessagesRequestSchema, cloudpeersEnvelopeSchema } from './schema.js';
|
|
10
|
+
export { TIERS, TASK_ROUTES, evaluateGate, gateEnforced, resolveChain } from './policy.js';
|
|
11
|
+
export { routeModel, routeMessages, routeMessagesStream } from './router.js';
|
|
12
|
+
export { capabilityMatrix } from './capabilities.js';
|
|
13
|
+
export { anthropicAdapter } from './adapters/anthropic.js';
|
|
14
|
+
export { geminiAdapter, anthropicToGeminiContents, geminiResponseToAnthropic, geminiChunksToSSE, geminiFinishToStopReason, } from './adapters/gemini.js';
|
|
15
|
+
export { selfhostedAdapter, messagesToPrompt } from './adapters/selfhosted.js';
|
|
16
|
+
import { anthropicAdapter } from './adapters/anthropic.js';
|
|
17
|
+
import { geminiAdapter } from './adapters/gemini.js';
|
|
18
|
+
import { selfhostedAdapter } from './adapters/selfhosted.js';
|
|
19
|
+
/** The approved backend set (A1.2): selfhosted, gemini, anthropic. No OpenAI. */
|
|
20
|
+
export function defaultBackendAdapters() {
|
|
21
|
+
return { selfhosted: selfhostedAdapter(), gemini: geminiAdapter(), anthropic: anthropicAdapter() };
|
|
22
|
+
}
|