freellm 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +89 -0
  2. package/bin/cli.js +25 -0
  3. package/client-dist/assets/index-aUFet1Mp.js +84 -0
  4. package/client-dist/index.html +16 -0
  5. package/dist/cli.d.ts +1 -0
  6. package/dist/cli.js +38 -0
  7. package/dist/crypto.d.ts +6 -0
  8. package/dist/crypto.js +56 -0
  9. package/dist/db/index.d.ts +62 -0
  10. package/dist/db/index.js +368 -0
  11. package/dist/index.d.ts +10 -0
  12. package/dist/index.js +100 -0
  13. package/dist/providers/cloudflare.d.ts +10 -0
  14. package/dist/providers/cloudflare.js +140 -0
  15. package/dist/providers/google.d.ts +8 -0
  16. package/dist/providers/google.js +181 -0
  17. package/dist/providers/index.d.ts +14 -0
  18. package/dist/providers/index.js +60 -0
  19. package/dist/providers/ollama.d.ts +10 -0
  20. package/dist/providers/ollama.js +99 -0
  21. package/dist/providers/openai-compat.d.ts +11 -0
  22. package/dist/providers/openai-compat.js +115 -0
  23. package/dist/providers/openai-image-compat.d.ts +10 -0
  24. package/dist/providers/openai-image-compat.js +40 -0
  25. package/dist/providers/openai-tts-compat.d.ts +10 -0
  26. package/dist/providers/openai-tts-compat.js +40 -0
  27. package/dist/providers/pollinations-image.d.ts +7 -0
  28. package/dist/providers/pollinations-image.js +50 -0
  29. package/dist/providers/pollinations.d.ts +9 -0
  30. package/dist/providers/pollinations.js +76 -0
  31. package/dist/providers/types.d.ts +132 -0
  32. package/dist/providers/types.js +66 -0
  33. package/dist/routes/api/analytics.d.ts +1 -0
  34. package/dist/routes/api/analytics.js +10 -0
  35. package/dist/routes/api/auth.d.ts +3 -0
  36. package/dist/routes/api/auth.js +60 -0
  37. package/dist/routes/api/keys.d.ts +1 -0
  38. package/dist/routes/api/keys.js +65 -0
  39. package/dist/routes/api/models.d.ts +1 -0
  40. package/dist/routes/api/models.js +26 -0
  41. package/dist/routes/v1/audio.d.ts +1 -0
  42. package/dist/routes/v1/audio.js +94 -0
  43. package/dist/routes/v1/chat.d.ts +1 -0
  44. package/dist/routes/v1/chat.js +131 -0
  45. package/dist/routes/v1/images.d.ts +1 -0
  46. package/dist/routes/v1/images.js +85 -0
  47. package/dist/routes/v1/models.d.ts +1 -0
  48. package/dist/routes/v1/models.js +59 -0
  49. package/dist/services/health.d.ts +2 -0
  50. package/dist/services/health.js +80 -0
  51. package/dist/services/ratelimit.d.ts +4 -0
  52. package/dist/services/ratelimit.js +88 -0
  53. package/dist/services/router.d.ts +14 -0
  54. package/dist/services/router.js +72 -0
  55. package/package.json +57 -0
@@ -0,0 +1,94 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.audioRouter = void 0;
4
+ const express_1 = require("express");
5
+ const router_1 = require("../../services/router");
6
+ const providers_1 = require("../../providers");
7
+ const ratelimit_1 = require("../../services/ratelimit");
8
+ const db_1 = require("../../db");
9
+ exports.audioRouter = (0, express_1.Router)();
10
+ const AUDIO_CONTENT_TYPES = {
11
+ mp3: 'audio/mpeg',
12
+ opus: 'audio/opus',
13
+ aac: 'audio/aac',
14
+ flac: 'audio/flac',
15
+ wav: 'audio/wav',
16
+ pcm: 'audio/l16',
17
+ };
18
+ exports.audioRouter.post('/audio/speech', async (req, res) => {
19
+ const body = req.body;
20
+ if (!body.input || typeof body.input !== 'string') {
21
+ return res.status(400).json({ error: { message: 'input is required', type: 'invalid_request_error' } });
22
+ }
23
+ const requestedModel = body.model || 'auto';
24
+ const candidates = (0, router_1.buildCandidates)(requestedModel, 'tts');
25
+ if (candidates.length === 0) {
26
+ return res.status(503).json({
27
+ error: {
28
+ message: 'No TTS models available. Add an API key for a TTS provider.',
29
+ type: 'service_unavailable',
30
+ code: 'no_tts_model',
31
+ },
32
+ });
33
+ }
34
+ for (let attempt = 0; attempt < candidates.length; attempt++) {
35
+ const candidate = candidates[attempt];
36
+ const provider = (0, providers_1.getTTSProvider)(candidate.model.platform);
37
+ if (!provider)
38
+ continue;
39
+ const platformModel = candidate.model.model_id;
40
+ const routeHeader = `${candidate.model.platform}/${platformModel}`;
41
+ const startMs = Date.now();
42
+ try {
43
+ const audioBuffer = await provider.textToSpeech(candidate.decryptedKey, {
44
+ ...body,
45
+ model: platformModel,
46
+ });
47
+ (0, ratelimit_1.recordRequest)(candidate.model.platform, candidate.providerKey?.id ?? 0, 0);
48
+ (0, db_1.logRequest)({
49
+ platform: candidate.model.platform,
50
+ model_id: platformModel,
51
+ prompt_tokens: 0,
52
+ complete_tokens: 0,
53
+ latency_ms: Date.now() - startMs,
54
+ success: true,
55
+ });
56
+ res.setHeader('Content-Type', AUDIO_CONTENT_TYPES[body.response_format ?? 'mp3'] ?? 'audio/mpeg');
57
+ res.setHeader('X-Routed-Via', routeHeader);
58
+ if (attempt > 0)
59
+ res.setHeader('X-Fallback-Attempts', String(attempt));
60
+ return res.send(Buffer.from(audioBuffer));
61
+ }
62
+ catch (e) {
63
+ const err = e;
64
+ (0, db_1.logRequest)({
65
+ platform: candidate.model.platform,
66
+ model_id: platformModel,
67
+ prompt_tokens: 0,
68
+ complete_tokens: 0,
69
+ latency_ms: Date.now() - startMs,
70
+ success: false,
71
+ error_code: String(err.status ?? 'unknown'),
72
+ });
73
+ if (err.status === 429 || err.status === 503 || err.status === 502) {
74
+ if (candidate.providerKey) {
75
+ (0, ratelimit_1.setCooldown)(candidate.model.platform, candidate.providerKey.id);
76
+ }
77
+ continue;
78
+ }
79
+ if (err.status === 401 || err.status === 403) {
80
+ continue;
81
+ }
82
+ if (attempt < candidates.length - 1)
83
+ continue;
84
+ }
85
+ }
86
+ if (!res.headersSent) {
87
+ return res.status(503).json({
88
+ error: {
89
+ message: 'All TTS providers exhausted or rate-limited. Try again later or add more API keys.',
90
+ type: 'service_unavailable',
91
+ },
92
+ });
93
+ }
94
+ });
@@ -0,0 +1 @@
1
+ export declare const chatRouter: import("express-serve-static-core").Router;
@@ -0,0 +1,131 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.chatRouter = void 0;
4
+ const express_1 = require("express");
5
+ const router_1 = require("../../services/router");
6
+ const providers_1 = require("../../providers");
7
+ const ratelimit_1 = require("../../services/ratelimit");
8
+ const db_1 = require("../../db");
9
+ exports.chatRouter = (0, express_1.Router)();
10
+ exports.chatRouter.post('/chat/completions', async (req, res) => {
11
+ const body = req.body;
12
+ const { messages, stream = false } = body;
13
+ if (!messages || !Array.isArray(messages) || messages.length === 0) {
14
+ return res.status(400).json({ error: { message: 'messages is required', type: 'invalid_request_error' } });
15
+ }
16
+ const requireVision = (0, router_1.hasVisionContent)(messages);
17
+ const requireTools = Array.isArray(body.tools) && body.tools.length > 0;
18
+ let requestedModel = body.model || 'auto';
19
+ // Normalize model alias: if request is 'auto' or any family name, keep as 'auto'
20
+ const candidates = (0, router_1.buildCandidates)(requestedModel, 'chat', requireVision, requireTools);
21
+ if (candidates.length === 0) {
22
+ return res.status(503).json({
23
+ error: {
24
+ message: requireVision
25
+ ? 'No vision-capable models available. Add an API key for a vision provider or enable a vision model.'
26
+ : 'No models available. Add at least one provider API key on the dashboard.',
27
+ type: 'service_unavailable',
28
+ code: requireVision ? 'no_vision_model' : 'no_model_available',
29
+ },
30
+ });
31
+ }
32
+ const MAX_ATTEMPTS = Math.min(candidates.length, 20);
33
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
34
+ const candidate = candidates[attempt];
35
+ const provider = (0, providers_1.getChatProvider)(candidate.model.platform);
36
+ if (!provider)
37
+ continue;
38
+ const platformModel = candidate.model.model_id;
39
+ const routeHeader = `${candidate.model.platform}/${platformModel}`;
40
+ const startMs = Date.now();
41
+ try {
42
+ if (stream) {
43
+ res.setHeader('Content-Type', 'text/event-stream');
44
+ res.setHeader('Cache-Control', 'no-cache');
45
+ res.setHeader('Connection', 'keep-alive');
46
+ res.setHeader('X-Routed-Via', routeHeader);
47
+ if (attempt > 0)
48
+ res.setHeader('X-Fallback-Attempts', String(attempt));
49
+ res.flushHeaders();
50
+ let promptTokens = 0;
51
+ let completionTokens = 0;
52
+ let done = false;
53
+ await provider.streamChat(candidate.decryptedKey, { ...body, model: platformModel }, (chunk) => {
54
+ completionTokens += chunk.choices?.[0]?.delta?.content?.length
55
+ ? Math.ceil(chunk.choices[0].delta.content.length / 4)
56
+ : 0;
57
+ res.write(`data: ${JSON.stringify(chunk)}\n\n`);
58
+ }, () => {
59
+ done = true;
60
+ res.write('data: [DONE]\n\n');
61
+ res.end();
62
+ }, (err) => {
63
+ if (err.status === 429 || err.status === 503) {
64
+ (0, ratelimit_1.setCooldown)(candidate.model.platform, candidate.providerKey?.id ?? 0);
65
+ }
66
+ if (!res.headersSent) {
67
+ // Try next
68
+ }
69
+ else {
70
+ res.end();
71
+ }
72
+ });
73
+ if (done) {
74
+ (0, ratelimit_1.recordRequest)(candidate.model.platform, candidate.providerKey?.id ?? 0, promptTokens + completionTokens);
75
+ (0, db_1.logRequest)({
76
+ platform: candidate.model.platform,
77
+ model_id: platformModel,
78
+ prompt_tokens: promptTokens,
79
+ complete_tokens: completionTokens,
80
+ latency_ms: Date.now() - startMs,
81
+ success: true,
82
+ });
83
+ return;
84
+ }
85
+ }
86
+ else {
87
+ const result = await provider.chat(candidate.decryptedKey, { ...body, model: platformModel });
88
+ const latency = Date.now() - startMs;
89
+ const promptTokens = result.usage?.prompt_tokens ?? 0;
90
+ const completionTokens = result.usage?.completion_tokens ?? 0;
91
+ (0, ratelimit_1.recordRequest)(candidate.model.platform, candidate.providerKey?.id ?? 0, promptTokens + completionTokens);
92
+ (0, db_1.logRequest)({ platform: candidate.model.platform, model_id: platformModel, prompt_tokens: promptTokens, complete_tokens: completionTokens, latency_ms: latency, success: true });
93
+ res.setHeader('X-Routed-Via', routeHeader);
94
+ if (attempt > 0)
95
+ res.setHeader('X-Fallback-Attempts', String(attempt));
96
+ return res.json(result);
97
+ }
98
+ }
99
+ catch (e) {
100
+ const err = e;
101
+ (0, db_1.logRequest)({
102
+ platform: candidate.model.platform,
103
+ model_id: platformModel,
104
+ prompt_tokens: 0,
105
+ complete_tokens: 0,
106
+ latency_ms: Date.now() - startMs,
107
+ success: false,
108
+ error_code: String(err.status ?? 'unknown'),
109
+ });
110
+ if (err.status === 429 || err.status === 503 || err.status === 502) {
111
+ if (candidate.providerKey) {
112
+ (0, ratelimit_1.setCooldown)(candidate.model.platform, candidate.providerKey.id);
113
+ }
114
+ continue; // try next candidate
115
+ }
116
+ if (err.status === 401 || err.status === 403) {
117
+ continue; // skip invalid key
118
+ }
119
+ if (attempt < MAX_ATTEMPTS - 1)
120
+ continue;
121
+ }
122
+ }
123
+ if (!res.headersSent) {
124
+ return res.status(503).json({
125
+ error: {
126
+ message: 'All providers exhausted or rate-limited. Try again later or add more API keys.',
127
+ type: 'service_unavailable',
128
+ },
129
+ });
130
+ }
131
+ });
@@ -0,0 +1 @@
1
+ export declare const imagesRouter: import("express-serve-static-core").Router;
@@ -0,0 +1,85 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.imagesRouter = void 0;
4
+ const express_1 = require("express");
5
+ const router_1 = require("../../services/router");
6
+ const providers_1 = require("../../providers");
7
+ const ratelimit_1 = require("../../services/ratelimit");
8
+ const db_1 = require("../../db");
9
+ exports.imagesRouter = (0, express_1.Router)();
10
+ exports.imagesRouter.post('/images/generations', async (req, res) => {
11
+ const body = req.body;
12
+ if (!body.prompt || typeof body.prompt !== 'string') {
13
+ return res.status(400).json({ error: { message: 'prompt is required', type: 'invalid_request_error' } });
14
+ }
15
+ const requestedModel = body.model || 'auto';
16
+ const candidates = (0, router_1.buildCandidates)(requestedModel, 'image');
17
+ if (candidates.length === 0) {
18
+ return res.status(503).json({
19
+ error: {
20
+ message: 'No image models available. Add an API key for an image generation provider.',
21
+ type: 'service_unavailable',
22
+ code: 'no_image_model',
23
+ },
24
+ });
25
+ }
26
+ for (let attempt = 0; attempt < candidates.length; attempt++) {
27
+ const candidate = candidates[attempt];
28
+ const provider = (0, providers_1.getImageProvider)(candidate.model.platform);
29
+ if (!provider)
30
+ continue;
31
+ const platformModel = candidate.model.model_id;
32
+ const routeHeader = `${candidate.model.platform}/${platformModel}`;
33
+ const startMs = Date.now();
34
+ try {
35
+ const result = await provider.generateImage(candidate.decryptedKey, {
36
+ ...body,
37
+ model: platformModel,
38
+ });
39
+ (0, ratelimit_1.recordRequest)(candidate.model.platform, candidate.providerKey?.id ?? 0, 0);
40
+ (0, db_1.logRequest)({
41
+ platform: candidate.model.platform,
42
+ model_id: platformModel,
43
+ prompt_tokens: 0,
44
+ complete_tokens: 0,
45
+ latency_ms: Date.now() - startMs,
46
+ success: true,
47
+ });
48
+ res.setHeader('X-Routed-Via', routeHeader);
49
+ if (attempt > 0)
50
+ res.setHeader('X-Fallback-Attempts', String(attempt));
51
+ return res.json(result);
52
+ }
53
+ catch (e) {
54
+ const err = e;
55
+ (0, db_1.logRequest)({
56
+ platform: candidate.model.platform,
57
+ model_id: platformModel,
58
+ prompt_tokens: 0,
59
+ complete_tokens: 0,
60
+ latency_ms: Date.now() - startMs,
61
+ success: false,
62
+ error_code: String(err.status ?? 'unknown'),
63
+ });
64
+ if (err.status === 429 || err.status === 503 || err.status === 502) {
65
+ if (candidate.providerKey) {
66
+ (0, ratelimit_1.setCooldown)(candidate.model.platform, candidate.providerKey.id);
67
+ }
68
+ continue;
69
+ }
70
+ if (err.status === 401 || err.status === 403) {
71
+ continue;
72
+ }
73
+ if (attempt < candidates.length - 1)
74
+ continue;
75
+ }
76
+ }
77
+ if (!res.headersSent) {
78
+ return res.status(503).json({
79
+ error: {
80
+ message: 'All image providers exhausted or rate-limited. Try again later or add more API keys.',
81
+ type: 'service_unavailable',
82
+ },
83
+ });
84
+ }
85
+ });
@@ -0,0 +1 @@
1
+ export declare const modelsRouter: import("express-serve-static-core").Router;
@@ -0,0 +1,59 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.modelsRouter = void 0;
4
+ const express_1 = require("express");
5
+ const db_1 = require("../../db");
6
+ exports.modelsRouter = (0, express_1.Router)();
7
+ exports.modelsRouter.get('/models', (_req, res) => {
8
+ const chatModels = (0, db_1.listEnabledModels)('chat');
9
+ const imageModels = (0, db_1.listEnabledModels)('image');
10
+ const ttsModels = (0, db_1.listEnabledModels)('tts');
11
+ const data = [
12
+ ...chatModels.map((m) => ({
13
+ id: m.model_id,
14
+ object: 'model',
15
+ created: 1677610602,
16
+ owned_by: m.platform,
17
+ platform: m.platform,
18
+ display_name: m.display_name,
19
+ type: 'chat',
20
+ supports_tools: !!m.supports_tools,
21
+ supports_vision: !!m.supports_vision,
22
+ })),
23
+ ...imageModels.map((m) => ({
24
+ id: m.model_id,
25
+ object: 'model',
26
+ created: 1677610602,
27
+ owned_by: m.platform,
28
+ platform: m.platform,
29
+ display_name: m.display_name,
30
+ type: 'image',
31
+ supports_tools: false,
32
+ supports_vision: false,
33
+ })),
34
+ ...ttsModels.map((m) => ({
35
+ id: m.model_id,
36
+ object: 'model',
37
+ created: 1677610602,
38
+ owned_by: m.platform,
39
+ platform: m.platform,
40
+ display_name: m.display_name,
41
+ type: 'tts',
42
+ supports_tools: false,
43
+ supports_vision: false,
44
+ })),
45
+ ];
46
+ // Include the virtual 'auto' model
47
+ data.unshift({
48
+ id: 'auto',
49
+ object: 'model',
50
+ created: 1677610602,
51
+ owned_by: 'freellm',
52
+ platform: 'freellm',
53
+ display_name: 'Auto (smart routing)',
54
+ type: 'chat',
55
+ supports_tools: true,
56
+ supports_vision: true,
57
+ });
58
+ return res.json({ object: 'list', data });
59
+ });
@@ -0,0 +1,2 @@
1
+ export declare function startHealthService(): void;
2
+ export declare function stopHealthService(): void;
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.startHealthService = startHealthService;
37
+ exports.stopHealthService = stopHealthService;
38
+ const db_1 = require("../db");
39
+ const providers_1 = require("../providers");
40
+ const INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
41
+ let timer = null;
42
+ function startHealthService() {
43
+ if (timer)
44
+ return;
45
+ // Initial check after 10s
46
+ setTimeout(runHealthChecks, 10000);
47
+ timer = setInterval(runHealthChecks, INTERVAL_MS);
48
+ }
49
+ function stopHealthService() {
50
+ if (timer) {
51
+ clearInterval(timer);
52
+ timer = null;
53
+ }
54
+ }
55
+ async function runHealthChecks() {
56
+ const keys = (0, db_1.listKeys)().filter((k) => k.enabled);
57
+ for (const k of keys) {
58
+ const provider = (0, providers_1.getChatProvider)(k.platform);
59
+ if (!provider)
60
+ continue;
61
+ try {
62
+ const { decrypt } = await Promise.resolve().then(() => __importStar(require('../crypto')));
63
+ const rawKey = decrypt(k.key_enc);
64
+ const ok = await provider.healthCheck(rawKey);
65
+ (0, db_1.updateKeyStatus)(k.id, ok ? 'healthy' : 'error');
66
+ }
67
+ catch (e) {
68
+ const err = e;
69
+ if (err.status === 401 || err.status === 403) {
70
+ (0, db_1.updateKeyStatus)(k.id, 'invalid');
71
+ }
72
+ else if (err.status === 429) {
73
+ (0, db_1.updateKeyStatus)(k.id, 'rate_limited');
74
+ }
75
+ else {
76
+ (0, db_1.updateKeyStatus)(k.id, 'error');
77
+ }
78
+ }
79
+ }
80
+ }
@@ -0,0 +1,4 @@
1
+ export declare function isOnCooldown(platform: string, keyId: number): boolean;
2
+ export declare function setCooldown(platform: string, keyId: number, ms?: number): void;
3
+ export declare function isUnderLimits(platform: string, keyId: number, estimatedTokens?: number): boolean;
4
+ export declare function recordRequest(platform: string, keyId: number, tokens?: number): void;
@@ -0,0 +1,88 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.isOnCooldown = isOnCooldown;
4
+ exports.setCooldown = setCooldown;
5
+ exports.isUnderLimits = isUnderLimits;
6
+ exports.recordRequest = recordRequest;
7
+ // Limits per platform
8
+ const LIMITS = {
9
+ google: { rpm: 15, rpd: 1500, tpm: 1000000, tpd: 1500000 },
10
+ groq: { rpm: 30, rpd: 1000, tpm: 6000 },
11
+ cerebras: { rpm: 30, rpd: 1000, tpm: 60000 },
12
+ mistral: { rpm: 5, rpd: 200, tpm: 500000, tpd: 1000000 },
13
+ openrouter: { rpm: 20, rpd: 200 },
14
+ github: { rpm: 15, rpd: 150 },
15
+ cohere: { rpm: 20, rpd: 1000 },
16
+ cloudflare: { rpm: 300, tpd: 10000 },
17
+ huggingface: { rpm: 20 },
18
+ ollama: {},
19
+ pollinations: { rpm: 60 },
20
+ llm7: { rpm: 60 },
21
+ ovh: { rpm: 60 },
22
+ nvidia: { rpm: 40 },
23
+ kilo: {},
24
+ opencode: { rpm: 20, rpd: 200 },
25
+ zhipu: { rpd: 1000000 },
26
+ };
27
+ const WINDOW_MS = {
28
+ rpm: 60000,
29
+ rpd: 86400000,
30
+ tpm: 60000,
31
+ tpd: 86400000,
32
+ };
33
+ // key: `${platform}:${keyId}:${window}`
34
+ const counters = new Map();
35
+ // Cooldown map: key `${platform}:${keyId}` → unix ms when available again
36
+ const cooldowns = new Map();
37
+ function isOnCooldown(platform, keyId) {
38
+ const until = cooldowns.get(`${platform}:${keyId}`);
39
+ if (!until)
40
+ return false;
41
+ if (Date.now() >= until) {
42
+ cooldowns.delete(`${platform}:${keyId}`);
43
+ return false;
44
+ }
45
+ return true;
46
+ }
47
+ function setCooldown(platform, keyId, ms = 60000) {
48
+ cooldowns.set(`${platform}:${keyId}`, Date.now() + ms);
49
+ }
50
+ function isUnderLimits(platform, keyId, estimatedTokens = 0) {
51
+ const limits = LIMITS[platform] ?? {};
52
+ for (const [win, limit] of Object.entries(limits)) {
53
+ const key = `${platform}:${keyId}:${win}`;
54
+ const counter = counters.get(key);
55
+ if (!counter || Date.now() >= counter.resetAt)
56
+ continue;
57
+ if (win === 'tpm' || win === 'tpd') {
58
+ if (counter.count + estimatedTokens > limit)
59
+ return false;
60
+ }
61
+ else {
62
+ if (counter.count >= limit)
63
+ return false;
64
+ }
65
+ }
66
+ return true;
67
+ }
68
+ function recordRequest(platform, keyId, tokens = 0) {
69
+ for (const win of ['rpm', 'rpd']) {
70
+ increment(platform, keyId, win, 1);
71
+ }
72
+ if (tokens > 0) {
73
+ for (const win of ['tpm', 'tpd']) {
74
+ increment(platform, keyId, win, tokens);
75
+ }
76
+ }
77
+ }
78
+ function increment(platform, keyId, win, amount) {
79
+ const key = `${platform}:${keyId}:${win}`;
80
+ const existing = counters.get(key);
81
+ const now = Date.now();
82
+ if (!existing || now >= existing.resetAt) {
83
+ counters.set(key, { count: amount, resetAt: now + WINDOW_MS[win] });
84
+ }
85
+ else {
86
+ existing.count += amount;
87
+ }
88
+ }
@@ -0,0 +1,14 @@
1
+ import { Model, ProviderKey } from '../db';
2
+ export type RouterEntry = {
3
+ model: Model;
4
+ providerKey: ProviderKey | null;
5
+ decryptedKey: string;
6
+ };
7
+ /**
8
+ * Returns an ordered list of (model, key) candidates for a request.
9
+ * Filters out disabled models, rate-limited/cooldown keys, and unhealthy keys.
10
+ */
11
+ export declare function buildCandidates(requestedModel: string, type?: string, requireVision?: boolean, requireTools?: boolean): RouterEntry[];
12
+ export declare function hasVisionContent(messages: Array<{
13
+ content: unknown;
14
+ }>): boolean;
@@ -0,0 +1,72 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.buildCandidates = buildCandidates;
4
+ exports.hasVisionContent = hasVisionContent;
5
+ const db_1 = require("../db");
6
+ const providers_1 = require("../providers");
7
+ const ratelimit_1 = require("./ratelimit");
8
+ const KEYLESS_PLATFORMS = new Set(['pollinations', 'llm7', 'ovh', 'ollama', 'kilo']);
9
+ /**
10
+ * Returns an ordered list of (model, key) candidates for a request.
11
+ * Filters out disabled models, rate-limited/cooldown keys, and unhealthy keys.
12
+ */
13
+ function buildCandidates(requestedModel, type = 'chat', requireVision = false, requireTools = false) {
14
+ const enabledModels = (0, db_1.listEnabledModels)(type);
15
+ const keys = (0, db_1.listKeys)().filter((k) => k.enabled && k.status !== 'invalid');
16
+ const keysByPlatform = {};
17
+ for (const k of keys) {
18
+ if (!keysByPlatform[k.platform])
19
+ keysByPlatform[k.platform] = [];
20
+ keysByPlatform[k.platform].push(k);
21
+ }
22
+ const candidates = [];
23
+ for (const model of enabledModels) {
24
+ // Filter by requested model
25
+ if (requestedModel !== 'auto') {
26
+ if (model.model_id !== requestedModel && model.platform + '/' + model.model_id !== requestedModel) {
27
+ continue;
28
+ }
29
+ }
30
+ if (type === 'chat') {
31
+ if (requireVision && !model.supports_vision)
32
+ continue;
33
+ if (requireTools && !model.supports_tools)
34
+ continue;
35
+ }
36
+ let provider;
37
+ if (type === 'chat')
38
+ provider = (0, providers_1.getChatProvider)(model.platform);
39
+ else if (type === 'image')
40
+ provider = (0, providers_1.getImageProvider)(model.platform);
41
+ else if (type === 'tts')
42
+ provider = (0, providers_1.getTTSProvider)(model.platform);
43
+ if (!provider)
44
+ continue;
45
+ // Keyless providers don't need a key in the DB
46
+ if (KEYLESS_PLATFORMS.has(model.platform)) {
47
+ candidates.push({ model, providerKey: null, decryptedKey: '' });
48
+ continue;
49
+ }
50
+ const platformKeys = keysByPlatform[model.platform] ?? [];
51
+ for (const pk of platformKeys) {
52
+ if ((0, ratelimit_1.isOnCooldown)(model.platform, pk.id))
53
+ continue;
54
+ if (!(0, ratelimit_1.isUnderLimits)(model.platform, pk.id))
55
+ continue;
56
+ candidates.push({
57
+ model,
58
+ providerKey: pk,
59
+ decryptedKey: (0, db_1.decryptKey)(pk.key_enc),
60
+ });
61
+ break; // one key per model is enough for the candidate list
62
+ }
63
+ }
64
+ return candidates;
65
+ }
66
+ function hasVisionContent(messages) {
67
+ return messages.some((m) => {
68
+ if (!Array.isArray(m.content))
69
+ return false;
70
+ return m.content.some((p) => p.type === 'image_url');
71
+ });
72
+ }