freellm 1.0.12 → 1.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client-dist/assets/index-DyoXu6aO.js +174 -0
- package/client-dist/index.html +1 -1
- package/dist/db/index.d.ts +16 -0
- package/dist/db/index.js +24 -2
- package/dist/providers/google.js +11 -2
- package/dist/providers/openai-compat.js +8 -4
- package/dist/providers/types.d.ts +9 -1
- package/dist/providers/types.js +3 -12
- package/dist/routes/api/analytics.js +8 -1
- package/dist/routes/api/keys.js +86 -6
- package/dist/routes/v1/chat.js +10 -4
- package/dist/services/router.d.ts +2 -0
- package/dist/services/router.js +3 -6
- package/package.json +12 -2
- package/client-dist/assets/index-nnyuHthF.js +0 -96
package/client-dist/index.html
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
9
9
|
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: #0f0f0f; color: #e1e1e1; }
|
|
10
10
|
</style>
|
|
11
|
-
<script type="module" crossorigin src="/assets/index-
|
|
11
|
+
<script type="module" crossorigin src="/assets/index-DyoXu6aO.js"></script>
|
|
12
12
|
</head>
|
|
13
13
|
<body>
|
|
14
14
|
<div id="root"></div>
|
package/dist/db/index.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export type Model = {
|
|
|
23
23
|
context_length: number | null;
|
|
24
24
|
fallback_order: number;
|
|
25
25
|
enabled: number;
|
|
26
|
+
available: number;
|
|
26
27
|
};
|
|
27
28
|
export declare function getSetting(key: string): string | undefined;
|
|
28
29
|
export declare function setSetting(key: string, value: string): void;
|
|
@@ -38,6 +39,8 @@ export declare function listModels(type?: string): Model[];
|
|
|
38
39
|
export declare function listEnabledModels(type?: string): Model[];
|
|
39
40
|
export declare function updateModelOrder(id: number, order: number): void;
|
|
40
41
|
export declare function toggleModel(id: number, enabled: boolean): void;
|
|
42
|
+
export declare function setModelAvailability(id: number, available: number): void;
|
|
43
|
+
export declare function resetPlatformAvailability(platform: string): void;
|
|
41
44
|
export declare function listModelsByPlatform(platform: string): Model[];
|
|
42
45
|
export declare function upsertModelsFromList(platform: string, models: Array<{
|
|
43
46
|
id: string;
|
|
@@ -73,6 +76,19 @@ export declare function getAnalytics(windowHours?: number): {
|
|
|
73
76
|
tokens: number;
|
|
74
77
|
}>;
|
|
75
78
|
};
|
|
79
|
+
export type RequestLog = {
|
|
80
|
+
id: number;
|
|
81
|
+
ts: number;
|
|
82
|
+
platform: string;
|
|
83
|
+
model_id: string;
|
|
84
|
+
prompt_tokens: number;
|
|
85
|
+
complete_tokens: number;
|
|
86
|
+
latency_ms: number | null;
|
|
87
|
+
success: number;
|
|
88
|
+
error_code: string | null;
|
|
89
|
+
};
|
|
90
|
+
export declare function listRequests(limit?: number): RequestLog[];
|
|
91
|
+
export declare function clearAnalytics(): void;
|
|
76
92
|
export declare function createSession(token: string, ttlSeconds?: number): void;
|
|
77
93
|
export declare function validateSession(token: string): boolean;
|
|
78
94
|
export declare function deleteSession(token: string): void;
|
package/dist/db/index.js
CHANGED
|
@@ -19,6 +19,8 @@ exports.listModels = listModels;
|
|
|
19
19
|
exports.listEnabledModels = listEnabledModels;
|
|
20
20
|
exports.updateModelOrder = updateModelOrder;
|
|
21
21
|
exports.toggleModel = toggleModel;
|
|
22
|
+
exports.setModelAvailability = setModelAvailability;
|
|
23
|
+
exports.resetPlatformAvailability = resetPlatformAvailability;
|
|
22
24
|
exports.listModelsByPlatform = listModelsByPlatform;
|
|
23
25
|
exports.upsertModelsFromList = upsertModelsFromList;
|
|
24
26
|
exports.listCustomProviders = listCustomProviders;
|
|
@@ -26,6 +28,8 @@ exports.addCustomProvider = addCustomProvider;
|
|
|
26
28
|
exports.deleteCustomProvider = deleteCustomProvider;
|
|
27
29
|
exports.logRequest = logRequest;
|
|
28
30
|
exports.getAnalytics = getAnalytics;
|
|
31
|
+
exports.listRequests = listRequests;
|
|
32
|
+
exports.clearAnalytics = clearAnalytics;
|
|
29
33
|
exports.createSession = createSession;
|
|
30
34
|
exports.validateSession = validateSession;
|
|
31
35
|
exports.deleteSession = deleteSession;
|
|
@@ -110,6 +114,10 @@ function migrate() {
|
|
|
110
114
|
CREATE INDEX IF NOT EXISTS idx_analytics_platform ON analytics(platform);
|
|
111
115
|
`);
|
|
112
116
|
seedModels();
|
|
117
|
+
const cols = db.prepare('PRAGMA table_info(models)').all();
|
|
118
|
+
if (!cols.some(c => c.name === 'available')) {
|
|
119
|
+
db.exec('ALTER TABLE models ADD COLUMN available INTEGER NOT NULL DEFAULT -1');
|
|
120
|
+
}
|
|
113
121
|
}
|
|
114
122
|
// ─── Settings ────────────────────────────────────────────────────────────────
|
|
115
123
|
function getSetting(key) {
|
|
@@ -165,6 +173,12 @@ function updateModelOrder(id, order) {
|
|
|
165
173
|
function toggleModel(id, enabled) {
|
|
166
174
|
db.prepare('UPDATE models SET enabled = ? WHERE id = ?').run(enabled ? 1 : 0, id);
|
|
167
175
|
}
|
|
176
|
+
function setModelAvailability(id, available) {
|
|
177
|
+
db.prepare('UPDATE models SET available = ? WHERE id = ?').run(available, id);
|
|
178
|
+
}
|
|
179
|
+
function resetPlatformAvailability(platform) {
|
|
180
|
+
db.prepare('UPDATE models SET available = -1 WHERE platform = ?').run(platform);
|
|
181
|
+
}
|
|
168
182
|
function listModelsByPlatform(platform) {
|
|
169
183
|
return db
|
|
170
184
|
.prepare('SELECT * FROM models WHERE platform = ? ORDER BY fallback_order, id')
|
|
@@ -204,8 +218,8 @@ function logRequest(data) {
|
|
|
204
218
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
205
219
|
`).run(data.platform, data.model_id, data.prompt_tokens, data.complete_tokens, data.latency_ms, data.success ? 1 : 0, data.error_code ?? null);
|
|
206
220
|
}
|
|
207
|
-
function getAnalytics(windowHours
|
|
208
|
-
const since = Math.floor(Date.now() / 1000) - windowHours * 3600;
|
|
221
|
+
function getAnalytics(windowHours) {
|
|
222
|
+
const since = windowHours ? Math.floor(Date.now() / 1000) - windowHours * 3600 : 0;
|
|
209
223
|
const rows = db
|
|
210
224
|
.prepare('SELECT * FROM analytics WHERE ts >= ?')
|
|
211
225
|
.all(since);
|
|
@@ -234,6 +248,14 @@ function getAnalytics(windowHours = 24) {
|
|
|
234
248
|
byPlatform,
|
|
235
249
|
};
|
|
236
250
|
}
|
|
251
|
+
function listRequests(limit = 5000) {
|
|
252
|
+
return db
|
|
253
|
+
.prepare('SELECT * FROM analytics ORDER BY id DESC LIMIT ?')
|
|
254
|
+
.all(limit);
|
|
255
|
+
}
|
|
256
|
+
function clearAnalytics() {
|
|
257
|
+
db.prepare('DELETE FROM analytics').run();
|
|
258
|
+
}
|
|
237
259
|
// ─── Sessions ─────────────────────────────────────────────────────────────────
|
|
238
260
|
function createSession(token, ttlSeconds = 86400 * 7) {
|
|
239
261
|
const expires = Math.floor(Date.now() / 1000) + ttlSeconds;
|
package/dist/providers/google.js
CHANGED
|
@@ -47,8 +47,12 @@ class GoogleProvider extends types_1.BaseProvider {
|
|
|
47
47
|
}
|
|
48
48
|
async listModels(apiKey) {
|
|
49
49
|
const res = await fetch(`${this.baseUrl}/models?key=${encodeURIComponent(apiKey)}`);
|
|
50
|
-
if (!res.ok)
|
|
51
|
-
|
|
50
|
+
if (!res.ok) {
|
|
51
|
+
const text = await res.text().catch(() => '');
|
|
52
|
+
const err = new Error(`${res.status}: ${text}`);
|
|
53
|
+
err.status = res.status;
|
|
54
|
+
throw err;
|
|
55
|
+
}
|
|
52
56
|
const data = await res.json();
|
|
53
57
|
return (data.models ?? [])
|
|
54
58
|
.filter((m) => !!m.name && (m.supportedGenerationMethods ?? []).includes('generateContent'))
|
|
@@ -171,6 +175,11 @@ class GoogleProvider extends types_1.BaseProvider {
|
|
|
171
175
|
finish_reason: finishReason?.toLowerCase() ?? null,
|
|
172
176
|
},
|
|
173
177
|
],
|
|
178
|
+
usage: data.usageMetadata ? {
|
|
179
|
+
prompt_tokens: data.usageMetadata.promptTokenCount ?? 0,
|
|
180
|
+
completion_tokens: data.usageMetadata.candidatesTokenCount ?? 0,
|
|
181
|
+
total_tokens: data.usageMetadata.totalTokenCount ?? 0,
|
|
182
|
+
} : undefined,
|
|
174
183
|
});
|
|
175
184
|
}
|
|
176
185
|
catch { /* skip */ }
|
|
@@ -18,8 +18,12 @@ class OpenAICompatProvider extends types_1.BaseProvider {
|
|
|
18
18
|
if (apiKey)
|
|
19
19
|
headers['Authorization'] = `Bearer ${apiKey}`;
|
|
20
20
|
const res = await fetch(`${this.baseUrl}/models`, { headers });
|
|
21
|
-
if (!res.ok)
|
|
22
|
-
|
|
21
|
+
if (!res.ok) {
|
|
22
|
+
const text = await res.text().catch(() => '');
|
|
23
|
+
const err = new Error(`${res.status}: ${text}`);
|
|
24
|
+
err.status = res.status;
|
|
25
|
+
throw err;
|
|
26
|
+
}
|
|
23
27
|
const data = await res.json();
|
|
24
28
|
return (data.data ?? [])
|
|
25
29
|
.filter((m) => !!m.id)
|
|
@@ -56,7 +60,7 @@ class OpenAICompatProvider extends types_1.BaseProvider {
|
|
|
56
60
|
res = await fetch(`${this.baseUrl}/chat/completions`, {
|
|
57
61
|
method: 'POST',
|
|
58
62
|
headers,
|
|
59
|
-
body: JSON.stringify({ ...req, stream: true }),
|
|
63
|
+
body: JSON.stringify({ ...req, stream: true, stream_options: { include_usage: true } }),
|
|
60
64
|
});
|
|
61
65
|
}
|
|
62
66
|
catch (e) {
|
|
@@ -120,7 +124,7 @@ exports.PROVIDERS = {
|
|
|
120
124
|
cohere: new OpenAICompatProvider('cohere', 'https://api.cohere.com/compatibility/v1', 'command-r'),
|
|
121
125
|
llm7: new OpenAICompatProvider('llm7', 'https://llm7.io/v1', 'gpt-4o-mini'),
|
|
122
126
|
ovh: new OpenAICompatProvider('ovh', 'https://oai.endpoints.kepler.ai.cloud.ovh.net/v1', 'Llama-3.3-70B-Instruct'),
|
|
123
|
-
nvidia: new OpenAICompatProvider('nvidia', 'https://integrate.api.nvidia.com/v1', 'deepseek-ai/deepseek-
|
|
127
|
+
nvidia: new OpenAICompatProvider('nvidia', 'https://integrate.api.nvidia.com/v1', 'deepseek-ai/deepseek-coder-6.7b-instruct'),
|
|
124
128
|
kilo: new OpenAICompatProvider('kilo', 'https://gateway.kilo.chat/v1', 'poolside/laguna-m.1:free'),
|
|
125
129
|
opencode: new OpenAICompatProvider('opencode', 'https://api.opencode.ai/v1', 'deepseek-v4-flash-free'),
|
|
126
130
|
zhipu: new OpenAICompatProvider('zhipu', 'https://open.bigmodel.cn/api/paas/v4', 'glm-4.7-flash'),
|
|
@@ -35,6 +35,9 @@ export type ChatRequest = {
|
|
|
35
35
|
model: string;
|
|
36
36
|
messages: ChatMessage[];
|
|
37
37
|
stream?: boolean;
|
|
38
|
+
stream_options?: {
|
|
39
|
+
include_usage?: boolean;
|
|
40
|
+
};
|
|
38
41
|
temperature?: number;
|
|
39
42
|
max_tokens?: number;
|
|
40
43
|
top_p?: number;
|
|
@@ -81,6 +84,11 @@ export type StreamChunk = {
|
|
|
81
84
|
};
|
|
82
85
|
finish_reason: string | null;
|
|
83
86
|
}>;
|
|
87
|
+
usage?: {
|
|
88
|
+
prompt_tokens: number;
|
|
89
|
+
completion_tokens: number;
|
|
90
|
+
total_tokens: number;
|
|
91
|
+
};
|
|
84
92
|
};
|
|
85
93
|
export declare abstract class BaseProvider {
|
|
86
94
|
abstract readonly platform: string;
|
|
@@ -88,7 +96,7 @@ export declare abstract class BaseProvider {
|
|
|
88
96
|
protected fetchJSON<T>(url: string, options: RequestInit): Promise<T>;
|
|
89
97
|
abstract chat(apiKey: string, req: ChatRequest): Promise<ChatResponse>;
|
|
90
98
|
abstract streamChat(apiKey: string, req: ChatRequest, onChunk: (chunk: StreamChunk) => void, onDone: () => void, onError: (err: Error) => void): Promise<void>;
|
|
91
|
-
/** Default implementation:
|
|
99
|
+
/** Default implementation: verify the key via the provider's /models endpoint; throws on failure */
|
|
92
100
|
healthCheck(apiKey: string): Promise<boolean>;
|
|
93
101
|
/** Fetch available models from the provider. Default: none (provider does not expose a list). */
|
|
94
102
|
listModels(_apiKey: string): Promise<Array<{
|
package/dist/providers/types.js
CHANGED
|
@@ -13,19 +13,10 @@ class BaseProvider {
|
|
|
13
13
|
}
|
|
14
14
|
return res.json();
|
|
15
15
|
}
|
|
16
|
-
/** Default implementation:
|
|
16
|
+
/** Default implementation: verify the key via the provider's /models endpoint; throws on failure */
|
|
17
17
|
async healthCheck(apiKey) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
model: this.defaultModel(),
|
|
21
|
-
messages: [{ role: 'user', content: 'hi' }],
|
|
22
|
-
max_tokens: 1,
|
|
23
|
-
});
|
|
24
|
-
return true;
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
return false;
|
|
28
|
-
}
|
|
18
|
+
await this.listModels(apiKey);
|
|
19
|
+
return true;
|
|
29
20
|
}
|
|
30
21
|
/** Fetch available models from the provider. Default: none (provider does not expose a list). */
|
|
31
22
|
async listModels(_apiKey) {
|
|
@@ -5,6 +5,13 @@ const express_1 = require("express");
|
|
|
5
5
|
const db_1 = require("../../db");
|
|
6
6
|
exports.analyticsRouter = (0, express_1.Router)();
|
|
7
7
|
exports.analyticsRouter.get('/analytics', (req, res) => {
|
|
8
|
-
const hours = Number(req.query.hours) ||
|
|
8
|
+
const hours = Number(req.query.hours) || undefined;
|
|
9
9
|
return res.json((0, db_1.getAnalytics)(hours));
|
|
10
10
|
});
|
|
11
|
+
exports.analyticsRouter.get('/analytics/requests', (_req, res) => {
|
|
12
|
+
return res.json((0, db_1.listRequests)());
|
|
13
|
+
});
|
|
14
|
+
exports.analyticsRouter.delete('/analytics', (_req, res) => {
|
|
15
|
+
(0, db_1.clearAnalytics)();
|
|
16
|
+
return res.json({ ok: true });
|
|
17
|
+
});
|
package/dist/routes/api/keys.js
CHANGED
|
@@ -1,24 +1,28 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
6
|
exports.keysRouter = void 0;
|
|
4
7
|
const express_1 = require("express");
|
|
8
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
5
9
|
const db_1 = require("../../db");
|
|
6
10
|
const db_2 = require("../../db");
|
|
7
|
-
const
|
|
11
|
+
const crypto_2 = require("../../crypto");
|
|
8
12
|
const providers_1 = require("../../providers");
|
|
9
13
|
exports.keysRouter = (0, express_1.Router)();
|
|
10
14
|
// Get unified API key (or generate one)
|
|
11
15
|
exports.keysRouter.get('/keys/unified', (_req, res) => {
|
|
12
16
|
let key = (0, db_2.getSetting)('unified_key');
|
|
13
17
|
if (!key) {
|
|
14
|
-
key = (0,
|
|
18
|
+
key = (0, crypto_2.generateUnifiedKey)();
|
|
15
19
|
(0, db_2.setSetting)('unified_key', key);
|
|
16
20
|
}
|
|
17
21
|
return res.json({ key });
|
|
18
22
|
});
|
|
19
23
|
// Regenerate unified key
|
|
20
24
|
exports.keysRouter.post('/keys/unified/regenerate', (_req, res) => {
|
|
21
|
-
const key = (0,
|
|
25
|
+
const key = (0, crypto_2.generateUnifiedKey)();
|
|
22
26
|
(0, db_2.setSetting)('unified_key', key);
|
|
23
27
|
return res.json({ key });
|
|
24
28
|
});
|
|
@@ -47,10 +51,13 @@ exports.keysRouter.get('/keys', (_req, res) => {
|
|
|
47
51
|
// Add provider key
|
|
48
52
|
exports.keysRouter.post('/keys', (req, res) => {
|
|
49
53
|
const { platform, key, label } = req.body;
|
|
50
|
-
if (!platform
|
|
51
|
-
return res.status(400).json({ error: 'platform
|
|
54
|
+
if (!platform) {
|
|
55
|
+
return res.status(400).json({ error: 'platform is required' });
|
|
52
56
|
}
|
|
53
|
-
|
|
57
|
+
if (!key && !KEYLESS_PLATFORMS.has(platform)) {
|
|
58
|
+
return res.status(400).json({ error: 'key is required for this platform' });
|
|
59
|
+
}
|
|
60
|
+
const id = (0, db_1.addKey)(platform, key ?? '', label);
|
|
54
61
|
return res.json({ id });
|
|
55
62
|
});
|
|
56
63
|
// Delete provider key
|
|
@@ -116,3 +123,76 @@ exports.keysRouter.post('/keys/:id/models/sync', async (req, res) => {
|
|
|
116
123
|
return res.status(500).json({ error: e.message });
|
|
117
124
|
}
|
|
118
125
|
});
|
|
126
|
+
const checkJobs = new Map();
|
|
127
|
+
const CHECK_PACE_MS = 1500;
|
|
128
|
+
const KEYLESS_PLATFORMS = new Set(['pollinations', 'llm7', 'ovh', 'ollama', 'kilo']);
|
|
129
|
+
// Start checking every chat model of this key against the provider, one by one
|
|
130
|
+
exports.keysRouter.post('/keys/:id/models/check', async (req, res) => {
|
|
131
|
+
const k = (0, db_1.getKey)(Number(req.params.id));
|
|
132
|
+
if (!k)
|
|
133
|
+
return res.status(404).json({ error: 'key not found' });
|
|
134
|
+
const provider = (0, providers_1.getChatProvider)(k.platform);
|
|
135
|
+
if (!provider)
|
|
136
|
+
return res.status(400).json({ error: `no provider for platform '${k.platform}'` });
|
|
137
|
+
for (const [id, job] of checkJobs) {
|
|
138
|
+
if (job.keyId === k.id && !job.done)
|
|
139
|
+
return res.json({ jobId: id, total: job.total });
|
|
140
|
+
}
|
|
141
|
+
const models = (0, db_1.listModelsByPlatform)(k.platform).filter((m) => m.type === 'chat');
|
|
142
|
+
const job = {
|
|
143
|
+
keyId: k.id,
|
|
144
|
+
total: models.length,
|
|
145
|
+
checked: 0,
|
|
146
|
+
current: '',
|
|
147
|
+
available: {},
|
|
148
|
+
done: false,
|
|
149
|
+
};
|
|
150
|
+
const jobId = crypto_1.default.randomBytes(8).toString('hex');
|
|
151
|
+
checkJobs.set(jobId, job);
|
|
152
|
+
const rawKey = (0, db_1.decryptKey)(k.key_enc);
|
|
153
|
+
(0, db_1.resetPlatformAvailability)(k.platform);
|
|
154
|
+
(async () => {
|
|
155
|
+
try {
|
|
156
|
+
for (const m of models) {
|
|
157
|
+
job.current = m.model_id;
|
|
158
|
+
try {
|
|
159
|
+
await Promise.race([
|
|
160
|
+
provider.chat(rawKey, {
|
|
161
|
+
model: m.model_id,
|
|
162
|
+
messages: [{ role: 'user', content: 'hi' }],
|
|
163
|
+
max_tokens: 1,
|
|
164
|
+
}),
|
|
165
|
+
new Promise((_resolve, reject) => setTimeout(() => reject(new Error('timeout')), 30000)),
|
|
166
|
+
]);
|
|
167
|
+
job.available[m.id] = 1;
|
|
168
|
+
(0, db_1.setModelAvailability)(m.id, 1);
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
job.available[m.id] = 0;
|
|
172
|
+
(0, db_1.setModelAvailability)(m.id, 0);
|
|
173
|
+
}
|
|
174
|
+
job.checked++;
|
|
175
|
+
await new Promise((r) => setTimeout(r, CHECK_PACE_MS));
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
finally {
|
|
179
|
+
job.done = true;
|
|
180
|
+
job.current = '';
|
|
181
|
+
setTimeout(() => checkJobs.delete(jobId), 10 * 60 * 1000);
|
|
182
|
+
}
|
|
183
|
+
})();
|
|
184
|
+
return res.json({ jobId, total: models.length });
|
|
185
|
+
});
|
|
186
|
+
// Poll check progress
|
|
187
|
+
exports.keysRouter.get('/keys/:id/models/check/:jobId', (_req, res) => {
|
|
188
|
+
const job = checkJobs.get(_req.params.jobId);
|
|
189
|
+
if (!job)
|
|
190
|
+
return res.status(404).json({ error: 'job not found' });
|
|
191
|
+
return res.json({
|
|
192
|
+
done: job.done,
|
|
193
|
+
total: job.total,
|
|
194
|
+
checked: job.checked,
|
|
195
|
+
current: job.current,
|
|
196
|
+
available: job.available,
|
|
197
|
+
});
|
|
198
|
+
});
|
package/dist/routes/v1/chat.js
CHANGED
|
@@ -50,10 +50,16 @@ exports.chatRouter.post('/chat/completions', async (req, res) => {
|
|
|
50
50
|
let promptTokens = 0;
|
|
51
51
|
let completionTokens = 0;
|
|
52
52
|
let done = false;
|
|
53
|
-
await provider.streamChat(candidate.decryptedKey, { ...body, model: platformModel }, (chunk) => {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
await provider.streamChat(candidate.decryptedKey, { ...body, model: platformModel, stream_options: { include_usage: true } }, (chunk) => {
|
|
54
|
+
if (chunk.usage) {
|
|
55
|
+
promptTokens = chunk.usage.prompt_tokens;
|
|
56
|
+
completionTokens = chunk.usage.completion_tokens;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
completionTokens += chunk.choices?.[0]?.delta?.content?.length
|
|
60
|
+
? Math.ceil(chunk.choices[0].delta.content.length / 4)
|
|
61
|
+
: 0;
|
|
62
|
+
}
|
|
57
63
|
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
|
58
64
|
}, () => {
|
|
59
65
|
done = true;
|
|
@@ -6,6 +6,8 @@ export type RouterEntry = {
|
|
|
6
6
|
};
|
|
7
7
|
/**
|
|
8
8
|
* Returns an ordered list of (model, key) candidates for a request.
|
|
9
|
+
* Follows the model priority list (fallback order) and only includes
|
|
10
|
+
* platforms that have an added, enabled, healthy key.
|
|
9
11
|
* Filters out disabled models, rate-limited/cooldown keys, and unhealthy keys.
|
|
10
12
|
*/
|
|
11
13
|
export declare function buildCandidates(requestedModel: string, type?: string, requireVision?: boolean, requireTools?: boolean): RouterEntry[];
|
package/dist/services/router.js
CHANGED
|
@@ -5,9 +5,10 @@ exports.hasVisionContent = hasVisionContent;
|
|
|
5
5
|
const db_1 = require("../db");
|
|
6
6
|
const providers_1 = require("../providers");
|
|
7
7
|
const ratelimit_1 = require("./ratelimit");
|
|
8
|
-
const KEYLESS_PLATFORMS = new Set(['pollinations', 'llm7', 'ovh', 'ollama', 'kilo']);
|
|
9
8
|
/**
|
|
10
9
|
* Returns an ordered list of (model, key) candidates for a request.
|
|
10
|
+
* Follows the model priority list (fallback order) and only includes
|
|
11
|
+
* platforms that have an added, enabled, healthy key.
|
|
11
12
|
* Filters out disabled models, rate-limited/cooldown keys, and unhealthy keys.
|
|
12
13
|
*/
|
|
13
14
|
function buildCandidates(requestedModel, type = 'chat', requireVision = false, requireTools = false) {
|
|
@@ -42,11 +43,7 @@ function buildCandidates(requestedModel, type = 'chat', requireVision = false, r
|
|
|
42
43
|
provider = (0, providers_1.getTTSProvider)(model.platform);
|
|
43
44
|
if (!provider)
|
|
44
45
|
continue;
|
|
45
|
-
//
|
|
46
|
-
if (KEYLESS_PLATFORMS.has(model.platform)) {
|
|
47
|
-
candidates.push({ model, providerKey: null, decryptedKey: '' });
|
|
48
|
-
continue;
|
|
49
|
-
}
|
|
46
|
+
// Providers are only used when a key has been added for them
|
|
50
47
|
const platformKeys = keysByPlatform[model.platform] ?? [];
|
|
51
48
|
for (const pk of platformKeys) {
|
|
52
49
|
if ((0, ratelimit_1.isOnCooldown)(model.platform, pk.id))
|
package/package.json
CHANGED
|
@@ -1,8 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "freellm",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.14",
|
|
4
4
|
"description": "One OpenAI-compatible endpoint for 15+ free LLM providers. Smart routing, failover, encrypted keys. Run with npx freellm.",
|
|
5
|
-
"keywords": [
|
|
5
|
+
"keywords": [
|
|
6
|
+
"llm",
|
|
7
|
+
"openai",
|
|
8
|
+
"proxy",
|
|
9
|
+
"groq",
|
|
10
|
+
"gemini",
|
|
11
|
+
"mistral",
|
|
12
|
+
"free",
|
|
13
|
+
"api-gateway"
|
|
14
|
+
],
|
|
6
15
|
"author": "",
|
|
7
16
|
"license": "MIT",
|
|
8
17
|
"bin": {
|
|
@@ -31,6 +40,7 @@
|
|
|
31
40
|
"cookie-parser": "^1.4.6",
|
|
32
41
|
"cors": "^2.8.5",
|
|
33
42
|
"express": "^4.18.3",
|
|
43
|
+
"marked": "^18.0.9",
|
|
34
44
|
"open": "^10.1.0",
|
|
35
45
|
"react": "^18.2.0",
|
|
36
46
|
"react-dom": "^18.2.0"
|