freegate 0.6.11 → 0.6.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/README.ru.md +13 -13
- package/bin/freegate.js +15 -15
- package/lib/clean.js +11 -1
- package/package.json +1 -1
- package/providers.json +20 -0
- package/server.js +208 -55
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# Freegate
|
|
2
2
|
|
|
3
|
-
[](https://github.com/Artur21101965/freegate/actions)
|
|
4
4
|
[](https://hub.docker.com/r/nik951751/freegate)
|
|
5
5
|
[](https://opensource.org/licenses/MIT)
|
|
6
|
-
[](https://www.npmjs.com/package/freegate)
|
|
7
|
+
[](https://github.com/Artur21101965/freegate)
|
|
8
8
|
|
|
9
9
|
**[Русская версия](README.ru.md) · Russian version**
|
|
10
10
|
|
|
@@ -39,9 +39,9 @@ never see "rate limit", and you never pay.
|
|
|
39
39
|
## Quick start — 30 seconds
|
|
40
40
|
|
|
41
41
|
```bash
|
|
42
|
-
npx
|
|
43
|
-
npx
|
|
44
|
-
npx
|
|
42
|
+
npx freegate init -i # wizard: password + each provider's key
|
|
43
|
+
npx freegate start # proxy on http://localhost:4000
|
|
44
|
+
npx freegate test # verify everything works
|
|
45
45
|
```
|
|
46
46
|
|
|
47
47
|
Dashboard: `http://localhost:4000/?key=your_password`
|
|
@@ -49,7 +49,7 @@ Dashboard: `http://localhost:4000/?key=your_password`
|
|
|
49
49
|
Or via Docker:
|
|
50
50
|
|
|
51
51
|
```bash
|
|
52
|
-
docker run -d --name
|
|
52
|
+
docker run -d --name freegate -p 4000:4000 \
|
|
53
53
|
-e PROVIDER_GROQ_APIKEY=... \
|
|
54
54
|
-e PROVIDER_MISTRAL_APIKEY=... \
|
|
55
55
|
-e AUTH=your-secret-key \
|
|
@@ -146,13 +146,13 @@ put it in `config.json` → `providers` (same format as `providers.json`).
|
|
|
146
146
|
## CLI commands
|
|
147
147
|
|
|
148
148
|
```bash
|
|
149
|
-
npx
|
|
150
|
-
npx
|
|
151
|
-
npx
|
|
152
|
-
npx
|
|
153
|
-
npx
|
|
154
|
-
npx
|
|
155
|
-
npx
|
|
149
|
+
npx freegate init # create config
|
|
150
|
+
npx freegate init -i # interactive wizard (keys, password)
|
|
151
|
+
npx freegate start # start proxy
|
|
152
|
+
npx freegate status # diagnostics: providers, limits, errors
|
|
153
|
+
npx freegate test # verify it works
|
|
154
|
+
npx freegate install-service # autostart at boot
|
|
155
|
+
npx freegate dashboard # open dashboard
|
|
156
156
|
```
|
|
157
157
|
|
|
158
158
|
## FAQ
|
package/README.ru.md
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# Freegate
|
|
2
2
|
|
|
3
|
-
[](https://github.com/Artur21101965/freegate/actions)
|
|
4
4
|
[](https://hub.docker.com/r/nik951751/freegate)
|
|
5
5
|
[](https://opensource.org/licenses/MIT)
|
|
6
|
-
[](https://www.npmjs.com/package/freegate)
|
|
7
7
|
|
|
8
8
|
**English version: [README.md](README.md)**
|
|
9
9
|
|
|
@@ -38,9 +38,9 @@ DeepSeek и локальные модели. Если один провайде
|
|
|
38
38
|
## Быстрый старт — 30 секунд
|
|
39
39
|
|
|
40
40
|
```bash
|
|
41
|
-
npx
|
|
42
|
-
npx
|
|
43
|
-
npx
|
|
41
|
+
npx freegate init -i # мастер спросит: пароль + ключи каждого провайдера
|
|
42
|
+
npx freegate start # прокси на http://localhost:4000
|
|
43
|
+
npx freegate test # проверить, что всё работает
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
Дашборд: `http://localhost:4000/?key=твой_пароль`
|
|
@@ -48,7 +48,7 @@ npx davil-cod test # проверить, что всё работает
|
|
|
48
48
|
Или через Docker:
|
|
49
49
|
|
|
50
50
|
```bash
|
|
51
|
-
docker run -d --name
|
|
51
|
+
docker run -d --name freegate -p 4000:4000 \
|
|
52
52
|
-e PROVIDER_GROQ_APIKEY=... \
|
|
53
53
|
-e PROVIDER_MISTRAL_APIKEY=... \
|
|
54
54
|
-e AUTH=your-secret-key \
|
|
@@ -145,13 +145,13 @@ PROVIDER_ZAI_APIKEY=... # ZAI
|
|
|
145
145
|
## Команды CLI
|
|
146
146
|
|
|
147
147
|
```bash
|
|
148
|
-
npx
|
|
149
|
-
npx
|
|
150
|
-
npx
|
|
151
|
-
npx
|
|
152
|
-
npx
|
|
153
|
-
npx
|
|
154
|
-
npx
|
|
148
|
+
npx freegate init # создать конфиг
|
|
149
|
+
npx freegate init -i # интерактивный мастер (ключи, пароль)
|
|
150
|
+
npx freegate start # запустить прокси
|
|
151
|
+
npx freegate status # диагностика: провайдеры, лимиты, ошибки
|
|
152
|
+
npx freegate test # проверить, что работает
|
|
153
|
+
npx freegate install-service # автозапуск при старте системы
|
|
154
|
+
npx freegate dashboard # открыть дашборд
|
|
155
155
|
```
|
|
156
156
|
|
|
157
157
|
## FAQ
|
package/bin/freegate.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// bin/
|
|
2
|
+
// bin/freegate.js — CLI wrapper + interactive setup wizard
|
|
3
3
|
const path = require('path');
|
|
4
4
|
const fs = require('fs');
|
|
5
5
|
const { spawn } = require('child_process');
|
|
@@ -74,7 +74,7 @@ async function initWizard() {
|
|
|
74
74
|
|
|
75
75
|
const filled = Object.values(keys).filter(Boolean).length;
|
|
76
76
|
console.log(`\nГотово! Провайдеров с ключами: ${filled} из ${envVars.size}`);
|
|
77
|
-
console.log(`Запуск: npx
|
|
77
|
+
console.log(`Запуск: npx freegate start (пароль: ${authKey})`);
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
function copyExample(name, example) {
|
|
@@ -95,8 +95,8 @@ if (cmd === 'init') {
|
|
|
95
95
|
// Non-interactive fallback: plain copy (safe — never overwrites)
|
|
96
96
|
copyExample('config.json', 'config.example.json');
|
|
97
97
|
copyExample('.env', '.env.example');
|
|
98
|
-
console.log('\nГотово! Заполни .env ключами, затем: npx
|
|
99
|
-
console.log('Совет: npx
|
|
98
|
+
console.log('\nГотово! Заполни .env ключами, затем: npx freegate start');
|
|
99
|
+
console.log('Совет: npx freegate init -i — интерактивный мастер с вопросами.');
|
|
100
100
|
}
|
|
101
101
|
} else if (cmd === 'start') {
|
|
102
102
|
// Spawn from the user's cwd (not package dir) so server.js picks up their
|
|
@@ -150,10 +150,10 @@ RestartSec=5
|
|
|
150
150
|
[Install]
|
|
151
151
|
WantedBy=default.target
|
|
152
152
|
`;
|
|
153
|
-
const unitPath = path.join(os.homedir(), '.config', 'systemd', 'user', '
|
|
153
|
+
const unitPath = path.join(os.homedir(), '.config', 'systemd', 'user', 'freegate.service');
|
|
154
154
|
fs.mkdirSync(path.dirname(unitPath), { recursive: true });
|
|
155
155
|
fs.writeFileSync(unitPath, unit);
|
|
156
|
-
execSync(`systemctl --user daemon-reload && systemctl --user enable
|
|
156
|
+
execSync(`systemctl --user daemon-reload && systemctl --user enable freegate && systemctl --user start freegate`);
|
|
157
157
|
console.log('✅ Служба автозапуска установлена (systemd)');
|
|
158
158
|
}
|
|
159
159
|
} else if (cmd === 'dashboard') {
|
|
@@ -174,7 +174,7 @@ WantedBy=default.target
|
|
|
174
174
|
});
|
|
175
175
|
}).on('error', (err) => {
|
|
176
176
|
console.log('❌ Прокси не запущен на ' + base);
|
|
177
|
-
console.log(' Запусти: npx
|
|
177
|
+
console.log(' Запусти: npx freegate start');
|
|
178
178
|
process.exit(1);
|
|
179
179
|
});
|
|
180
180
|
} else if (cmd === 'status') {
|
|
@@ -189,7 +189,7 @@ WantedBy=default.target
|
|
|
189
189
|
res.on('end', () => {
|
|
190
190
|
if (res.statusCode !== 200) {
|
|
191
191
|
console.log('❌ Прокси не отвечает (HTTP ' + res.statusCode + ')');
|
|
192
|
-
console.log(' Запусти: npx
|
|
192
|
+
console.log(' Запусти: npx freegate start');
|
|
193
193
|
process.exit(1);
|
|
194
194
|
}
|
|
195
195
|
try {
|
|
@@ -211,7 +211,7 @@ WantedBy=default.target
|
|
|
211
211
|
});
|
|
212
212
|
}).on('error', () => {
|
|
213
213
|
console.log('❌ Прокси не запущен на ' + base);
|
|
214
|
-
console.log(' Запусти: npx
|
|
214
|
+
console.log(' Запусти: npx freegate start');
|
|
215
215
|
process.exit(1);
|
|
216
216
|
});
|
|
217
217
|
} else if (cmd === 'version' || cmd === '-v' || cmd === '--version') {
|
|
@@ -220,10 +220,10 @@ WantedBy=default.target
|
|
|
220
220
|
} else {
|
|
221
221
|
console.log('Freegate — бесплатный LLM-прокси с failover');
|
|
222
222
|
console.log('Команды:');
|
|
223
|
-
console.log(' npx
|
|
224
|
-
console.log(' npx
|
|
225
|
-
console.log(' npx
|
|
226
|
-
console.log(' npx
|
|
227
|
-
console.log(' npx
|
|
228
|
-
console.log(' npx
|
|
223
|
+
console.log(' npx freegate init интерактивная настройка (ключи, пароль)');
|
|
224
|
+
console.log(' npx freegate start запустить прокси');
|
|
225
|
+
console.log(' npx freegate status диагностика: провайдеры, лимиты, ошибки');
|
|
226
|
+
console.log(' npx freegate test проверить, что прокси работает');
|
|
227
|
+
console.log(' npx freegate dashboard открыть дашборд');
|
|
228
|
+
console.log(' npx freegate install-service автозапуск при старте системы');
|
|
229
229
|
}
|
package/lib/clean.js
CHANGED
|
@@ -65,4 +65,14 @@ function hasContent(data) {
|
|
|
65
65
|
return content.length > 0 || reasoning.length > 0;
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
|
|
68
|
+
// Ответ короче MIN_ANSWER_LEN символов считается мусором (обрыв/один токен).
|
|
69
|
+
const MIN_ANSWER_LEN = 5;
|
|
70
|
+
function isTooShort(data) {
|
|
71
|
+
if (!data || !Array.isArray(data.choices)) return true;
|
|
72
|
+
const msg = data.choices[0]?.message;
|
|
73
|
+
const content = typeof msg?.content === 'string' ? msg.content.trim() : '';
|
|
74
|
+
const reasoning = typeof msg?.reasoning === 'string' ? msg.reasoning.trim() : '';
|
|
75
|
+
return (content + reasoning).length < MIN_ANSWER_LEN;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
module.exports = { stripThink, cleanMessage, cleanDelta, fixReasoningMessage, hasContent, isTooShort, MIN_ANSWER_LEN };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "freegate",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.13",
|
|
4
4
|
"description": "Free multi-provider LLM gateway with automatic failover. One OpenAI-compatible endpoint routes to 25 free models (Groq, Mistral, Gemini, NIM, OpenRouter, ZAI, Cerebras, DeepSeek). Never pay for LLMs.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"bin": {
|
package/providers.json
CHANGED
|
@@ -374,5 +374,25 @@
|
|
|
374
374
|
"envVar": "PROVIDER_HF_APIKEY",
|
|
375
375
|
"free": true,
|
|
376
376
|
"category": "general"
|
|
377
|
+
},
|
|
378
|
+
"or-minimax-m3-free": {
|
|
379
|
+
"endpoint": "https://openrouter.ai/api/v1/chat/completions",
|
|
380
|
+
"model": "minimax/minimax-m3:free",
|
|
381
|
+
"priority": 20,
|
|
382
|
+
"dailyLimit": 50,
|
|
383
|
+
"keyHint": "openrouter.ai → Keys (автодобавлено, vision)",
|
|
384
|
+
"envVar": "PROVIDER_OPENROUTER_APIKEY",
|
|
385
|
+
"free": true,
|
|
386
|
+
"category": "vision"
|
|
387
|
+
},
|
|
388
|
+
"or-minimax-m2-7-free": {
|
|
389
|
+
"endpoint": "https://openrouter.ai/api/v1/chat/completions",
|
|
390
|
+
"model": "minimax/minimax-m2.7:free",
|
|
391
|
+
"priority": 20,
|
|
392
|
+
"dailyLimit": 50,
|
|
393
|
+
"keyHint": "openrouter.ai → Keys (автодобавлено, reasoning)",
|
|
394
|
+
"envVar": "PROVIDER_OPENROUTER_APIKEY",
|
|
395
|
+
"free": true,
|
|
396
|
+
"category": "reasoning"
|
|
377
397
|
}
|
|
378
398
|
}
|
package/server.js
CHANGED
|
@@ -8,7 +8,7 @@ const { loadState, initHealth, isCircuitOpen, recordSuccess, recordFailure, reco
|
|
|
8
8
|
const { checkRateLimit } = require('./lib/rateLimit');
|
|
9
9
|
const { handleDashboard } = require('./lib/dashboard');
|
|
10
10
|
const { acquire, stats: poolStats } = require('./lib/pool');
|
|
11
|
-
const { stripThink, cleanDelta, cleanMessage, fixReasoningMessage,
|
|
11
|
+
const { stripThink, cleanDelta, cleanMessage, fixReasoningMessage, isTooShort, MIN_ANSWER_LEN } = require('./lib/clean');
|
|
12
12
|
const { classifyComplexity, maybeUpgradeTier, classifyVisionComplexity } = require('./lib/routing');
|
|
13
13
|
const logger = require('./lib/logger');
|
|
14
14
|
|
|
@@ -106,17 +106,33 @@ async function checkProvider(key, provider) {
|
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
+
const PROBE_CAP = 8;
|
|
110
|
+
|
|
109
111
|
async function healthCheck() {
|
|
110
112
|
const now = Date.now();
|
|
111
|
-
// Параллельный
|
|
112
|
-
//
|
|
113
|
+
// Параллельный пробинг с лимитом: пакетами по PROBE_CAP, чтобы не создавать
|
|
114
|
+
// десятки одновременных fetch-запросов при 30+ провайдерах.
|
|
113
115
|
const due = Object.entries(PROVIDERS)
|
|
114
116
|
.filter(([key, provider]) => provider.enabled && !(healthIntervals[key] && healthIntervals[key].nextCheck > now));
|
|
115
|
-
|
|
117
|
+
for (let i = 0; i < due.length; i += PROBE_CAP) {
|
|
118
|
+
const batch = due.slice(i, i + PROBE_CAP);
|
|
119
|
+
await Promise.allSettled(batch.map(([key, provider]) => checkProvider(key, provider)));
|
|
120
|
+
}
|
|
116
121
|
}
|
|
117
122
|
setInterval(healthCheck, 30000);
|
|
118
123
|
setTimeout(healthCheck, 1000);
|
|
119
124
|
|
|
125
|
+
// Извлекает все значения content из SSE-чанка. Возвращает true, если есть
|
|
126
|
+
// хотя бы одно непустое (реальный токен, а не пустая дельта).
|
|
127
|
+
function chunkHasToken(str) {
|
|
128
|
+
const re = /"content"\s*:\s*"((?:[^"\\]|\\.)*)"/g;
|
|
129
|
+
let m;
|
|
130
|
+
while ((m = re.exec(str)) !== null) {
|
|
131
|
+
if (m[1].trim().length > 0) return true;
|
|
132
|
+
}
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
|
|
120
136
|
// Chat completion handler
|
|
121
137
|
async function handleChatCompletion(req, res, body) {
|
|
122
138
|
const requestedModel = body.model || 'tier-splus';
|
|
@@ -358,65 +374,149 @@ async function handleChatCompletion(req, res, body) {
|
|
|
358
374
|
fixReasoningMessage(result.data.choices[0].message);
|
|
359
375
|
cleanMessage(result.data.choices[0].message);
|
|
360
376
|
}
|
|
361
|
-
if (
|
|
362
|
-
//
|
|
363
|
-
const msg = key + ': empty response';
|
|
377
|
+
if (isTooShort(result.data)) {
|
|
378
|
+
// Пустой/мусорный ответ (провайдер-глитч) НЕ считается успехом — пробуем следующего.
|
|
379
|
+
const msg = key + ': empty or too short response';
|
|
364
380
|
errors.push(msg);
|
|
365
381
|
recordFailure(key, 0);
|
|
366
382
|
recordRequest(key, false, msg);
|
|
367
383
|
recordRecent({ model: requestedModel, provider: key, status: 204, latency: result.latency, cached: false });
|
|
368
|
-
logger.warn('Empty response, trying next provider', { key });
|
|
384
|
+
logger.warn('Empty or too-short response, trying next provider', { key });
|
|
369
385
|
continue;
|
|
370
386
|
}
|
|
371
387
|
}
|
|
372
388
|
|
|
373
|
-
recordSuccess(key);
|
|
374
|
-
recordRequest(key, true);
|
|
375
|
-
logger.request({ model: requestedModel, provider: key, status: 200, latency: result.latency, stream: isStreaming });
|
|
376
|
-
recordRecent({ model: requestedModel, provider: key, status: 200, latency: result.latency, cached: false });
|
|
377
|
-
recordSelection(key, provider.model, requestedModel);
|
|
378
|
-
|
|
379
389
|
if (isStreaming && result.stream) {
|
|
380
|
-
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' });
|
|
381
|
-
const { Transform } = require('stream');
|
|
382
|
-
// Accumulate content deltas so we can cache the final answer for
|
|
383
|
-
// identical repeat prompts (opencode always streams).
|
|
384
390
|
const chunks = [];
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
const delta = obj.choices?.[0]?.delta?.content;
|
|
396
|
-
if (typeof delta === 'string') chunks.push(stripThink(delta, false));
|
|
397
|
-
} catch {}
|
|
391
|
+
|
|
392
|
+
// Очистка SSE-строки: убрать nvext, logprobs, think-блоки из дельт.
|
|
393
|
+
const cleanStr = (str) => str.replace(/^data: (.+)$/gm, (match, jsonStr) => {
|
|
394
|
+
if (jsonStr.trim() === '[DONE]') return match;
|
|
395
|
+
try {
|
|
396
|
+
const obj = JSON.parse(jsonStr);
|
|
397
|
+
delete obj.nvext;
|
|
398
|
+
if (obj.choices?.[0]) {
|
|
399
|
+
delete obj.choices[0].logprobs;
|
|
400
|
+
cleanDelta(obj.choices[0].delta);
|
|
398
401
|
}
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
402
|
+
return 'data: ' + JSON.stringify(obj);
|
|
403
|
+
} catch { return match; }
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
// Сбор контент-токенов для кэша (strip think).
|
|
407
|
+
const collect = (str) => {
|
|
408
|
+
const lines = str.split('\n');
|
|
409
|
+
for (const line of lines) {
|
|
410
|
+
const m = line.match(/^data: (.+)$/);
|
|
411
|
+
if (!m || m[1].trim() === '[DONE]') continue;
|
|
412
|
+
try {
|
|
413
|
+
const obj = JSON.parse(m[1]);
|
|
414
|
+
const delta = obj.choices?.[0]?.delta?.content;
|
|
415
|
+
if (typeof delta === 'string') chunks.push(stripThink(delta, false));
|
|
416
|
+
} catch {}
|
|
412
417
|
}
|
|
418
|
+
};
|
|
419
|
+
|
|
420
|
+
// Reasoning-модели (ox-alpha) думают 10с+ до первого токена — стримим сразу.
|
|
421
|
+
// Успех записываем ДО любого токена намеренно: fallback по таймауту 5с
|
|
422
|
+
// нанёс бы лишний дабл-счёт, если бы success фиксировался после первого токена.
|
|
423
|
+
if (provider.reasoning) {
|
|
424
|
+
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' });
|
|
425
|
+
recordSuccess(key);
|
|
426
|
+
recordRequest(key, true);
|
|
427
|
+
logger.request({ model: requestedModel, provider: key, status: 200, latency: result.latency, stream: isStreaming });
|
|
428
|
+
recordRecent({ model: requestedModel, provider: key, status: 200, latency: result.latency, cached: false });
|
|
429
|
+
recordSelection(key, provider.model, requestedModel);
|
|
430
|
+
const { Transform } = require('stream');
|
|
431
|
+
const cleaner = new Transform({
|
|
432
|
+
transform(chunk, encoding, callback) {
|
|
433
|
+
const str = chunk.toString();
|
|
434
|
+
collect(str);
|
|
435
|
+
callback(null, cleanStr(str));
|
|
436
|
+
}
|
|
437
|
+
});
|
|
438
|
+
result.stream.on('end', () => {
|
|
439
|
+
const full = chunks.join('');
|
|
440
|
+
if (full.trim().length >= MIN_ANSWER_LEN) {
|
|
441
|
+
cache.set(effectiveModel, body.messages, body.temperature, {
|
|
442
|
+
id: 'chatcmpl-cached',
|
|
443
|
+
object: 'chat.completion',
|
|
444
|
+
created: Math.floor(Date.now() / 1000),
|
|
445
|
+
model: provider.model,
|
|
446
|
+
choices: [{ index: 0, message: { role: 'assistant', content: full }, finish_reason: 'stop' }],
|
|
447
|
+
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
res.end();
|
|
451
|
+
});
|
|
452
|
+
result.stream.on('error', (err) => { logger.error('Stream error', { key, error: err.message }); res.end(); });
|
|
453
|
+
result.stream.pipe(cleaner).pipe(res);
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Обычные модели: буферизуем до первого токена (макс 5 сек).
|
|
458
|
+
// Заголовки не пишем сразу — если токена нет за 5 сек, fallback.
|
|
459
|
+
const rawBuf = [];
|
|
460
|
+
const firstToken = new Promise((resolve) => {
|
|
461
|
+
let done = false;
|
|
462
|
+
const timer = setTimeout(() => { if (!done) { done = true; resolve(false); } }, 5000);
|
|
463
|
+
const finish = (ok) => { if (!done) { done = true; clearTimeout(timer); resolve(ok); } };
|
|
464
|
+
result.stream.on('data', (chunk) => {
|
|
465
|
+
const str = chunk.toString();
|
|
466
|
+
rawBuf.push(str);
|
|
467
|
+
collect(str);
|
|
468
|
+
// Первый контент-токен: хотя бы одно непустое `"content":"..."` в чанке.
|
|
469
|
+
if (chunkHasToken(str)) {
|
|
470
|
+
finish(true);
|
|
471
|
+
}
|
|
472
|
+
});
|
|
473
|
+
result.stream.once('end', () => finish(false));
|
|
474
|
+
result.stream.once('error', () => finish(false));
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
// Клиент отключился во время ожидания первого токена — прерываем.
|
|
478
|
+
const onClientClose = () => {
|
|
479
|
+
try { result.stream.destroy(); } catch {}
|
|
480
|
+
};
|
|
481
|
+
req.once('close', onClientClose);
|
|
482
|
+
|
|
483
|
+
const gotFirst = await firstToken;
|
|
484
|
+
if (!gotFirst) {
|
|
485
|
+
const msg = key + ': no first token within 5s';
|
|
486
|
+
errors.push(msg);
|
|
487
|
+
recordFailure(key, 0);
|
|
488
|
+
recordRequest(key, false, msg);
|
|
489
|
+
recordRecent({ model: requestedModel, provider: key, status: 204, latency: result.latency, cached: false });
|
|
490
|
+
logger.warn('Streaming fallback: no first token', { key });
|
|
491
|
+
try { result.stream.destroy(); } catch {}
|
|
492
|
+
continue; // РАБОТАЕТ — мы внутри for-цикла провайдеров.
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// Первый токен пришёл: пишем заголовки, промываем буфер, дальше стримим.
|
|
496
|
+
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' });
|
|
497
|
+
recordSuccess(key);
|
|
498
|
+
recordRequest(key, true);
|
|
499
|
+
logger.request({ model: requestedModel, provider: key, status: 200, latency: result.latency, stream: isStreaming });
|
|
500
|
+
recordRecent({ model: requestedModel, provider: key, status: 200, latency: result.latency, cached: false });
|
|
501
|
+
recordSelection(key, provider.model, requestedModel);
|
|
502
|
+
res.on('error', (err) => {
|
|
503
|
+
logger.error('Client stream error', { key, error: err.message });
|
|
504
|
+
try { result.stream.destroy(); } catch {}
|
|
505
|
+
});
|
|
506
|
+
for (const b of rawBuf) res.write(cleanStr(b));
|
|
507
|
+
rawBuf.length = 0;
|
|
508
|
+
|
|
509
|
+
// Убираем наш 'data'-слушатель (он больше не нужен — данные уже
|
|
510
|
+
// буферизованы в rawBuf и промыты). Дальше обрабатываем вручную.
|
|
511
|
+
result.stream.removeAllListeners('data');
|
|
512
|
+
result.stream.on('data', (chunk) => {
|
|
513
|
+
const str = chunk.toString();
|
|
514
|
+
collect(str);
|
|
515
|
+
res.write(cleanStr(str));
|
|
413
516
|
});
|
|
414
517
|
result.stream.on('end', () => {
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
if (chunks.length > 0) {
|
|
418
|
-
const full = chunks.join('');
|
|
419
|
-
if (full.trim().length > 0) {
|
|
518
|
+
const full = chunks.join('');
|
|
519
|
+
if (full.trim().length >= MIN_ANSWER_LEN) {
|
|
420
520
|
cache.set(effectiveModel, body.messages, body.temperature, {
|
|
421
521
|
id: 'chatcmpl-cached',
|
|
422
522
|
object: 'chat.completion',
|
|
@@ -425,19 +525,23 @@ async function handleChatCompletion(req, res, body) {
|
|
|
425
525
|
choices: [{ index: 0, message: { role: 'assistant', content: full }, finish_reason: 'stop' }],
|
|
426
526
|
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
|
427
527
|
});
|
|
428
|
-
}
|
|
429
528
|
}
|
|
529
|
+
res.end();
|
|
430
530
|
});
|
|
431
531
|
result.stream.on('error', (err) => {
|
|
432
532
|
logger.error('Stream error', { key, error: err.message });
|
|
433
533
|
res.end();
|
|
434
534
|
});
|
|
435
|
-
result.stream.pipe(cleaner).pipe(res);
|
|
436
535
|
return;
|
|
437
536
|
}
|
|
438
537
|
|
|
439
538
|
if (!isStreaming && result.data) {
|
|
440
539
|
// content already verified non-empty above
|
|
540
|
+
recordSuccess(key);
|
|
541
|
+
recordRequest(key, true);
|
|
542
|
+
logger.request({ model: requestedModel, provider: key, status: 200, latency: result.latency, stream: isStreaming });
|
|
543
|
+
recordRecent({ model: requestedModel, provider: key, status: 200, latency: result.latency, cached: false });
|
|
544
|
+
recordSelection(key, provider.model, requestedModel);
|
|
441
545
|
cache.set(effectiveModel, body.messages, body.temperature, result.data);
|
|
442
546
|
recordTokens(key, result.usage);
|
|
443
547
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
@@ -496,15 +600,16 @@ async function handleChatCompletion(req, res, body) {
|
|
|
496
600
|
fixReasoningMessage(result.data.choices[0].message);
|
|
497
601
|
cleanMessage(result.data.choices[0].message);
|
|
498
602
|
}
|
|
499
|
-
if (
|
|
603
|
+
if (isTooShort(result.data)) {
|
|
500
604
|
recordFailure(key, 0);
|
|
501
|
-
recordRequest(key, false, key + ': empty response (retry)');
|
|
502
|
-
logger.warn('Empty response in retry, trying next', { key });
|
|
605
|
+
recordRequest(key, false, key + ': empty or too short response (retry)');
|
|
606
|
+
logger.warn('Empty or too-short response in retry, trying next', { key });
|
|
503
607
|
continue;
|
|
504
608
|
}
|
|
505
609
|
recordSuccess(key);
|
|
506
610
|
recordRequest(key, true);
|
|
507
611
|
recordRecent({ model: requestedModel, provider: key, status: 200, latency: result.latency, cached: false });
|
|
612
|
+
recordSelection(key, provider.model, requestedModel);
|
|
508
613
|
cache.set(effectiveModel, body.messages, body.temperature, result.data);
|
|
509
614
|
recordTokens(key, result.usage);
|
|
510
615
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
@@ -515,8 +620,56 @@ async function handleChatCompletion(req, res, body) {
|
|
|
515
620
|
recordSuccess(key);
|
|
516
621
|
recordRequest(key, true);
|
|
517
622
|
recordRecent({ model: requestedModel, provider: key, status: 200, latency: result.latency, cached: false });
|
|
623
|
+
recordSelection(key, provider.model, requestedModel);
|
|
518
624
|
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' });
|
|
519
|
-
|
|
625
|
+
const chunks = [];
|
|
626
|
+
const collectRetry = (str) => {
|
|
627
|
+
const lines = str.split('\n');
|
|
628
|
+
for (const line of lines) {
|
|
629
|
+
const m = line.match(/^data: (.+)$/);
|
|
630
|
+
if (!m || m[1].trim() === '[DONE]') continue;
|
|
631
|
+
try {
|
|
632
|
+
const obj = JSON.parse(m[1]);
|
|
633
|
+
const delta = obj.choices?.[0]?.delta?.content;
|
|
634
|
+
if (typeof delta === 'string') chunks.push(stripThink(delta, false));
|
|
635
|
+
} catch {}
|
|
636
|
+
}
|
|
637
|
+
};
|
|
638
|
+
const cleanRetry = (str) => str.replace(/^data: (.+)$/gm, (match, jsonStr) => {
|
|
639
|
+
if (jsonStr.trim() === '[DONE]') return match;
|
|
640
|
+
try {
|
|
641
|
+
const obj = JSON.parse(jsonStr);
|
|
642
|
+
delete obj.nvext;
|
|
643
|
+
if (obj.choices?.[0]) {
|
|
644
|
+
delete obj.choices[0].logprobs;
|
|
645
|
+
cleanDelta(obj.choices[0].delta);
|
|
646
|
+
}
|
|
647
|
+
return 'data: ' + JSON.stringify(obj);
|
|
648
|
+
} catch { return match; }
|
|
649
|
+
});
|
|
650
|
+
result.stream.on('data', (chunk) => {
|
|
651
|
+
const str = chunk.toString();
|
|
652
|
+
collectRetry(str);
|
|
653
|
+
res.write(cleanRetry(str));
|
|
654
|
+
});
|
|
655
|
+
result.stream.on('end', () => {
|
|
656
|
+
const full = chunks.join('');
|
|
657
|
+
if (full.trim().length >= MIN_ANSWER_LEN) {
|
|
658
|
+
cache.set(effectiveModel, body.messages, body.temperature, {
|
|
659
|
+
id: 'chatcmpl-cached',
|
|
660
|
+
object: 'chat.completion',
|
|
661
|
+
created: Math.floor(Date.now() / 1000),
|
|
662
|
+
model: provider.model,
|
|
663
|
+
choices: [{ index: 0, message: { role: 'assistant', content: full }, finish_reason: 'stop' }],
|
|
664
|
+
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
res.end();
|
|
668
|
+
});
|
|
669
|
+
result.stream.on('error', (err) => {
|
|
670
|
+
logger.error('Stream error (retry)', { key, error: err.message });
|
|
671
|
+
res.end();
|
|
672
|
+
});
|
|
520
673
|
return;
|
|
521
674
|
}
|
|
522
675
|
} catch (err2) {
|