freegate 0.6.12 → 0.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/README.ru.md +13 -13
- package/bin/freegate.js +15 -15
- package/lib/bandit.js +72 -0
- package/lib/health.js +11 -1
- package/package.json +1 -1
- package/server.js +42 -35
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# Freegate
|
|
2
2
|
|
|
3
|
-
[](https://github.com/Artur21101965/freegate/actions)
|
|
4
4
|
[](https://hub.docker.com/r/nik951751/freegate)
|
|
5
5
|
[](https://opensource.org/licenses/MIT)
|
|
6
|
-
[](https://www.npmjs.com/package/freegate)
|
|
7
|
+
[](https://github.com/Artur21101965/freegate)
|
|
8
8
|
|
|
9
9
|
**[Русская версия](README.ru.md) · Russian version**
|
|
10
10
|
|
|
@@ -39,9 +39,9 @@ never see "rate limit", and you never pay.
|
|
|
39
39
|
## Quick start — 30 seconds
|
|
40
40
|
|
|
41
41
|
```bash
|
|
42
|
-
npx
|
|
43
|
-
npx
|
|
44
|
-
npx
|
|
42
|
+
npx freegate init -i # wizard: password + each provider's key
|
|
43
|
+
npx freegate start # proxy on http://localhost:4000
|
|
44
|
+
npx freegate test # verify everything works
|
|
45
45
|
```
|
|
46
46
|
|
|
47
47
|
Dashboard: `http://localhost:4000/?key=your_password`
|
|
@@ -49,7 +49,7 @@ Dashboard: `http://localhost:4000/?key=your_password`
|
|
|
49
49
|
Or via Docker:
|
|
50
50
|
|
|
51
51
|
```bash
|
|
52
|
-
docker run -d --name
|
|
52
|
+
docker run -d --name freegate -p 4000:4000 \
|
|
53
53
|
-e PROVIDER_GROQ_APIKEY=... \
|
|
54
54
|
-e PROVIDER_MISTRAL_APIKEY=... \
|
|
55
55
|
-e AUTH=your-secret-key \
|
|
@@ -146,13 +146,13 @@ put it in `config.json` → `providers` (same format as `providers.json`).
|
|
|
146
146
|
## CLI commands
|
|
147
147
|
|
|
148
148
|
```bash
|
|
149
|
-
npx
|
|
150
|
-
npx
|
|
151
|
-
npx
|
|
152
|
-
npx
|
|
153
|
-
npx
|
|
154
|
-
npx
|
|
155
|
-
npx
|
|
149
|
+
npx freegate init # create config
|
|
150
|
+
npx freegate init -i # interactive wizard (keys, password)
|
|
151
|
+
npx freegate start # start proxy
|
|
152
|
+
npx freegate status # diagnostics: providers, limits, errors
|
|
153
|
+
npx freegate test # verify it works
|
|
154
|
+
npx freegate install-service # autostart at boot
|
|
155
|
+
npx freegate dashboard # open dashboard
|
|
156
156
|
```
|
|
157
157
|
|
|
158
158
|
## FAQ
|
package/README.ru.md
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# Freegate
|
|
2
2
|
|
|
3
|
-
[](https://github.com/Artur21101965/freegate/actions)
|
|
4
4
|
[](https://hub.docker.com/r/nik951751/freegate)
|
|
5
5
|
[](https://opensource.org/licenses/MIT)
|
|
6
|
-
[](https://www.npmjs.com/package/freegate)
|
|
7
7
|
|
|
8
8
|
**English version: [README.md](README.md)**
|
|
9
9
|
|
|
@@ -38,9 +38,9 @@ DeepSeek и локальные модели. Если один провайде
|
|
|
38
38
|
## Быстрый старт — 30 секунд
|
|
39
39
|
|
|
40
40
|
```bash
|
|
41
|
-
npx
|
|
42
|
-
npx
|
|
43
|
-
npx
|
|
41
|
+
npx freegate init -i # мастер спросит: пароль + ключи каждого провайдера
|
|
42
|
+
npx freegate start # прокси на http://localhost:4000
|
|
43
|
+
npx freegate test # проверить, что всё работает
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
Дашборд: `http://localhost:4000/?key=твой_пароль`
|
|
@@ -48,7 +48,7 @@ npx davil-cod test # проверить, что всё работает
|
|
|
48
48
|
Или через Docker:
|
|
49
49
|
|
|
50
50
|
```bash
|
|
51
|
-
docker run -d --name
|
|
51
|
+
docker run -d --name freegate -p 4000:4000 \
|
|
52
52
|
-e PROVIDER_GROQ_APIKEY=... \
|
|
53
53
|
-e PROVIDER_MISTRAL_APIKEY=... \
|
|
54
54
|
-e AUTH=your-secret-key \
|
|
@@ -145,13 +145,13 @@ PROVIDER_ZAI_APIKEY=... # ZAI
|
|
|
145
145
|
## Команды CLI
|
|
146
146
|
|
|
147
147
|
```bash
|
|
148
|
-
npx
|
|
149
|
-
npx
|
|
150
|
-
npx
|
|
151
|
-
npx
|
|
152
|
-
npx
|
|
153
|
-
npx
|
|
154
|
-
npx
|
|
148
|
+
npx freegate init # создать конфиг
|
|
149
|
+
npx freegate init -i # интерактивный мастер (ключи, пароль)
|
|
150
|
+
npx freegate start # запустить прокси
|
|
151
|
+
npx freegate status # диагностика: провайдеры, лимиты, ошибки
|
|
152
|
+
npx freegate test # проверить, что работает
|
|
153
|
+
npx freegate install-service # автозапуск при старте системы
|
|
154
|
+
npx freegate dashboard # открыть дашборд
|
|
155
155
|
```
|
|
156
156
|
|
|
157
157
|
## FAQ
|
package/bin/freegate.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// bin/
|
|
2
|
+
// bin/freegate.js — CLI wrapper + interactive setup wizard
|
|
3
3
|
const path = require('path');
|
|
4
4
|
const fs = require('fs');
|
|
5
5
|
const { spawn } = require('child_process');
|
|
@@ -74,7 +74,7 @@ async function initWizard() {
|
|
|
74
74
|
|
|
75
75
|
const filled = Object.values(keys).filter(Boolean).length;
|
|
76
76
|
console.log(`\nГотово! Провайдеров с ключами: ${filled} из ${envVars.size}`);
|
|
77
|
-
console.log(`Запуск: npx
|
|
77
|
+
console.log(`Запуск: npx freegate start (пароль: ${authKey})`);
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
function copyExample(name, example) {
|
|
@@ -95,8 +95,8 @@ if (cmd === 'init') {
|
|
|
95
95
|
// Non-interactive fallback: plain copy (safe — never overwrites)
|
|
96
96
|
copyExample('config.json', 'config.example.json');
|
|
97
97
|
copyExample('.env', '.env.example');
|
|
98
|
-
console.log('\nГотово! Заполни .env ключами, затем: npx
|
|
99
|
-
console.log('Совет: npx
|
|
98
|
+
console.log('\nГотово! Заполни .env ключами, затем: npx freegate start');
|
|
99
|
+
console.log('Совет: npx freegate init -i — интерактивный мастер с вопросами.');
|
|
100
100
|
}
|
|
101
101
|
} else if (cmd === 'start') {
|
|
102
102
|
// Spawn from the user's cwd (not package dir) so server.js picks up their
|
|
@@ -150,10 +150,10 @@ RestartSec=5
|
|
|
150
150
|
[Install]
|
|
151
151
|
WantedBy=default.target
|
|
152
152
|
`;
|
|
153
|
-
const unitPath = path.join(os.homedir(), '.config', 'systemd', 'user', '
|
|
153
|
+
const unitPath = path.join(os.homedir(), '.config', 'systemd', 'user', 'freegate.service');
|
|
154
154
|
fs.mkdirSync(path.dirname(unitPath), { recursive: true });
|
|
155
155
|
fs.writeFileSync(unitPath, unit);
|
|
156
|
-
execSync(`systemctl --user daemon-reload && systemctl --user enable
|
|
156
|
+
execSync(`systemctl --user daemon-reload && systemctl --user enable freegate && systemctl --user start freegate`);
|
|
157
157
|
console.log('✅ Служба автозапуска установлена (systemd)');
|
|
158
158
|
}
|
|
159
159
|
} else if (cmd === 'dashboard') {
|
|
@@ -174,7 +174,7 @@ WantedBy=default.target
|
|
|
174
174
|
});
|
|
175
175
|
}).on('error', (err) => {
|
|
176
176
|
console.log('❌ Прокси не запущен на ' + base);
|
|
177
|
-
console.log(' Запусти: npx
|
|
177
|
+
console.log(' Запусти: npx freegate start');
|
|
178
178
|
process.exit(1);
|
|
179
179
|
});
|
|
180
180
|
} else if (cmd === 'status') {
|
|
@@ -189,7 +189,7 @@ WantedBy=default.target
|
|
|
189
189
|
res.on('end', () => {
|
|
190
190
|
if (res.statusCode !== 200) {
|
|
191
191
|
console.log('❌ Прокси не отвечает (HTTP ' + res.statusCode + ')');
|
|
192
|
-
console.log(' Запусти: npx
|
|
192
|
+
console.log(' Запусти: npx freegate start');
|
|
193
193
|
process.exit(1);
|
|
194
194
|
}
|
|
195
195
|
try {
|
|
@@ -211,7 +211,7 @@ WantedBy=default.target
|
|
|
211
211
|
});
|
|
212
212
|
}).on('error', () => {
|
|
213
213
|
console.log('❌ Прокси не запущен на ' + base);
|
|
214
|
-
console.log(' Запусти: npx
|
|
214
|
+
console.log(' Запусти: npx freegate start');
|
|
215
215
|
process.exit(1);
|
|
216
216
|
});
|
|
217
217
|
} else if (cmd === 'version' || cmd === '-v' || cmd === '--version') {
|
|
@@ -220,10 +220,10 @@ WantedBy=default.target
|
|
|
220
220
|
} else {
|
|
221
221
|
console.log('Freegate — бесплатный LLM-прокси с failover');
|
|
222
222
|
console.log('Команды:');
|
|
223
|
-
console.log(' npx
|
|
224
|
-
console.log(' npx
|
|
225
|
-
console.log(' npx
|
|
226
|
-
console.log(' npx
|
|
227
|
-
console.log(' npx
|
|
228
|
-
console.log(' npx
|
|
223
|
+
console.log(' npx freegate init интерактивная настройка (ключи, пароль)');
|
|
224
|
+
console.log(' npx freegate start запустить прокси');
|
|
225
|
+
console.log(' npx freegate status диагностика: провайдеры, лимиты, ошибки');
|
|
226
|
+
console.log(' npx freegate test проверить, что прокси работает');
|
|
227
|
+
console.log(' npx freegate dashboard открыть дашборд');
|
|
228
|
+
console.log(' npx freegate install-service автозапуск при старте системы');
|
|
229
229
|
}
|
package/lib/bandit.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Thompson sampling (Beta-Bernoulli) bandit для выбора провайдера.
|
|
2
|
+
// Каждый провайдер в каждом бакете сложности имеет приоры Beta(a, b).
|
|
3
|
+
// Выбор: рисуем сэмпл из Beta(a, b) для каждого, берём максимум.
|
|
4
|
+
// Исход: успех → a+=1, фейл → b+=1.
|
|
5
|
+
|
|
6
|
+
const BUCKET_THRESHOLDS = [0.4, 0.7];
|
|
7
|
+
|
|
8
|
+
function bucket(complexity) {
|
|
9
|
+
if (complexity < BUCKET_THRESHOLDS[0]) return 'low';
|
|
10
|
+
if (complexity < BUCKET_THRESHOLDS[1]) return 'med';
|
|
11
|
+
return 'high';
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// Standard normal via polar method (Box-Muller).
|
|
15
|
+
function stdNormal() {
|
|
16
|
+
let u1, u2, s;
|
|
17
|
+
do { u1 = Math.random() * 2 - 1; u2 = Math.random() * 2 - 1; s = u1 * u1 + u2 * u2; } while (s >= 1 || s === 0);
|
|
18
|
+
return u1 * Math.sqrt(-2 * Math.log(s) / s);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// Marsaglia-Tsang sampling of Gamma(shape, scale=1). Valid for shape >= 1.
|
|
22
|
+
function sampleGamma(shape) {
|
|
23
|
+
if (shape < 1) {
|
|
24
|
+
return sampleGamma(shape + 1) * Math.pow(Math.random(), 1 / shape);
|
|
25
|
+
}
|
|
26
|
+
const d = shape - 1 / 3;
|
|
27
|
+
const c = 1 / Math.sqrt(9 * d);
|
|
28
|
+
for (;;) {
|
|
29
|
+
let x, v;
|
|
30
|
+
do { x = stdNormal(); } while (1 + c * x <= 0);
|
|
31
|
+
v = 1 + c * x;
|
|
32
|
+
v = v * v * v;
|
|
33
|
+
const u = Math.random();
|
|
34
|
+
if (u < 1 - 0.0331 * x * x * x * x) return d * v;
|
|
35
|
+
if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) return d * v;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Сэмпл из Beta(alpha, beta). Beta(1,1) = uniform.
|
|
40
|
+
function sampleBeta(alpha, beta) {
|
|
41
|
+
if (alpha <= 0 || beta <= 0) return 0.5;
|
|
42
|
+
const x = sampleGamma(alpha);
|
|
43
|
+
const y = sampleGamma(beta);
|
|
44
|
+
const denom = x + y;
|
|
45
|
+
return denom > 0 ? x / denom : 0.5;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Выбор провайдера. Сэмпл из Beta(prior) умножается на weight (safety-штрафы
|
|
49
|
+
// типа ratelimited ×0.05 должны действовать и при холодном старте). Приоры
|
|
50
|
+
// bandit'а доминируют по мере накопления исходов.
|
|
51
|
+
function pick(scored, bucketPriors) {
|
|
52
|
+
let best = null;
|
|
53
|
+
let bestVal = -Infinity;
|
|
54
|
+
for (const item of scored) {
|
|
55
|
+
const key = item.key;
|
|
56
|
+
const weight = typeof item.weight === 'number' ? item.weight : 1;
|
|
57
|
+
const prior = (bucketPriors && bucketPriors[key]) || { a: 1, b: 1 };
|
|
58
|
+
const sample = sampleBeta(prior.a + 1, prior.b + 1);
|
|
59
|
+
const val = sample * weight;
|
|
60
|
+
if (val > bestVal) { bestVal = val; best = key; }
|
|
61
|
+
}
|
|
62
|
+
return best;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Обновление приоров после исхода.
|
|
66
|
+
function recordOutcome(bucketPriors, key, success) {
|
|
67
|
+
bucketPriors[key] = bucketPriors[key] || { a: 1, b: 1 };
|
|
68
|
+
if (success) bucketPriors[key].a += 1;
|
|
69
|
+
else bucketPriors[key].b += 1;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
module.exports = { bucket, sampleBeta, pick, recordOutcome, BUCKET_THRESHOLDS };
|
package/lib/health.js
CHANGED
|
@@ -7,7 +7,7 @@ const STATE_PATH = path.join(__dirname, '..', 'state.json');
|
|
|
7
7
|
|
|
8
8
|
let health = {};
|
|
9
9
|
let circuitBreakers = {};
|
|
10
|
-
let stats = { totalRequests: 0, successfulRequests: 0, failedRequests: 0, providerUsage: {}, errors: {}, startTime: Date.now(), tokenUsage: {} };
|
|
10
|
+
let stats = { totalRequests: 0, successfulRequests: 0, failedRequests: 0, providerUsage: {}, errors: {}, startTime: Date.now(), tokenUsage: {}, bandit: { low: {}, med: {}, high: {} } };
|
|
11
11
|
let lastSelection = null;
|
|
12
12
|
|
|
13
13
|
const MAX_RECENT = 50;
|
|
@@ -31,6 +31,7 @@ function loadState() {
|
|
|
31
31
|
tokenUsage: saved.tokenUsage || {},
|
|
32
32
|
dailyUsage: saved.dailyUsage || {},
|
|
33
33
|
reliability: saved.reliability || {},
|
|
34
|
+
bandit: saved.bandit || { low: {}, med: {}, high: {} },
|
|
34
35
|
};
|
|
35
36
|
logger.info('State loaded', { healthKeys: Object.keys(health) });
|
|
36
37
|
} catch {
|
|
@@ -176,6 +177,14 @@ function getRecent() { return recent; }
|
|
|
176
177
|
function getRpm() { return rpm; }
|
|
177
178
|
function getDailyUsage() { return stats.dailyUsage || {}; }
|
|
178
179
|
|
|
180
|
+
function getBandit() { return stats.bandit; }
|
|
181
|
+
function recordBandit(bucketName, key, success) {
|
|
182
|
+
stats.bandit[bucketName] = stats.bandit[bucketName] || {};
|
|
183
|
+
stats.bandit[bucketName][key] = stats.bandit[bucketName][key] || { a: 1, b: 1 };
|
|
184
|
+
if (success) stats.bandit[bucketName][key].a += 1;
|
|
185
|
+
else stats.bandit[bucketName][key].b += 1;
|
|
186
|
+
}
|
|
187
|
+
|
|
179
188
|
// Auto-save every 30 seconds
|
|
180
189
|
const _saveTimer = setInterval(saveState, 30000);
|
|
181
190
|
|
|
@@ -188,5 +197,6 @@ module.exports = {
|
|
|
188
197
|
recordRecent, recordRpm, getRecent, getRpm,
|
|
189
198
|
getDailyUsage, getReliability,
|
|
190
199
|
recordSelection, getLastSelection,
|
|
200
|
+
getBandit, recordBandit,
|
|
191
201
|
_stopTimers,
|
|
192
202
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "freegate",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.14",
|
|
4
4
|
"description": "Free multi-provider LLM gateway with automatic failover. One OpenAI-compatible endpoint routes to 25 free models (Groq, Mistral, Gemini, NIM, OpenRouter, ZAI, Cerebras, DeepSeek). Never pay for LLMs.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"bin": {
|
package/server.js
CHANGED
|
@@ -4,12 +4,13 @@ const fs = require('fs');
|
|
|
4
4
|
const path = require('path');
|
|
5
5
|
const { LRUCache } = require('./lib/cache');
|
|
6
6
|
const { PROVIDERS, MODEL_MAP, callProvider } = require('./lib/providers');
|
|
7
|
-
const { loadState, initHealth, isCircuitOpen, recordSuccess, recordFailure, recordRequest, recordTokens, getHealth, getStats, recordRecent, recordRpm, getRecent, getRpm, recordSelection, getLastSelection } = require('./lib/health');
|
|
7
|
+
const { loadState, initHealth, isCircuitOpen, recordSuccess, recordFailure, recordRequest, recordTokens, getHealth, getStats, recordRecent, recordRpm, getRecent, getRpm, recordSelection, getLastSelection, getBandit, recordBandit } = require('./lib/health');
|
|
8
8
|
const { checkRateLimit } = require('./lib/rateLimit');
|
|
9
9
|
const { handleDashboard } = require('./lib/dashboard');
|
|
10
10
|
const { acquire, stats: poolStats } = require('./lib/pool');
|
|
11
11
|
const { stripThink, cleanDelta, cleanMessage, fixReasoningMessage, isTooShort, MIN_ANSWER_LEN } = require('./lib/clean');
|
|
12
12
|
const { classifyComplexity, maybeUpgradeTier, classifyVisionComplexity } = require('./lib/routing');
|
|
13
|
+
const { bucket, pick: banditPick } = require('./lib/bandit');
|
|
13
14
|
const logger = require('./lib/logger');
|
|
14
15
|
|
|
15
16
|
// Load persisted state
|
|
@@ -138,7 +139,9 @@ async function handleChatCompletion(req, res, body) {
|
|
|
138
139
|
const requestedModel = body.model || 'tier-splus';
|
|
139
140
|
// Умный роутинг: сложные задачи с лёгкого тира поднимаем на более мощный.
|
|
140
141
|
// Классифицируем ПОСЛЕ того, как определён requestedModel, ДО выбора провайдера.
|
|
141
|
-
|
|
142
|
+
const complexity = classifyComplexity(body.messages);
|
|
143
|
+
let complexityBucket = bucket(complexity);
|
|
144
|
+
let effectiveModel = maybeUpgradeTier(requestedModel, complexity);
|
|
142
145
|
let targetProviderKey = MODEL_MAP[effectiveModel] || MODEL_MAP[requestedModel] || 'zai';
|
|
143
146
|
const isStreaming = body.stream === true;
|
|
144
147
|
|
|
@@ -209,6 +212,8 @@ async function handleChatCompletion(req, res, body) {
|
|
|
209
212
|
// Умный vision-роутинг: скриншот с кодом/ошибкой поднимает тир.
|
|
210
213
|
const vc = classifyVisionComplexity(cleaned);
|
|
211
214
|
if (vc > 0) effectiveModel = maybeUpgradeTier(effectiveModel, vc);
|
|
215
|
+
// Vision-текст может изменить сложность — пересчитываем бакет.
|
|
216
|
+
complexityBucket = bucket(classifyVisionComplexity(cleaned));
|
|
212
217
|
// Пересчитываем target-провайдера — vision-апгрейд мог сменить тир.
|
|
213
218
|
targetProviderKey = MODEL_MAP[effectiveModel] || MODEL_MAP[requestedModel] || 'zai';
|
|
214
219
|
// Replace image content with the extracted text as context,
|
|
@@ -274,47 +279,29 @@ async function handleChatCompletion(req, res, body) {
|
|
|
274
279
|
const scored = pool.map(([key, provider]) => {
|
|
275
280
|
const h = getHealth()[key];
|
|
276
281
|
let score = h.score || 50;
|
|
277
|
-
// Latency is only reliable after real requests; unmeasured/zero latency
|
|
278
|
-
// must NOT balloon a provider's weight. Treat <100ms as neutral.
|
|
279
282
|
const rawLat = h.latency || 0;
|
|
280
283
|
const lat = rawLat > 0 ? Math.max(rawLat, 100) : 500;
|
|
281
284
|
let weight = score / lat;
|
|
282
|
-
// Rate-limited providers are last resort — heavy penalty
|
|
283
285
|
if (h.status === 'ratelimited') weight *= 0.05;
|
|
284
|
-
// Mapped (target) provider gets a SLIGHT preference, but the pool must
|
|
285
|
-
// still be able to pick faster/healthier providers for tier aliases —
|
|
286
|
-
// otherwise Freegate always routes tier-s to Codestral and never varies.
|
|
287
286
|
if (key === targetProviderKey) weight *= 1.15;
|
|
288
287
|
const dailyLimit = provider.dailyLimit || 1000;
|
|
289
288
|
const usedToday = getStats().providerUsage[key] || 0;
|
|
290
289
|
if (usedToday >= dailyLimit * 0.9) weight *= 0.5;
|
|
291
|
-
// Providers with a history of failures lose weight (stability first).
|
|
292
|
-
// Use TODAY's failure count (reliability.fail) so overloaded providers
|
|
293
|
-
// are excluded now but recover next day. Heavy count → near-exclusion.
|
|
294
|
-
const relToday = getStats().reliability?.[key];
|
|
295
|
-
const todayFails = relToday?.day === today ? (relToday.fail || 0) : 0;
|
|
296
|
-
if (todayFails > 5) weight *= 0.4;
|
|
297
|
-
if (todayFails > 20) weight *= 0.05;
|
|
298
|
-
// Today's reliability: providers that have been 100% successful get a boost
|
|
299
|
-
const rel = getStats().reliability?.[key];
|
|
300
|
-
if (rel && rel.success + rel.fail >= 3) {
|
|
301
|
-
const ratio = rel.success / (rel.success + rel.fail);
|
|
302
|
-
if (ratio === 1) weight *= 1.3;
|
|
303
|
-
else if (ratio < 0.5) weight *= 0.5;
|
|
304
|
-
}
|
|
305
290
|
return { key, provider, weight };
|
|
306
|
-
})
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
//
|
|
310
|
-
//
|
|
311
|
-
//
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
// Bandit weight contract: bandit's pick() multiplies the Beta sample by
|
|
294
|
+
// `weight`, so safety-штрафы (ratelimited ×0.05, target ×1.15) действуют и
|
|
295
|
+
// при холодном старте. score/latency держит вес ~0.01-1.0; приоры bandit'а
|
|
296
|
+
// (a,b ~1+) со временем начинают доминировать. Не добавляй нормализацию
|
|
297
|
+
// здесь, пока измеренные веса не превысят ~5.
|
|
298
|
+
// Thompson sampling: рисуем сэмпл Beta(a+1, b+1) для каждого, умножаем на
|
|
299
|
+
// weight, выбираем максимум. Приоры из бакета сложности (bandit обучается).
|
|
300
|
+
const priors = getBandit()[complexityBucket] || {};
|
|
301
|
+
const bestKey = banditPick(scored, priors);
|
|
302
|
+
const bestProvider = scored.find((p) => p.key === bestKey);
|
|
303
|
+
if (bestProvider) selected = [bestProvider];
|
|
304
|
+
else if (scored.length > 0) selected = [scored[0]];
|
|
318
305
|
}
|
|
319
306
|
|
|
320
307
|
// Weighted-random picked ONE provider as the primary; append the rest of the
|
|
@@ -380,6 +367,7 @@ async function handleChatCompletion(req, res, body) {
|
|
|
380
367
|
errors.push(msg);
|
|
381
368
|
recordFailure(key, 0);
|
|
382
369
|
recordRequest(key, false, msg);
|
|
370
|
+
recordBandit(complexityBucket, key, false);
|
|
383
371
|
recordRecent({ model: requestedModel, provider: key, status: 204, latency: result.latency, cached: false });
|
|
384
372
|
logger.warn('Empty or too-short response, trying next provider', { key });
|
|
385
373
|
continue;
|
|
@@ -437,6 +425,8 @@ async function handleChatCompletion(req, res, body) {
|
|
|
437
425
|
});
|
|
438
426
|
result.stream.on('end', () => {
|
|
439
427
|
const full = chunks.join('');
|
|
428
|
+
// Bandit учится по качеству: пустой/мусорный стрим = фейл.
|
|
429
|
+
recordBandit(complexityBucket, key, full.trim().length >= MIN_ANSWER_LEN);
|
|
440
430
|
if (full.trim().length >= MIN_ANSWER_LEN) {
|
|
441
431
|
cache.set(effectiveModel, body.messages, body.temperature, {
|
|
442
432
|
id: 'chatcmpl-cached',
|
|
@@ -449,7 +439,11 @@ async function handleChatCompletion(req, res, body) {
|
|
|
449
439
|
}
|
|
450
440
|
res.end();
|
|
451
441
|
});
|
|
452
|
-
result.stream.on('error', (err) => {
|
|
442
|
+
result.stream.on('error', (err) => {
|
|
443
|
+
logger.error('Stream error', { key, error: err.message });
|
|
444
|
+
recordBandit(complexityBucket, key, false);
|
|
445
|
+
res.end();
|
|
446
|
+
});
|
|
453
447
|
result.stream.pipe(cleaner).pipe(res);
|
|
454
448
|
return;
|
|
455
449
|
}
|
|
@@ -486,6 +480,7 @@ async function handleChatCompletion(req, res, body) {
|
|
|
486
480
|
errors.push(msg);
|
|
487
481
|
recordFailure(key, 0);
|
|
488
482
|
recordRequest(key, false, msg);
|
|
483
|
+
recordBandit(complexityBucket, key, false);
|
|
489
484
|
recordRecent({ model: requestedModel, provider: key, status: 204, latency: result.latency, cached: false });
|
|
490
485
|
logger.warn('Streaming fallback: no first token', { key });
|
|
491
486
|
try { result.stream.destroy(); } catch {}
|
|
@@ -516,6 +511,8 @@ async function handleChatCompletion(req, res, body) {
|
|
|
516
511
|
});
|
|
517
512
|
result.stream.on('end', () => {
|
|
518
513
|
const full = chunks.join('');
|
|
514
|
+
// Bandit учится по качеству: обрыв/мусорный стрим = фейл.
|
|
515
|
+
recordBandit(complexityBucket, key, full.trim().length >= MIN_ANSWER_LEN);
|
|
519
516
|
if (full.trim().length >= MIN_ANSWER_LEN) {
|
|
520
517
|
cache.set(effectiveModel, body.messages, body.temperature, {
|
|
521
518
|
id: 'chatcmpl-cached',
|
|
@@ -530,6 +527,7 @@ async function handleChatCompletion(req, res, body) {
|
|
|
530
527
|
});
|
|
531
528
|
result.stream.on('error', (err) => {
|
|
532
529
|
logger.error('Stream error', { key, error: err.message });
|
|
530
|
+
recordBandit(complexityBucket, key, false);
|
|
533
531
|
res.end();
|
|
534
532
|
});
|
|
535
533
|
return;
|
|
@@ -539,6 +537,7 @@ async function handleChatCompletion(req, res, body) {
|
|
|
539
537
|
// content already verified non-empty above
|
|
540
538
|
recordSuccess(key);
|
|
541
539
|
recordRequest(key, true);
|
|
540
|
+
recordBandit(complexityBucket, key, true);
|
|
542
541
|
logger.request({ model: requestedModel, provider: key, status: 200, latency: result.latency, stream: isStreaming });
|
|
543
542
|
recordRecent({ model: requestedModel, provider: key, status: 200, latency: result.latency, cached: false });
|
|
544
543
|
recordSelection(key, provider.model, requestedModel);
|
|
@@ -552,6 +551,7 @@ async function handleChatCompletion(req, res, body) {
|
|
|
552
551
|
const statusCode = err.statusCode || 502;
|
|
553
552
|
errors.push(err.message);
|
|
554
553
|
recordRequest(key, false, err.message);
|
|
554
|
+
recordBandit(complexityBucket, key, false);
|
|
555
555
|
recordRecent({ model: requestedModel, provider: key, status: statusCode, latency: 0, cached: false });
|
|
556
556
|
initHealth(key);
|
|
557
557
|
// Do NOT flip provider to 'error' on a single failed request — transient
|
|
@@ -603,11 +603,13 @@ async function handleChatCompletion(req, res, body) {
|
|
|
603
603
|
if (isTooShort(result.data)) {
|
|
604
604
|
recordFailure(key, 0);
|
|
605
605
|
recordRequest(key, false, key + ': empty or too short response (retry)');
|
|
606
|
+
recordBandit(complexityBucket, key, false);
|
|
606
607
|
logger.warn('Empty or too-short response in retry, trying next', { key });
|
|
607
608
|
continue;
|
|
608
609
|
}
|
|
609
610
|
recordSuccess(key);
|
|
610
611
|
recordRequest(key, true);
|
|
612
|
+
recordBandit(complexityBucket, key, true);
|
|
611
613
|
recordRecent({ model: requestedModel, provider: key, status: 200, latency: result.latency, cached: false });
|
|
612
614
|
recordSelection(key, provider.model, requestedModel);
|
|
613
615
|
cache.set(effectiveModel, body.messages, body.temperature, result.data);
|
|
@@ -654,6 +656,8 @@ async function handleChatCompletion(req, res, body) {
|
|
|
654
656
|
});
|
|
655
657
|
result.stream.on('end', () => {
|
|
656
658
|
const full = chunks.join('');
|
|
659
|
+
// Bandit учится по качеству в ретрае тоже.
|
|
660
|
+
recordBandit(complexityBucket, key, full.trim().length >= MIN_ANSWER_LEN);
|
|
657
661
|
if (full.trim().length >= MIN_ANSWER_LEN) {
|
|
658
662
|
cache.set(effectiveModel, body.messages, body.temperature, {
|
|
659
663
|
id: 'chatcmpl-cached',
|
|
@@ -668,12 +672,14 @@ async function handleChatCompletion(req, res, body) {
|
|
|
668
672
|
});
|
|
669
673
|
result.stream.on('error', (err) => {
|
|
670
674
|
logger.error('Stream error (retry)', { key, error: err.message });
|
|
675
|
+
recordBandit(complexityBucket, key, false);
|
|
671
676
|
res.end();
|
|
672
677
|
});
|
|
673
678
|
return;
|
|
674
679
|
}
|
|
675
680
|
} catch (err2) {
|
|
676
681
|
recordRequest(key, false, err2.message);
|
|
682
|
+
recordBandit(complexityBucket, key, false);
|
|
677
683
|
recordFailure(key, err2.statusCode);
|
|
678
684
|
}
|
|
679
685
|
}
|
|
@@ -748,6 +754,7 @@ const server = http.createServer(async (req, res) => {
|
|
|
748
754
|
limits,
|
|
749
755
|
pool: poolStats(),
|
|
750
756
|
last_selection: getLastSelection(),
|
|
757
|
+
bandit: getBandit(),
|
|
751
758
|
}));
|
|
752
759
|
return;
|
|
753
760
|
}
|