ldrouter 1.11.17 → 1.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/server/app.js +32 -7
- package/dist/server/auth/backup-crypto.js +31 -0
- package/dist/server/auth/crypto.js +3 -0
- package/dist/server/config/index.js +3 -1
- package/dist/server/gateway/runner.js +197 -13
- package/dist/server/logging/debug.js +271 -55
- package/dist/server/routes/admin/backup.js +38 -4
- package/dist/server/routes/gateway/anthropic.js +38 -1
- package/dist/server/routes/gateway/openai.js +38 -2
- package/dist/server/routing/combo.js +37 -7
- package/dist/server/upstream/client.js +30 -2
- package/dist/web/assets/index-CBMHkVXC.js +330 -0
- package/dist/web/assets/index-Dswaxg_c.css +1 -0
- package/dist/web/index.html +2 -2
- package/package.json +1 -3
- package/dist/web/assets/index-CqVMNQ5F.css +0 -1
- package/dist/web/assets/index-DhNkAaqo.js +0 -330
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,33 @@ All notable changes to this project are documented here. The format follows
|
|
|
4
4
|
[Keep a Changelog](https://keepachangelog.com/) and the project adheres to
|
|
5
5
|
[Semantic Versioning](https://semver.org/).
|
|
6
6
|
|
|
7
|
+
## [1.13.1] - 2026-09-11
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
|
|
11
|
+
- **Update notification**: force a fresh npm registry check when the admin UI loads, so the update button is not hidden by the 15-minute cache.
|
|
12
|
+
|
|
13
|
+
## [1.13.0] - 2026-09-11
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- **Passphrase-protected full backups**: backups now include administrator state and a master-key envelope protected by a user-entered six-digit passphrase; `/setup` can import them without recreating the admin account.
|
|
18
|
+
|
|
19
|
+
## [1.12.0] - 2026-09-05
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- **Request-lifecycle debug logging** (`docs/13-LOGGING.md`): full observability of every request as it passes through the gateway, emitted to stdout/stderr so `docker logs` collects it.
|
|
24
|
+
- Per-request ID (reuses `x-request-id` or generates one) tagged on every line — `docker logs ldrouter 2>&1 | grep req_xxx` reconstructs the whole lifecycle.
|
|
25
|
+
- `[INCOMING]` → `[BODY SUMMARY]` → `[MESSAGES]` → `[TOOLS]` → `[MODEL RESOLVE]` → `[CAPABILITIES REQUIRED]` → `[CAPABILITY REJECT]` → `[ATTEMPT]` → `[UPSTREAM REQUEST]` → `[UPSTREAM RESPONSE]` → `[STREAM START/END/ERROR]` → `[DONE]`.
|
|
26
|
+
- Per-candidate rejection reasons for the `No combo member satisfies...` error (why each member was filtered: `member_disabled`, `model_not_found`, `upstream_unavailable`, `circuit_open`, `tools`, `reasoning`, `streaming`, etc.).
|
|
27
|
+
- Nested `error.cause` / undici stack capture for `UPSTREAM FETCH ERROR` (no more bare `fetch failed`).
|
|
28
|
+
- Stream counters (chunks/bytes/first-chunk-ms) and client-disconnect detection.
|
|
29
|
+
- `[CONFIG]` line at startup printing body limit + active debug flags.
|
|
30
|
+
- Replaced `uncaughtException`/`unhandledRejection` no-op `{}` handlers with full message/stack/cause logging.
|
|
31
|
+
- **Debug env flags**: `DEBUG_HTTP`, `DEBUG_HTTP_BODY`, `DEBUG_UPSTREAM`, `DEBUG_STREAM`, plus `LOG_LEVEL` alias for `LATEDEV_LOG_LEVEL` (all default off; enable for one-shot reproduction).
|
|
32
|
+
- **Secret redaction**: all debug output routes through the existing `redact.ts` — provider keys are fingerprinted, never logged; `Authorization`/`x-api-key`/cookies always masked.
|
|
33
|
+
|
|
7
34
|
## [1.11.16] - 2026-09-04
|
|
8
35
|
|
|
9
36
|
### Fixed
|
package/dist/server/app.js
CHANGED
|
@@ -21,6 +21,7 @@ import { registerGatewayRoutes } from './routes/gateway.js';
|
|
|
21
21
|
import { registerHealthRoutes } from './routes/health.js';
|
|
22
22
|
import { registerAdminIpGate } from './security/admin-ip-gate.js';
|
|
23
23
|
import { metricsRegistry } from './metrics/registry.js';
|
|
24
|
+
import { fatal, lifecycle, formatError, getDebugFlags, errorLine } from './logging/debug.js';
|
|
24
25
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
25
26
|
export async function buildApp(opts = {}) {
|
|
26
27
|
const cfg = loadConfig();
|
|
@@ -63,11 +64,21 @@ export async function buildApp(opts = {}) {
|
|
|
63
64
|
name: normalized instanceof Error ? normalized.name : undefined,
|
|
64
65
|
message: errMsg,
|
|
65
66
|
stack: normalized instanceof Error ? normalized.stack : undefined,
|
|
67
|
+
cause: normalized instanceof Error ? normalized.cause : undefined,
|
|
66
68
|
};
|
|
67
69
|
if (status >= 500)
|
|
68
70
|
log.error({ requestId, url: req.url, err: errDetail }, 'request error');
|
|
69
71
|
else
|
|
70
72
|
log.warn({ requestId, url: req.url, err: { type: (g?.type ?? 'error'), message: errMsg } }, 'request rejected');
|
|
73
|
+
// docs/13 §19: server-side stack with request context (never to client).
|
|
74
|
+
if (status >= 500) {
|
|
75
|
+
errorLine(requestId, 'SERVER ERROR', [
|
|
76
|
+
`route=${req.routeOptions?.url ?? req.url}`,
|
|
77
|
+
`method=${req.method}`,
|
|
78
|
+
`statusCode=${status}`,
|
|
79
|
+
...formatError(normalized),
|
|
80
|
+
]);
|
|
81
|
+
}
|
|
71
82
|
const accept = (req.headers['accept'] ?? '').toString();
|
|
72
83
|
const isAnthropic = accept.includes('application/vnd.anthropic') || req.url.includes('/v1/messages');
|
|
73
84
|
if (g) {
|
|
@@ -82,9 +93,21 @@ export async function buildApp(opts = {}) {
|
|
|
82
93
|
registerAdminIpGate(app);
|
|
83
94
|
// Operational routes (always available)
|
|
84
95
|
await registerHealthRoutes(app);
|
|
85
|
-
//
|
|
86
|
-
|
|
87
|
-
|
|
96
|
+
// docs/13 §20: print effective config so operators know body limits and
|
|
97
|
+
// which debug flags are active for the running process.
|
|
98
|
+
const dbg = getDebugFlags();
|
|
99
|
+
lifecycle('-', 'CONFIG', [
|
|
100
|
+
`host=${cfg.host}`,
|
|
101
|
+
`port=${cfg.port}`,
|
|
102
|
+
`dataDir=${cfg.dataDir}`,
|
|
103
|
+
`logLevel=${cfg.logLevel}`,
|
|
104
|
+
`trustProxyHops=${cfg.trustProxyHops}`,
|
|
105
|
+
`bodyLimit=${64 * 1024 * 1024} bytes`,
|
|
106
|
+
`debugHttp=${dbg.http}`,
|
|
107
|
+
`debugHttpBody=${dbg.httpBody}`,
|
|
108
|
+
`debugUpstream=${dbg.upstream}`,
|
|
109
|
+
`debugStream=${dbg.stream}`,
|
|
110
|
+
]);
|
|
88
111
|
// Admin + gateway routes MUST be registered BEFORE static files to avoid
|
|
89
112
|
// 404s falling through to SPA index.html or static assets being served instead
|
|
90
113
|
await registerAdminRoutes(app);
|
|
@@ -112,15 +135,17 @@ export async function buildApp(opts = {}) {
|
|
|
112
135
|
reply.code(404).send(toOpenAIError(err, req.id));
|
|
113
136
|
});
|
|
114
137
|
// Process-level crash prevention
|
|
115
|
-
process.on('uncaughtException', () => {
|
|
138
|
+
process.on('uncaughtException', (err) => {
|
|
139
|
+
fatal('FATAL', ['uncaughtException', ...formatError(err)]);
|
|
116
140
|
const log = getLogger();
|
|
117
|
-
log.error({ err: {} }, 'uncaught exception');
|
|
141
|
+
log.error({ err: { name: err.name, message: err.message, stack: err.stack } }, 'uncaught exception');
|
|
118
142
|
// Don't exit immediately - let Fastify error handler process
|
|
119
143
|
setTimeout(() => process.exit(1), 1000);
|
|
120
144
|
});
|
|
121
|
-
process.on('unhandledRejection', (
|
|
145
|
+
process.on('unhandledRejection', (reason) => {
|
|
146
|
+
fatal('FATAL', ['unhandledRejection', ...formatError(reason)]);
|
|
122
147
|
const log = getLogger();
|
|
123
|
-
log.error({ reason:
|
|
148
|
+
log.error({ err: { reason: String(reason) } }, 'unhandled rejection');
|
|
124
149
|
});
|
|
125
150
|
// On startup: ensure settings row + detect master key status
|
|
126
151
|
app.addHook('onReady', async () => {
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
function passphraseKey(passphrase, salt) {
|
|
3
|
+
if (!/^\d{6}$/.test(passphrase))
|
|
4
|
+
throw new Error('Backup passphrase must contain exactly six digits');
|
|
5
|
+
return crypto.scryptSync(passphrase, salt, 32, { N: 16_384, r: 8, p: 1 });
|
|
6
|
+
}
|
|
7
|
+
export function encryptBackupMasterKey(masterKey, passphrase) {
|
|
8
|
+
const salt = crypto.randomBytes(16);
|
|
9
|
+
const nonce = crypto.randomBytes(12);
|
|
10
|
+
const cipher = crypto.createCipheriv('aes-256-gcm', passphraseKey(passphrase, salt), nonce);
|
|
11
|
+
const ciphertext = Buffer.concat([cipher.update(masterKey), cipher.final()]);
|
|
12
|
+
return {
|
|
13
|
+
algorithm: 'scrypt-aes-256-gcm',
|
|
14
|
+
salt: salt.toString('base64'),
|
|
15
|
+
nonce: nonce.toString('base64'),
|
|
16
|
+
ciphertext: ciphertext.toString('base64'),
|
|
17
|
+
tag: cipher.getAuthTag().toString('base64'),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export function decryptBackupMasterKey(envelope, passphrase) {
|
|
21
|
+
if (envelope.algorithm !== 'scrypt-aes-256-gcm')
|
|
22
|
+
throw new Error('Unsupported backup key encryption');
|
|
23
|
+
try {
|
|
24
|
+
const decipher = crypto.createDecipheriv('aes-256-gcm', passphraseKey(passphrase, Buffer.from(envelope.salt, 'base64')), Buffer.from(envelope.nonce, 'base64'));
|
|
25
|
+
decipher.setAuthTag(Buffer.from(envelope.tag, 'base64'));
|
|
26
|
+
return Buffer.concat([decipher.update(Buffer.from(envelope.ciphertext, 'base64')), decipher.final()]);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
throw new Error('Invalid backup passphrase');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -40,6 +40,9 @@ export function getMasterKey() {
|
|
|
40
40
|
export function isMasterKeyConfigured() {
|
|
41
41
|
return Boolean(loadConfig().masterKey);
|
|
42
42
|
}
|
|
43
|
+
export function resetMasterKeyCache() {
|
|
44
|
+
cachedKey = null;
|
|
45
|
+
}
|
|
43
46
|
export function masterKeyVersion() {
|
|
44
47
|
return cachedKeyVersion;
|
|
45
48
|
}
|
|
@@ -11,6 +11,8 @@ const EnvSchema = z.object({
|
|
|
11
11
|
LATEDEV_MASTER_KEY: z.string().optional(),
|
|
12
12
|
LATEDEV_TRUST_PROXY: z.coerce.number().int().min(0).max(8).default(0),
|
|
13
13
|
LATEDEV_LOG_LEVEL: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'),
|
|
14
|
+
// Alias used by docs/13 debug tooling: LOG_LEVEL=debug enables lifecycle lines.
|
|
15
|
+
LOG_LEVEL: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).optional(),
|
|
14
16
|
LATEDEV_DB_URL: z.string().optional(),
|
|
15
17
|
NODE_ENV: z.enum(['development', 'production', 'test']).default('production'),
|
|
16
18
|
});
|
|
@@ -54,7 +56,7 @@ export function loadConfig(env = process.env, argv = process.argv) {
|
|
|
54
56
|
dbFile,
|
|
55
57
|
masterKey: parsed.LATEDEV_MASTER_KEY ?? readMasterKeyFile(dataDir),
|
|
56
58
|
trustProxyHops: parsed.LATEDEV_TRUST_PROXY,
|
|
57
|
-
logLevel: parsed.LATEDEV_LOG_LEVEL,
|
|
59
|
+
logLevel: parsed.LOG_LEVEL ?? parsed.LATEDEV_LOG_LEVEL,
|
|
58
60
|
env: parsed.NODE_ENV,
|
|
59
61
|
isContainer,
|
|
60
62
|
appVersion: getAppVersion(),
|
|
@@ -18,6 +18,7 @@ import { getSettings } from '../db/repositories/settings.js';
|
|
|
18
18
|
import { buildCacheKey, lookupCache, storeCache, cacheAllowed } from '../caching/store.js';
|
|
19
19
|
import { metrics } from '../metrics/registry.js';
|
|
20
20
|
import { emitRequestLogged, emitRequestStarted } from './events.js';
|
|
21
|
+
import { debugHttp, debugBody, debugUpstream, debugStream, errorLine, formatError, getDebugFlags, truncate } from '../logging/debug.js';
|
|
21
22
|
export class GatewayRunner {
|
|
22
23
|
async execute(req, ctx) {
|
|
23
24
|
const start = Date.now();
|
|
@@ -31,6 +32,17 @@ export class GatewayRunner {
|
|
|
31
32
|
let comboPlan = null;
|
|
32
33
|
if (resolved.kind === 'combo')
|
|
33
34
|
comboPlan = loadCombo(resolved.comboId);
|
|
35
|
+
// docs/13 §9: full resolution chain requested -> alias -> combo/model
|
|
36
|
+
if (getDebugFlags().http) {
|
|
37
|
+
const lines = [`requested=${req.canonical.model}`, `type=${resolved.kind}`];
|
|
38
|
+
if (target.kind === 'alias')
|
|
39
|
+
lines.push(`viaAlias=${target.alias}`);
|
|
40
|
+
if (resolved.kind === 'model')
|
|
41
|
+
lines.push(`providerModelId=${resolved.modelId}`, `publicModelId=${resolved.publicModelId}`);
|
|
42
|
+
if (resolved.kind === 'combo')
|
|
43
|
+
lines.push(`comboId=${resolved.comboId}`, `comboPublicId=${resolved.publicModelId}`);
|
|
44
|
+
debugHttp(ctx.requestId, 'MODEL RESOLVE', lines);
|
|
45
|
+
}
|
|
34
46
|
// --- ACL check ---
|
|
35
47
|
if (ctx.key) {
|
|
36
48
|
if (resolved.kind === 'model') {
|
|
@@ -74,20 +86,52 @@ export class GatewayRunner {
|
|
|
74
86
|
try {
|
|
75
87
|
// --- Capability requirements ---
|
|
76
88
|
const required = deriveRequiredCapabilities(req.canonical);
|
|
89
|
+
debugHttp(ctx.requestId, 'CAPABILITIES REQUIRED', [
|
|
90
|
+
`tools=${required.tools}`,
|
|
91
|
+
`stream=${required.streaming}`,
|
|
92
|
+
`vision=${required.imageInput}`,
|
|
93
|
+
`reasoning=${required.reasoning}`,
|
|
94
|
+
`json=${required.structuredOutput}`,
|
|
95
|
+
`structuredOutput=${required.structuredOutput}`,
|
|
96
|
+
`parallelToolCalls=undefined(derived-from-request-not-gated)`,
|
|
97
|
+
`audioInput=${required.audioInput}`,
|
|
98
|
+
]);
|
|
77
99
|
// --- Determine candidates ---
|
|
78
100
|
let candidates = [];
|
|
79
101
|
let selectionReasons = [];
|
|
80
102
|
if (resolved.kind === 'model') {
|
|
81
|
-
candidates = await this.loadModelCandidate(resolved.modelId, required);
|
|
103
|
+
candidates = await this.loadModelCandidate(resolved.modelId, required, ctx.requestId);
|
|
82
104
|
selectionReasons.push('direct_model');
|
|
105
|
+
if (candidates.length === 0) {
|
|
106
|
+
debugHttp(ctx.requestId, 'CAPABILITY REJECT', [
|
|
107
|
+
`model=${resolved.publicModelId}`,
|
|
108
|
+
`reason=direct_model_unavailable_or_capability_mismatch`,
|
|
109
|
+
'(direct model candidates rejected: not found / provider disabled / model disabled / upstream unavailable / circuit open / capability mismatch)',
|
|
110
|
+
]);
|
|
111
|
+
}
|
|
83
112
|
}
|
|
84
113
|
else if (comboPlan) {
|
|
85
114
|
const all = await this.loadAllModels();
|
|
86
|
-
const
|
|
115
|
+
const rejected = [];
|
|
116
|
+
const filtered = selectCandidates(comboPlan, all, required, (c, reason) => {
|
|
117
|
+
rejected.push({ publicModelId: c.publicModelId, reason });
|
|
118
|
+
debugHttp(ctx.requestId, 'CAPABILITY REJECT', [`model=${c.publicModelId}`, `reason=${reason}`]);
|
|
119
|
+
});
|
|
120
|
+
// docs/13 §10: always report why EACH member was filtered out
|
|
121
|
+
debugHttp(ctx.requestId, 'CANDIDATE FILTER', [
|
|
122
|
+
`comboMembers=${comboPlan.members.length}`,
|
|
123
|
+
`afterFilter=${filtered.length}`,
|
|
124
|
+
`rejectedCount=${rejected.length}`,
|
|
125
|
+
...rejected.map((r) => `rejected: ${r.publicModelId} reason=${r.reason}`),
|
|
126
|
+
]);
|
|
87
127
|
if (filtered.length === 0) {
|
|
88
128
|
throw new GatewayError('capability_not_supported', 'No combo member satisfies the request capabilities or availability', { status: 400 });
|
|
89
129
|
}
|
|
90
130
|
candidates = orderCandidates(comboPlan, filtered);
|
|
131
|
+
debugHttp(ctx.requestId, 'CANDIDATES ORDERED', [
|
|
132
|
+
`mode=${comboPlan.mode}`,
|
|
133
|
+
...candidates.map((c, i) => `candidate[${i}]: providerModelId=${c.modelId} publicModelId=${c.publicModelId}`),
|
|
134
|
+
]);
|
|
91
135
|
selectionReasons.push('combo');
|
|
92
136
|
}
|
|
93
137
|
if (candidates.length === 0) {
|
|
@@ -137,11 +181,13 @@ export class GatewayRunner {
|
|
|
137
181
|
finalModelId = candidate.modelId;
|
|
138
182
|
const provider = getDb().select().from(schema.providers).where(eq(schema.providers.id, candidate.providerId)).get();
|
|
139
183
|
if (!provider || !provider.enabled) {
|
|
184
|
+
debugHttp(ctx.requestId, 'ATTEMPT SKIP', [`attempt=${i + 1} model=${candidate.publicModelId} reason=skipped_disabled_provider`]);
|
|
140
185
|
attempts.push(this.failedAttempt(i + 1, candidate, provider?.name ?? '', 'skipped_disabled_provider', null, null, 0, null, null));
|
|
141
186
|
continue;
|
|
142
187
|
}
|
|
143
188
|
const eff = getEffectiveState(provider.id, provider.cbCooldownSeconds);
|
|
144
189
|
if (eff === 'open' && !halfOpenProbeAllowed(provider.id)) {
|
|
190
|
+
debugHttp(ctx.requestId, 'ATTEMPT SKIP', [`attempt=${i + 1} model=${candidate.publicModelId} provider=${provider.name} reason=circuit_open`]);
|
|
145
191
|
attempts.push(this.failedAttempt(i + 1, candidate, provider.name, 'circuit_open', null, null, 0, null, null));
|
|
146
192
|
if (comboPlan && shouldFallback(comboPlan, { type: 'connection_error' })) {
|
|
147
193
|
metrics.fallbackCount.inc();
|
|
@@ -152,6 +198,19 @@ export class GatewayRunner {
|
|
|
152
198
|
}
|
|
153
199
|
const cfg = providerToUpstreamConfig(provider);
|
|
154
200
|
const attemptStart = Date.now();
|
|
201
|
+
// docs/13 §11–§12: provider/account + upstream request summary
|
|
202
|
+
const upstreamModel = candidate.publicModelId.split('/').slice(1).join('/');
|
|
203
|
+
debugHttp(ctx.requestId, 'ATTEMPT', [
|
|
204
|
+
`attempt=${i + 1}/${maxAttempts}`,
|
|
205
|
+
`provider=${provider.name}`,
|
|
206
|
+
`providerId=${provider.id}`,
|
|
207
|
+
`model=${candidate.publicModelId}`,
|
|
208
|
+
`upstreamModel=${upstreamModel}`,
|
|
209
|
+
`upstreamType=${cfg.type}`,
|
|
210
|
+
`baseUrl=${cfg.baseUrl}`,
|
|
211
|
+
`stream=${req.canonical.stream}`,
|
|
212
|
+
`providerKeyFingerprint=${apiKeyFingerprint(cfg.apiKey)}`,
|
|
213
|
+
]);
|
|
155
214
|
const attempt = {
|
|
156
215
|
attemptNumber: i + 1,
|
|
157
216
|
providerId: provider.id,
|
|
@@ -206,6 +265,14 @@ export class GatewayRunner {
|
|
|
206
265
|
}
|
|
207
266
|
catch (e) {
|
|
208
267
|
const err = e instanceof GatewayError ? e : new GatewayError('upstream_error', e.message, { cause: e });
|
|
268
|
+
debugUpstream(ctx.requestId, 'ATTEMPT ERROR', [
|
|
269
|
+
`attempt=${i + 1}`,
|
|
270
|
+
`provider=${provider.name}`,
|
|
271
|
+
`model=${candidate.publicModelId}`,
|
|
272
|
+
`type=${err.type}`,
|
|
273
|
+
`status=${err.status}`,
|
|
274
|
+
`willFallback=${comboPlan ? shouldFallback(comboPlan, { type: classifyFailure(err), status: err.status }) : false}`,
|
|
275
|
+
]);
|
|
209
276
|
const shouldRetry = comboPlan ? shouldFallback(comboPlan, { type: classifyFailure(err), status: err.status }) : false;
|
|
210
277
|
attempt.statusCode = err.status;
|
|
211
278
|
attempt.success = false;
|
|
@@ -288,14 +355,26 @@ export class GatewayRunner {
|
|
|
288
355
|
metrics.activeRequests.dec();
|
|
289
356
|
}
|
|
290
357
|
}
|
|
291
|
-
async loadModelCandidate(modelId, required) {
|
|
358
|
+
async loadModelCandidate(modelId, required, requestId) {
|
|
292
359
|
const db = getDb();
|
|
360
|
+
const reject = (reason) => {
|
|
361
|
+
if (requestId)
|
|
362
|
+
debugHttp(requestId, 'CAPABILITY REJECT', [`modelId=${modelId}`, `reason=${reason}`]);
|
|
363
|
+
};
|
|
293
364
|
const m = db.select().from(schema.models).where(eq(schema.models.id, modelId)).get();
|
|
294
|
-
if (!m)
|
|
365
|
+
if (!m) {
|
|
366
|
+
reject('model_not_found');
|
|
295
367
|
return [];
|
|
368
|
+
}
|
|
296
369
|
const p = db.select().from(schema.providers).where(eq(schema.providers.id, m.providerId)).get();
|
|
297
|
-
if (!p
|
|
370
|
+
if (!p) {
|
|
371
|
+
reject('provider_not_found');
|
|
298
372
|
return [];
|
|
373
|
+
}
|
|
374
|
+
if (!p.enabled) {
|
|
375
|
+
reject('provider_disabled');
|
|
376
|
+
return [];
|
|
377
|
+
}
|
|
299
378
|
const caps = safeJson(m.capabilitiesJson);
|
|
300
379
|
const candidate = {
|
|
301
380
|
modelId: m.id,
|
|
@@ -306,12 +385,30 @@ export class GatewayRunner {
|
|
|
306
385
|
circuitOpen: isOpen(m.providerId),
|
|
307
386
|
capabilities: caps,
|
|
308
387
|
};
|
|
309
|
-
if (!m.enabled
|
|
388
|
+
if (!m.enabled) {
|
|
389
|
+
reject('model_disabled');
|
|
310
390
|
return [];
|
|
311
|
-
|
|
391
|
+
}
|
|
392
|
+
if (!m.upstreamAvailable) {
|
|
393
|
+
reject('upstream_unavailable');
|
|
394
|
+
return [];
|
|
395
|
+
}
|
|
396
|
+
if (candidate.circuitOpen) {
|
|
397
|
+
reject('circuit_open');
|
|
312
398
|
return [];
|
|
313
|
-
|
|
399
|
+
}
|
|
400
|
+
if (!modelMeets(caps, required)) {
|
|
401
|
+
reject('capability_mismatch');
|
|
314
402
|
return [];
|
|
403
|
+
}
|
|
404
|
+
debugHttp(requestId ?? '-', 'CAPABILITY CANDIDATE', [
|
|
405
|
+
`model=${m.publicModelId}`,
|
|
406
|
+
`caps.tools=${caps.tools}`,
|
|
407
|
+
`caps.streaming=${caps.streaming}`,
|
|
408
|
+
`caps.reasoning=${caps.reasoning}`,
|
|
409
|
+
`caps.image_input=${caps.image_input}`,
|
|
410
|
+
`caps.structured_output=${caps.structured_output}`,
|
|
411
|
+
]);
|
|
315
412
|
return [candidate];
|
|
316
413
|
}
|
|
317
414
|
async loadAllModels() {
|
|
@@ -333,18 +430,20 @@ export class GatewayRunner {
|
|
|
333
430
|
if (req.canonical.stream) {
|
|
334
431
|
return this.runStreamingAttempt(req, ctx, candidate, cfg, onStreamStart, onFirstToken);
|
|
335
432
|
}
|
|
336
|
-
return this.runNonStreamingAttempt(req, candidate, cfg);
|
|
433
|
+
return this.runNonStreamingAttempt(req, candidate, cfg, ctx);
|
|
337
434
|
}
|
|
338
|
-
async runNonStreamingAttempt(req, candidate, cfg) {
|
|
435
|
+
async runNonStreamingAttempt(req, candidate, cfg, ctx) {
|
|
339
436
|
const upstreamModel = candidate.publicModelId.split('/').slice(1).join('/');
|
|
340
437
|
let call;
|
|
341
438
|
if (cfg.type === 'openai') {
|
|
342
439
|
const payload = canonicalToOpenAIRequest(req.canonical, upstreamModel);
|
|
343
|
-
|
|
440
|
+
logUpstreamRequest(ctx.requestId, cfg, upstreamUrl(cfg, '/v1/chat/completions'), payload, req.canonical.stream);
|
|
441
|
+
call = await callUpstreamNonStreaming(cfg, upstreamUrl(cfg, '/v1/chat/completions'), payload, ctx.requestId);
|
|
344
442
|
}
|
|
345
443
|
else {
|
|
346
444
|
const payload = canonicalToAnthropicRequest(req.canonical, upstreamModel);
|
|
347
|
-
|
|
445
|
+
logUpstreamRequest(ctx.requestId, cfg, upstreamUrl(cfg, '/v1/messages'), payload, req.canonical.stream);
|
|
446
|
+
call = await callUpstreamNonStreaming(cfg, upstreamUrl(cfg, '/v1/messages'), payload, ctx.requestId);
|
|
348
447
|
}
|
|
349
448
|
if (!call.ok) {
|
|
350
449
|
if (call.status === 429)
|
|
@@ -402,7 +501,39 @@ export class GatewayRunner {
|
|
|
402
501
|
const toolBuf = [];
|
|
403
502
|
let finishReason = null;
|
|
404
503
|
const usage = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, reasoning: 0, total: 0 };
|
|
504
|
+
// docs/13 §15–§16: stream counters + client disconnect tracking
|
|
505
|
+
let chunkCount = 0;
|
|
506
|
+
let bytesReceived = 0;
|
|
507
|
+
let firstChunkTime = null;
|
|
508
|
+
let lastChunkTime = null;
|
|
509
|
+
const loggedFirstChunks = [];
|
|
510
|
+
let clientDisconnected = false;
|
|
511
|
+
// Guard: the admin test-stream endpoint passes a minimal `fakeRaw` that has
|
|
512
|
+
// writeHead/write/end but no `.on` — skip disconnect tracking in that case.
|
|
513
|
+
if (typeof pipe.on === 'function') {
|
|
514
|
+
pipe.on('close', () => {
|
|
515
|
+
// Distinguish client disconnect from normal end: 'close' fires on the raw
|
|
516
|
+
// socket when the client (or upstream) side goes away.
|
|
517
|
+
if (!pipe.writableEnded) {
|
|
518
|
+
clientDisconnected = true;
|
|
519
|
+
debugStream(ctx.requestId, 'CLIENT DISCONNECT', [
|
|
520
|
+
`afterMs=${Date.now() - streamStartTs}`,
|
|
521
|
+
`streaming=${streamStarted}`,
|
|
522
|
+
`chunksSoFar=${chunkCount}`,
|
|
523
|
+
]);
|
|
524
|
+
}
|
|
525
|
+
});
|
|
526
|
+
}
|
|
405
527
|
const chunkHandler = (chunk, isFirst) => {
|
|
528
|
+
chunkCount++;
|
|
529
|
+
bytesReceived += chunk.data.length;
|
|
530
|
+
lastChunkTime = Date.now();
|
|
531
|
+
if (isFirst)
|
|
532
|
+
firstChunkTime = Date.now() - streamStartTs;
|
|
533
|
+
if (loggedFirstChunks.length < 3) {
|
|
534
|
+
loggedFirstChunks.push(chunk.data);
|
|
535
|
+
debugStream(ctx.requestId, `STREAM FIRST CHUNK ${loggedFirstChunks.length}`, [truncate(chunk.data, 2000)]);
|
|
536
|
+
}
|
|
406
537
|
try {
|
|
407
538
|
const obj = JSON.parse(chunk.data);
|
|
408
539
|
if (cfg.type === 'openai') {
|
|
@@ -473,12 +604,23 @@ export class GatewayRunner {
|
|
|
473
604
|
try {
|
|
474
605
|
const url = cfg.type === 'openai' ? upstreamUrl(cfg, '/v1/chat/completions') : upstreamUrl(cfg, '/v1/messages');
|
|
475
606
|
const payload = cfg.type === 'openai' ? canonicalToOpenAIRequest(req.canonical, upstreamModel) : canonicalToAnthropicRequest(req.canonical, upstreamModel);
|
|
476
|
-
|
|
607
|
+
logUpstreamRequest(ctx.requestId, cfg, url, payload, true);
|
|
608
|
+
debugStream(ctx.requestId, 'STREAM START', [`upstreamConnected=true`]);
|
|
609
|
+
const meta = await callUpstreamStreaming(cfg, url, payload, chunkHandler, ctx.requestId);
|
|
477
610
|
// Upstream completed cleanly: ensure head + terminator are written.
|
|
478
611
|
if (!headWritten)
|
|
479
612
|
writeHead();
|
|
480
613
|
pipe.write('data: [DONE]\n\n');
|
|
481
614
|
pipe.end();
|
|
615
|
+
debugStream(ctx.requestId, 'STREAM END', [
|
|
616
|
+
`chunks=${chunkCount}`,
|
|
617
|
+
`bytes=${bytesReceived}`,
|
|
618
|
+
`firstChunkMs=${firstChunkTime ?? 'null'}`,
|
|
619
|
+
`lastChunkMs=${lastChunkTime ? lastChunkTime - streamStartTs : 'null'}`,
|
|
620
|
+
`durationMs=${Date.now() - streamStartTs}`,
|
|
621
|
+
`finishedNormally=true`,
|
|
622
|
+
`clientDisconnected=${clientDisconnected}`,
|
|
623
|
+
]);
|
|
482
624
|
if (!usage.total)
|
|
483
625
|
usage.total = usage.input + usage.output;
|
|
484
626
|
return {
|
|
@@ -497,8 +639,20 @@ export class GatewayRunner {
|
|
|
497
639
|
writeHead();
|
|
498
640
|
pipe.end();
|
|
499
641
|
const err = e instanceof GatewayError ? e : new GatewayError('upstream_error', e.message);
|
|
642
|
+
errorLine(ctx.requestId, 'STREAM ERROR', [
|
|
643
|
+
`chunksBeforeError=${chunkCount}`,
|
|
644
|
+
`bytesBeforeError=${bytesReceived}`,
|
|
645
|
+
`clientDisconnected=${clientDisconnected}`,
|
|
646
|
+
...formatErrorPublic(e),
|
|
647
|
+
]);
|
|
500
648
|
throw err;
|
|
501
649
|
}
|
|
650
|
+
errorLine(ctx.requestId, 'STREAM ERROR (before first chunk)', [
|
|
651
|
+
`chunksBeforeError=${chunkCount}`,
|
|
652
|
+
`bytesBeforeError=${bytesReceived}`,
|
|
653
|
+
`clientDisconnected=${clientDisconnected}`,
|
|
654
|
+
...formatErrorPublic(e),
|
|
655
|
+
]);
|
|
502
656
|
throw e;
|
|
503
657
|
}
|
|
504
658
|
}
|
|
@@ -732,6 +886,36 @@ function hasTools(req) {
|
|
|
732
886
|
function estimateTokens(s) {
|
|
733
887
|
return Math.ceil(s.length / 4);
|
|
734
888
|
}
|
|
889
|
+
// docs/13 §11: safe provider key fingerprint — never the key itself.
|
|
890
|
+
function apiKeyFingerprint(key) {
|
|
891
|
+
let h = 5381;
|
|
892
|
+
for (let i = 0; i < key.length; i++) {
|
|
893
|
+
h = ((h << 5) + h + key.charCodeAt(i)) >>> 0;
|
|
894
|
+
}
|
|
895
|
+
return `fp_${h.toString(16).padStart(8, '0')}…${key.slice(-4).replace(/./g, '*')}${key.length}ch`;
|
|
896
|
+
}
|
|
897
|
+
// docs/13 §12: upstream request summary + optional full body
|
|
898
|
+
function logUpstreamRequest(requestId, cfg, url, payload, stream) {
|
|
899
|
+
const json = JSON.stringify(payload ?? {});
|
|
900
|
+
debugUpstream(requestId, 'UPSTREAM REQUEST', [
|
|
901
|
+
`method=POST`,
|
|
902
|
+
`url=${url}`,
|
|
903
|
+
`type=${cfg.type}`,
|
|
904
|
+
`stream=${stream}`,
|
|
905
|
+
`contentLength=${json.length}`,
|
|
906
|
+
`customHeaders=${JSON.stringify(Object.keys(cfg.customHeaders ?? {}))}`,
|
|
907
|
+
]);
|
|
908
|
+
if (getDebugFlags().httpBody) {
|
|
909
|
+
const LIMIT = 512 * 1024;
|
|
910
|
+
debugBody(requestId, 'UPSTREAM BODY', [
|
|
911
|
+
json.length > LIMIT ? `${json.slice(0, LIMIT)}…(+${json.length - LIMIT} chars, truncated)` : json,
|
|
912
|
+
]);
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
/** Public alias so error formatting from debug.ts is available in this module. */
|
|
916
|
+
function formatErrorPublic(e) {
|
|
917
|
+
return formatError(e);
|
|
918
|
+
}
|
|
735
919
|
/**
|
|
736
920
|
* Safely parse capabilities JSON. Returns minimal default if parsing fails.
|
|
737
921
|
* CRITICAL: Must return a complete default object with all capability fields,
|