ldrouter 1.11.17 → 1.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -4,6 +4,33 @@ All notable changes to this project are documented here. The format follows
4
4
  [Keep a Changelog](https://keepachangelog.com/) and the project adheres to
5
5
  [Semantic Versioning](https://semver.org/).
6
6
 
7
+ ## [1.13.1] - 2026-09-11
8
+
9
+ ### Fixed
10
+
11
+ - **Update notification**: force a fresh npm registry check when the admin UI loads, so the update button is not hidden by the 15-minute cache.
12
+
13
+ ## [1.13.0] - 2026-09-11
14
+
15
+ ### Added
16
+
17
+ - **Passphrase-protected full backups**: backups now include administrator state and a master-key envelope protected by a user-entered six-digit passphrase; `/setup` can import them without recreating the admin account.
18
+
19
+ ## [1.12.0] - 2026-09-05
20
+
21
+ ### Added
22
+
23
+ - **Request-lifecycle debug logging** (`docs/13-LOGGING.md`): full observability of every request as it passes through the gateway, emitted to stdout/stderr so `docker logs` collects it.
24
+ - Per-request ID (reuses `x-request-id` or generates one) tagged on every line — `docker logs ldrouter 2>&1 | grep req_xxx` reconstructs the whole lifecycle.
25
+ - `[INCOMING]` → `[BODY SUMMARY]` → `[MESSAGES]` → `[TOOLS]` → `[MODEL RESOLVE]` → `[CAPABILITIES REQUIRED]` → `[CAPABILITY REJECT]` → `[ATTEMPT]` → `[UPSTREAM REQUEST]` → `[UPSTREAM RESPONSE]` → `[STREAM START/END/ERROR]` → `[DONE]`.
26
+ - Per-candidate rejection reasons for the `No combo member satisfies...` error (why each member was filtered: `member_disabled`, `model_not_found`, `upstream_unavailable`, `circuit_open`, `tools`, `reasoning`, `streaming`, etc.).
27
+ - Nested `error.cause` / undici stack capture for `UPSTREAM FETCH ERROR` (no more bare `fetch failed`).
28
+ - Stream counters (chunks/bytes/first-chunk-ms) and client-disconnect detection.
29
+ - `[CONFIG]` line at startup printing body limit + active debug flags.
30
+ - Replaced `uncaughtException`/`unhandledRejection` no-op `{}` handlers with full message/stack/cause logging.
31
+ - **Debug env flags**: `DEBUG_HTTP`, `DEBUG_HTTP_BODY`, `DEBUG_UPSTREAM`, `DEBUG_STREAM`, plus `LOG_LEVEL` alias for `LATEDEV_LOG_LEVEL` (all default off; enable for one-shot reproduction).
32
+ - **Secret redaction**: all debug output routes through the existing `redact.ts` — provider keys are fingerprinted, never logged; `Authorization`/`x-api-key`/cookies always masked.
33
+
7
34
  ## [1.11.16] - 2026-09-04
8
35
 
9
36
  ### Fixed
@@ -21,6 +21,7 @@ import { registerGatewayRoutes } from './routes/gateway.js';
21
21
  import { registerHealthRoutes } from './routes/health.js';
22
22
  import { registerAdminIpGate } from './security/admin-ip-gate.js';
23
23
  import { metricsRegistry } from './metrics/registry.js';
24
+ import { fatal, lifecycle, formatError, getDebugFlags, errorLine } from './logging/debug.js';
24
25
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
25
26
  export async function buildApp(opts = {}) {
26
27
  const cfg = loadConfig();
@@ -63,11 +64,21 @@ export async function buildApp(opts = {}) {
63
64
  name: normalized instanceof Error ? normalized.name : undefined,
64
65
  message: errMsg,
65
66
  stack: normalized instanceof Error ? normalized.stack : undefined,
67
+ cause: normalized instanceof Error ? normalized.cause : undefined,
66
68
  };
67
69
  if (status >= 500)
68
70
  log.error({ requestId, url: req.url, err: errDetail }, 'request error');
69
71
  else
70
72
  log.warn({ requestId, url: req.url, err: { type: (g?.type ?? 'error'), message: errMsg } }, 'request rejected');
73
+ // docs/13 §19: server-side stack with request context (never to client).
74
+ if (status >= 500) {
75
+ errorLine(requestId, 'SERVER ERROR', [
76
+ `route=${req.routeOptions?.url ?? req.url}`,
77
+ `method=${req.method}`,
78
+ `statusCode=${status}`,
79
+ ...formatError(normalized),
80
+ ]);
81
+ }
71
82
  const accept = (req.headers['accept'] ?? '').toString();
72
83
  const isAnthropic = accept.includes('application/vnd.anthropic') || req.url.includes('/v1/messages');
73
84
  if (g) {
@@ -82,9 +93,21 @@ export async function buildApp(opts = {}) {
82
93
  registerAdminIpGate(app);
83
94
  // Operational routes (always available)
84
95
  await registerHealthRoutes(app);
85
- // Debug logging for request lifecycle (before routes)
86
- const { registerDebugHook } = await import('./logging/debug.js');
87
- registerDebugHook(app);
96
+ // docs/13 §20: print effective config so operators know body limits and
97
+ // which debug flags are active for the running process.
98
+ const dbg = getDebugFlags();
99
+ lifecycle('-', 'CONFIG', [
100
+ `host=${cfg.host}`,
101
+ `port=${cfg.port}`,
102
+ `dataDir=${cfg.dataDir}`,
103
+ `logLevel=${cfg.logLevel}`,
104
+ `trustProxyHops=${cfg.trustProxyHops}`,
105
+ `bodyLimit=${64 * 1024 * 1024} bytes`,
106
+ `debugHttp=${dbg.http}`,
107
+ `debugHttpBody=${dbg.httpBody}`,
108
+ `debugUpstream=${dbg.upstream}`,
109
+ `debugStream=${dbg.stream}`,
110
+ ]);
88
111
  // Admin + gateway routes MUST be registered BEFORE static files to avoid
89
112
  // 404s falling through to SPA index.html or static assets being served instead
90
113
  await registerAdminRoutes(app);
@@ -112,15 +135,17 @@ export async function buildApp(opts = {}) {
112
135
  reply.code(404).send(toOpenAIError(err, req.id));
113
136
  });
114
137
  // Process-level crash prevention
115
- process.on('uncaughtException', () => {
138
+ process.on('uncaughtException', (err) => {
139
+ fatal('FATAL', ['uncaughtException', ...formatError(err)]);
116
140
  const log = getLogger();
117
- log.error({ err: {} }, 'uncaught exception');
141
+ log.error({ err: { name: err.name, message: err.message, stack: err.stack } }, 'uncaught exception');
118
142
  // Don't exit immediately - let Fastify error handler process
119
143
  setTimeout(() => process.exit(1), 1000);
120
144
  });
121
- process.on('unhandledRejection', (_reason) => {
145
+ process.on('unhandledRejection', (reason) => {
146
+ fatal('FATAL', ['unhandledRejection', ...formatError(reason)]);
122
147
  const log = getLogger();
123
- log.error({ reason: '' }, 'unhandled rejection');
148
+ log.error({ err: { reason: String(reason) } }, 'unhandled rejection');
124
149
  });
125
150
  // On startup: ensure settings row + detect master key status
126
151
  app.addHook('onReady', async () => {
@@ -0,0 +1,31 @@
1
+ import crypto from 'node:crypto';
2
+ function passphraseKey(passphrase, salt) {
3
+ if (!/^\d{6}$/.test(passphrase))
4
+ throw new Error('Backup passphrase must contain exactly six digits');
5
+ return crypto.scryptSync(passphrase, salt, 32, { N: 16_384, r: 8, p: 1 });
6
+ }
7
+ export function encryptBackupMasterKey(masterKey, passphrase) {
8
+ const salt = crypto.randomBytes(16);
9
+ const nonce = crypto.randomBytes(12);
10
+ const cipher = crypto.createCipheriv('aes-256-gcm', passphraseKey(passphrase, salt), nonce);
11
+ const ciphertext = Buffer.concat([cipher.update(masterKey), cipher.final()]);
12
+ return {
13
+ algorithm: 'scrypt-aes-256-gcm',
14
+ salt: salt.toString('base64'),
15
+ nonce: nonce.toString('base64'),
16
+ ciphertext: ciphertext.toString('base64'),
17
+ tag: cipher.getAuthTag().toString('base64'),
18
+ };
19
+ }
20
+ export function decryptBackupMasterKey(envelope, passphrase) {
21
+ if (envelope.algorithm !== 'scrypt-aes-256-gcm')
22
+ throw new Error('Unsupported backup key encryption');
23
+ try {
24
+ const decipher = crypto.createDecipheriv('aes-256-gcm', passphraseKey(passphrase, Buffer.from(envelope.salt, 'base64')), Buffer.from(envelope.nonce, 'base64'));
25
+ decipher.setAuthTag(Buffer.from(envelope.tag, 'base64'));
26
+ return Buffer.concat([decipher.update(Buffer.from(envelope.ciphertext, 'base64')), decipher.final()]);
27
+ }
28
+ catch {
29
+ throw new Error('Invalid backup passphrase');
30
+ }
31
+ }
@@ -40,6 +40,9 @@ export function getMasterKey() {
40
40
  export function isMasterKeyConfigured() {
41
41
  return Boolean(loadConfig().masterKey);
42
42
  }
43
+ export function resetMasterKeyCache() {
44
+ cachedKey = null;
45
+ }
43
46
  export function masterKeyVersion() {
44
47
  return cachedKeyVersion;
45
48
  }
@@ -11,6 +11,8 @@ const EnvSchema = z.object({
11
11
  LATEDEV_MASTER_KEY: z.string().optional(),
12
12
  LATEDEV_TRUST_PROXY: z.coerce.number().int().min(0).max(8).default(0),
13
13
  LATEDEV_LOG_LEVEL: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'),
14
+ // Alias used by docs/13 debug tooling: LOG_LEVEL=debug enables lifecycle lines.
15
+ LOG_LEVEL: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).optional(),
14
16
  LATEDEV_DB_URL: z.string().optional(),
15
17
  NODE_ENV: z.enum(['development', 'production', 'test']).default('production'),
16
18
  });
@@ -54,7 +56,7 @@ export function loadConfig(env = process.env, argv = process.argv) {
54
56
  dbFile,
55
57
  masterKey: parsed.LATEDEV_MASTER_KEY ?? readMasterKeyFile(dataDir),
56
58
  trustProxyHops: parsed.LATEDEV_TRUST_PROXY,
57
- logLevel: parsed.LATEDEV_LOG_LEVEL,
59
+ logLevel: parsed.LOG_LEVEL ?? parsed.LATEDEV_LOG_LEVEL,
58
60
  env: parsed.NODE_ENV,
59
61
  isContainer,
60
62
  appVersion: getAppVersion(),
@@ -18,6 +18,7 @@ import { getSettings } from '../db/repositories/settings.js';
18
18
  import { buildCacheKey, lookupCache, storeCache, cacheAllowed } from '../caching/store.js';
19
19
  import { metrics } from '../metrics/registry.js';
20
20
  import { emitRequestLogged, emitRequestStarted } from './events.js';
21
+ import { debugHttp, debugBody, debugUpstream, debugStream, errorLine, formatError, getDebugFlags, truncate } from '../logging/debug.js';
21
22
  export class GatewayRunner {
22
23
  async execute(req, ctx) {
23
24
  const start = Date.now();
@@ -31,6 +32,17 @@ export class GatewayRunner {
31
32
  let comboPlan = null;
32
33
  if (resolved.kind === 'combo')
33
34
  comboPlan = loadCombo(resolved.comboId);
35
+ // docs/13 §9: full resolution chain requested -> alias -> combo/model
36
+ if (getDebugFlags().http) {
37
+ const lines = [`requested=${req.canonical.model}`, `type=${resolved.kind}`];
38
+ if (target.kind === 'alias')
39
+ lines.push(`viaAlias=${target.alias}`);
40
+ if (resolved.kind === 'model')
41
+ lines.push(`providerModelId=${resolved.modelId}`, `publicModelId=${resolved.publicModelId}`);
42
+ if (resolved.kind === 'combo')
43
+ lines.push(`comboId=${resolved.comboId}`, `comboPublicId=${resolved.publicModelId}`);
44
+ debugHttp(ctx.requestId, 'MODEL RESOLVE', lines);
45
+ }
34
46
  // --- ACL check ---
35
47
  if (ctx.key) {
36
48
  if (resolved.kind === 'model') {
@@ -74,20 +86,52 @@ export class GatewayRunner {
74
86
  try {
75
87
  // --- Capability requirements ---
76
88
  const required = deriveRequiredCapabilities(req.canonical);
89
+ debugHttp(ctx.requestId, 'CAPABILITIES REQUIRED', [
90
+ `tools=${required.tools}`,
91
+ `stream=${required.streaming}`,
92
+ `vision=${required.imageInput}`,
93
+ `reasoning=${required.reasoning}`,
94
+ `json=${required.structuredOutput}`,
95
+ `structuredOutput=${required.structuredOutput}`,
96
+ `parallelToolCalls=undefined(derived-from-request-not-gated)`,
97
+ `audioInput=${required.audioInput}`,
98
+ ]);
77
99
  // --- Determine candidates ---
78
100
  let candidates = [];
79
101
  let selectionReasons = [];
80
102
  if (resolved.kind === 'model') {
81
- candidates = await this.loadModelCandidate(resolved.modelId, required);
103
+ candidates = await this.loadModelCandidate(resolved.modelId, required, ctx.requestId);
82
104
  selectionReasons.push('direct_model');
105
+ if (candidates.length === 0) {
106
+ debugHttp(ctx.requestId, 'CAPABILITY REJECT', [
107
+ `model=${resolved.publicModelId}`,
108
+ `reason=direct_model_unavailable_or_capability_mismatch`,
109
+ '(direct model candidates rejected: not found / provider disabled / model disabled / upstream unavailable / circuit open / capability mismatch)',
110
+ ]);
111
+ }
83
112
  }
84
113
  else if (comboPlan) {
85
114
  const all = await this.loadAllModels();
86
- const filtered = selectCandidates(comboPlan, all, required);
115
+ const rejected = [];
116
+ const filtered = selectCandidates(comboPlan, all, required, (c, reason) => {
117
+ rejected.push({ publicModelId: c.publicModelId, reason });
118
+ debugHttp(ctx.requestId, 'CAPABILITY REJECT', [`model=${c.publicModelId}`, `reason=${reason}`]);
119
+ });
120
+ // docs/13 §10: always report why EACH member was filtered out
121
+ debugHttp(ctx.requestId, 'CANDIDATE FILTER', [
122
+ `comboMembers=${comboPlan.members.length}`,
123
+ `afterFilter=${filtered.length}`,
124
+ `rejectedCount=${rejected.length}`,
125
+ ...rejected.map((r) => `rejected: ${r.publicModelId} reason=${r.reason}`),
126
+ ]);
87
127
  if (filtered.length === 0) {
88
128
  throw new GatewayError('capability_not_supported', 'No combo member satisfies the request capabilities or availability', { status: 400 });
89
129
  }
90
130
  candidates = orderCandidates(comboPlan, filtered);
131
+ debugHttp(ctx.requestId, 'CANDIDATES ORDERED', [
132
+ `mode=${comboPlan.mode}`,
133
+ ...candidates.map((c, i) => `candidate[${i}]: providerModelId=${c.modelId} publicModelId=${c.publicModelId}`),
134
+ ]);
91
135
  selectionReasons.push('combo');
92
136
  }
93
137
  if (candidates.length === 0) {
@@ -137,11 +181,13 @@ export class GatewayRunner {
137
181
  finalModelId = candidate.modelId;
138
182
  const provider = getDb().select().from(schema.providers).where(eq(schema.providers.id, candidate.providerId)).get();
139
183
  if (!provider || !provider.enabled) {
184
+ debugHttp(ctx.requestId, 'ATTEMPT SKIP', [`attempt=${i + 1} model=${candidate.publicModelId} reason=skipped_disabled_provider`]);
140
185
  attempts.push(this.failedAttempt(i + 1, candidate, provider?.name ?? '', 'skipped_disabled_provider', null, null, 0, null, null));
141
186
  continue;
142
187
  }
143
188
  const eff = getEffectiveState(provider.id, provider.cbCooldownSeconds);
144
189
  if (eff === 'open' && !halfOpenProbeAllowed(provider.id)) {
190
+ debugHttp(ctx.requestId, 'ATTEMPT SKIP', [`attempt=${i + 1} model=${candidate.publicModelId} provider=${provider.name} reason=circuit_open`]);
145
191
  attempts.push(this.failedAttempt(i + 1, candidate, provider.name, 'circuit_open', null, null, 0, null, null));
146
192
  if (comboPlan && shouldFallback(comboPlan, { type: 'connection_error' })) {
147
193
  metrics.fallbackCount.inc();
@@ -152,6 +198,19 @@ export class GatewayRunner {
152
198
  }
153
199
  const cfg = providerToUpstreamConfig(provider);
154
200
  const attemptStart = Date.now();
201
+ // docs/13 §11–§12: provider/account + upstream request summary
202
+ const upstreamModel = candidate.publicModelId.split('/').slice(1).join('/');
203
+ debugHttp(ctx.requestId, 'ATTEMPT', [
204
+ `attempt=${i + 1}/${maxAttempts}`,
205
+ `provider=${provider.name}`,
206
+ `providerId=${provider.id}`,
207
+ `model=${candidate.publicModelId}`,
208
+ `upstreamModel=${upstreamModel}`,
209
+ `upstreamType=${cfg.type}`,
210
+ `baseUrl=${cfg.baseUrl}`,
211
+ `stream=${req.canonical.stream}`,
212
+ `providerKeyFingerprint=${apiKeyFingerprint(cfg.apiKey)}`,
213
+ ]);
155
214
  const attempt = {
156
215
  attemptNumber: i + 1,
157
216
  providerId: provider.id,
@@ -206,6 +265,14 @@ export class GatewayRunner {
206
265
  }
207
266
  catch (e) {
208
267
  const err = e instanceof GatewayError ? e : new GatewayError('upstream_error', e.message, { cause: e });
268
+ debugUpstream(ctx.requestId, 'ATTEMPT ERROR', [
269
+ `attempt=${i + 1}`,
270
+ `provider=${provider.name}`,
271
+ `model=${candidate.publicModelId}`,
272
+ `type=${err.type}`,
273
+ `status=${err.status}`,
274
+ `willFallback=${comboPlan ? shouldFallback(comboPlan, { type: classifyFailure(err), status: err.status }) : false}`,
275
+ ]);
209
276
  const shouldRetry = comboPlan ? shouldFallback(comboPlan, { type: classifyFailure(err), status: err.status }) : false;
210
277
  attempt.statusCode = err.status;
211
278
  attempt.success = false;
@@ -288,14 +355,26 @@ export class GatewayRunner {
288
355
  metrics.activeRequests.dec();
289
356
  }
290
357
  }
291
- async loadModelCandidate(modelId, required) {
358
+ async loadModelCandidate(modelId, required, requestId) {
292
359
  const db = getDb();
360
+ const reject = (reason) => {
361
+ if (requestId)
362
+ debugHttp(requestId, 'CAPABILITY REJECT', [`modelId=${modelId}`, `reason=${reason}`]);
363
+ };
293
364
  const m = db.select().from(schema.models).where(eq(schema.models.id, modelId)).get();
294
- if (!m)
365
+ if (!m) {
366
+ reject('model_not_found');
295
367
  return [];
368
+ }
296
369
  const p = db.select().from(schema.providers).where(eq(schema.providers.id, m.providerId)).get();
297
- if (!p || !p.enabled)
370
+ if (!p) {
371
+ reject('provider_not_found');
298
372
  return [];
373
+ }
374
+ if (!p.enabled) {
375
+ reject('provider_disabled');
376
+ return [];
377
+ }
299
378
  const caps = safeJson(m.capabilitiesJson);
300
379
  const candidate = {
301
380
  modelId: m.id,
@@ -306,12 +385,30 @@ export class GatewayRunner {
306
385
  circuitOpen: isOpen(m.providerId),
307
386
  capabilities: caps,
308
387
  };
309
- if (!m.enabled || !m.upstreamAvailable)
388
+ if (!m.enabled) {
389
+ reject('model_disabled');
310
390
  return [];
311
- if (candidate.circuitOpen)
391
+ }
392
+ if (!m.upstreamAvailable) {
393
+ reject('upstream_unavailable');
394
+ return [];
395
+ }
396
+ if (candidate.circuitOpen) {
397
+ reject('circuit_open');
312
398
  return [];
313
- if (!modelMeets(caps, required))
399
+ }
400
+ if (!modelMeets(caps, required)) {
401
+ reject('capability_mismatch');
314
402
  return [];
403
+ }
404
+ debugHttp(requestId ?? '-', 'CAPABILITY CANDIDATE', [
405
+ `model=${m.publicModelId}`,
406
+ `caps.tools=${caps.tools}`,
407
+ `caps.streaming=${caps.streaming}`,
408
+ `caps.reasoning=${caps.reasoning}`,
409
+ `caps.image_input=${caps.image_input}`,
410
+ `caps.structured_output=${caps.structured_output}`,
411
+ ]);
315
412
  return [candidate];
316
413
  }
317
414
  async loadAllModels() {
@@ -333,18 +430,20 @@ export class GatewayRunner {
333
430
  if (req.canonical.stream) {
334
431
  return this.runStreamingAttempt(req, ctx, candidate, cfg, onStreamStart, onFirstToken);
335
432
  }
336
- return this.runNonStreamingAttempt(req, candidate, cfg);
433
+ return this.runNonStreamingAttempt(req, candidate, cfg, ctx);
337
434
  }
338
- async runNonStreamingAttempt(req, candidate, cfg) {
435
+ async runNonStreamingAttempt(req, candidate, cfg, ctx) {
339
436
  const upstreamModel = candidate.publicModelId.split('/').slice(1).join('/');
340
437
  let call;
341
438
  if (cfg.type === 'openai') {
342
439
  const payload = canonicalToOpenAIRequest(req.canonical, upstreamModel);
343
- call = await callUpstreamNonStreaming(cfg, upstreamUrl(cfg, '/v1/chat/completions'), payload);
440
+ logUpstreamRequest(ctx.requestId, cfg, upstreamUrl(cfg, '/v1/chat/completions'), payload, req.canonical.stream);
441
+ call = await callUpstreamNonStreaming(cfg, upstreamUrl(cfg, '/v1/chat/completions'), payload, ctx.requestId);
344
442
  }
345
443
  else {
346
444
  const payload = canonicalToAnthropicRequest(req.canonical, upstreamModel);
347
- call = await callUpstreamNonStreaming(cfg, upstreamUrl(cfg, '/v1/messages'), payload);
445
+ logUpstreamRequest(ctx.requestId, cfg, upstreamUrl(cfg, '/v1/messages'), payload, req.canonical.stream);
446
+ call = await callUpstreamNonStreaming(cfg, upstreamUrl(cfg, '/v1/messages'), payload, ctx.requestId);
348
447
  }
349
448
  if (!call.ok) {
350
449
  if (call.status === 429)
@@ -402,7 +501,39 @@ export class GatewayRunner {
402
501
  const toolBuf = [];
403
502
  let finishReason = null;
404
503
  const usage = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, reasoning: 0, total: 0 };
504
+ // docs/13 §15–§16: stream counters + client disconnect tracking
505
+ let chunkCount = 0;
506
+ let bytesReceived = 0;
507
+ let firstChunkTime = null;
508
+ let lastChunkTime = null;
509
+ const loggedFirstChunks = [];
510
+ let clientDisconnected = false;
511
+ // Guard: the admin test-stream endpoint passes a minimal `fakeRaw` that has
512
+ // writeHead/write/end but no `.on` — skip disconnect tracking in that case.
513
+ if (typeof pipe.on === 'function') {
514
+ pipe.on('close', () => {
515
+ // Distinguish client disconnect from normal end: 'close' fires on the raw
516
+ // socket when the client (or upstream) side goes away.
517
+ if (!pipe.writableEnded) {
518
+ clientDisconnected = true;
519
+ debugStream(ctx.requestId, 'CLIENT DISCONNECT', [
520
+ `afterMs=${Date.now() - streamStartTs}`,
521
+ `streaming=${streamStarted}`,
522
+ `chunksSoFar=${chunkCount}`,
523
+ ]);
524
+ }
525
+ });
526
+ }
405
527
  const chunkHandler = (chunk, isFirst) => {
528
+ chunkCount++;
529
+ bytesReceived += chunk.data.length;
530
+ lastChunkTime = Date.now();
531
+ if (isFirst)
532
+ firstChunkTime = Date.now() - streamStartTs;
533
+ if (loggedFirstChunks.length < 3) {
534
+ loggedFirstChunks.push(chunk.data);
535
+ debugStream(ctx.requestId, `STREAM FIRST CHUNK ${loggedFirstChunks.length}`, [truncate(chunk.data, 2000)]);
536
+ }
406
537
  try {
407
538
  const obj = JSON.parse(chunk.data);
408
539
  if (cfg.type === 'openai') {
@@ -473,12 +604,23 @@ export class GatewayRunner {
473
604
  try {
474
605
  const url = cfg.type === 'openai' ? upstreamUrl(cfg, '/v1/chat/completions') : upstreamUrl(cfg, '/v1/messages');
475
606
  const payload = cfg.type === 'openai' ? canonicalToOpenAIRequest(req.canonical, upstreamModel) : canonicalToAnthropicRequest(req.canonical, upstreamModel);
476
- const meta = await callUpstreamStreaming(cfg, url, payload, chunkHandler);
607
+ logUpstreamRequest(ctx.requestId, cfg, url, payload, true);
608
+ debugStream(ctx.requestId, 'STREAM START', [`upstreamConnected=true`]);
609
+ const meta = await callUpstreamStreaming(cfg, url, payload, chunkHandler, ctx.requestId);
477
610
  // Upstream completed cleanly: ensure head + terminator are written.
478
611
  if (!headWritten)
479
612
  writeHead();
480
613
  pipe.write('data: [DONE]\n\n');
481
614
  pipe.end();
615
+ debugStream(ctx.requestId, 'STREAM END', [
616
+ `chunks=${chunkCount}`,
617
+ `bytes=${bytesReceived}`,
618
+ `firstChunkMs=${firstChunkTime ?? 'null'}`,
619
+ `lastChunkMs=${lastChunkTime ? lastChunkTime - streamStartTs : 'null'}`,
620
+ `durationMs=${Date.now() - streamStartTs}`,
621
+ `finishedNormally=true`,
622
+ `clientDisconnected=${clientDisconnected}`,
623
+ ]);
482
624
  if (!usage.total)
483
625
  usage.total = usage.input + usage.output;
484
626
  return {
@@ -497,8 +639,20 @@ export class GatewayRunner {
497
639
  writeHead();
498
640
  pipe.end();
499
641
  const err = e instanceof GatewayError ? e : new GatewayError('upstream_error', e.message);
642
+ errorLine(ctx.requestId, 'STREAM ERROR', [
643
+ `chunksBeforeError=${chunkCount}`,
644
+ `bytesBeforeError=${bytesReceived}`,
645
+ `clientDisconnected=${clientDisconnected}`,
646
+ ...formatErrorPublic(e),
647
+ ]);
500
648
  throw err;
501
649
  }
650
+ errorLine(ctx.requestId, 'STREAM ERROR (before first chunk)', [
651
+ `chunksBeforeError=${chunkCount}`,
652
+ `bytesBeforeError=${bytesReceived}`,
653
+ `clientDisconnected=${clientDisconnected}`,
654
+ ...formatErrorPublic(e),
655
+ ]);
502
656
  throw e;
503
657
  }
504
658
  }
@@ -732,6 +886,36 @@ function hasTools(req) {
732
886
  function estimateTokens(s) {
733
887
  return Math.ceil(s.length / 4);
734
888
  }
889
+ // docs/13 §11: safe provider key fingerprint — never the key itself.
890
+ function apiKeyFingerprint(key) {
891
+ let h = 5381;
892
+ for (let i = 0; i < key.length; i++) {
893
+ h = ((h << 5) + h + key.charCodeAt(i)) >>> 0;
894
+ }
895
+ return `fp_${h.toString(16).padStart(8, '0')}…${key.slice(-4).replace(/./g, '*')}${key.length}ch`;
896
+ }
897
+ // docs/13 §12: upstream request summary + optional full body
898
+ function logUpstreamRequest(requestId, cfg, url, payload, stream) {
899
+ const json = JSON.stringify(payload ?? {});
900
+ debugUpstream(requestId, 'UPSTREAM REQUEST', [
901
+ `method=POST`,
902
+ `url=${url}`,
903
+ `type=${cfg.type}`,
904
+ `stream=${stream}`,
905
+ `contentLength=${json.length}`,
906
+ `customHeaders=${JSON.stringify(Object.keys(cfg.customHeaders ?? {}))}`,
907
+ ]);
908
+ if (getDebugFlags().httpBody) {
909
+ const LIMIT = 512 * 1024;
910
+ debugBody(requestId, 'UPSTREAM BODY', [
911
+ json.length > LIMIT ? `${json.slice(0, LIMIT)}…(+${json.length - LIMIT} chars, truncated)` : json,
912
+ ]);
913
+ }
914
+ }
915
+ /** Public alias so error formatting from debug.ts is available in this module. */
916
+ function formatErrorPublic(e) {
917
+ return formatError(e);
918
+ }
735
919
  /**
736
920
  * Safely parse capabilities JSON. Returns minimal default if parsing fails.
737
921
  * CRITICAL: Must return a complete default object with all capability fields,