ldrouter 1.11.17 → 1.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/server/app.js +32 -7
- package/dist/server/auth/backup-crypto.js +31 -0
- package/dist/server/auth/crypto.js +3 -0
- package/dist/server/config/index.js +3 -1
- package/dist/server/gateway/runner.js +197 -13
- package/dist/server/logging/debug.js +271 -55
- package/dist/server/routes/admin/backup.js +38 -4
- package/dist/server/routes/gateway/anthropic.js +38 -1
- package/dist/server/routes/gateway/openai.js +38 -2
- package/dist/server/routing/combo.js +37 -7
- package/dist/server/upstream/client.js +30 -2
- package/dist/web/assets/index-CBMHkVXC.js +330 -0
- package/dist/web/assets/index-Dswaxg_c.css +1 -0
- package/dist/web/index.html +2 -2
- package/package.json +1 -3
- package/dist/web/assets/index-CqVMNQ5F.css +0 -1
- package/dist/web/assets/index-DhNkAaqo.js +0 -330
|
@@ -1,62 +1,278 @@
|
|
|
1
|
-
//
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
//
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
1
|
+
// Request-lifecycle debug logging.
|
|
2
|
+
//
|
|
3
|
+
// All output goes to stdout/stderr so `docker logs` collects it (docs/13).
|
|
4
|
+
// Every line is prefixed with the requestId so the whole lifecycle of one
|
|
5
|
+
// request can be extracted with:
|
|
6
|
+
// docker logs ldrouter 2>&1 | grep req_xxx
|
|
7
|
+
//
|
|
8
|
+
// Env flags (all default off; enabled per docs/13 §22):
|
|
9
|
+
// DEBUG_HTTP — incoming request + body summary + messages/tools structure
|
|
10
|
+
// DEBUG_HTTP_BODY — full sanitized JSON bodies (incoming + upstream)
|
|
11
|
+
// DEBUG_UPSTREAM — upstream fetch/response/error detail
|
|
12
|
+
// DEBUG_STREAM — SSE stream lifecycle (start/first chunks/end/error)
|
|
13
|
+
// LOG_LEVEL gates lifecycle INFO-level lines ([INCOMING]/[DONE]/errors) —
|
|
14
|
+
// they emit at debug, so set LOG_LEVEL=debug to see them.
|
|
15
|
+
import process from 'node:process';
|
|
16
|
+
import { redactValue } from '../security/redact.js';
|
|
17
|
+
function envFlag(name) {
|
|
18
|
+
const v = process.env[name];
|
|
19
|
+
return v === '1' || v === 'true' || v === 'yes';
|
|
20
|
+
}
|
|
21
|
+
let flags = null;
|
|
22
|
+
/** Debug flags are read once per process (docs/13 §22). */
|
|
23
|
+
export function getDebugFlags() {
|
|
24
|
+
if (!flags) {
|
|
25
|
+
flags = {
|
|
26
|
+
http: envFlag('DEBUG_HTTP'),
|
|
27
|
+
httpBody: envFlag('DEBUG_HTTP_BODY'),
|
|
28
|
+
upstream: envFlag('DEBUG_UPSTREAM'),
|
|
29
|
+
stream: envFlag('DEBUG_STREAM'),
|
|
30
|
+
};
|
|
10
31
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
32
|
+
return flags;
|
|
33
|
+
}
|
|
34
|
+
export function resetDebugFlagsForTests() {
|
|
35
|
+
flags = null;
|
|
36
|
+
}
|
|
37
|
+
// --- Output -------------------------------------------------------------
|
|
38
|
+
// Direct console use (not pino) keeps the human-readable
|
|
39
|
+
// [timestamp] [req_xxx] [TAG] line format that docs/13 §23 asks for, and is
|
|
40
|
+
// trivially visible in `docker logs -f`. Errors go to stderr.
|
|
41
|
+
function emit(level, requestId, tag, lines) {
|
|
42
|
+
const ts = new Date().toISOString();
|
|
43
|
+
const stream = level === 'error' ? process.stderr : process.stdout;
|
|
44
|
+
for (const body of lines) {
|
|
45
|
+
const first = body.split('\n')[0] ?? '';
|
|
46
|
+
const rest = body.split('\n').slice(1).join('\n');
|
|
47
|
+
const head = `[${ts}] [${requestId}] [${tag}] ${first}`;
|
|
48
|
+
stream.write(rest ? head + '\n' + rest + '\n' : head + '\n');
|
|
14
49
|
}
|
|
15
50
|
}
|
|
16
|
-
|
|
17
|
-
|
|
51
|
+
/** Lifecycle INFO line — always on (gated by LOG_LEVEL=debug via pino parity, but kept unconditional so operators never lose the trail). */
|
|
52
|
+
export function lifecycle(requestId, tag, lines) {
|
|
53
|
+
emit('info', requestId, tag, lines);
|
|
54
|
+
}
|
|
55
|
+
export function debugHttp(requestId, tag, lines) {
|
|
56
|
+
if (getDebugFlags().http)
|
|
57
|
+
emit('info', requestId, tag, lines);
|
|
58
|
+
}
|
|
59
|
+
export function debugBody(requestId, tag, lines) {
|
|
60
|
+
if (getDebugFlags().httpBody)
|
|
61
|
+
emit('info', requestId, tag, lines);
|
|
62
|
+
}
|
|
63
|
+
export function debugUpstream(requestId, tag, lines) {
|
|
64
|
+
if (getDebugFlags().upstream)
|
|
65
|
+
emit('info', requestId, tag, lines);
|
|
66
|
+
}
|
|
67
|
+
export function debugStream(requestId, tag, lines) {
|
|
68
|
+
if (getDebugFlags().stream)
|
|
69
|
+
emit('info', requestId, tag, lines);
|
|
70
|
+
}
|
|
71
|
+
export function errorLine(requestId, tag, lines) {
|
|
72
|
+
emit('error', requestId, tag, lines);
|
|
73
|
+
}
|
|
74
|
+
/** Fatal process-level line (no requestId). */
|
|
75
|
+
export function fatal(tag, lines) {
|
|
76
|
+
emit('error', '-', tag, lines);
|
|
77
|
+
}
|
|
78
|
+
// --- Sanitization --------------------------------------------------------
|
|
79
|
+
/** Sanitize an arbitrary value for logging: deep-redact secrets. */
|
|
80
|
+
export function sanitize(value) {
|
|
81
|
+
return redactValue(value);
|
|
82
|
+
}
|
|
83
|
+
/** Sanitized JSON string; on circular/unserializable falls back to a marker. */
|
|
84
|
+
export function sanitizeJson(value) {
|
|
18
85
|
try {
|
|
19
|
-
|
|
20
|
-
}
|
|
21
|
-
catch
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
86
|
+
return JSON.stringify(redactValue(value));
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
try {
|
|
90
|
+
return JSON.stringify({ bodyLogError: 'unserializable' });
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return '{"bodyLogError":"unserializable"}';
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/** Truncate a string to `max` chars, marking truncation (docs/13 §5). */
|
|
98
|
+
export function truncate(s, max) {
|
|
99
|
+
if (s.length <= max)
|
|
100
|
+
return s;
|
|
101
|
+
return `${s.slice(0, max)}…(+${s.length - max} chars, truncated)`;
|
|
102
|
+
}
|
|
103
|
+
export function summarizeBody(body) {
|
|
104
|
+
const lines = [];
|
|
105
|
+
const p = (k, v) => {
|
|
106
|
+
lines.push(`${k}=${v === undefined ? 'undefined' : JSON.stringify(v)}`);
|
|
107
|
+
};
|
|
108
|
+
p('model', body['model']);
|
|
109
|
+
p('stream', body['stream']);
|
|
110
|
+
const keys = Object.keys(body);
|
|
111
|
+
lines.push(`bodyKeys=[${keys.map((k) => JSON.stringify(k)).join(', ')}]`);
|
|
112
|
+
const messages = Array.isArray(body['messages']) ? body['messages'] : null;
|
|
113
|
+
lines.push(`messages=${messages ? messages.length : 'undefined'}`);
|
|
114
|
+
const tools = Array.isArray(body['tools']) ? body['tools'] : null;
|
|
115
|
+
if (tools) {
|
|
116
|
+
lines.push(`toolsCount=${tools.length}`);
|
|
117
|
+
lines.push(`serializedToolsSize=${safeSize(tools)}`);
|
|
118
|
+
}
|
|
119
|
+
p('max_tokens', body['max_tokens']);
|
|
120
|
+
p('max_completion_tokens', body['max_completion_tokens']);
|
|
121
|
+
p('temperature', body['temperature']);
|
|
122
|
+
p('top_p', body['top_p']);
|
|
123
|
+
p('reasoning_effort', body['reasoning_effort']);
|
|
124
|
+
p('reasoning', body['reasoning']);
|
|
125
|
+
p('thinking', body['thinking']);
|
|
126
|
+
p('tool_choice', body['tool_choice']);
|
|
127
|
+
p('parallel_tool_calls', body['parallel_tool_calls']);
|
|
128
|
+
p('response_format', body['response_format']);
|
|
129
|
+
p('stream_options', body['stream_options']);
|
|
130
|
+
if (messages) {
|
|
131
|
+
lines.push(`messageRoles=[${messages.map((m) => (isRec(m) ? String(m['role'] ?? '?') : '?')).join(',')}]`);
|
|
132
|
+
}
|
|
133
|
+
return lines;
|
|
134
|
+
}
|
|
135
|
+
export function summarizeMessages(body) {
|
|
136
|
+
const messages = Array.isArray(body['messages']) ? body['messages'] : [];
|
|
137
|
+
const lines = [];
|
|
138
|
+
messages.forEach((m, i) => {
|
|
139
|
+
if (!isRec(m)) {
|
|
140
|
+
lines.push(`#${i} <non-object: ${typeof m}>`);
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
const role = m['role'];
|
|
144
|
+
const content = m['content'];
|
|
145
|
+
const contentDesc = content === null
|
|
146
|
+
? 'contentType=null contentLength=0'
|
|
147
|
+
: typeof content === 'string'
|
|
148
|
+
? `contentType=string contentLength=${content.length}`
|
|
149
|
+
: Array.isArray(content)
|
|
150
|
+
? `contentType=array contentParts=${content.length}`
|
|
151
|
+
: content === undefined
|
|
152
|
+
? 'contentType=undefined'
|
|
153
|
+
: `contentType=${typeof content}`;
|
|
154
|
+
lines.push(`#${i} role=${role} ${contentDesc}`);
|
|
155
|
+
const toolCalls = m['tool_calls'];
|
|
156
|
+
if (Array.isArray(toolCalls))
|
|
157
|
+
lines.push(` toolCalls=${toolCalls.length}`);
|
|
158
|
+
if (m['tool_call_id'])
|
|
159
|
+
lines.push(` toolCallId=${String(m['tool_call_id'])}`);
|
|
160
|
+
if ('reasoning_content' in m)
|
|
161
|
+
lines.push(` reasoningContent=present`);
|
|
162
|
+
if (Array.isArray(content)) {
|
|
163
|
+
const partTypes = content.map((c) => (isRec(c) ? String(c['type'] ?? '?') : '?')).join(',');
|
|
164
|
+
lines.push(` partTypes=[${partTypes}]`);
|
|
165
|
+
}
|
|
57
166
|
});
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
167
|
+
return lines;
|
|
168
|
+
}
|
|
169
|
+
export function summarizeTools(body) {
|
|
170
|
+
const tools = Array.isArray(body['tools']) ? body['tools'] : [];
|
|
171
|
+
const lines = [`count=${tools.length}`, `serializedSize=${safeSize(tools)}`];
|
|
172
|
+
tools.forEach((t, i) => {
|
|
173
|
+
if (!isRec(t)) {
|
|
174
|
+
lines.push(`tool[${i}]: <non-object>`);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
const fn = isRec(t['function']) ? t['function'] : null;
|
|
178
|
+
const name = fn ? String(fn['name'] ?? '?') : String(t['name'] ?? '?');
|
|
179
|
+
const descLen = fn && typeof fn['description'] === 'string' ? fn['description'].length : fn?.['description'] !== undefined ? -1 : 0;
|
|
180
|
+
const schema = fn ? fn['parameters'] : t['input_schema'];
|
|
181
|
+
const schemaSize = schema !== undefined ? safeSize(schema) : 0;
|
|
182
|
+
lines.push(`tool[${i}]: name=${name} descriptionLength=${descLen} schemaSize=${schemaSize}`);
|
|
61
183
|
});
|
|
184
|
+
return lines;
|
|
185
|
+
}
|
|
186
|
+
// --- Error formatting (docs/13 §13) ---------------------------------------
|
|
187
|
+
/** Full error detail including nested undici `cause` chains. */
|
|
188
|
+
export function formatError(e) {
|
|
189
|
+
const lines = [];
|
|
190
|
+
const seen = new Set();
|
|
191
|
+
let depth = 0;
|
|
192
|
+
let cur = e;
|
|
193
|
+
while (cur instanceof Error && depth < 4) {
|
|
194
|
+
if (seen.has(cur))
|
|
195
|
+
break;
|
|
196
|
+
seen.add(cur);
|
|
197
|
+
const prefix = depth === 0 ? '' : 'cause.';
|
|
198
|
+
lines.push(`${prefix}name=${cur.name}`);
|
|
199
|
+
lines.push(`${prefix}message=${cur.message}`);
|
|
200
|
+
const code = cur.code;
|
|
201
|
+
if (code)
|
|
202
|
+
lines.push(`${prefix}code=${code}`);
|
|
203
|
+
if (depth === 0 && cur.stack)
|
|
204
|
+
lines.push(`stack=${truncate(cur.stack, 4000)}`);
|
|
205
|
+
cur = cur.cause;
|
|
206
|
+
depth++;
|
|
207
|
+
}
|
|
208
|
+
if (cur !== undefined && cur !== null && !(cur instanceof Error)) {
|
|
209
|
+
lines.push(`cause(raw)=${truncate(safeStringify(cur), 2000)}`);
|
|
210
|
+
}
|
|
211
|
+
if (lines.length === 0)
|
|
212
|
+
lines.push(`value=${truncate(safeStringify(e), 2000)}`);
|
|
213
|
+
return lines;
|
|
214
|
+
}
|
|
215
|
+
// --- Helpers --------------------------------------------------------------
|
|
216
|
+
function isRec(v) {
|
|
217
|
+
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
218
|
+
}
|
|
219
|
+
function safeSize(v) {
|
|
220
|
+
try {
|
|
221
|
+
return JSON.stringify(v)?.length ?? 0;
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
return -1;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
function safeStringify(v) {
|
|
228
|
+
try {
|
|
229
|
+
return JSON.stringify(v);
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
return '[unserializable]';
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
// --- Header sanitization (docs/13 §2–§3) ----------------------------------
|
|
236
|
+
const SENSITIVE_HEADERS = new Set([
|
|
237
|
+
'authorization',
|
|
238
|
+
'x-api-key',
|
|
239
|
+
'cookie',
|
|
240
|
+
'set-cookie',
|
|
241
|
+
'x-goog-api-key',
|
|
242
|
+
'proxy-authorization',
|
|
243
|
+
]);
|
|
244
|
+
const INTERESTING_HEADERS = [
|
|
245
|
+
'content-type',
|
|
246
|
+
'content-length',
|
|
247
|
+
'user-agent',
|
|
248
|
+
'host',
|
|
249
|
+
'x-forwarded-for',
|
|
250
|
+
'cf-ray',
|
|
251
|
+
'cf-connecting-ip',
|
|
252
|
+
'accept',
|
|
253
|
+
'accept-encoding',
|
|
254
|
+
'connection',
|
|
255
|
+
'anthropic-version',
|
|
256
|
+
];
|
|
257
|
+
/** Sanitized incoming-request header lines for the [INCOMING] block. */
|
|
258
|
+
export function summarizeHeaders(headers) {
|
|
259
|
+
const lines = [];
|
|
260
|
+
for (const name of INTERESTING_HEADERS) {
|
|
261
|
+
const v = headers[name];
|
|
262
|
+
if (v !== undefined)
|
|
263
|
+
lines.push(`${name}=${Array.isArray(v) ? v.join(',') : String(v)}`);
|
|
264
|
+
}
|
|
265
|
+
const auth = headers['authorization'];
|
|
266
|
+
if (auth !== undefined)
|
|
267
|
+
lines.push(`authorization=Bearer ***REDACTED***`);
|
|
268
|
+
const apiKey = headers['x-api-key'];
|
|
269
|
+
if (apiKey !== undefined)
|
|
270
|
+
lines.push(`x-api-key=***REDACTED***`);
|
|
271
|
+
// Report presence of any other sensitive headers without values.
|
|
272
|
+
for (const k of Object.keys(headers)) {
|
|
273
|
+
if (SENSITIVE_HEADERS.has(k.toLowerCase()) && !lines.some((l) => l.startsWith(`${k.toLowerCase()}=`) || l.startsWith(`${k}=`))) {
|
|
274
|
+
lines.push(`${k}=***REDACTED***`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
return lines;
|
|
62
278
|
}
|
|
@@ -7,11 +7,13 @@ import { getDb, closeDb, openDb, schema } from '../../db/index.js';
|
|
|
7
7
|
import { requireAdminAuth } from '../../auth/middleware.js';
|
|
8
8
|
import { sha256Hex } from '../../auth/ids.js';
|
|
9
9
|
import { recordAudit } from '../../db/repositories/audit.js';
|
|
10
|
-
import { loadConfig } from '../../config/index.js';
|
|
10
|
+
import { loadConfig, setConfigMasterKey } from '../../config/index.js';
|
|
11
11
|
import { getSettings } from '../../db/repositories/settings.js';
|
|
12
12
|
import { GatewayError } from '../../errors.js';
|
|
13
13
|
import { getAppVersion } from '../../version.js';
|
|
14
14
|
import { eq, sql } from 'drizzle-orm';
|
|
15
|
+
import { decryptBackupMasterKey, encryptBackupMasterKey } from '../../auth/backup-crypto.js';
|
|
16
|
+
import { parseMasterKey, resetMasterKeyCache } from '../../auth/crypto.js';
|
|
15
17
|
const BACKUP_VERSION = 1;
|
|
16
18
|
/** Reopen the in-process SQLite connection on the (possibly just-replaced)
|
|
17
19
|
* database file. The old connection must already be closed: a hot restore
|
|
@@ -52,8 +54,16 @@ function assertDatabaseUsable(expectedSchemaVersion) {
|
|
|
52
54
|
}
|
|
53
55
|
}
|
|
54
56
|
export async function registerBackupRoutes(app) {
|
|
55
|
-
app.addHook('preHandler',
|
|
57
|
+
app.addHook('preHandler', async (req, reply) => {
|
|
58
|
+
if (req.url === '/api/admin/backup/restore' && !getSettings().setupComplete)
|
|
59
|
+
return;
|
|
60
|
+
return requireAdminAuth(req, reply);
|
|
61
|
+
});
|
|
56
62
|
app.post('/api/admin/backup/create', async (req, reply) => {
|
|
63
|
+
const passphrase = req.body?.passphrase ?? '';
|
|
64
|
+
if (!/^\d{6}$/.test(passphrase)) {
|
|
65
|
+
throw new GatewayError('invalid_request_error', 'Backup passphrase must contain exactly six digits', { status: 400 });
|
|
66
|
+
}
|
|
57
67
|
const cfg = loadConfig();
|
|
58
68
|
const temp = path.join(cfg.dataDir, `.backup-${Date.now()}.sqlite`);
|
|
59
69
|
const Database = (await import('better-sqlite3')).default;
|
|
@@ -69,6 +79,7 @@ export async function registerBackupRoutes(app) {
|
|
|
69
79
|
const compressed = zlib.gzipSync(buf, { level: 6 });
|
|
70
80
|
const checksum = crypto.createHash('sha256').update(compressed).digest('hex');
|
|
71
81
|
const settings = getSettings();
|
|
82
|
+
const masterKey = parseMasterKey(cfg.masterKey ?? fs.readFileSync(path.join(cfg.dataDir, 'master.key'), 'utf8').trim());
|
|
72
83
|
const envelope = {
|
|
73
84
|
format: 'latedev-backup',
|
|
74
85
|
version: BACKUP_VERSION,
|
|
@@ -78,6 +89,7 @@ export async function registerBackupRoutes(app) {
|
|
|
78
89
|
createdAt: new Date().toISOString(),
|
|
79
90
|
payload: compressed.toString('base64'),
|
|
80
91
|
checksum,
|
|
92
|
+
keyEnvelope: encryptBackupMasterKey(masterKey, passphrase),
|
|
81
93
|
};
|
|
82
94
|
const envBuf = Buffer.from(JSON.stringify(envelope), 'utf8');
|
|
83
95
|
const fileName = `latedev-backup-${new Date().toISOString().replace(/[:.]/g, '-')}.ldb.json`;
|
|
@@ -88,12 +100,17 @@ export async function registerBackupRoutes(app) {
|
|
|
88
100
|
});
|
|
89
101
|
app.post('/api/admin/backup/restore', async (req, reply) => {
|
|
90
102
|
const cfg = loadConfig();
|
|
91
|
-
|
|
103
|
+
const incoming = req.body;
|
|
104
|
+
const wrapped = Boolean(incoming && 'backup' in incoming);
|
|
105
|
+
const wrappedBody = incoming;
|
|
106
|
+
const passphrase = wrapped ? wrappedBody.passphrase : undefined;
|
|
92
107
|
let envelope;
|
|
93
108
|
try {
|
|
94
|
-
envelope =
|
|
109
|
+
envelope = (wrapped ? wrappedBody.backup : incoming);
|
|
95
110
|
if (!envelope || envelope.format !== 'latedev-backup')
|
|
96
111
|
throw new Error('not a backup envelope');
|
|
112
|
+
if (envelope.keyEnvelope && !/^\d{6}$/.test(passphrase ?? ''))
|
|
113
|
+
throw new Error('backup passphrase required');
|
|
97
114
|
}
|
|
98
115
|
catch {
|
|
99
116
|
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'invalid_envelope' } });
|
|
@@ -122,6 +139,17 @@ export async function registerBackupRoutes(app) {
|
|
|
122
139
|
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'not_sqlite' } });
|
|
123
140
|
throw new GatewayError('invalid_request_error', 'Backup does not contain a valid SQLite database', { status: 400 });
|
|
124
141
|
}
|
|
142
|
+
let restoredMasterKey;
|
|
143
|
+
if (envelope.keyEnvelope) {
|
|
144
|
+
try {
|
|
145
|
+
restoredMasterKey = decryptBackupMasterKey(envelope.keyEnvelope, passphrase);
|
|
146
|
+
parseMasterKey(restoredMasterKey.toString('base64'));
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'invalid_backup_passphrase' } });
|
|
150
|
+
throw new GatewayError('invalid_request_error', 'Invalid backup passphrase', { status: 400 });
|
|
151
|
+
}
|
|
152
|
+
}
|
|
125
153
|
const liveDb = cfg.dbFile;
|
|
126
154
|
// Snapshot current DB before restore (kept for manual rollback).
|
|
127
155
|
const snapshot = path.join(cfg.dataDir, `pre-restore-${Date.now()}.sqlite`);
|
|
@@ -184,6 +212,12 @@ export async function registerBackupRoutes(app) {
|
|
|
184
212
|
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'validation_failed', err: e.message } });
|
|
185
213
|
throw new GatewayError('gateway_error', `Restore failed: ${e.message}`, { status: 500 });
|
|
186
214
|
}
|
|
215
|
+
if (restoredMasterKey) {
|
|
216
|
+
const restoredKey = restoredMasterKey.toString('base64');
|
|
217
|
+
fs.writeFileSync(path.join(cfg.dataDir, 'master.key'), restoredKey, { mode: 0o600, encoding: 'utf8' });
|
|
218
|
+
setConfigMasterKey(restoredKey);
|
|
219
|
+
resetMasterKeyCache();
|
|
220
|
+
}
|
|
187
221
|
// The swap invalidates the previous admin session (its row lived in the
|
|
188
222
|
// old database). Re-create the current session in the restored database so
|
|
189
223
|
// the admin stays logged in across the hot restore.
|
|
@@ -6,6 +6,7 @@ import { anthropicToCanonical } from '../../protocols/anthropic.js';
|
|
|
6
6
|
import { GatewayError, toAnthropicError } from '../../errors.js';
|
|
7
7
|
import { GatewayRunner } from '../../gateway/runner.js';
|
|
8
8
|
import { uuid } from '../../auth/ids.js';
|
|
9
|
+
import { lifecycle, debugHttp, debugBody, getDebugFlags, summarizeMessages, summarizeTools, summarizeHeaders, sanitizeJson, truncate } from '../../logging/debug.js';
|
|
9
10
|
const MessagesBody = z.object({
|
|
10
11
|
model: z.string().min(1),
|
|
11
12
|
messages: z.array(z.any()).min(1),
|
|
@@ -27,15 +28,21 @@ export async function registerAnthropicRoutes(app) {
|
|
|
27
28
|
reply.code(405).send(toAnthropicError(new GatewayError('invalid_request_error', 'Use POST /v1/messages', { status: 405 }), ''));
|
|
28
29
|
});
|
|
29
30
|
app.post('/v1/messages', async (req, reply) => {
|
|
31
|
+
const requestId = req.id || uuid();
|
|
32
|
+
lifecycle(requestId, 'INCOMING', [
|
|
33
|
+
`POST ${req.url}`,
|
|
34
|
+
...summarizeHeaders(req.headers),
|
|
35
|
+
]);
|
|
30
36
|
const key = authenticateGatewayHeaders(req);
|
|
31
37
|
const body = MessagesBody.parse(req.body);
|
|
38
|
+
logIncomingMessagesBody(requestId, body);
|
|
32
39
|
const ar = body;
|
|
33
40
|
if (!ar.max_tokens) {
|
|
34
41
|
throw new GatewayError('invalid_request_error', 'max_tokens is required', { status: 400 });
|
|
35
42
|
}
|
|
36
43
|
const canonical = anthropicToCanonical(ar);
|
|
37
44
|
const ctx = {
|
|
38
|
-
requestId
|
|
45
|
+
requestId,
|
|
39
46
|
clientIp: resolveClientIp(req),
|
|
40
47
|
protocol: 'anthropic',
|
|
41
48
|
endpoint: 'messages',
|
|
@@ -60,9 +67,11 @@ export async function registerAnthropicRoutes(app) {
|
|
|
60
67
|
}
|
|
61
68
|
if (!outcome.success) {
|
|
62
69
|
const g = new GatewayError(outcome.errorType ?? 'gateway_error', outcome.errorMessage ?? 'Gateway error', { status: outcome.httpStatus });
|
|
70
|
+
lifecycle(requestId, 'DONE', [`status=${outcome.httpStatus} durationMs=${outcome.latencyMs} error=true type=${g.type}`]);
|
|
63
71
|
reply.code(outcome.httpStatus).send(toAnthropicError(g, ctx.requestId));
|
|
64
72
|
return;
|
|
65
73
|
}
|
|
74
|
+
lifecycle(requestId, 'DONE', [`status=200 durationMs=${outcome.latencyMs} finishReason=${outcome.finishReason ?? 'null'}`]);
|
|
66
75
|
reply.header('x-request-id', ctx.requestId);
|
|
67
76
|
reply.send({
|
|
68
77
|
id: `msg_${ctx.requestId}`,
|
|
@@ -101,6 +110,34 @@ export async function registerAnthropicRoutes(app) {
|
|
|
101
110
|
reply.send({ input_tokens: inputTokens });
|
|
102
111
|
});
|
|
103
112
|
}
|
|
113
|
+
// docs/13 §4–§7: incoming /v1/messages body structure logging.
|
|
114
|
+
function logIncomingMessagesBody(requestId, body) {
|
|
115
|
+
debugHttp(requestId, 'BODY SUMMARY', [
|
|
116
|
+
`model=${JSON.stringify(body['model'])}`,
|
|
117
|
+
`stream=${JSON.stringify(body['stream'])}`,
|
|
118
|
+
`max_tokens=${JSON.stringify(body['max_tokens'])}`,
|
|
119
|
+
`temperature=${JSON.stringify(body['temperature'])}`,
|
|
120
|
+
`top_p=${JSON.stringify(body['top_p'])}`,
|
|
121
|
+
`thinking=${JSON.stringify(body['thinking'])}`,
|
|
122
|
+
`tool_choice=${JSON.stringify(body['tool_choice'])}`,
|
|
123
|
+
`systemType=${Array.isArray(body['system']) ? `array(${body['system'].length})` : typeof body['system']}`,
|
|
124
|
+
`bodyKeys=[${Object.keys(body).map((k) => JSON.stringify(k)).join(', ')}]`,
|
|
125
|
+
`messages=${Array.isArray(body['messages']) ? body['messages'].length : 'undefined'}`,
|
|
126
|
+
`tools=${Array.isArray(body['tools']) ? body['tools'].length : 'undefined'}`,
|
|
127
|
+
]);
|
|
128
|
+
debugHttp(requestId, 'MESSAGES', summarizeMessages(body));
|
|
129
|
+
if (body['tools'] !== undefined)
|
|
130
|
+
debugHttp(requestId, 'TOOLS', summarizeTools(body));
|
|
131
|
+
if (getDebugFlags().httpBody) {
|
|
132
|
+
const json = sanitizeJson(body);
|
|
133
|
+
const bodySize = json.length;
|
|
134
|
+
const LIMIT = 512 * 1024; // generous debug-only cap (docs/13 §5)
|
|
135
|
+
debugBody(requestId, 'INCOMING BODY', [
|
|
136
|
+
`bodySize=${bodySize} bytes bodyTruncated=${bodySize > LIMIT}`,
|
|
137
|
+
bodySize > LIMIT ? truncate(json, LIMIT) : json,
|
|
138
|
+
]);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
104
141
|
function authenticateGatewayHeaders(req) {
|
|
105
142
|
const key = authenticateGatewayKey(req);
|
|
106
143
|
if (!key)
|
|
@@ -8,6 +8,7 @@ import { openAIToCanonical, openAIModelList } from '../../protocols/canonical.js
|
|
|
8
8
|
import { GatewayError, toOpenAIError } from '../../errors.js';
|
|
9
9
|
import { GatewayRunner } from '../../gateway/runner.js';
|
|
10
10
|
import { uuid } from '../../auth/ids.js';
|
|
11
|
+
import { lifecycle, debugHttp, debugBody, getDebugFlags, summarizeBody, summarizeMessages, summarizeTools, summarizeHeaders, sanitizeJson, truncate } from '../../logging/debug.js';
|
|
11
12
|
const ChatBody = z.object({
|
|
12
13
|
model: z.string().min(1),
|
|
13
14
|
messages: z.array(z.any()).min(1),
|
|
@@ -52,12 +53,18 @@ export async function registerOpenAIRoutes(app) {
|
|
|
52
53
|
return openAIModelList(ids.map((id) => ({ publicModelId: id, upstreamModelId: id })));
|
|
53
54
|
});
|
|
54
55
|
app.post('/v1/chat/completions', async (req, reply) => {
|
|
56
|
+
const requestId = req.id || uuid();
|
|
57
|
+
lifecycle(requestId, 'INCOMING', [
|
|
58
|
+
`POST ${req.url}`,
|
|
59
|
+
...summarizeHeaders(req.headers),
|
|
60
|
+
]);
|
|
55
61
|
const key = authenticateGatewayHeaders(req);
|
|
56
62
|
const body = ChatBody.parse(req.body);
|
|
63
|
+
logIncomingChatBody(requestId, body);
|
|
57
64
|
const req1 = body;
|
|
58
65
|
const canonical = openAIToCanonical(req1);
|
|
59
66
|
const ctx = {
|
|
60
|
-
requestId
|
|
67
|
+
requestId,
|
|
61
68
|
clientIp: resolveClientIp(req),
|
|
62
69
|
protocol: 'openai',
|
|
63
70
|
endpoint: 'chat/completions',
|
|
@@ -89,9 +96,11 @@ export async function registerOpenAIRoutes(app) {
|
|
|
89
96
|
}
|
|
90
97
|
if (!outcome.success) {
|
|
91
98
|
const g = new GatewayError(outcome.errorType ?? 'gateway_error', outcome.errorMessage ?? 'Gateway error', { status: outcome.httpStatus });
|
|
99
|
+
lifecycle(requestId, 'DONE', [`status=${outcome.httpStatus} durationMs=${outcome.latencyMs} error=true type=${g.type}`]);
|
|
92
100
|
reply.code(outcome.httpStatus).send(toOpenAIError(g, ctx.requestId));
|
|
93
101
|
return;
|
|
94
102
|
}
|
|
103
|
+
lifecycle(requestId, 'DONE', [`status=200 durationMs=${outcome.latencyMs} finishReason=${outcome.finishReason ?? 'null'}`]);
|
|
95
104
|
reply.header('x-request-id', ctx.requestId);
|
|
96
105
|
reply.send({
|
|
97
106
|
id: `chatcmpl-${ctx.requestId}`,
|
|
@@ -129,9 +138,19 @@ export async function registerOpenAIRoutes(app) {
|
|
|
129
138
|
}
|
|
130
139
|
});
|
|
131
140
|
app.post('/v1/responses', async (req, reply) => {
|
|
141
|
+
const requestId = req.id || uuid();
|
|
142
|
+
lifecycle(requestId, 'INCOMING', [
|
|
143
|
+
`POST ${req.url}`,
|
|
144
|
+
...summarizeHeaders(req.headers),
|
|
145
|
+
]);
|
|
132
146
|
const key = authenticateGatewayHeaders(req);
|
|
133
147
|
// v1 subset: accept Responses-style input, flatten to chat-completions messages.
|
|
134
148
|
const body = ResponsesBody.parse(req.body);
|
|
149
|
+
debugHttp(requestId, 'BODY SUMMARY', [
|
|
150
|
+
`model=${body.model}`,
|
|
151
|
+
`stream=${body.stream ?? 'undefined'}`,
|
|
152
|
+
`inputType=${Array.isArray(body.input) ? `array(${body.input.length})` : typeof body.input}`,
|
|
153
|
+
]);
|
|
135
154
|
const flat = responsesInputToChat(body.input);
|
|
136
155
|
const chatBody = {
|
|
137
156
|
model: body.model,
|
|
@@ -141,7 +160,7 @@ export async function registerOpenAIRoutes(app) {
|
|
|
141
160
|
};
|
|
142
161
|
const canonical = openAIToCanonical(chatBody);
|
|
143
162
|
const ctx = {
|
|
144
|
-
requestId
|
|
163
|
+
requestId,
|
|
145
164
|
clientIp: resolveClientIp(req),
|
|
146
165
|
protocol: 'openai',
|
|
147
166
|
endpoint: 'responses',
|
|
@@ -198,6 +217,23 @@ export async function registerOpenAIRoutes(app) {
|
|
|
198
217
|
}
|
|
199
218
|
});
|
|
200
219
|
}
|
|
220
|
+
// docs/13 §4–§7: incoming chat body structure logging. Summary always
|
|
221
|
+
// available under DEBUG_HTTP; full sanitized body under DEBUG_HTTP_BODY.
|
|
222
|
+
function logIncomingChatBody(requestId, body) {
|
|
223
|
+
debugHttp(requestId, 'BODY SUMMARY', summarizeBody(body));
|
|
224
|
+
debugHttp(requestId, 'MESSAGES', summarizeMessages(body));
|
|
225
|
+
if (body['tools'] !== undefined)
|
|
226
|
+
debugHttp(requestId, 'TOOLS', summarizeTools(body));
|
|
227
|
+
if (getDebugFlags().httpBody) {
|
|
228
|
+
const json = sanitizeJson(body);
|
|
229
|
+
const bodySize = json.length;
|
|
230
|
+
const LIMIT = 512 * 1024; // generous debug-only cap (docs/13 §5)
|
|
231
|
+
debugBody(requestId, 'INCOMING BODY', [
|
|
232
|
+
`bodySize=${bodySize} bytes bodyTruncated=${bodySize > LIMIT}`,
|
|
233
|
+
bodySize > LIMIT ? truncate(json, LIMIT) : json,
|
|
234
|
+
]);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
201
237
|
export function listRoutableModelIds(models, key, db) {
|
|
202
238
|
const modelIds = new Set(models.map((m) => m.id));
|
|
203
239
|
const modelById = new Map(models.map((m) => [m.id, m.publicModelId]));
|
|
@@ -23,28 +23,58 @@ export function loadCombo(comboId) {
|
|
|
23
23
|
},
|
|
24
24
|
};
|
|
25
25
|
}
|
|
26
|
-
export function selectCandidates(combo, allModels, req) {
|
|
26
|
+
export function selectCandidates(combo, allModels, req, onReject) {
|
|
27
27
|
// Resolve each combo member to a candidate and apply filters
|
|
28
28
|
const map = new Map(allModels.map((m) => [m.modelId, m]));
|
|
29
29
|
const candidates = [];
|
|
30
30
|
for (const m of combo.members) {
|
|
31
|
-
if (!m.enabled)
|
|
31
|
+
if (!m.enabled) {
|
|
32
|
+
onReject?.({ modelId: m.modelId, publicModelId: map.get(m.modelId)?.publicModelId ?? m.modelId }, 'member_disabled');
|
|
32
33
|
continue;
|
|
34
|
+
}
|
|
33
35
|
const c = map.get(m.modelId);
|
|
34
|
-
if (!c)
|
|
36
|
+
if (!c) {
|
|
37
|
+
onReject?.({ modelId: m.modelId, publicModelId: m.modelId }, 'model_not_found');
|
|
35
38
|
continue;
|
|
36
|
-
|
|
39
|
+
}
|
|
40
|
+
if (!c.enabled) {
|
|
41
|
+
onReject?.(c, 'model_disabled');
|
|
37
42
|
continue;
|
|
38
|
-
|
|
43
|
+
}
|
|
44
|
+
if (!c.upstreamAvailable) {
|
|
45
|
+
onReject?.(c, 'upstream_unavailable');
|
|
39
46
|
continue;
|
|
40
|
-
|
|
47
|
+
}
|
|
48
|
+
if (c.circuitOpen) {
|
|
49
|
+
onReject?.(c, 'circuit_open');
|
|
41
50
|
continue;
|
|
42
|
-
|
|
51
|
+
}
|
|
52
|
+
if (!modelMeets(c.capabilities, req)) {
|
|
53
|
+
onReject?.(c, capabilityRejection(c.capabilities, req));
|
|
43
54
|
continue;
|
|
55
|
+
}
|
|
44
56
|
candidates.push(c);
|
|
45
57
|
}
|
|
46
58
|
return candidates;
|
|
47
59
|
}
|
|
60
|
+
/** First capability that explicitly failed (undefined = unknown caps never reject). */
|
|
61
|
+
function capabilityRejection(caps, req) {
|
|
62
|
+
if (req.streaming && caps.streaming === false)
|
|
63
|
+
return 'streaming';
|
|
64
|
+
if (req.tools && caps.tools === false)
|
|
65
|
+
return 'tools';
|
|
66
|
+
if (req.structuredOutput && caps.structured_output === false)
|
|
67
|
+
return 'structured_output';
|
|
68
|
+
if (req.imageInput && caps.image_input === false)
|
|
69
|
+
return 'image_input';
|
|
70
|
+
if (req.audioInput && caps.audio_input === false)
|
|
71
|
+
return 'audio_input';
|
|
72
|
+
if (req.reasoning && caps.reasoning === false)
|
|
73
|
+
return 'reasoning';
|
|
74
|
+
if (req.responses && caps.responses === false)
|
|
75
|
+
return 'responses';
|
|
76
|
+
return 'capability_mismatch';
|
|
77
|
+
}
|
|
48
78
|
export function orderCandidates(combo, candidates) {
|
|
49
79
|
if (combo.mode === 'fallback') {
|
|
50
80
|
// Preserve declared position order
|