ldrouter 1.18.2 → 1.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  // Drizzle ORM schema for LateDev Router SQLite database.
2
2
  // Source of truth: docs/02-DATA-MODEL.md
3
3
  import { sql } from 'drizzle-orm';
4
- import { sqliteTable, text, integer, index, uniqueIndex } from 'drizzle-orm/sqlite-core';
4
+ import { sqliteTable, text, integer, index, uniqueIndex, primaryKey } from 'drizzle-orm/sqlite-core';
5
5
  // ============================================================================
6
6
  // Application settings (singleton)
7
7
  // ============================================================================
@@ -381,6 +381,23 @@ export const requestAttempts = sqliteTable('request_attempts', {
381
381
  qoderAccountIdx: index('idx_attempt_qoder_account').on(t.qoderAccountId, t.startedAt),
382
382
  }));
383
383
  // ============================================================================
384
+ export const dailyModelStats = sqliteTable('daily_model_stats', {
385
+ day: text('day').notNull(), modelId: text('model_id').notNull().default(''), requests: integer('requests').notNull().default(0),
386
+ successfulRequests: integer('successful_requests').notNull().default(0), failedRequests: integer('failed_requests').notNull().default(0),
387
+ inputTokens: integer('input_tokens').notNull().default(0), outputTokens: integer('output_tokens').notNull().default(0), cacheReadTokens: integer('cache_read_tokens').notNull().default(0),
388
+ cacheWriteTokens: integer('cache_write_tokens').notNull().default(0), reasoningTokens: integer('reasoning_tokens').notNull().default(0), gatewayCacheHits: integer('gateway_cache_hits').notNull().default(0),
389
+ fallbackRequests: integer('fallback_requests').notNull().default(0), latencySuccessSumMs: integer('latency_success_sum_ms').notNull().default(0), latencySuccessCount: integer('latency_success_count').notNull().default(0),
390
+ ttftSumMs: integer('ttft_sum_ms').notNull().default(0), ttftCount: integer('ttft_count').notNull().default(0),
391
+ }, (t) => ({ pk: primaryKey({ columns: [t.day, t.modelId] }) }));
392
+ export const dailyProviderStats = sqliteTable('daily_provider_stats', {
393
+ day: text('day').notNull(), providerId: text('provider_id').notNull(), attempts: integer('attempts').notNull().default(0), successfulAttempts: integer('successful_attempts').notNull().default(0),
394
+ failedAttempts: integer('failed_attempts').notNull().default(0), inputTokens: integer('input_tokens').notNull().default(0), outputTokens: integer('output_tokens').notNull().default(0), cacheReadTokens: integer('cache_read_tokens').notNull().default(0),
395
+ cacheWriteTokens: integer('cache_write_tokens').notNull().default(0), reasoningTokens: integer('reasoning_tokens').notNull().default(0), latencySuccessSumMs: integer('latency_success_sum_ms').notNull().default(0), latencySuccessCount: integer('latency_success_count').notNull().default(0),
396
+ }, (t) => ({ pk: primaryKey({ columns: [t.day, t.providerId] }) }));
397
+ export const dailyApiKeyStats = sqliteTable('daily_api_key_stats', {
398
+ day: text('day').notNull(), apiKeyId: text('api_key_id').notNull().default(''), requests: integer('requests').notNull().default(0), successfulRequests: integer('successful_requests').notNull().default(0), failedRequests: integer('failed_requests').notNull().default(0),
399
+ inputTokens: integer('input_tokens').notNull().default(0), outputTokens: integer('output_tokens').notNull().default(0), totalTokens: integer('total_tokens').notNull().default(0),
400
+ }, (t) => ({ pk: primaryKey({ columns: [t.day, t.apiKeyId] }) }));
384
401
  // Audit logs (immutable, excluded from request retention)
385
402
  // ============================================================================
386
403
  export const auditLogs = sqliteTable('audit_logs', {
@@ -1,5 +1,5 @@
1
1
  // Gateway execution pipeline: auth -> resolve -> ACL -> limits -> route -> attempt -> persist.
2
- import { getDb, schema } from '../db/index.js';
2
+ import { getDb, getRawDb, schema } from '../db/index.js';
3
3
  import { eq } from 'drizzle-orm';
4
4
  import { GatewayError } from '../errors.js';
5
5
  import { resolveRequestedModel, unwrapAlias } from '../routing/resolver.js';
@@ -23,6 +23,7 @@ import { uuid } from '../auth/ids.js';
23
23
  import { redactString, redactValue } from '../security/redact.js';
24
24
  import { getSettings } from '../db/repositories/settings.js';
25
25
  import { buildCacheKey, lookupCache, storeCache, cacheAllowed } from '../caching/store.js';
26
+ import { upsertDailyAggregates } from '../backup/stats.js';
26
27
  import { metrics } from '../metrics/registry.js';
27
28
  import { emitRequestLogged, emitRequestStarted } from './events.js';
28
29
  import { debugHttp, debugBody, debugUpstream, debugStream, errorLine, formatError, getDebugFlags, truncate } from '../logging/debug.js';
@@ -945,65 +946,78 @@ export class GatewayRunner {
945
946
  if (settings.contentLogMode === 'prompt_and_response') {
946
947
  responsePayload = JSON.stringify(redactValue({ text: outcome.text, toolCalls: outcome.toolCalls, finishReason: outcome.finishReason }));
947
948
  }
948
- db.insert(schema.requests).values({
949
- id: requestId,
950
- createdAt: now,
951
- completedAt: now,
952
- apiKeyId: ctx.key?.id ?? null,
953
- keyPrefixSnapshot: ctx.key?.keyPrefix ?? null,
954
- clientIp: ctx.clientIp,
955
- protocol: ctx.protocol,
956
- endpoint: req.endpoint,
957
- requestedModel: req.canonical.model,
958
- resolvedTargetKind: outcome.resolvedTargetKind,
959
- resolvedTargetId: outcome.resolvedTargetId,
960
- finalModelId: outcome.finalModelId,
961
- streaming: req.canonical.stream,
962
- httpStatus: outcome.httpStatus,
963
- success: outcome.success ? 1 : 0,
964
- totalLatencyMs: outcome.latencyMs,
965
- ttftMs: outcome.ttftMs,
966
- inputTokens: usage.input,
967
- outputTokens: usage.output,
968
- cacheReadTokens: usage.cacheRead,
969
- cacheWriteTokens: usage.cacheWrite,
970
- reasoningTokens: usage.reasoning,
971
- totalTokens: usage.total,
972
- attemptsCount: attempts.length,
973
- errorType: outcome.errorType,
974
- errorMessage: outcome.errorMessage ? redactString(outcome.errorMessage) : null,
975
- requestPayloadJson: requestPayload,
976
- responsePayloadJson: responsePayload,
977
- gatewayCacheHit: cacheHit ? true : false,
978
- }).run();
979
- for (const a of attempts) {
980
- db.insert(schema.requestAttempts).values({
981
- id: uuid(),
982
- requestId,
983
- attemptNumber: a.attemptNumber,
984
- providerId: a.providerId,
985
- modelId: a.modelId,
986
- codexAccountId: a.codexAccountId ?? null,
987
- qoderAccountId: a.qoderAccountId ?? null,
988
- startedAt: a.startedAt,
989
- completedAt: a.completedAt,
990
- statusCode: a.statusCode,
991
- success: a.success,
992
- latencyMs: a.latencyMs,
993
- ttftMs: a.ttftMs,
994
- inputTokens: a.usage.input,
995
- outputTokens: a.usage.output,
996
- cacheReadTokens: a.usage.cacheRead,
997
- cacheWriteTokens: a.usage.cacheWrite,
998
- reasoningTokens: a.usage.reasoning,
999
- streamStarted: a.streamStarted ? 1 : 0,
1000
- partialResponse: a.partialResponse ? 1 : 0,
1001
- selectionReason: a.selectionReason,
1002
- failureReason: a.failureReason,
1003
- errorMessage: a.sanitizedError ? redactString(a.sanitizedError) : null,
1004
- upstreamRequestId: a.upstreamRequestId,
949
+ const persistedAttempts = attempts.map((a) => ({
950
+ id: uuid(),
951
+ requestId,
952
+ attemptNumber: a.attemptNumber,
953
+ providerId: a.providerId,
954
+ modelId: a.modelId,
955
+ codexAccountId: a.codexAccountId ?? null,
956
+ qoderAccountId: a.qoderAccountId ?? null,
957
+ startedAt: a.startedAt,
958
+ completedAt: a.completedAt,
959
+ statusCode: a.statusCode,
960
+ success: a.success,
961
+ latencyMs: a.latencyMs,
962
+ ttftMs: a.ttftMs,
963
+ inputTokens: a.usage.input,
964
+ outputTokens: a.usage.output,
965
+ cacheReadTokens: a.usage.cacheRead,
966
+ cacheWriteTokens: a.usage.cacheWrite,
967
+ reasoningTokens: a.usage.reasoning,
968
+ streamStarted: a.streamStarted ? 1 : 0,
969
+ partialResponse: a.partialResponse ? 1 : 0,
970
+ selectionReason: a.selectionReason,
971
+ failureReason: a.failureReason,
972
+ errorMessage: a.sanitizedError ? redactString(a.sanitizedError) : null,
973
+ upstreamRequestId: a.upstreamRequestId,
974
+ }));
975
+ const tx = getRawDb().transaction(() => {
976
+ db.insert(schema.requests).values({
977
+ id: requestId,
978
+ createdAt: now,
979
+ completedAt: now,
980
+ apiKeyId: ctx.key?.id ?? null,
981
+ keyPrefixSnapshot: ctx.key?.keyPrefix ?? null,
982
+ clientIp: ctx.clientIp,
983
+ protocol: ctx.protocol,
984
+ endpoint: req.endpoint,
985
+ requestedModel: req.canonical.model,
986
+ resolvedTargetKind: outcome.resolvedTargetKind,
987
+ resolvedTargetId: outcome.resolvedTargetId,
988
+ finalModelId: outcome.finalModelId,
989
+ streaming: req.canonical.stream,
990
+ httpStatus: outcome.httpStatus,
991
+ success: outcome.success ? 1 : 0,
992
+ totalLatencyMs: outcome.latencyMs,
993
+ ttftMs: outcome.ttftMs,
994
+ inputTokens: usage.input,
995
+ outputTokens: usage.output,
996
+ cacheReadTokens: usage.cacheRead,
997
+ cacheWriteTokens: usage.cacheWrite,
998
+ reasoningTokens: usage.reasoning,
999
+ totalTokens: usage.total,
1000
+ attemptsCount: attempts.length,
1001
+ errorType: outcome.errorType,
1002
+ errorMessage: outcome.errorMessage ? redactString(outcome.errorMessage) : null,
1003
+ requestPayloadJson: requestPayload,
1004
+ responsePayloadJson: responsePayload,
1005
+ gatewayCacheHit: cacheHit ? true : false,
1005
1006
  }).run();
1006
- }
1007
+ for (const attempt of persistedAttempts)
1008
+ db.insert(schema.requestAttempts).values(attempt).run();
1009
+ upsertDailyAggregates(getRawDb(), {
1010
+ day: now.slice(0, 10), modelId: outcome.finalModelId, apiKeyId: ctx.key?.id ?? null,
1011
+ success: outcome.success, inputTokens: usage.input, outputTokens: usage.output,
1012
+ cacheReadTokens: usage.cacheRead, cacheWriteTokens: usage.cacheWrite, reasoningTokens: usage.reasoning,
1013
+ cacheHit, attemptsCount: attempts.length, latencyMs: outcome.latencyMs, ttftMs: outcome.ttftMs,
1014
+ }, attempts.map((a) => ({
1015
+ day: a.startedAt.slice(0, 10), providerId: a.providerId, success: a.success,
1016
+ inputTokens: a.usage.input, outputTokens: a.usage.output, cacheReadTokens: a.usage.cacheRead,
1017
+ cacheWriteTokens: a.usage.cacheWrite, reasoningTokens: a.usage.reasoning, latencyMs: a.latencyMs,
1018
+ })));
1019
+ });
1020
+ tx();
1007
1021
  emitRequestLogged(requestId);
1008
1022
  }
1009
1023
  }
@@ -78,6 +78,7 @@ export function openAIToCanonical(req) {
78
78
  maxOutputTokens: req.max_tokens,
79
79
  stop: Array.isArray(req.stop) ? req.stop : req.stop ? [req.stop] : undefined,
80
80
  stream: Boolean(req.stream),
81
+ streamIncludeUsage: Boolean(req.stream_options?.include_usage),
81
82
  responseFormat,
82
83
  reasoning: req.reasoning_effort ? { effort: req.reasoning_effort } : undefined,
83
84
  };
@@ -88,6 +89,11 @@ export function canonicalToOpenAIRequest(req, targetModel) {
88
89
  stream: req.stream,
89
90
  messages: [],
90
91
  };
92
+ // OpenAI-compatible upstreams omit `usage` from a streamed answer unless the
93
+ // request opts in. Without this the gateway persisted 0 tokens for every
94
+ // streamed request to such a provider (live: "Tokens 0" on the Requests page).
95
+ if (req.stream)
96
+ out.stream_options = { include_usage: true };
91
97
  if (req.system)
92
98
  out.messages.push({ role: 'system', content: req.system });
93
99
  for (const m of req.messages) {
@@ -1,7 +1,6 @@
1
1
  // Admin API: database backup / restore.
2
2
  import fs from 'node:fs';
3
3
  import path from 'node:path';
4
- import zlib from 'node:zlib';
5
4
  import crypto from 'node:crypto';
6
5
  import { getDb, closeDb, openDb, schema } from '../../db/index.js';
7
6
  import { requireAdminAuth, requireAdminCsrf } from '../../auth/middleware.js';
@@ -10,11 +9,10 @@ import { recordAudit } from '../../db/repositories/audit.js';
10
9
  import { loadConfig, setConfigMasterKey } from '../../config/index.js';
11
10
  import { getSettings } from '../../db/repositories/settings.js';
12
11
  import { GatewayError } from '../../errors.js';
13
- import { getAppVersion } from '../../version.js';
14
12
  import { eq, sql } from 'drizzle-orm';
15
- import { decryptBackupMasterKey, encryptBackupMasterKey } from '../../auth/backup-crypto.js';
16
- import { parseMasterKey, resetMasterKeyCache } from '../../auth/crypto.js';
17
- const BACKUP_VERSION = 1;
13
+ import { resetMasterKeyCache } from '../../auth/crypto.js';
14
+ import { createBackupExport } from '../../backup/export.js';
15
+ import { receiveBackupUpload, validateUploadedBackup, writeLegacyEnvelopeUpload } from '../../backup/restore-upload.js';
18
16
  /** Reopen the in-process SQLite connection on the (possibly just-replaced)
19
17
  * database file. The old connection must already be closed: a hot restore
20
18
  * swaps the file on disk, then the gateway keeps serving from the new data
@@ -53,6 +51,144 @@ function assertDatabaseUsable(expectedSchemaVersion) {
53
51
  throw new Error(`restored database schema version mismatch: expected ${expectedSchemaVersion}, got ${row.schemaVersion}`);
54
52
  }
55
53
  }
54
+ /** Raw filesystem/multipart failures become safe fixed-message errors; validated
55
+ * failures keep their own safe GatewayError. Internal paths never reach the browser. */
56
+ function safeRestoreError(error) {
57
+ if (error instanceof GatewayError)
58
+ return error;
59
+ return new GatewayError('gateway_error', 'Restore failed', { status: 500 });
60
+ }
61
+ function auditRestoreFailure(req, error) {
62
+ const reason = error instanceof GatewayError ? (error.code ?? error.type) : 'restore_error';
63
+ recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason } });
64
+ }
65
+ /** Accept both the wrapped `{ backup, passphrase }` legacy body and the bare envelope. */
66
+ function legacyRequestBody(body) {
67
+ const incoming = body;
68
+ const wrapped = Boolean(incoming && typeof incoming === 'object' && 'backup' in incoming);
69
+ const wrappedBody = incoming;
70
+ return { envelope: (wrapped ? wrappedBody.backup : incoming), passphrase: wrapped ? wrappedBody.passphrase : undefined };
71
+ }
72
+ /**
73
+ * Atomic swap of an already-validated restore payload. Both the v2 multipart path and the legacy
74
+ * v1 JSON path feed this, so the snapshot/rollback, hot reopen, session/CSRF re-seed, and audit
75
+ * behaviour stay identical regardless of container format.
76
+ */
77
+ export async function restoreValidatedDatabase(validated, req) {
78
+ const cfg = loadConfig();
79
+ const liveDb = cfg.dbFile;
80
+ const previousMasterKey = cfg.masterKey;
81
+ const restoreKeyPath = path.join(cfg.dataDir, 'master.key.restore');
82
+ const previousRestoreKey = fs.existsSync(restoreKeyPath) ? fs.readFileSync(restoreKeyPath) : null;
83
+ const restoredKey = validated.masterKey?.toString('base64');
84
+ // Snapshot current DB before restore (kept for manual rollback).
85
+ const snapshot = path.join(cfg.dataDir, `pre-restore-${Date.now()}.sqlite`);
86
+ const Database = (await import('better-sqlite3')).default;
87
+ const live = new Database(liveDb);
88
+ try {
89
+ await live.backup(snapshot);
90
+ }
91
+ catch {
92
+ live.close();
93
+ recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'snapshot_failed' } });
94
+ throw new GatewayError('gateway_error', 'Could not snapshot current database', { status: 500 });
95
+ }
96
+ live.close();
97
+ // Close the in-process connection before swapping the file. The gateway
98
+ // keeps serving (no restart) but the file must be free: on Windows a
99
+ // rename fails while a handle is open. Reopening runs migrations + the
100
+ // app_settings bootstrap automatically.
101
+ getDb();
102
+ closeDb();
103
+ // Atomic replace.
104
+ const tempDb = `${liveDb}.restore-${Date.now()}`;
105
+ fs.copyFileSync(validated.sqlitePath, tempDb);
106
+ try {
107
+ fs.renameSync(tempDb, liveDb);
108
+ }
109
+ catch {
110
+ fs.rmSync(tempDb, { force: true });
111
+ recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'rename_failed' } });
112
+ throw new GatewayError('gateway_error', 'Restore atomic replace failed', { status: 500 });
113
+ }
114
+ // The backup key is authoritative. Apply it before reopening the restored
115
+ // database so every post-restore read uses the key that encrypted it.
116
+ if (restoredKey) {
117
+ fs.writeFileSync(restoreKeyPath, restoredKey, { mode: 0o600, encoding: 'utf8' });
118
+ setConfigMasterKey(restoredKey);
119
+ resetMasterKeyCache();
120
+ }
121
+ // Hot-reload the database in-process: reopen the replaced file, validate
122
+ // it, and keep serving. No gateway restart needed.
123
+ try {
124
+ reopenDatabase(liveDb);
125
+ assertDatabaseUsable(validated.schemaVersion);
126
+ }
127
+ catch (e) {
128
+ // Roll back to the snapshot taken before the restore so the gateway
129
+ // never stays on a broken database.
130
+ try {
131
+ closeDb();
132
+ }
133
+ catch { /* ignore */ }
134
+ try {
135
+ for (const suffix of ['-wal', '-shm']) {
136
+ const stale = path.join(path.dirname(liveDb), `${path.basename(liveDb)}${suffix}`);
137
+ if (fs.existsSync(stale))
138
+ fs.unlinkSync(stale);
139
+ }
140
+ fs.copyFileSync(snapshot, liveDb);
141
+ openDb(liveDb);
142
+ try {
143
+ if (previousRestoreKey)
144
+ fs.writeFileSync(restoreKeyPath, previousRestoreKey, { mode: 0o600 });
145
+ else
146
+ fs.unlinkSync(restoreKeyPath);
147
+ }
148
+ catch { /* no prior override */ }
149
+ if (previousMasterKey)
150
+ setConfigMasterKey(previousMasterKey);
151
+ resetMasterKeyCache();
152
+ }
153
+ catch (rollbackErr) {
154
+ const err = e.message;
155
+ const rerr = rollbackErr.message;
156
+ recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'rollback_failed', err, rollbackErr: rerr } });
157
+ throw new GatewayError('gateway_error', `Restore failed (${err}) and automatic rollback also failed (${rerr}). Please restart the gateway.`, { status: 500 });
158
+ }
159
+ recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'validation_failed', err: e.message } });
160
+ throw new GatewayError('gateway_error', `Restore failed: ${e.message}`, { status: 500 });
161
+ }
162
+ // The swap invalidates the previous admin session (its row lived in the
163
+ // old database). Re-create the current session in the restored database so
164
+ // the admin stays logged in across the hot restore.
165
+ const sessionToken = req.cookies['ld_session'];
166
+ const sessionId = req.adminSessionId;
167
+ if (sessionId && sessionToken) {
168
+ const db = getDb();
169
+ const expiresAt = new Date(Date.now() + 12 * 3600 * 1000).toISOString();
170
+ db.delete(schema.adminSessions).where(sql `id = ${sessionId}`).run();
171
+ db.insert(schema.adminSessions).values({
172
+ id: sessionId,
173
+ tokenDigest: sha256Hex(sessionToken),
174
+ expiresAt,
175
+ lastSeenAt: new Date().toISOString(),
176
+ ip: req.ip,
177
+ }).run();
178
+ // Re-seed the CSRF token that was bound to the old session.
179
+ const csrfRow = db.select().from(schema.csrfTokens).where(eq(schema.csrfTokens.sessionId, sessionId)).get();
180
+ if (!csrfRow) {
181
+ db.insert(schema.csrfTokens).values({
182
+ id: crypto.randomUUID(),
183
+ sessionId,
184
+ token: crypto.randomBytes(32).toString('base64url'),
185
+ expiresAt,
186
+ }).run();
187
+ }
188
+ }
189
+ recordAudit({ action: 'db.restore', success: true, ip: req.ip, metadata: { profile: validated.profile, schemaVersion: validated.schemaVersion } });
190
+ return { profile: validated.profile, message: 'Database restored. The gateway continues running with the restored data — no restart needed.' };
191
+ }
56
192
  export async function registerBackupRoutes(app) {
57
193
  // This module is registered in its own scope (routes/admin.ts) precisely so this is the only
58
194
  // auth hook on the backup routes. A preHandler cannot cancel a later one, so the first-run
@@ -64,208 +200,58 @@ export async function registerBackupRoutes(app) {
64
200
  if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(req.method))
65
201
  await requireAdminCsrf(req);
66
202
  });
203
+ /** UI preflight: the effective server-side maximum upload size. */
204
+ app.get('/api/admin/backup/capabilities', async () => ({ maxUploadBytes: loadConfig().maxBackupUploadBytes }));
67
205
  app.post('/api/admin/backup/create', async (req, reply) => {
68
- const passphrase = req.body?.passphrase ?? '';
206
+ const body = req.body ?? {};
207
+ const passphrase = body.passphrase ?? '';
69
208
  if (!/^\d{6}$/.test(passphrase)) {
70
209
  throw new GatewayError('invalid_request_error', 'Backup passphrase must contain exactly six digits', { status: 400 });
71
210
  }
72
- const cfg = loadConfig();
73
- const temp = path.join(cfg.dataDir, `.backup-${Date.now()}.sqlite`);
74
- const Database = (await import('better-sqlite3')).default;
75
- const source = new Database(cfg.dbFile, { readonly: true });
76
- try {
77
- await source.backup(temp);
78
- }
79
- finally {
80
- source.close();
81
- }
82
- const buf = fs.readFileSync(temp);
83
- fs.unlinkSync(temp);
84
- const compressed = zlib.gzipSync(buf, { level: 6 });
85
- const checksum = crypto.createHash('sha256').update(compressed).digest('hex');
86
- const settings = getSettings();
87
- const masterKey = parseMasterKey(cfg.masterKey ?? fs.readFileSync(path.join(cfg.dataDir, 'master.key'), 'utf8').trim());
88
- const envelope = {
89
- format: 'latedev-backup',
90
- version: BACKUP_VERSION,
91
- appVersion: getAppVersion(),
92
- schemaVersion: settings.schemaVersion,
93
- masterKeyConfigured: settings.masterKeyConfigured,
94
- createdAt: new Date().toISOString(),
95
- payload: compressed.toString('base64'),
96
- checksum,
97
- keyEnvelope: encryptBackupMasterKey(masterKey, passphrase),
98
- };
99
- const envBuf = Buffer.from(JSON.stringify(envelope), 'utf8');
100
- const fileName = `latedev-backup-${new Date().toISOString().replace(/[:.]/g, '-')}.ldb.json`;
101
- reply.header('content-type', 'application/json');
211
+ const profile = body.profile === 'full' ? 'full' : 'compact';
212
+ const { manifest, body: payload } = await createBackupExport({ profile, passphrase, cfg: loadConfig() });
213
+ const fileName = `latedev-backup-${new Date().toISOString().replace(/[:.]/g, '-')}.ldb`;
214
+ reply.header('content-type', 'application/octet-stream');
102
215
  reply.header('content-disposition', `attachment; filename="${fileName}"`);
103
- recordAudit({ action: 'db.backup_download', success: true, ip: req.ip, metadata: { sizeBytes: envBuf.length, checksum } });
104
- return reply.send(envBuf);
216
+ recordAudit({
217
+ action: 'db.backup_download',
218
+ success: true,
219
+ ip: req.ip,
220
+ metadata: { profile, sizeBytes: manifest.compressedBytes, checksum: manifest.checksum, excludedTables: manifest.excludedTables },
221
+ });
222
+ return reply.send(payload);
105
223
  });
106
224
  app.post('/api/admin/backup/restore', async (req, reply) => {
107
225
  const cfg = loadConfig();
108
- const incoming = req.body;
109
- const wrapped = Boolean(incoming && 'backup' in incoming);
110
- const wrappedBody = incoming;
111
- const passphrase = wrapped ? wrappedBody.passphrase : undefined;
112
- let envelope;
113
- try {
114
- envelope = (wrapped ? wrappedBody.backup : incoming);
115
- if (!envelope || envelope.format !== 'latedev-backup')
116
- throw new Error('not a backup envelope');
117
- if (envelope.keyEnvelope && !/^\d{6}$/.test(passphrase ?? ''))
118
- throw new Error('backup passphrase required');
119
- }
120
- catch {
121
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'invalid_envelope' } });
122
- throw new GatewayError('invalid_request_error', 'Invalid backup file format', { status: 400 });
123
- }
124
- if (envelope.version > BACKUP_VERSION) {
125
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'future_version', version: envelope.version } });
126
- throw new GatewayError('invalid_request_error', `Backup version ${envelope.version} is not supported (max ${BACKUP_VERSION})`, { status: 400 });
127
- }
128
- const compressed = Buffer.from(envelope.payload, 'base64');
129
- const checksum = crypto.createHash('sha256').update(compressed).digest('hex');
130
- if (checksum !== envelope.checksum) {
131
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'checksum_mismatch' } });
132
- throw new GatewayError('invalid_request_error', 'Backup checksum mismatch', { status: 400 });
133
- }
134
- let buf;
226
+ const contentType = String(req.headers['content-type'] ?? '');
227
+ let upload;
135
228
  try {
136
- buf = zlib.gunzipSync(compressed);
137
- }
138
- catch {
139
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'decompress_failed' } });
140
- throw new GatewayError('invalid_request_error', 'Backup decompression failed', { status: 400 });
141
- }
142
- // Validate SQLite header
143
- if (!(buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4c && buf[3] === 0x69 && buf[4] === 0x74 && buf[5] === 0x65)) {
144
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'not_sqlite' } });
145
- throw new GatewayError('invalid_request_error', 'Backup does not contain a valid SQLite database', { status: 400 });
146
- }
147
- let restoredMasterKey;
148
- if (envelope.keyEnvelope) {
149
- try {
150
- restoredMasterKey = decryptBackupMasterKey(envelope.keyEnvelope, passphrase);
151
- parseMasterKey(restoredMasterKey.toString('base64'));
229
+ if (contentType.includes('multipart/form-data')) {
230
+ upload = await receiveBackupUpload(req, { cfg, maxBytes: cfg.maxBackupUploadBytes });
152
231
  }
153
- catch {
154
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'invalid_backup_passphrase' } });
155
- throw new GatewayError('invalid_request_error', 'Invalid backup passphrase', { status: 400 });
232
+ else {
233
+ const { envelope, passphrase } = legacyRequestBody(req.body);
234
+ upload = await writeLegacyEnvelopeUpload(envelope, cfg, passphrase);
156
235
  }
157
236
  }
158
- const liveDb = cfg.dbFile;
159
- const previousMasterKey = cfg.masterKey;
160
- const restoreKeyPath = path.join(cfg.dataDir, 'master.key.restore');
161
- const previousRestoreKey = fs.existsSync(restoreKeyPath) ? fs.readFileSync(restoreKeyPath) : null;
162
- const restoredKey = restoredMasterKey?.toString('base64');
163
- // Snapshot current DB before restore (kept for manual rollback).
164
- const snapshot = path.join(cfg.dataDir, `pre-restore-${Date.now()}.sqlite`);
165
- const Database = (await import('better-sqlite3')).default;
166
- const live = new Database(liveDb);
167
- try {
168
- await live.backup(snapshot);
169
- }
170
- catch (e) {
171
- live.close();
172
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'snapshot_failed', err: String(e) } });
173
- throw new GatewayError('gateway_error', 'Could not snapshot current database', { status: 500 });
237
+ catch (error) {
238
+ auditRestoreFailure(req, error);
239
+ throw safeRestoreError(error);
174
240
  }
175
- live.close();
176
- // Close the in-process connection before swapping the file. The gateway
177
- // keeps serving (no restart) but the file must be free: on Windows a
178
- // rename fails while a handle is open. Reopening runs migrations + the
179
- // app_settings bootstrap automatically.
180
- getDb();
181
- closeDb();
182
- // Atomic replace.
183
- const tempDb = `${liveDb}.restore-${Date.now()}`;
184
- fs.writeFileSync(tempDb, buf);
241
+ let validated;
185
242
  try {
186
- fs.renameSync(tempDb, liveDb);
243
+ validated = await validateUploadedBackup(upload, upload.passphrase);
244
+ const result = await restoreValidatedDatabase(validated, req);
245
+ return reply.code(200).send({ ok: true, profile: result.profile, message: result.message });
187
246
  }
188
- catch (e) {
189
- fs.unlinkSync(tempDb);
190
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'rename_failed', err: String(e) } });
191
- throw new GatewayError('gateway_error', 'Restore atomic replace failed', { status: 500 });
247
+ catch (error) {
248
+ auditRestoreFailure(req, error);
249
+ throw safeRestoreError(error);
192
250
  }
193
- // The backup key is authoritative. Apply it before reopening the restored
194
- // database so every post-restore read uses the key that encrypted it.
195
- if (restoredKey) {
196
- fs.writeFileSync(restoreKeyPath, restoredKey, { mode: 0o600, encoding: 'utf8' });
197
- setConfigMasterKey(restoredKey);
198
- resetMasterKeyCache();
199
- }
200
- // Hot-reload the database in-process: reopen the replaced file, validate
201
- // it, and keep serving. No gateway restart needed.
202
- try {
203
- reopenDatabase(liveDb);
204
- assertDatabaseUsable(envelope.schemaVersion);
205
- }
206
- catch (e) {
207
- // Roll back to the snapshot taken before the restore so the gateway
208
- // never stays on a broken database.
209
- try {
210
- closeDb();
211
- }
212
- catch { /* ignore */ }
213
- try {
214
- for (const suffix of ['-wal', '-shm']) {
215
- const stale = path.join(path.dirname(liveDb), `${path.basename(liveDb)}${suffix}`);
216
- if (fs.existsSync(stale))
217
- fs.unlinkSync(stale);
218
- }
219
- fs.copyFileSync(snapshot, liveDb);
220
- openDb(liveDb);
221
- try {
222
- if (previousRestoreKey)
223
- fs.writeFileSync(restoreKeyPath, previousRestoreKey, { mode: 0o600 });
224
- else
225
- fs.unlinkSync(restoreKeyPath);
226
- }
227
- catch { /* no prior override */ }
228
- if (previousMasterKey)
229
- setConfigMasterKey(previousMasterKey);
230
- resetMasterKeyCache();
231
- }
232
- catch (rollbackErr) {
233
- const err = e.message;
234
- const rerr = rollbackErr.message;
235
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'rollback_failed', err, rollbackErr: rerr } });
236
- throw new GatewayError('gateway_error', `Restore failed (${err}) and automatic rollback also failed (${rerr}). Please restart the gateway.`, { status: 500 });
237
- }
238
- recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'validation_failed', err: e.message } });
239
- throw new GatewayError('gateway_error', `Restore failed: ${e.message}`, { status: 500 });
240
- }
241
- // The swap invalidates the previous admin session (its row lived in the
242
- // old database). Re-create the current session in the restored database so
243
- // the admin stays logged in across the hot restore.
244
- const sessionToken = req.cookies['ld_session'];
245
- const sessionId = req.adminSessionId;
246
- if (sessionId && sessionToken) {
247
- const db = getDb();
248
- const expiresAt = new Date(Date.now() + 12 * 3600 * 1000).toISOString();
249
- db.delete(schema.adminSessions).where(sql `id = ${sessionId}`).run();
250
- db.insert(schema.adminSessions).values({
251
- id: sessionId,
252
- tokenDigest: sha256Hex(sessionToken),
253
- expiresAt,
254
- lastSeenAt: new Date().toISOString(),
255
- ip: req.ip,
256
- }).run();
257
- // Re-seed the CSRF token that was bound to the old session.
258
- const csrfRow = db.select().from(schema.csrfTokens).where(eq(schema.csrfTokens.sessionId, sessionId)).get();
259
- if (!csrfRow) {
260
- db.insert(schema.csrfTokens).values({
261
- id: crypto.randomUUID(),
262
- sessionId,
263
- token: crypto.randomBytes(32).toString('base64url'),
264
- expiresAt,
265
- }).run();
266
- }
251
+ finally {
252
+ // Temp upload + decompressed payload are removed on success, failure, and abort.
253
+ await validated?.cleanup();
254
+ await upload.cleanup();
267
255
  }
268
- recordAudit({ action: 'db.restore', success: true, ip: req.ip, metadata: { schemaVersion: envelope.schemaVersion } });
269
- return reply.code(200).send({ ok: true, message: 'Database restored. The gateway continues running with the restored data — no restart needed.' });
270
256
  });
271
257
  }