ldrouter 1.18.2 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +15 -1
- package/dist/server/backup/container.js +99 -0
- package/dist/server/backup/export.js +99 -0
- package/dist/server/backup/restore-upload.js +333 -0
- package/dist/server/backup/stats.js +15 -0
- package/dist/server/config/index.js +4 -0
- package/dist/server/db/schema.js +18 -1
- package/dist/server/gateway/runner.js +73 -59
- package/dist/server/routes/admin/backup.js +177 -191
- package/dist/server/routes/admin/stats.js +129 -61
- package/dist/web/assets/index-BdbskFHA.js +349 -0
- package/dist/web/index.html +1 -1
- package/migrations/0011_daily_aggregate_stats.sql +60 -0
- package/package.json +1 -1
- package/dist/web/assets/index-DYrnLQwk.js +0 -349
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Gateway execution pipeline: auth -> resolve -> ACL -> limits -> route -> attempt -> persist.
|
|
2
|
-
import { getDb, schema } from '../db/index.js';
|
|
2
|
+
import { getDb, getRawDb, schema } from '../db/index.js';
|
|
3
3
|
import { eq } from 'drizzle-orm';
|
|
4
4
|
import { GatewayError } from '../errors.js';
|
|
5
5
|
import { resolveRequestedModel, unwrapAlias } from '../routing/resolver.js';
|
|
@@ -23,6 +23,7 @@ import { uuid } from '../auth/ids.js';
|
|
|
23
23
|
import { redactString, redactValue } from '../security/redact.js';
|
|
24
24
|
import { getSettings } from '../db/repositories/settings.js';
|
|
25
25
|
import { buildCacheKey, lookupCache, storeCache, cacheAllowed } from '../caching/store.js';
|
|
26
|
+
import { upsertDailyAggregates } from '../backup/stats.js';
|
|
26
27
|
import { metrics } from '../metrics/registry.js';
|
|
27
28
|
import { emitRequestLogged, emitRequestStarted } from './events.js';
|
|
28
29
|
import { debugHttp, debugBody, debugUpstream, debugStream, errorLine, formatError, getDebugFlags, truncate } from '../logging/debug.js';
|
|
@@ -945,65 +946,78 @@ export class GatewayRunner {
|
|
|
945
946
|
if (settings.contentLogMode === 'prompt_and_response') {
|
|
946
947
|
responsePayload = JSON.stringify(redactValue({ text: outcome.text, toolCalls: outcome.toolCalls, finishReason: outcome.finishReason }));
|
|
947
948
|
}
|
|
948
|
-
|
|
949
|
-
id:
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
949
|
+
const persistedAttempts = attempts.map((a) => ({
|
|
950
|
+
id: uuid(),
|
|
951
|
+
requestId,
|
|
952
|
+
attemptNumber: a.attemptNumber,
|
|
953
|
+
providerId: a.providerId,
|
|
954
|
+
modelId: a.modelId,
|
|
955
|
+
codexAccountId: a.codexAccountId ?? null,
|
|
956
|
+
qoderAccountId: a.qoderAccountId ?? null,
|
|
957
|
+
startedAt: a.startedAt,
|
|
958
|
+
completedAt: a.completedAt,
|
|
959
|
+
statusCode: a.statusCode,
|
|
960
|
+
success: a.success,
|
|
961
|
+
latencyMs: a.latencyMs,
|
|
962
|
+
ttftMs: a.ttftMs,
|
|
963
|
+
inputTokens: a.usage.input,
|
|
964
|
+
outputTokens: a.usage.output,
|
|
965
|
+
cacheReadTokens: a.usage.cacheRead,
|
|
966
|
+
cacheWriteTokens: a.usage.cacheWrite,
|
|
967
|
+
reasoningTokens: a.usage.reasoning,
|
|
968
|
+
streamStarted: a.streamStarted ? 1 : 0,
|
|
969
|
+
partialResponse: a.partialResponse ? 1 : 0,
|
|
970
|
+
selectionReason: a.selectionReason,
|
|
971
|
+
failureReason: a.failureReason,
|
|
972
|
+
errorMessage: a.sanitizedError ? redactString(a.sanitizedError) : null,
|
|
973
|
+
upstreamRequestId: a.upstreamRequestId,
|
|
974
|
+
}));
|
|
975
|
+
const tx = getRawDb().transaction(() => {
|
|
976
|
+
db.insert(schema.requests).values({
|
|
977
|
+
id: requestId,
|
|
978
|
+
createdAt: now,
|
|
979
|
+
completedAt: now,
|
|
980
|
+
apiKeyId: ctx.key?.id ?? null,
|
|
981
|
+
keyPrefixSnapshot: ctx.key?.keyPrefix ?? null,
|
|
982
|
+
clientIp: ctx.clientIp,
|
|
983
|
+
protocol: ctx.protocol,
|
|
984
|
+
endpoint: req.endpoint,
|
|
985
|
+
requestedModel: req.canonical.model,
|
|
986
|
+
resolvedTargetKind: outcome.resolvedTargetKind,
|
|
987
|
+
resolvedTargetId: outcome.resolvedTargetId,
|
|
988
|
+
finalModelId: outcome.finalModelId,
|
|
989
|
+
streaming: req.canonical.stream,
|
|
990
|
+
httpStatus: outcome.httpStatus,
|
|
991
|
+
success: outcome.success ? 1 : 0,
|
|
992
|
+
totalLatencyMs: outcome.latencyMs,
|
|
993
|
+
ttftMs: outcome.ttftMs,
|
|
994
|
+
inputTokens: usage.input,
|
|
995
|
+
outputTokens: usage.output,
|
|
996
|
+
cacheReadTokens: usage.cacheRead,
|
|
997
|
+
cacheWriteTokens: usage.cacheWrite,
|
|
998
|
+
reasoningTokens: usage.reasoning,
|
|
999
|
+
totalTokens: usage.total,
|
|
1000
|
+
attemptsCount: attempts.length,
|
|
1001
|
+
errorType: outcome.errorType,
|
|
1002
|
+
errorMessage: outcome.errorMessage ? redactString(outcome.errorMessage) : null,
|
|
1003
|
+
requestPayloadJson: requestPayload,
|
|
1004
|
+
responsePayloadJson: responsePayload,
|
|
1005
|
+
gatewayCacheHit: cacheHit ? true : false,
|
|
1005
1006
|
}).run();
|
|
1006
|
-
|
|
1007
|
+
for (const attempt of persistedAttempts)
|
|
1008
|
+
db.insert(schema.requestAttempts).values(attempt).run();
|
|
1009
|
+
upsertDailyAggregates(getRawDb(), {
|
|
1010
|
+
day: now.slice(0, 10), modelId: outcome.finalModelId, apiKeyId: ctx.key?.id ?? null,
|
|
1011
|
+
success: outcome.success, inputTokens: usage.input, outputTokens: usage.output,
|
|
1012
|
+
cacheReadTokens: usage.cacheRead, cacheWriteTokens: usage.cacheWrite, reasoningTokens: usage.reasoning,
|
|
1013
|
+
cacheHit, attemptsCount: attempts.length, latencyMs: outcome.latencyMs, ttftMs: outcome.ttftMs,
|
|
1014
|
+
}, attempts.map((a) => ({
|
|
1015
|
+
day: a.startedAt.slice(0, 10), providerId: a.providerId, success: a.success,
|
|
1016
|
+
inputTokens: a.usage.input, outputTokens: a.usage.output, cacheReadTokens: a.usage.cacheRead,
|
|
1017
|
+
cacheWriteTokens: a.usage.cacheWrite, reasoningTokens: a.usage.reasoning, latencyMs: a.latencyMs,
|
|
1018
|
+
})));
|
|
1019
|
+
});
|
|
1020
|
+
tx();
|
|
1007
1021
|
emitRequestLogged(requestId);
|
|
1008
1022
|
}
|
|
1009
1023
|
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
// Admin API: database backup / restore.
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
|
-
import zlib from 'node:zlib';
|
|
5
4
|
import crypto from 'node:crypto';
|
|
6
5
|
import { getDb, closeDb, openDb, schema } from '../../db/index.js';
|
|
7
6
|
import { requireAdminAuth, requireAdminCsrf } from '../../auth/middleware.js';
|
|
@@ -10,11 +9,10 @@ import { recordAudit } from '../../db/repositories/audit.js';
|
|
|
10
9
|
import { loadConfig, setConfigMasterKey } from '../../config/index.js';
|
|
11
10
|
import { getSettings } from '../../db/repositories/settings.js';
|
|
12
11
|
import { GatewayError } from '../../errors.js';
|
|
13
|
-
import { getAppVersion } from '../../version.js';
|
|
14
12
|
import { eq, sql } from 'drizzle-orm';
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
|
|
13
|
+
import { resetMasterKeyCache } from '../../auth/crypto.js';
|
|
14
|
+
import { createBackupExport } from '../../backup/export.js';
|
|
15
|
+
import { receiveBackupUpload, validateUploadedBackup, writeLegacyEnvelopeUpload } from '../../backup/restore-upload.js';
|
|
18
16
|
/** Reopen the in-process SQLite connection on the (possibly just-replaced)
|
|
19
17
|
* database file. The old connection must already be closed: a hot restore
|
|
20
18
|
* swaps the file on disk, then the gateway keeps serving from the new data
|
|
@@ -53,6 +51,144 @@ function assertDatabaseUsable(expectedSchemaVersion) {
|
|
|
53
51
|
throw new Error(`restored database schema version mismatch: expected ${expectedSchemaVersion}, got ${row.schemaVersion}`);
|
|
54
52
|
}
|
|
55
53
|
}
|
|
54
|
+
/** Raw filesystem/multipart failures become safe fixed-message errors; validated
|
|
55
|
+
* failures keep their own safe GatewayError. Internal paths never reach the browser. */
|
|
56
|
+
function safeRestoreError(error) {
|
|
57
|
+
if (error instanceof GatewayError)
|
|
58
|
+
return error;
|
|
59
|
+
return new GatewayError('gateway_error', 'Restore failed', { status: 500 });
|
|
60
|
+
}
|
|
61
|
+
function auditRestoreFailure(req, error) {
|
|
62
|
+
const reason = error instanceof GatewayError ? (error.code ?? error.type) : 'restore_error';
|
|
63
|
+
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason } });
|
|
64
|
+
}
|
|
65
|
+
/** Accept both the wrapped `{ backup, passphrase }` legacy body and the bare envelope. */
|
|
66
|
+
function legacyRequestBody(body) {
|
|
67
|
+
const incoming = body;
|
|
68
|
+
const wrapped = Boolean(incoming && typeof incoming === 'object' && 'backup' in incoming);
|
|
69
|
+
const wrappedBody = incoming;
|
|
70
|
+
return { envelope: (wrapped ? wrappedBody.backup : incoming), passphrase: wrapped ? wrappedBody.passphrase : undefined };
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Atomic swap of an already-validated restore payload. Both the v2 multipart path and the legacy
|
|
74
|
+
* v1 JSON path feed this, so the snapshot/rollback, hot reopen, session/CSRF re-seed, and audit
|
|
75
|
+
* behaviour stay identical regardless of container format.
|
|
76
|
+
*/
|
|
77
|
+
export async function restoreValidatedDatabase(validated, req) {
|
|
78
|
+
const cfg = loadConfig();
|
|
79
|
+
const liveDb = cfg.dbFile;
|
|
80
|
+
const previousMasterKey = cfg.masterKey;
|
|
81
|
+
const restoreKeyPath = path.join(cfg.dataDir, 'master.key.restore');
|
|
82
|
+
const previousRestoreKey = fs.existsSync(restoreKeyPath) ? fs.readFileSync(restoreKeyPath) : null;
|
|
83
|
+
const restoredKey = validated.masterKey?.toString('base64');
|
|
84
|
+
// Snapshot current DB before restore (kept for manual rollback).
|
|
85
|
+
const snapshot = path.join(cfg.dataDir, `pre-restore-${Date.now()}.sqlite`);
|
|
86
|
+
const Database = (await import('better-sqlite3')).default;
|
|
87
|
+
const live = new Database(liveDb);
|
|
88
|
+
try {
|
|
89
|
+
await live.backup(snapshot);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
live.close();
|
|
93
|
+
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'snapshot_failed' } });
|
|
94
|
+
throw new GatewayError('gateway_error', 'Could not snapshot current database', { status: 500 });
|
|
95
|
+
}
|
|
96
|
+
live.close();
|
|
97
|
+
// Close the in-process connection before swapping the file. The gateway
|
|
98
|
+
// keeps serving (no restart) but the file must be free: on Windows a
|
|
99
|
+
// rename fails while a handle is open. Reopening runs migrations + the
|
|
100
|
+
// app_settings bootstrap automatically.
|
|
101
|
+
getDb();
|
|
102
|
+
closeDb();
|
|
103
|
+
// Atomic replace.
|
|
104
|
+
const tempDb = `${liveDb}.restore-${Date.now()}`;
|
|
105
|
+
fs.copyFileSync(validated.sqlitePath, tempDb);
|
|
106
|
+
try {
|
|
107
|
+
fs.renameSync(tempDb, liveDb);
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
fs.rmSync(tempDb, { force: true });
|
|
111
|
+
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'rename_failed' } });
|
|
112
|
+
throw new GatewayError('gateway_error', 'Restore atomic replace failed', { status: 500 });
|
|
113
|
+
}
|
|
114
|
+
// The backup key is authoritative. Apply it before reopening the restored
|
|
115
|
+
// database so every post-restore read uses the key that encrypted it.
|
|
116
|
+
if (restoredKey) {
|
|
117
|
+
fs.writeFileSync(restoreKeyPath, restoredKey, { mode: 0o600, encoding: 'utf8' });
|
|
118
|
+
setConfigMasterKey(restoredKey);
|
|
119
|
+
resetMasterKeyCache();
|
|
120
|
+
}
|
|
121
|
+
// Hot-reload the database in-process: reopen the replaced file, validate
|
|
122
|
+
// it, and keep serving. No gateway restart needed.
|
|
123
|
+
try {
|
|
124
|
+
reopenDatabase(liveDb);
|
|
125
|
+
assertDatabaseUsable(validated.schemaVersion);
|
|
126
|
+
}
|
|
127
|
+
catch (e) {
|
|
128
|
+
// Roll back to the snapshot taken before the restore so the gateway
|
|
129
|
+
// never stays on a broken database.
|
|
130
|
+
try {
|
|
131
|
+
closeDb();
|
|
132
|
+
}
|
|
133
|
+
catch { /* ignore */ }
|
|
134
|
+
try {
|
|
135
|
+
for (const suffix of ['-wal', '-shm']) {
|
|
136
|
+
const stale = path.join(path.dirname(liveDb), `${path.basename(liveDb)}${suffix}`);
|
|
137
|
+
if (fs.existsSync(stale))
|
|
138
|
+
fs.unlinkSync(stale);
|
|
139
|
+
}
|
|
140
|
+
fs.copyFileSync(snapshot, liveDb);
|
|
141
|
+
openDb(liveDb);
|
|
142
|
+
try {
|
|
143
|
+
if (previousRestoreKey)
|
|
144
|
+
fs.writeFileSync(restoreKeyPath, previousRestoreKey, { mode: 0o600 });
|
|
145
|
+
else
|
|
146
|
+
fs.unlinkSync(restoreKeyPath);
|
|
147
|
+
}
|
|
148
|
+
catch { /* no prior override */ }
|
|
149
|
+
if (previousMasterKey)
|
|
150
|
+
setConfigMasterKey(previousMasterKey);
|
|
151
|
+
resetMasterKeyCache();
|
|
152
|
+
}
|
|
153
|
+
catch (rollbackErr) {
|
|
154
|
+
const err = e.message;
|
|
155
|
+
const rerr = rollbackErr.message;
|
|
156
|
+
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'rollback_failed', err, rollbackErr: rerr } });
|
|
157
|
+
throw new GatewayError('gateway_error', `Restore failed (${err}) and automatic rollback also failed (${rerr}). Please restart the gateway.`, { status: 500 });
|
|
158
|
+
}
|
|
159
|
+
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'validation_failed', err: e.message } });
|
|
160
|
+
throw new GatewayError('gateway_error', `Restore failed: ${e.message}`, { status: 500 });
|
|
161
|
+
}
|
|
162
|
+
// The swap invalidates the previous admin session (its row lived in the
|
|
163
|
+
// old database). Re-create the current session in the restored database so
|
|
164
|
+
// the admin stays logged in across the hot restore.
|
|
165
|
+
const sessionToken = req.cookies['ld_session'];
|
|
166
|
+
const sessionId = req.adminSessionId;
|
|
167
|
+
if (sessionId && sessionToken) {
|
|
168
|
+
const db = getDb();
|
|
169
|
+
const expiresAt = new Date(Date.now() + 12 * 3600 * 1000).toISOString();
|
|
170
|
+
db.delete(schema.adminSessions).where(sql `id = ${sessionId}`).run();
|
|
171
|
+
db.insert(schema.adminSessions).values({
|
|
172
|
+
id: sessionId,
|
|
173
|
+
tokenDigest: sha256Hex(sessionToken),
|
|
174
|
+
expiresAt,
|
|
175
|
+
lastSeenAt: new Date().toISOString(),
|
|
176
|
+
ip: req.ip,
|
|
177
|
+
}).run();
|
|
178
|
+
// Re-seed the CSRF token that was bound to the old session.
|
|
179
|
+
const csrfRow = db.select().from(schema.csrfTokens).where(eq(schema.csrfTokens.sessionId, sessionId)).get();
|
|
180
|
+
if (!csrfRow) {
|
|
181
|
+
db.insert(schema.csrfTokens).values({
|
|
182
|
+
id: crypto.randomUUID(),
|
|
183
|
+
sessionId,
|
|
184
|
+
token: crypto.randomBytes(32).toString('base64url'),
|
|
185
|
+
expiresAt,
|
|
186
|
+
}).run();
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
recordAudit({ action: 'db.restore', success: true, ip: req.ip, metadata: { profile: validated.profile, schemaVersion: validated.schemaVersion } });
|
|
190
|
+
return { profile: validated.profile, message: 'Database restored. The gateway continues running with the restored data — no restart needed.' };
|
|
191
|
+
}
|
|
56
192
|
export async function registerBackupRoutes(app) {
|
|
57
193
|
// This module is registered in its own scope (routes/admin.ts) precisely so this is the only
|
|
58
194
|
// auth hook on the backup routes. A preHandler cannot cancel a later one, so the first-run
|
|
@@ -64,208 +200,58 @@ export async function registerBackupRoutes(app) {
|
|
|
64
200
|
if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(req.method))
|
|
65
201
|
await requireAdminCsrf(req);
|
|
66
202
|
});
|
|
203
|
+
/** UI preflight: the effective server-side maximum upload size. */
|
|
204
|
+
app.get('/api/admin/backup/capabilities', async () => ({ maxUploadBytes: loadConfig().maxBackupUploadBytes }));
|
|
67
205
|
app.post('/api/admin/backup/create', async (req, reply) => {
|
|
68
|
-
const
|
|
206
|
+
const body = req.body ?? {};
|
|
207
|
+
const passphrase = body.passphrase ?? '';
|
|
69
208
|
if (!/^\d{6}$/.test(passphrase)) {
|
|
70
209
|
throw new GatewayError('invalid_request_error', 'Backup passphrase must contain exactly six digits', { status: 400 });
|
|
71
210
|
}
|
|
72
|
-
const
|
|
73
|
-
const
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
try {
|
|
77
|
-
await source.backup(temp);
|
|
78
|
-
}
|
|
79
|
-
finally {
|
|
80
|
-
source.close();
|
|
81
|
-
}
|
|
82
|
-
const buf = fs.readFileSync(temp);
|
|
83
|
-
fs.unlinkSync(temp);
|
|
84
|
-
const compressed = zlib.gzipSync(buf, { level: 6 });
|
|
85
|
-
const checksum = crypto.createHash('sha256').update(compressed).digest('hex');
|
|
86
|
-
const settings = getSettings();
|
|
87
|
-
const masterKey = parseMasterKey(cfg.masterKey ?? fs.readFileSync(path.join(cfg.dataDir, 'master.key'), 'utf8').trim());
|
|
88
|
-
const envelope = {
|
|
89
|
-
format: 'latedev-backup',
|
|
90
|
-
version: BACKUP_VERSION,
|
|
91
|
-
appVersion: getAppVersion(),
|
|
92
|
-
schemaVersion: settings.schemaVersion,
|
|
93
|
-
masterKeyConfigured: settings.masterKeyConfigured,
|
|
94
|
-
createdAt: new Date().toISOString(),
|
|
95
|
-
payload: compressed.toString('base64'),
|
|
96
|
-
checksum,
|
|
97
|
-
keyEnvelope: encryptBackupMasterKey(masterKey, passphrase),
|
|
98
|
-
};
|
|
99
|
-
const envBuf = Buffer.from(JSON.stringify(envelope), 'utf8');
|
|
100
|
-
const fileName = `latedev-backup-${new Date().toISOString().replace(/[:.]/g, '-')}.ldb.json`;
|
|
101
|
-
reply.header('content-type', 'application/json');
|
|
211
|
+
const profile = body.profile === 'full' ? 'full' : 'compact';
|
|
212
|
+
const { manifest, body: payload } = await createBackupExport({ profile, passphrase, cfg: loadConfig() });
|
|
213
|
+
const fileName = `latedev-backup-${new Date().toISOString().replace(/[:.]/g, '-')}.ldb`;
|
|
214
|
+
reply.header('content-type', 'application/octet-stream');
|
|
102
215
|
reply.header('content-disposition', `attachment; filename="${fileName}"`);
|
|
103
|
-
recordAudit({
|
|
104
|
-
|
|
216
|
+
recordAudit({
|
|
217
|
+
action: 'db.backup_download',
|
|
218
|
+
success: true,
|
|
219
|
+
ip: req.ip,
|
|
220
|
+
metadata: { profile, sizeBytes: manifest.compressedBytes, checksum: manifest.checksum, excludedTables: manifest.excludedTables },
|
|
221
|
+
});
|
|
222
|
+
return reply.send(payload);
|
|
105
223
|
});
|
|
106
224
|
app.post('/api/admin/backup/restore', async (req, reply) => {
|
|
107
225
|
const cfg = loadConfig();
|
|
108
|
-
const
|
|
109
|
-
|
|
110
|
-
const wrappedBody = incoming;
|
|
111
|
-
const passphrase = wrapped ? wrappedBody.passphrase : undefined;
|
|
112
|
-
let envelope;
|
|
113
|
-
try {
|
|
114
|
-
envelope = (wrapped ? wrappedBody.backup : incoming);
|
|
115
|
-
if (!envelope || envelope.format !== 'latedev-backup')
|
|
116
|
-
throw new Error('not a backup envelope');
|
|
117
|
-
if (envelope.keyEnvelope && !/^\d{6}$/.test(passphrase ?? ''))
|
|
118
|
-
throw new Error('backup passphrase required');
|
|
119
|
-
}
|
|
120
|
-
catch {
|
|
121
|
-
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'invalid_envelope' } });
|
|
122
|
-
throw new GatewayError('invalid_request_error', 'Invalid backup file format', { status: 400 });
|
|
123
|
-
}
|
|
124
|
-
if (envelope.version > BACKUP_VERSION) {
|
|
125
|
-
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'future_version', version: envelope.version } });
|
|
126
|
-
throw new GatewayError('invalid_request_error', `Backup version ${envelope.version} is not supported (max ${BACKUP_VERSION})`, { status: 400 });
|
|
127
|
-
}
|
|
128
|
-
const compressed = Buffer.from(envelope.payload, 'base64');
|
|
129
|
-
const checksum = crypto.createHash('sha256').update(compressed).digest('hex');
|
|
130
|
-
if (checksum !== envelope.checksum) {
|
|
131
|
-
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'checksum_mismatch' } });
|
|
132
|
-
throw new GatewayError('invalid_request_error', 'Backup checksum mismatch', { status: 400 });
|
|
133
|
-
}
|
|
134
|
-
let buf;
|
|
226
|
+
const contentType = String(req.headers['content-type'] ?? '');
|
|
227
|
+
let upload;
|
|
135
228
|
try {
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
catch {
|
|
139
|
-
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'decompress_failed' } });
|
|
140
|
-
throw new GatewayError('invalid_request_error', 'Backup decompression failed', { status: 400 });
|
|
141
|
-
}
|
|
142
|
-
// Validate SQLite header
|
|
143
|
-
if (!(buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4c && buf[3] === 0x69 && buf[4] === 0x74 && buf[5] === 0x65)) {
|
|
144
|
-
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'not_sqlite' } });
|
|
145
|
-
throw new GatewayError('invalid_request_error', 'Backup does not contain a valid SQLite database', { status: 400 });
|
|
146
|
-
}
|
|
147
|
-
let restoredMasterKey;
|
|
148
|
-
if (envelope.keyEnvelope) {
|
|
149
|
-
try {
|
|
150
|
-
restoredMasterKey = decryptBackupMasterKey(envelope.keyEnvelope, passphrase);
|
|
151
|
-
parseMasterKey(restoredMasterKey.toString('base64'));
|
|
229
|
+
if (contentType.includes('multipart/form-data')) {
|
|
230
|
+
upload = await receiveBackupUpload(req, { cfg, maxBytes: cfg.maxBackupUploadBytes });
|
|
152
231
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
232
|
+
else {
|
|
233
|
+
const { envelope, passphrase } = legacyRequestBody(req.body);
|
|
234
|
+
upload = await writeLegacyEnvelopeUpload(envelope, cfg, passphrase);
|
|
156
235
|
}
|
|
157
236
|
}
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
const previousRestoreKey = fs.existsSync(restoreKeyPath) ? fs.readFileSync(restoreKeyPath) : null;
|
|
162
|
-
const restoredKey = restoredMasterKey?.toString('base64');
|
|
163
|
-
// Snapshot current DB before restore (kept for manual rollback).
|
|
164
|
-
const snapshot = path.join(cfg.dataDir, `pre-restore-${Date.now()}.sqlite`);
|
|
165
|
-
const Database = (await import('better-sqlite3')).default;
|
|
166
|
-
const live = new Database(liveDb);
|
|
167
|
-
try {
|
|
168
|
-
await live.backup(snapshot);
|
|
169
|
-
}
|
|
170
|
-
catch (e) {
|
|
171
|
-
live.close();
|
|
172
|
-
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'snapshot_failed', err: String(e) } });
|
|
173
|
-
throw new GatewayError('gateway_error', 'Could not snapshot current database', { status: 500 });
|
|
237
|
+
catch (error) {
|
|
238
|
+
auditRestoreFailure(req, error);
|
|
239
|
+
throw safeRestoreError(error);
|
|
174
240
|
}
|
|
175
|
-
|
|
176
|
-
// Close the in-process connection before swapping the file. The gateway
|
|
177
|
-
// keeps serving (no restart) but the file must be free: on Windows a
|
|
178
|
-
// rename fails while a handle is open. Reopening runs migrations + the
|
|
179
|
-
// app_settings bootstrap automatically.
|
|
180
|
-
getDb();
|
|
181
|
-
closeDb();
|
|
182
|
-
// Atomic replace.
|
|
183
|
-
const tempDb = `${liveDb}.restore-${Date.now()}`;
|
|
184
|
-
fs.writeFileSync(tempDb, buf);
|
|
241
|
+
let validated;
|
|
185
242
|
try {
|
|
186
|
-
|
|
243
|
+
validated = await validateUploadedBackup(upload, upload.passphrase);
|
|
244
|
+
const result = await restoreValidatedDatabase(validated, req);
|
|
245
|
+
return reply.code(200).send({ ok: true, profile: result.profile, message: result.message });
|
|
187
246
|
}
|
|
188
|
-
catch (
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
throw new GatewayError('gateway_error', 'Restore atomic replace failed', { status: 500 });
|
|
247
|
+
catch (error) {
|
|
248
|
+
auditRestoreFailure(req, error);
|
|
249
|
+
throw safeRestoreError(error);
|
|
192
250
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
setConfigMasterKey(restoredKey);
|
|
198
|
-
resetMasterKeyCache();
|
|
199
|
-
}
|
|
200
|
-
// Hot-reload the database in-process: reopen the replaced file, validate
|
|
201
|
-
// it, and keep serving. No gateway restart needed.
|
|
202
|
-
try {
|
|
203
|
-
reopenDatabase(liveDb);
|
|
204
|
-
assertDatabaseUsable(envelope.schemaVersion);
|
|
205
|
-
}
|
|
206
|
-
catch (e) {
|
|
207
|
-
// Roll back to the snapshot taken before the restore so the gateway
|
|
208
|
-
// never stays on a broken database.
|
|
209
|
-
try {
|
|
210
|
-
closeDb();
|
|
211
|
-
}
|
|
212
|
-
catch { /* ignore */ }
|
|
213
|
-
try {
|
|
214
|
-
for (const suffix of ['-wal', '-shm']) {
|
|
215
|
-
const stale = path.join(path.dirname(liveDb), `${path.basename(liveDb)}${suffix}`);
|
|
216
|
-
if (fs.existsSync(stale))
|
|
217
|
-
fs.unlinkSync(stale);
|
|
218
|
-
}
|
|
219
|
-
fs.copyFileSync(snapshot, liveDb);
|
|
220
|
-
openDb(liveDb);
|
|
221
|
-
try {
|
|
222
|
-
if (previousRestoreKey)
|
|
223
|
-
fs.writeFileSync(restoreKeyPath, previousRestoreKey, { mode: 0o600 });
|
|
224
|
-
else
|
|
225
|
-
fs.unlinkSync(restoreKeyPath);
|
|
226
|
-
}
|
|
227
|
-
catch { /* no prior override */ }
|
|
228
|
-
if (previousMasterKey)
|
|
229
|
-
setConfigMasterKey(previousMasterKey);
|
|
230
|
-
resetMasterKeyCache();
|
|
231
|
-
}
|
|
232
|
-
catch (rollbackErr) {
|
|
233
|
-
const err = e.message;
|
|
234
|
-
const rerr = rollbackErr.message;
|
|
235
|
-
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'rollback_failed', err, rollbackErr: rerr } });
|
|
236
|
-
throw new GatewayError('gateway_error', `Restore failed (${err}) and automatic rollback also failed (${rerr}). Please restart the gateway.`, { status: 500 });
|
|
237
|
-
}
|
|
238
|
-
recordAudit({ action: 'db.restore', success: false, ip: req.ip, metadata: { reason: 'validation_failed', err: e.message } });
|
|
239
|
-
throw new GatewayError('gateway_error', `Restore failed: ${e.message}`, { status: 500 });
|
|
240
|
-
}
|
|
241
|
-
// The swap invalidates the previous admin session (its row lived in the
|
|
242
|
-
// old database). Re-create the current session in the restored database so
|
|
243
|
-
// the admin stays logged in across the hot restore.
|
|
244
|
-
const sessionToken = req.cookies['ld_session'];
|
|
245
|
-
const sessionId = req.adminSessionId;
|
|
246
|
-
if (sessionId && sessionToken) {
|
|
247
|
-
const db = getDb();
|
|
248
|
-
const expiresAt = new Date(Date.now() + 12 * 3600 * 1000).toISOString();
|
|
249
|
-
db.delete(schema.adminSessions).where(sql `id = ${sessionId}`).run();
|
|
250
|
-
db.insert(schema.adminSessions).values({
|
|
251
|
-
id: sessionId,
|
|
252
|
-
tokenDigest: sha256Hex(sessionToken),
|
|
253
|
-
expiresAt,
|
|
254
|
-
lastSeenAt: new Date().toISOString(),
|
|
255
|
-
ip: req.ip,
|
|
256
|
-
}).run();
|
|
257
|
-
// Re-seed the CSRF token that was bound to the old session.
|
|
258
|
-
const csrfRow = db.select().from(schema.csrfTokens).where(eq(schema.csrfTokens.sessionId, sessionId)).get();
|
|
259
|
-
if (!csrfRow) {
|
|
260
|
-
db.insert(schema.csrfTokens).values({
|
|
261
|
-
id: crypto.randomUUID(),
|
|
262
|
-
sessionId,
|
|
263
|
-
token: crypto.randomBytes(32).toString('base64url'),
|
|
264
|
-
expiresAt,
|
|
265
|
-
}).run();
|
|
266
|
-
}
|
|
251
|
+
finally {
|
|
252
|
+
// Temp upload + decompressed payload are removed on success, failure, and abort.
|
|
253
|
+
await validated?.cleanup();
|
|
254
|
+
await upload.cleanup();
|
|
267
255
|
}
|
|
268
|
-
recordAudit({ action: 'db.restore', success: true, ip: req.ip, metadata: { schemaVersion: envelope.schemaVersion } });
|
|
269
|
-
return reply.code(200).send({ ok: true, message: 'Database restored. The gateway continues running with the restored data — no restart needed.' });
|
|
270
256
|
});
|
|
271
257
|
}
|