@hearthlog/ledger 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +235 -0
- package/README.md +173 -2
- package/contracts/append-hash.json +30 -0
- package/docs/database-structure.md +176 -0
- package/docs/event-store-service-design.md +124 -0
- package/docs/event-store-service.http +182 -0
- package/docs/event-store-service.openapi.yaml +1651 -0
- package/package.json +43 -3
- package/scripts/logical-backup.ts +656 -0
- package/src/action-batches.ts +65 -0
- package/src/app-registry.ts +479 -0
- package/src/auth.ts +186 -0
- package/src/contracts.ts +251 -0
- package/src/domain.ts +245 -0
- package/src/errors.ts +50 -0
- package/src/event-validation.ts +181 -0
- package/src/fence.ts +76 -0
- package/src/http.ts +656 -0
- package/src/index.ts +3 -0
- package/src/ledger-app.ts +2057 -0
- package/src/rpc.ts +42 -0
- package/src/schema.ts +236 -0
- package/src/schemas.ts +429 -0
- package/src/sqlite-export.ts +136 -0
- package/src/transfer.ts +219 -0
package/package.json
CHANGED
|
@@ -1,6 +1,46 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hearthlog/ledger",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Hearthlog's Ledger: the event store Worker (Elysia over SQLite Durable Objects), its contracts and tests, and the logical backup tooling",
|
|
5
|
+
"license": "AGPL-3.0-only",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/hearthlog/hearthlog.git",
|
|
9
|
+
"directory": "packages/ledger"
|
|
10
|
+
},
|
|
11
|
+
"publishConfig": {
|
|
12
|
+
"access": "public",
|
|
13
|
+
"provenance": true
|
|
14
|
+
},
|
|
15
|
+
"files": [
|
|
16
|
+
"src",
|
|
17
|
+
"contracts",
|
|
18
|
+
"scripts",
|
|
19
|
+
"docs",
|
|
20
|
+
"!**/*.test.ts"
|
|
21
|
+
],
|
|
22
|
+
"type": "module",
|
|
23
|
+
"exports": {
|
|
24
|
+
"./*.js": "./src/*.ts",
|
|
25
|
+
"./scripts/*.js": "./scripts/*.ts"
|
|
26
|
+
},
|
|
27
|
+
"imports": {
|
|
28
|
+
"#generated/event-schemas": "@hearthlog/generated/event-schemas"
|
|
29
|
+
},
|
|
30
|
+
"dependencies": {
|
|
31
|
+
"@sinclair/typebox": "0.34.52",
|
|
32
|
+
"elysia": "1.4.30",
|
|
33
|
+
"@hearthlog/contracts": "0.1.0"
|
|
34
|
+
},
|
|
35
|
+
"peerDependencies": {
|
|
36
|
+
"@hearthlog/generated": "*"
|
|
37
|
+
},
|
|
38
|
+
"peerDependenciesMeta": {
|
|
39
|
+
"@hearthlog/generated": {
|
|
40
|
+
"optional": true
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"devDependencies": {
|
|
44
|
+
"@hearthlog/generated": "0.1.0"
|
|
45
|
+
}
|
|
6
46
|
}
|
|
@@ -0,0 +1,656 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { actionBatches, authorRuns, restoreBatchRequest } from '../src/action-batches.js';
|
|
3
|
+
|
|
4
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
5
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
|
|
8
|
+
import type {
|
|
9
|
+
AdminEventsResponse,
|
|
10
|
+
ApplicationTokenResponse,
|
|
11
|
+
AppendEventsResponse,
|
|
12
|
+
CreateLogResponse,
|
|
13
|
+
EventBatchResponse,
|
|
14
|
+
JsonObject,
|
|
15
|
+
LogListResponse,
|
|
16
|
+
LogSummary,
|
|
17
|
+
SessionResponse,
|
|
18
|
+
StoredEvent,
|
|
19
|
+
} from '../src/schemas.js';
|
|
20
|
+
|
|
21
|
+
const FORMAT = 'ledger-logical-export';
|
|
22
|
+
const FORMAT_VERSION = 1;
|
|
23
|
+
const PAGE_SIZE = 512;
|
|
24
|
+
|
|
25
|
+
const [command, ...rawArgs] = process.argv.slice(2);
|
|
26
|
+
const options = parseOptions(rawArgs);
|
|
27
|
+
|
|
28
|
+
interface ApiOptions {
|
|
29
|
+
method?: string;
|
|
30
|
+
body?: unknown;
|
|
31
|
+
headers?: Record<string, string>;
|
|
32
|
+
allowFailure?: boolean;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface ExportPage {
|
|
36
|
+
file: string;
|
|
37
|
+
afterVersion: number;
|
|
38
|
+
throughVersion: number;
|
|
39
|
+
count: number;
|
|
40
|
+
sha256: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
interface ExportedLog {
|
|
44
|
+
sourceLogId: string;
|
|
45
|
+
name: string;
|
|
46
|
+
headVersion: number;
|
|
47
|
+
parentSourceLogId?: string | null;
|
|
48
|
+
sourceVersion?: number | null;
|
|
49
|
+
/** The log was hidden; a restore hides it again (at the restore time). */
|
|
50
|
+
hidden?: boolean;
|
|
51
|
+
pages: ExportPage[];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface UnsignedManifest {
|
|
55
|
+
format: typeof FORMAT;
|
|
56
|
+
version: typeof FORMAT_VERSION;
|
|
57
|
+
exportedAt: string;
|
|
58
|
+
sourceAppId: string;
|
|
59
|
+
pageSize: typeof PAGE_SIZE;
|
|
60
|
+
logs: ExportedLog[];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface ExportManifest extends UnsignedManifest {
|
|
64
|
+
manifestSha256: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
try {
|
|
68
|
+
if (command === 'export') {
|
|
69
|
+
await exportLedger();
|
|
70
|
+
} else if (command === 'restore') {
|
|
71
|
+
await restoreLedger();
|
|
72
|
+
} else {
|
|
73
|
+
usage();
|
|
74
|
+
process.exitCode = 2;
|
|
75
|
+
}
|
|
76
|
+
} catch (error) {
|
|
77
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
78
|
+
process.exitCode = 1;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Reads one application's logs and events. With `--app <id>` it reads through
|
|
83
|
+
* the administrator API (LEDGER_ADMIN_TOKEN), so the application's own token is
|
|
84
|
+
* neither needed nor rotated; otherwise it reads with LEDGER_AUTH_TOKEN.
|
|
85
|
+
*/
|
|
86
|
+
interface ExportSource {
|
|
87
|
+
appId: string;
|
|
88
|
+
logs: LogSummary[];
|
|
89
|
+
/** Events after `afterVersion`, ascending, at most up to the log head. */
|
|
90
|
+
events(log: LogSummary, afterVersion: number): Promise<StoredEvent[]>;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function appTokenSource(baseUrl: string, token: string): Promise<ExportSource> {
|
|
94
|
+
const session = await api<SessionResponse>(baseUrl, '/session', token);
|
|
95
|
+
const { logs } = await api<LogListResponse>(baseUrl, '/logs', token);
|
|
96
|
+
return {
|
|
97
|
+
appId: session.appId,
|
|
98
|
+
logs,
|
|
99
|
+
async events(log, afterVersion) {
|
|
100
|
+
const response = await api<EventBatchResponse>(
|
|
101
|
+
baseUrl,
|
|
102
|
+
`/logs/${log.id}/events?afterVersion=${afterVersion}&limit=${PAGE_SIZE}`,
|
|
103
|
+
token,
|
|
104
|
+
);
|
|
105
|
+
return response.events.filter((event) => event.version <= log.headVersion);
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function administratorSource(baseUrl: string, adminToken: string, appId: string): Promise<ExportSource> {
|
|
111
|
+
const { logs } = await api<LogListResponse>(baseUrl, `/admin/v1/apps/${appId}/logs`, adminToken);
|
|
112
|
+
const cache = new Map<string, StoredEvent[]>();
|
|
113
|
+
return {
|
|
114
|
+
appId,
|
|
115
|
+
logs,
|
|
116
|
+
async events(log, afterVersion) {
|
|
117
|
+
let all = cache.get(log.id);
|
|
118
|
+
if (!all) {
|
|
119
|
+
// The administrator API pages newest first, 100 at a time.
|
|
120
|
+
all = [];
|
|
121
|
+
let before: number | null = log.headVersion + 1;
|
|
122
|
+
while (before !== null && before > 1) {
|
|
123
|
+
const response: AdminEventsResponse = await api<AdminEventsResponse>(
|
|
124
|
+
baseUrl,
|
|
125
|
+
`/admin/v1/apps/${appId}/logs/${log.id}/events?beforeVersion=${before}&limit=100`,
|
|
126
|
+
adminToken,
|
|
127
|
+
);
|
|
128
|
+
all.push(...response.events);
|
|
129
|
+
before = response.events.length === 0 ? null : response.nextBeforeVersion;
|
|
130
|
+
}
|
|
131
|
+
all = all.filter((event) => event.version <= log.headVersion).sort((a, b) => a.version - b.version);
|
|
132
|
+
cache.set(log.id, all);
|
|
133
|
+
}
|
|
134
|
+
return all.filter((event) => event.version > afterVersion).slice(0, PAGE_SIZE);
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function exportLedger(): Promise<void> {
|
|
140
|
+
const baseUrl = requiredEnv('LEDGER_BASE_URL').replace(/\/$/, '');
|
|
141
|
+
const source = options.app
|
|
142
|
+
? await administratorSource(baseUrl, requiredEnv('LEDGER_ADMIN_TOKEN'), options.app)
|
|
143
|
+
: await appTokenSource(baseUrl, requiredEnv('LEDGER_AUTH_TOKEN'));
|
|
144
|
+
const output = path.resolve(options.output ?? `ledger-export-${Date.now()}`);
|
|
145
|
+
await mkdir(output);
|
|
146
|
+
|
|
147
|
+
const logs = source.logs;
|
|
148
|
+
const manifest: UnsignedManifest = {
|
|
149
|
+
format: FORMAT,
|
|
150
|
+
version: FORMAT_VERSION,
|
|
151
|
+
exportedAt: new Date().toISOString(),
|
|
152
|
+
sourceAppId: source.appId,
|
|
153
|
+
pageSize: PAGE_SIZE,
|
|
154
|
+
logs: [],
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
const skippedLogIds: string[] = [];
|
|
158
|
+
for (const [logIndex, log] of logs.entries()) {
|
|
159
|
+
// A fork waiting for its files cannot be read yet and has no children.
|
|
160
|
+
if (log.status === 'preparing') {
|
|
161
|
+
console.warn(`skipping fork ${log.id}: it is still being prepared`);
|
|
162
|
+
skippedLogIds.push(log.id);
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
const exportedLog: ExportedLog = {
|
|
166
|
+
sourceLogId: log.id,
|
|
167
|
+
name: log.name,
|
|
168
|
+
headVersion: log.headVersion,
|
|
169
|
+
parentSourceLogId: log.sourceLogId ?? null,
|
|
170
|
+
sourceVersion: log.sourceVersion ?? null,
|
|
171
|
+
...(log.hiddenAt ? { hidden: true } : {}),
|
|
172
|
+
pages: [],
|
|
173
|
+
};
|
|
174
|
+
let cursor = 0;
|
|
175
|
+
let pageIndex = 0;
|
|
176
|
+
while (cursor < log.headVersion) {
|
|
177
|
+
const events = await source.events(log, cursor);
|
|
178
|
+
assertContiguous(events, cursor, log.headVersion);
|
|
179
|
+
if (events.length === 0) {
|
|
180
|
+
throw new Error(`export stopped before ${log.id} reached version ${log.headVersion}`);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const content = encodePage(events);
|
|
184
|
+
const file = `log-${logIndex + 1}-page-${pageIndex + 1}.ndjson`;
|
|
185
|
+
await writeFile(path.join(output, file), content, { flag: 'wx', mode: 0o600 });
|
|
186
|
+
const throughVersion = events.at(-1)!.version;
|
|
187
|
+
exportedLog.pages.push({
|
|
188
|
+
file,
|
|
189
|
+
afterVersion: cursor,
|
|
190
|
+
throughVersion,
|
|
191
|
+
count: events.length,
|
|
192
|
+
sha256: sha256(content),
|
|
193
|
+
});
|
|
194
|
+
cursor = throughVersion;
|
|
195
|
+
pageIndex++;
|
|
196
|
+
}
|
|
197
|
+
manifest.logs.push(exportedLog);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const manifestSha256 = sha256(canonicalJson(manifest));
|
|
201
|
+
await writeFile(
|
|
202
|
+
path.join(output, 'manifest.json'),
|
|
203
|
+
`${JSON.stringify({ ...manifest, manifestSha256 }, null, 2)}\n`,
|
|
204
|
+
{ flag: 'wx', mode: 0o600 },
|
|
205
|
+
);
|
|
206
|
+
console.log(JSON.stringify({ output, manifestSha256, logs: manifest.logs.length, skippedLogIds }, null, 2));
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async function restoreLedger(): Promise<void> {
|
|
210
|
+
const baseUrl = requiredEnv('LEDGER_BASE_URL').replace(/\/$/, '');
|
|
211
|
+
const adminToken = requiredEnv('LEDGER_ADMIN_TOKEN');
|
|
212
|
+
const input = path.resolve(requiredOption('input'));
|
|
213
|
+
const { manifest, pagesByLog } = await validateExport(input);
|
|
214
|
+
const name = options.name ?? `restore-${manifest.sourceAppId}-${Date.now()}`;
|
|
215
|
+
|
|
216
|
+
let targetApp: ApplicationTokenResponse | undefined;
|
|
217
|
+
try {
|
|
218
|
+
targetApp = await api<ApplicationTokenResponse>(
|
|
219
|
+
baseUrl,
|
|
220
|
+
'/admin/v1/apps',
|
|
221
|
+
adminToken,
|
|
222
|
+
{
|
|
223
|
+
method: 'POST',
|
|
224
|
+
body: { name },
|
|
225
|
+
},
|
|
226
|
+
);
|
|
227
|
+
const bootstrapToken = targetApp.token;
|
|
228
|
+
const existing = await api<LogListResponse>(
|
|
229
|
+
baseUrl,
|
|
230
|
+
'/logs',
|
|
231
|
+
bootstrapToken,
|
|
232
|
+
);
|
|
233
|
+
if (existing.logs.length !== 0) throw new Error('restore target is not empty');
|
|
234
|
+
|
|
235
|
+
const logMappings: Array<{
|
|
236
|
+
sourceLogId: string;
|
|
237
|
+
targetLogId: string;
|
|
238
|
+
}> = [];
|
|
239
|
+
for (const [logIndex, sourceLog] of manifest.logs.entries()) {
|
|
240
|
+
const targetLog = await api<CreateLogResponse>(
|
|
241
|
+
baseUrl,
|
|
242
|
+
'/logs',
|
|
243
|
+
bootstrapToken,
|
|
244
|
+
{
|
|
245
|
+
method: 'POST',
|
|
246
|
+
body: { name: sourceLog.name },
|
|
247
|
+
},
|
|
248
|
+
);
|
|
249
|
+
logMappings.push({ sourceLogId: sourceLog.sourceLogId, targetLogId: targetLog.id });
|
|
250
|
+
|
|
251
|
+
const sourcePages = pagesByLog[logIndex];
|
|
252
|
+
if (!sourcePages) {
|
|
253
|
+
throw new Error(`missing export pages for ${sourceLog.sourceLogId}`);
|
|
254
|
+
}
|
|
255
|
+
for (const batch of actionBatches(sourcePages.flat())) {
|
|
256
|
+
for (const events of authorRuns(batch)) {
|
|
257
|
+
const response = await api<AppendEventsResponse>(
|
|
258
|
+
baseUrl,
|
|
259
|
+
`/logs/${targetLog.id}/events`,
|
|
260
|
+
bootstrapToken,
|
|
261
|
+
{
|
|
262
|
+
method: 'POST',
|
|
263
|
+
body: restoreBatchRequest(events, randomUUID()),
|
|
264
|
+
// A run is one replayed request, so its events share an author.
|
|
265
|
+
...(events[0]?.actor ? { headers: { 'X-Ledger-Actor': events[0].actor } } : {}),
|
|
266
|
+
},
|
|
267
|
+
);
|
|
268
|
+
const expectedVersion = events.at(-1)!.version;
|
|
269
|
+
if (response.newVersion !== expectedVersion) {
|
|
270
|
+
throw new Error(
|
|
271
|
+
`restore version mismatch for ${sourceLog.sourceLogId}: ` +
|
|
272
|
+
`expected ${expectedVersion}, received ${response.newVersion}`,
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
await verifyRestoredLog(baseUrl, bootstrapToken, targetLog.id, sourceLog);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const mappedLogs = new Map(logMappings.map(({ sourceLogId, targetLogId }) => [sourceLogId, targetLogId]));
|
|
281
|
+
for (const sourceLog of manifest.logs) {
|
|
282
|
+
if (!sourceLog.parentSourceLogId || sourceLog.sourceVersion === null || sourceLog.sourceVersion === undefined) continue;
|
|
283
|
+
const targetLogId = mappedLogs.get(sourceLog.sourceLogId);
|
|
284
|
+
const parentLogId = mappedLogs.get(sourceLog.parentSourceLogId);
|
|
285
|
+
if (!targetLogId || !parentLogId) throw new Error(`missing fork lineage for ${sourceLog.sourceLogId}`);
|
|
286
|
+
await api<LogSummary>(baseUrl,
|
|
287
|
+
`/admin/v1/apps/${targetApp.app.id}/logs/${targetLogId}/lineage`, adminToken,
|
|
288
|
+
{ method: 'POST', body: { sourceLogId: parentLogId, sourceVersion: sourceLog.sourceVersion } },
|
|
289
|
+
);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
for (const sourceLog of manifest.logs) {
|
|
293
|
+
if (!sourceLog.hidden) continue;
|
|
294
|
+
const targetLogId = mappedLogs.get(sourceLog.sourceLogId)!;
|
|
295
|
+
const hidden = await api<LogSummary>(baseUrl, `/logs/${targetLogId}/visibility`, bootstrapToken,
|
|
296
|
+
{ method: 'POST', body: { hidden: true } });
|
|
297
|
+
if (!hidden.hiddenAt) throw new Error(`restored log ${targetLogId} did not stay hidden`);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const activated = await api<ApplicationTokenResponse>(
|
|
301
|
+
baseUrl,
|
|
302
|
+
`/admin/v1/apps/${targetApp.app.id}/rotate-token`,
|
|
303
|
+
adminToken,
|
|
304
|
+
{ method: 'POST' },
|
|
305
|
+
);
|
|
306
|
+
console.log(JSON.stringify({
|
|
307
|
+
app: activated.app,
|
|
308
|
+
token: activated.token,
|
|
309
|
+
sourceAppId: manifest.sourceAppId,
|
|
310
|
+
manifestSha256: manifest.manifestSha256,
|
|
311
|
+
logs: logMappings,
|
|
312
|
+
}, null, 2));
|
|
313
|
+
} catch (error) {
|
|
314
|
+
if (targetApp?.app?.id) {
|
|
315
|
+
await api(
|
|
316
|
+
baseUrl,
|
|
317
|
+
`/admin/v1/apps/${targetApp.app.id}/revoke`,
|
|
318
|
+
adminToken,
|
|
319
|
+
{ method: 'POST', allowFailure: true },
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
throw error;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
async function validateExport(input: string): Promise<{
|
|
327
|
+
manifest: ExportManifest;
|
|
328
|
+
pagesByLog: StoredEvent[][][];
|
|
329
|
+
}> {
|
|
330
|
+
const manifest = parseManifest(
|
|
331
|
+
JSON.parse(
|
|
332
|
+
await readFile(path.join(input, 'manifest.json'), 'utf8'),
|
|
333
|
+
) as unknown,
|
|
334
|
+
);
|
|
335
|
+
if (manifest.format !== FORMAT || manifest.version !== FORMAT_VERSION) {
|
|
336
|
+
throw new Error('unsupported logical export format');
|
|
337
|
+
}
|
|
338
|
+
const { manifestSha256, ...unsignedManifest } = manifest;
|
|
339
|
+
if (sha256(canonicalJson(unsignedManifest)) !== manifestSha256) {
|
|
340
|
+
throw new Error('manifest checksum mismatch');
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const logById = new Map(manifest.logs.map(log => [log.sourceLogId, log]));
|
|
344
|
+
for (const log of manifest.logs) {
|
|
345
|
+
if (!log.parentSourceLogId) {
|
|
346
|
+
if (log.sourceVersion !== undefined && log.sourceVersion !== null) {
|
|
347
|
+
throw new Error(`fork source version lacks a parent: ${log.sourceLogId}`);
|
|
348
|
+
}
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
const parent = logById.get(log.parentSourceLogId);
|
|
352
|
+
if (!parent || log.sourceVersion === undefined || log.sourceVersion === null
|
|
353
|
+
|| log.sourceVersion > parent.headVersion || log.sourceVersion > log.headVersion) {
|
|
354
|
+
throw new Error(`invalid fork lineage: ${log.sourceLogId}`);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const pagesByLog: StoredEvent[][][] = [];
|
|
359
|
+
for (const log of manifest.logs) {
|
|
360
|
+
let cursor = 0;
|
|
361
|
+
const pages: StoredEvent[][] = [];
|
|
362
|
+
for (const page of log.pages) {
|
|
363
|
+
const content = await readFile(safeExportPath(input, page.file), 'utf8');
|
|
364
|
+
if (sha256(content) !== page.sha256) {
|
|
365
|
+
throw new Error(`page checksum mismatch: ${page.file}`);
|
|
366
|
+
}
|
|
367
|
+
const events = content
|
|
368
|
+
.split('\n')
|
|
369
|
+
.filter(Boolean)
|
|
370
|
+
.map((line) => parseStoredEvent(JSON.parse(line) as unknown));
|
|
371
|
+
if (
|
|
372
|
+
page.afterVersion !== cursor ||
|
|
373
|
+
page.count !== events.length ||
|
|
374
|
+
page.throughVersion !== events.at(-1)?.version
|
|
375
|
+
) {
|
|
376
|
+
throw new Error(`invalid page metadata: ${page.file}`);
|
|
377
|
+
}
|
|
378
|
+
assertContiguous(events, cursor, log.headVersion);
|
|
379
|
+
cursor = page.throughVersion;
|
|
380
|
+
pages.push(events);
|
|
381
|
+
}
|
|
382
|
+
if (cursor !== log.headVersion) {
|
|
383
|
+
throw new Error(`log ${log.sourceLogId} ends at ${cursor}, expected ${log.headVersion}`);
|
|
384
|
+
}
|
|
385
|
+
// Validate complete grouping before provisioning the restore destination.
|
|
386
|
+
for (const _batch of actionBatches(pages.flat())) { /* validate */ }
|
|
387
|
+
pagesByLog.push(pages);
|
|
388
|
+
}
|
|
389
|
+
return { manifest, pagesByLog };
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
async function verifyRestoredLog(
|
|
393
|
+
baseUrl: string,
|
|
394
|
+
token: string,
|
|
395
|
+
targetLogId: string,
|
|
396
|
+
sourceLog: ExportedLog,
|
|
397
|
+
): Promise<void> {
|
|
398
|
+
for (const page of sourceLog.pages) {
|
|
399
|
+
const response = await api<EventBatchResponse>(
|
|
400
|
+
baseUrl,
|
|
401
|
+
`/logs/${targetLogId}/events?afterVersion=${page.afterVersion}&limit=${page.count}`,
|
|
402
|
+
token,
|
|
403
|
+
);
|
|
404
|
+
if (response.events.length !== page.count || sha256(encodePage(response.events)) !== page.sha256) {
|
|
405
|
+
throw new Error(`post-restore verification failed for ${page.file}`);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
const target = await api<LogSummary>(
|
|
409
|
+
baseUrl,
|
|
410
|
+
`/logs/${targetLogId}`,
|
|
411
|
+
token,
|
|
412
|
+
);
|
|
413
|
+
if (target.headVersion !== sourceLog.headVersion) {
|
|
414
|
+
throw new Error(`restored log head mismatch for ${sourceLog.sourceLogId}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
async function api<T>(
|
|
419
|
+
baseUrl: string,
|
|
420
|
+
endpoint: string,
|
|
421
|
+
token: string,
|
|
422
|
+
options: ApiOptions = {},
|
|
423
|
+
): Promise<T> {
|
|
424
|
+
const response = await fetch(`${baseUrl}${endpoint}`, {
|
|
425
|
+
method: options.method ?? 'GET',
|
|
426
|
+
headers: {
|
|
427
|
+
Authorization: `Bearer ${token}`,
|
|
428
|
+
"X-Ledger-Replay-Version": "2",
|
|
429
|
+
...options.headers,
|
|
430
|
+
...(options.body === undefined ? {} : { 'Content-Type': 'application/json' }),
|
|
431
|
+
},
|
|
432
|
+
body: options.body === undefined ? undefined : JSON.stringify(options.body),
|
|
433
|
+
});
|
|
434
|
+
if (!response.ok) {
|
|
435
|
+
if (options.allowFailure) return null as T;
|
|
436
|
+
const details = await response.text();
|
|
437
|
+
throw new Error(`${options.method ?? 'GET'} ${endpoint} failed (${response.status}): ${details}`);
|
|
438
|
+
}
|
|
439
|
+
if (response.status === 204) return null as T;
|
|
440
|
+
return (await response.json()) as T;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function assertContiguous(
|
|
444
|
+
events: readonly StoredEvent[],
|
|
445
|
+
afterVersion: number,
|
|
446
|
+
maximumVersion: number,
|
|
447
|
+
): void {
|
|
448
|
+
for (const [index, event] of events.entries()) {
|
|
449
|
+
const expected = afterVersion + index + 1;
|
|
450
|
+
if (!Number.isSafeInteger(event.version) || event.version !== expected) {
|
|
451
|
+
throw new Error(`non-contiguous export page: expected ${expected}, received ${event.version}`);
|
|
452
|
+
}
|
|
453
|
+
if (event.version > maximumVersion) {
|
|
454
|
+
throw new Error(`event version ${event.version} exceeds snapshot head ${maximumVersion}`);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
function encodePage(events: readonly StoredEvent[]): string {
|
|
460
|
+
return events.map(canonicalJson).join('\n') + (events.length > 0 ? '\n' : '');
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
function canonicalJson(value: unknown): string {
|
|
464
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
|
|
465
|
+
if (value && typeof value === 'object') {
|
|
466
|
+
const record = value as Record<string, unknown>;
|
|
467
|
+
return `{${Object.keys(record).sort().map((key) => (
|
|
468
|
+
`${JSON.stringify(key)}:${canonicalJson(record[key])}`
|
|
469
|
+
)).join(',')}}`;
|
|
470
|
+
}
|
|
471
|
+
const serialized = JSON.stringify(value);
|
|
472
|
+
if (serialized === undefined) throw new Error('value is not JSON serializable');
|
|
473
|
+
return serialized;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
function sha256(value: string): string {
|
|
477
|
+
return createHash('sha256').update(value).digest('hex');
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
function safeExportPath(root: string, file: string): string {
|
|
481
|
+
const resolved = path.resolve(root, file);
|
|
482
|
+
if (path.dirname(resolved) !== root || path.basename(file) !== file) {
|
|
483
|
+
throw new Error(`unsafe export page path: ${file}`);
|
|
484
|
+
}
|
|
485
|
+
return resolved;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function parseOptions(args: string[]): Record<string, string> {
|
|
489
|
+
const parsed: Record<string, string> = {};
|
|
490
|
+
for (let index = 0; index < args.length; index += 2) {
|
|
491
|
+
const key = args[index];
|
|
492
|
+
const value = args[index + 1];
|
|
493
|
+
if (!key?.startsWith('--') || value === undefined) {
|
|
494
|
+
throw new Error(`invalid option list near ${key ?? '<end>'}`);
|
|
495
|
+
}
|
|
496
|
+
parsed[key.slice(2)] = value;
|
|
497
|
+
}
|
|
498
|
+
return parsed;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
function parseManifest(value: unknown): ExportManifest {
|
|
502
|
+
const manifest = recordValue(value, 'manifest');
|
|
503
|
+
if (manifest.format !== FORMAT || manifest.version !== FORMAT_VERSION) {
|
|
504
|
+
throw new Error('unsupported logical export format');
|
|
505
|
+
}
|
|
506
|
+
if (manifest.pageSize !== PAGE_SIZE) {
|
|
507
|
+
throw new Error('unsupported logical export page size');
|
|
508
|
+
}
|
|
509
|
+
return {
|
|
510
|
+
format: FORMAT,
|
|
511
|
+
version: FORMAT_VERSION,
|
|
512
|
+
exportedAt: stringValue(manifest.exportedAt, 'manifest.exportedAt'),
|
|
513
|
+
sourceAppId: stringValue(manifest.sourceAppId, 'manifest.sourceAppId'),
|
|
514
|
+
pageSize: PAGE_SIZE,
|
|
515
|
+
logs: arrayValue(manifest.logs, 'manifest.logs').map(
|
|
516
|
+
(log, index) => parseExportedLog(log, index),
|
|
517
|
+
),
|
|
518
|
+
manifestSha256: stringValue(
|
|
519
|
+
manifest.manifestSha256,
|
|
520
|
+
'manifest.manifestSha256',
|
|
521
|
+
),
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function parseExportedLog(value: unknown, index: number): ExportedLog {
|
|
526
|
+
const label = `manifest.logs[${index}]`;
|
|
527
|
+
const log = recordValue(value, label);
|
|
528
|
+
return {
|
|
529
|
+
sourceLogId: stringValue(log.sourceLogId, `${label}.sourceLogId`),
|
|
530
|
+
name: stringValue(log.name, `${label}.name`),
|
|
531
|
+
headVersion: nonnegativeInteger(
|
|
532
|
+
log.headVersion,
|
|
533
|
+
`${label}.headVersion`,
|
|
534
|
+
),
|
|
535
|
+
...(log.parentSourceLogId !== undefined ? {
|
|
536
|
+
parentSourceLogId: log.parentSourceLogId === null ? null
|
|
537
|
+
: stringValue(log.parentSourceLogId, `${label}.parentSourceLogId`),
|
|
538
|
+
} : {}),
|
|
539
|
+
...(log.sourceVersion !== undefined ? {
|
|
540
|
+
sourceVersion: log.sourceVersion === null ? null
|
|
541
|
+
: nonnegativeInteger(log.sourceVersion, `${label}.sourceVersion`),
|
|
542
|
+
} : {}),
|
|
543
|
+
...(log.hidden !== undefined ? { hidden: booleanValue(log.hidden, `${label}.hidden`) } : {}),
|
|
544
|
+
pages: arrayValue(log.pages, `${label}.pages`).map(
|
|
545
|
+
(page, pageIndex) => parseExportPage(page, label, pageIndex),
|
|
546
|
+
),
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
function parseExportPage(
|
|
551
|
+
value: unknown,
|
|
552
|
+
logLabel: string,
|
|
553
|
+
index: number,
|
|
554
|
+
): ExportPage {
|
|
555
|
+
const label = `${logLabel}.pages[${index}]`;
|
|
556
|
+
const page = recordValue(value, label);
|
|
557
|
+
return {
|
|
558
|
+
file: stringValue(page.file, `${label}.file`),
|
|
559
|
+
afterVersion: nonnegativeInteger(
|
|
560
|
+
page.afterVersion,
|
|
561
|
+
`${label}.afterVersion`,
|
|
562
|
+
),
|
|
563
|
+
throughVersion: nonnegativeInteger(
|
|
564
|
+
page.throughVersion,
|
|
565
|
+
`${label}.throughVersion`,
|
|
566
|
+
),
|
|
567
|
+
count: nonnegativeInteger(page.count, `${label}.count`),
|
|
568
|
+
sha256: stringValue(page.sha256, `${label}.sha256`),
|
|
569
|
+
};
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function parseStoredEvent(value: unknown): StoredEvent {
|
|
573
|
+
const event = recordValue(value, 'stored event');
|
|
574
|
+
const action = event.action === undefined ? undefined : recordValue(event.action, 'stored event.action');
|
|
575
|
+
return {
|
|
576
|
+
...(action ? { action: {
|
|
577
|
+
id: stringValue(action.id, 'action.id'),
|
|
578
|
+
index: nonnegativeInteger(action.index, 'action.index'),
|
|
579
|
+
size: positiveInteger(action.size, 'action.size'),
|
|
580
|
+
} } : {}),
|
|
581
|
+
version: positiveInteger(event.version, 'stored event.version'),
|
|
582
|
+
id: stringValue(event.id, 'stored event.id'),
|
|
583
|
+
type: stringValue(event.type, 'stored event.type'),
|
|
584
|
+
entityId: stringValue(event.entityId, 'stored event.entityId'),
|
|
585
|
+
entityVersion: positiveInteger(
|
|
586
|
+
event.entityVersion,
|
|
587
|
+
'stored event.entityVersion',
|
|
588
|
+
),
|
|
589
|
+
timestamp: nonnegativeInteger(
|
|
590
|
+
event.timestamp,
|
|
591
|
+
'stored event.timestamp',
|
|
592
|
+
),
|
|
593
|
+
payload: recordValue(
|
|
594
|
+
event.payload,
|
|
595
|
+
'stored event.payload',
|
|
596
|
+
) as JsonObject,
|
|
597
|
+
...(event.actor === undefined ? {} : { actor: stringValue(event.actor, 'stored event.actor') }),
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
function recordValue(
|
|
602
|
+
value: unknown,
|
|
603
|
+
label: string,
|
|
604
|
+
): Record<string, unknown> {
|
|
605
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
606
|
+
throw new Error(`${label} must be an object`);
|
|
607
|
+
}
|
|
608
|
+
return value as Record<string, unknown>;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
function arrayValue(value: unknown, label: string): unknown[] {
|
|
612
|
+
if (!Array.isArray(value)) throw new Error(`${label} must be an array`);
|
|
613
|
+
return value;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
function booleanValue(value: unknown, label: string): boolean {
|
|
617
|
+
if (typeof value !== 'boolean') throw new Error(`${label} must be a boolean`);
|
|
618
|
+
return value;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
function stringValue(value: unknown, label: string): string {
|
|
622
|
+
if (typeof value !== 'string') throw new Error(`${label} must be a string`);
|
|
623
|
+
return value;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
function nonnegativeInteger(value: unknown, label: string): number {
|
|
627
|
+
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
|
628
|
+
throw new Error(`${label} must be a non-negative safe integer`);
|
|
629
|
+
}
|
|
630
|
+
return value as number;
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
function positiveInteger(value: unknown, label: string): number {
|
|
634
|
+
const parsed = nonnegativeInteger(value, label);
|
|
635
|
+
if (parsed < 1) throw new Error(`${label} must be positive`);
|
|
636
|
+
return parsed;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
function requiredEnv(name: string): string {
|
|
640
|
+
const value = process.env[name];
|
|
641
|
+
if (!value) throw new Error(`${name} is required`);
|
|
642
|
+
return value;
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function requiredOption(name: string): string {
|
|
646
|
+
const value = options[name];
|
|
647
|
+
if (!value) throw new Error(`--${name} is required`);
|
|
648
|
+
return value;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
function usage(): void {
|
|
652
|
+
console.error(`Usage:
|
|
653
|
+
LEDGER_BASE_URL=... LEDGER_AUTH_TOKEN=... pnpm ledger:export --output DIR
|
|
654
|
+
LEDGER_BASE_URL=... LEDGER_ADMIN_TOKEN=... pnpm ledger:export --app APP_ID --output DIR
|
|
655
|
+
LEDGER_BASE_URL=... LEDGER_ADMIN_TOKEN=... pnpm ledger:restore --input DIR [--name NAME]`);
|
|
656
|
+
}
|