@hearthlog/ledger 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/package.json +2 -2
- package/src/ledger-app.ts +9 -5
- package/src/log-maintenance.ts +159 -168
- package/src/schema.ts +21 -0
package/README.md
CHANGED
|
@@ -208,8 +208,10 @@ Set `LEDGER_BASE_URL` and `LEDGER_AUTH_TOKEN` in the process environment; the CL
|
|
|
208
208
|
Use `--request-id` to retry an explicit operation and `--once` for one bounded work step.
|
|
209
209
|
|
|
210
210
|
A started job is carried to completion by the application object's alarm, with no caller or open browser. Each step
|
|
211
|
-
|
|
212
|
-
|
|
211
|
+
works for a few seconds, folding source events into small per-entity replay slots (source versions only). The whole job's
|
|
212
|
+
slots live in one private R2 object that is read and written once per step, so a snapshot costs a handful of SQL writes
|
|
213
|
+
however many records the log holds. The retained events are then written as checksummed pages. Archiving rewrites each
|
|
214
|
+
archived event row once (its payload moves to R2); the archived prefix is tracked per log rather than indexed per event. A later snapshot starts from the previous one's retained facts instead of re-reading
|
|
213
215
|
archived history, unless a restore/removal marker has been added since. A failing step is retried with backoff; the
|
|
214
216
|
error and attempt count stay visible, and after five attempts the job stops as `failed` until it is explicitly retried
|
|
215
217
|
(the maintenance panel's Retry, `POST /maintenance/step`, or the CLI). A job that uses its daily budget is `paused`
|
|
@@ -237,7 +239,7 @@ payloads into the existing portable format. A SQLite-only export returns `409 ar
|
|
|
237
239
|
external checkpoint/archive objects are required, avoiding a misleading incomplete backup. Rollback after
|
|
238
240
|
compaction needs a compatible reader or a full logical restore first.
|
|
239
241
|
|
|
240
|
-
The first format bounds a single entity's replay state to about 1.5 MB and stores page descriptors in SQLite.
|
|
242
|
+
The first format bounds a single entity's replay state to about 1.5 MB and a job's work state to 64 MB and stores page descriptors in SQLite.
|
|
241
243
|
Exceeding a snapshot work/metadata bound fails the job while source history stays readable. Measure maintenance
|
|
242
244
|
work and storage on representative data; retained identity metadata still grows, and physical SQLite file size
|
|
243
245
|
need not shrink immediately when payload space becomes reusable.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hearthlog/ledger",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.1",
|
|
4
4
|
"description": "Hearthlog's Ledger: the event store Worker (Elysia over SQLite Durable Objects), its contracts and tests, and the logical backup tooling",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"repository": {
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"dependencies": {
|
|
31
31
|
"@sinclair/typebox": "0.34.52",
|
|
32
32
|
"elysia": "1.4.30",
|
|
33
|
-
"@hearthlog/contracts": "0.11.
|
|
33
|
+
"@hearthlog/contracts": "0.11.1"
|
|
34
34
|
},
|
|
35
35
|
"peerDependencies": {
|
|
36
36
|
"@hearthlog/generated": "*"
|
package/src/ledger-app.ts
CHANGED
|
@@ -485,7 +485,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
485
485
|
|
|
486
486
|
async administratorExportSqlite(appId: string): Promise<Response> {
|
|
487
487
|
this.administratorApplication(appId);
|
|
488
|
-
if (this.ctx.storage.sql.exec('SELECT 1 FROM
|
|
488
|
+
if (this.ctx.storage.sql.exec('SELECT 1 FROM log_storage WHERE archived_through > 0 LIMIT 1').toArray().length
|
|
489
489
|
|| this.ctx.storage.sql.exec('SELECT 1 FROM log_storage WHERE checkpoint IS NOT NULL LIMIT 1').toArray().length) {
|
|
490
490
|
return jsonError('archive_bundle_required', 'Use the complete logical export or application transfer; archived payloads are outside SQLite.', 409);
|
|
491
491
|
}
|
|
@@ -910,7 +910,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
910
910
|
throw transferError("not_importing", "Only an import in progress can be aborted");
|
|
911
911
|
}
|
|
912
912
|
this.ctx.storage.transactionSync(() => {
|
|
913
|
-
for (const table of ["
|
|
913
|
+
for (const table of ["log_checkpoint_entities", "log_storage", "log_archive_pages", "log_maintenance_jobs", "gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
|
|
914
914
|
this.ctx.storage.sql.exec(`DELETE FROM ${table}`);
|
|
915
915
|
}
|
|
916
916
|
});
|
|
@@ -1164,6 +1164,12 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1164
1164
|
FROM log_events WHERE log_id = ? AND version <= ? ORDER BY version`,
|
|
1165
1165
|
id, sourceLogId, targetVersion,
|
|
1166
1166
|
);
|
|
1167
|
+
// A fork shares its source's archived prefix up to the fork point.
|
|
1168
|
+
this.ctx.storage.sql.exec(
|
|
1169
|
+
`INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint, archived_through)
|
|
1170
|
+
SELECT ?, 0, 0, NULL, MIN(archived_through, ?) FROM log_storage WHERE log_id = ? AND archived_through > 0`,
|
|
1171
|
+
id, targetVersion, sourceLogId,
|
|
1172
|
+
);
|
|
1167
1173
|
this.ctx.storage.sql.exec(
|
|
1168
1174
|
`INSERT INTO entity_claims (log_id, entity_id, current_version, version_created, created_at, updated_at)
|
|
1169
1175
|
SELECT ?, source.entity_id, source.entity_version, first_version.first_version, ?, ?
|
|
@@ -1224,8 +1230,6 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1224
1230
|
}
|
|
1225
1231
|
for (const query of [
|
|
1226
1232
|
"DELETE FROM log_checkpoint_entities WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
|
|
1227
|
-
"DELETE FROM log_checkpoint_work WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
|
|
1228
|
-
"DELETE FROM log_checkpoint_versions WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
|
|
1229
1233
|
"DELETE FROM log_fork_requests WHERE fork_log_id = ?",
|
|
1230
1234
|
"DELETE FROM append_requests WHERE log_id = ?",
|
|
1231
1235
|
"DELETE FROM log_storage WHERE log_id = ?",
|
|
@@ -1658,7 +1662,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1658
1662
|
return;
|
|
1659
1663
|
}
|
|
1660
1664
|
const head = this.logHead(attachment.logId);
|
|
1661
|
-
const archived = this.ctx.storage.sql.exec<{
|
|
1665
|
+
const archived = this.ctx.storage.sql.exec<{ archived_through: number }>('SELECT archived_through FROM log_storage WHERE log_id = ?', attachment.logId).toArray()[0]?.archived_through ?? 0;
|
|
1662
1666
|
if (attachment.cursor < archived) {
|
|
1663
1667
|
this.sendFrame(ws, { type: 'resync_required', headVersion: head, reason: 'gap_too_large' });
|
|
1664
1668
|
ws.close(1008, 'baseline resync required');
|
package/src/log-maintenance.ts
CHANGED
|
@@ -6,11 +6,14 @@ import { LedgerError, validationError } from './errors';
|
|
|
6
6
|
|
|
7
7
|
/** Events archived per compaction step. */
|
|
8
8
|
const PAGE = 128;
|
|
9
|
-
/** Source events
|
|
9
|
+
/** Source events per read while folding (also bounded by the byte window of the reader). */
|
|
10
10
|
const FOLD_EVENTS = 512;
|
|
11
|
-
/**
|
|
12
|
-
const
|
|
13
|
-
const
|
|
11
|
+
/** Retained events read per query while writing snapshot pages. */
|
|
12
|
+
const READ_VERSIONS = 256;
|
|
13
|
+
const CLAIMS_PER_PAGE = 4096;
|
|
14
|
+
/** Wall time one snapshot step keeps working before it saves its state. */
|
|
15
|
+
const STEP_MS = 4000;
|
|
16
|
+
const MAX_WORK_BYTES = 64_000_000;
|
|
14
17
|
const MAX_PAGE_BYTES = 1_500_000;
|
|
15
18
|
const MAX_STATE_BYTES = 1_500_000;
|
|
16
19
|
const MAX_MARKER_BYTES = 1_000_000;
|
|
@@ -24,7 +27,7 @@ const SELECTORS: readonly CheckpointSelector[] = [
|
|
|
24
27
|
event => typeof event.payload.thumbnailRef === 'string' ? 'thumbnail:string' : null,
|
|
25
28
|
];
|
|
26
29
|
type Row = Record<string, SqlStorageValue>;
|
|
27
|
-
type Phase = '
|
|
30
|
+
type Phase = 'seed' | 'fold' | 'pages';
|
|
28
31
|
interface JobRow extends Row {
|
|
29
32
|
id: string; log_id: string; kind: MaintenanceJob['kind']; through_version: number;
|
|
30
33
|
status: MaintenanceJob['status']; cursor: number; entity_cursor: string; pages: string;
|
|
@@ -117,19 +120,11 @@ export class LogMaintenanceStore {
|
|
|
117
120
|
if (kind === 'compact' && (!checkpoint || checkpoint.throughVersion <= this.layout(logId).compacted_through)) {
|
|
118
121
|
throw new LedgerError(409, { error: 'Conflict', code: 'snapshot_required', message: 'Create a newer verified snapshot before compacting.' });
|
|
119
122
|
}
|
|
120
|
-
|
|
121
|
-
// instead of re-reading the whole (archived) history, unless a
|
|
122
|
-
// replay directive since then could reach back behind it.
|
|
123
|
-
let phase: Phase = 'fold', cursor = 0;
|
|
124
|
-
if (kind === 'snapshot' && checkpoint && checkpoint.throughVersion > 0 && checkpoint.throughVersion <= log.version
|
|
125
|
-
&& !this.query(`SELECT 1 FROM log_events WHERE log_id = ? AND version > ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') LIMIT 1`,
|
|
126
|
-
logId, checkpoint.throughVersion).length) {
|
|
127
|
-
phase = 'seed'; cursor = checkpoint.throughVersion;
|
|
128
|
-
}
|
|
123
|
+
const start = kind === 'snapshot' ? this.initialBuild(logId, log.version) : { phase: null, cursor: 0 };
|
|
129
124
|
this.query(`INSERT INTO log_maintenance_jobs (id, log_id, kind, through_version, status, cursor, entity_cursor, pages, updated_at, day, writes, phase)
|
|
130
125
|
VALUES (?, ?, ?, ?, ?, ?, '', '[]', ?, ?, 0, ?)`, id, logId, kind,
|
|
131
|
-
kind === 'compact' ? checkpoint!.throughVersion : log.version, kind === 'compact' ? 'archiving' : 'building', cursor,
|
|
132
|
-
new Date().toISOString(), day(),
|
|
126
|
+
kind === 'compact' ? checkpoint!.throughVersion : log.version, kind === 'compact' ? 'archiving' : 'building', start.cursor,
|
|
127
|
+
new Date().toISOString(), day(), start.phase);
|
|
133
128
|
});
|
|
134
129
|
return this.state(logId, id);
|
|
135
130
|
}
|
|
@@ -213,7 +208,7 @@ export class LogMaintenanceStore {
|
|
|
213
208
|
}
|
|
214
209
|
try {
|
|
215
210
|
if (job.status === 'building') await this.build(job);
|
|
216
|
-
else if (job.status === 'cleaning') this.clean(job);
|
|
211
|
+
else if (job.status === 'cleaning') await this.clean(job);
|
|
217
212
|
else if (job.status === 'archiving' || job.status === 'reclaiming') await this.archive(job);
|
|
218
213
|
if (job.attempts > 0 || job.error) {
|
|
219
214
|
this.query('UPDATE log_maintenance_jobs SET attempts = 0, error = NULL, retry_at = NULL WHERE id = ?', id);
|
|
@@ -283,26 +278,23 @@ export class LogMaintenanceStore {
|
|
|
283
278
|
}
|
|
284
279
|
|
|
285
280
|
private async build(job: JobRow): Promise<void> {
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
case 'fold':
|
|
290
|
-
if (job.cursor < job.through_version) return this.fold(job);
|
|
291
|
-
this.setPhase(job, 'index');
|
|
292
|
-
return this.index({ ...job, phase: 'index' });
|
|
293
|
-
case 'index': return this.index(job);
|
|
294
|
-
case 'pages': return this.pages(job);
|
|
295
|
-
}
|
|
281
|
+
if (!job.phase || !['seed', 'fold', 'pages'].includes(job.phase)) return this.reset(job);
|
|
282
|
+
if (job.phase === 'pages') return this.pages(job);
|
|
283
|
+
return this.fold(job);
|
|
296
284
|
}
|
|
297
285
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
286
|
+
/** Where a new snapshot starts: from the previous one's retained facts when no later replay directive can reach behind it. */
|
|
287
|
+
private initialBuild(logId: string, through: number): { phase: Phase; cursor: number } {
|
|
288
|
+
const checkpoint = this.state(logId).checkpoint;
|
|
289
|
+
if (checkpoint && checkpoint.throughVersion > 0 && checkpoint.throughVersion <= through
|
|
290
|
+
&& !this.query(`SELECT 1 FROM log_events WHERE log_id = ? AND version > ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') LIMIT 1`,
|
|
291
|
+
logId, checkpoint.throughVersion).length) {
|
|
292
|
+
return { phase: 'seed', cursor: checkpoint.throughVersion };
|
|
293
|
+
}
|
|
294
|
+
return { phase: 'fold', cursor: 0 };
|
|
303
295
|
}
|
|
304
296
|
|
|
305
|
-
/** Discard partial work (including work files of
|
|
297
|
+
/** Discard partial work (including work files of earlier builders) and start the build again. */
|
|
306
298
|
private async reset(job: JobRow): Promise<void> {
|
|
307
299
|
const legacy = this.query<{ entity_id: string; object_key: string }>('SELECT entity_id, object_key FROM log_checkpoint_entities WHERE job_id = ? LIMIT 16', job.id);
|
|
308
300
|
if (legacy.length) {
|
|
@@ -313,168 +305,161 @@ export class LogMaintenanceStore {
|
|
|
313
305
|
}
|
|
314
306
|
return;
|
|
315
307
|
}
|
|
316
|
-
|
|
317
|
-
this.
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
/** Delete a bounded batch of work rows; true while any remain. */
|
|
321
|
-
private deleteWork(jobId: string): boolean {
|
|
308
|
+
const reference = workReference(job.entity_cursor);
|
|
309
|
+
if (reference) await this.deleteObject(reference.key);
|
|
310
|
+
const start = this.initialBuild(job.log_id, job.through_version);
|
|
322
311
|
this.assertWritable();
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
this.query('DELETE FROM log_checkpoint_work WHERE job_id = ? AND entity_id IN (SELECT value FROM json_each(?))', jobId, JSON.stringify(entities.map(row => row.entity_id)));
|
|
326
|
-
return true;
|
|
327
|
-
}
|
|
328
|
-
const versions = this.query<{ version: number }>('SELECT version FROM log_checkpoint_versions WHERE job_id = ? LIMIT ?', jobId, CLEAN_ROWS * 2);
|
|
329
|
-
if (versions.length) {
|
|
330
|
-
this.query('DELETE FROM log_checkpoint_versions WHERE job_id = ? AND version IN (SELECT value FROM json_each(?))', jobId, JSON.stringify(versions.map(row => row.version)));
|
|
331
|
-
return true;
|
|
332
|
-
}
|
|
333
|
-
return false;
|
|
312
|
+
this.query(`UPDATE log_maintenance_jobs SET phase = ?, cursor = ?, version_cursor = 0, entity_cursor = '', pages = '[]', markers = NULL, updated_at = ?
|
|
313
|
+
WHERE id = ?`, start.phase, start.cursor, new Date().toISOString(), job.id);
|
|
334
314
|
}
|
|
335
315
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
316
|
+
/**
|
|
317
|
+
* The whole job's replay slots live in one private object, read and written once per step,
|
|
318
|
+
* so SQL writes grow with steps rather than with entities. null when it was lost (the build restarts).
|
|
319
|
+
*/
|
|
320
|
+
private async loadState(job: JobRow): Promise<Map<string, WorkState> | null> {
|
|
321
|
+
const reference = workReference(job.entity_cursor);
|
|
322
|
+
if (!reference) return new Map();
|
|
323
|
+
const object = await this.getObject(reference.key);
|
|
324
|
+
const value = object ? await object.text() : null;
|
|
325
|
+
if (value === null || await checkpointDigest(value) !== reference.sha256) {
|
|
326
|
+
console.warn('ledger_maintenance_state_lost', { appId: this.appId, logId: job.log_id, jobId: job.id });
|
|
327
|
+
this.assertWritable();
|
|
328
|
+
this.query("UPDATE log_maintenance_jobs SET entity_cursor = '', phase = NULL WHERE id = ?", job.id);
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
return new Map(Object.entries(JSON.parse(value) as Record<string, WorkState>));
|
|
340
332
|
}
|
|
341
333
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
for (const [
|
|
346
|
-
|
|
347
|
-
|
|
334
|
+
private async saveState(job: JobRow, states: Map<string, WorkState>, next: { phase: Phase; cursor: number; versionCursor: number }): Promise<void> {
|
|
335
|
+
let value = '{';
|
|
336
|
+
let first = true;
|
|
337
|
+
for (const [entityId, state] of states) {
|
|
338
|
+
const encoded = JSON.stringify(state);
|
|
339
|
+
if (encoded.length > MAX_STATE_BYTES) throw new Error('Entity replay state exceeds the snapshot work budget');
|
|
340
|
+
value += `${first ? '' : ','}${JSON.stringify(entityId)}:${encoded}`;
|
|
341
|
+
first = false;
|
|
348
342
|
}
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
343
|
+
value += '}';
|
|
344
|
+
if (value.length > MAX_WORK_BYTES) throw new Error('Snapshot work state exceeds the snapshot work budget');
|
|
345
|
+
const key = `v1/${this.appId}/${job.log_id}/work/${job.id}/${next.phase}-${next.cursor}-${next.versionCursor}.json`;
|
|
346
|
+
const sha256 = await this.put(key, value);
|
|
347
|
+
this.assertWritable();
|
|
348
|
+
this.query('UPDATE log_maintenance_jobs SET phase = ?, cursor = ?, version_cursor = ?, entity_cursor = ?, updated_at = ? WHERE id = ?',
|
|
349
|
+
next.phase, next.cursor, next.versionCursor, `${key}#${sha256}`, new Date().toISOString(), job.id);
|
|
350
|
+
const previous = workReference(job.entity_cursor);
|
|
351
|
+
if (previous && previous.key !== key) await this.deleteObject(previous.key);
|
|
352
352
|
}
|
|
353
353
|
|
|
354
|
-
/**
|
|
355
|
-
private async
|
|
356
|
-
const
|
|
357
|
-
if (!
|
|
358
|
-
|
|
359
|
-
this.setPhase(job, 'fold');
|
|
360
|
-
return;
|
|
361
|
-
}
|
|
362
|
-
const chunk = await this.chunk(job.log_id, job.version_cursor);
|
|
363
|
-
const touched = new Set<string>([...chunk.events.map(event => event.entityId), ...chunk.claims.map(claim => claim.entityId)]);
|
|
364
|
-
const states = this.loadWork(job.id, [...touched]);
|
|
354
|
+
/** Seed from the previous snapshot, then fold source events, for as long as one step allows. */
|
|
355
|
+
private async fold(job: JobRow): Promise<void> {
|
|
356
|
+
const states = await this.loadState(job);
|
|
357
|
+
if (!states) return;
|
|
358
|
+
const started = Date.now();
|
|
365
359
|
const work = (entityId: string): WorkState => {
|
|
366
360
|
let state = states.get(entityId);
|
|
367
361
|
if (!state) { state = { s: newCheckpointEntityState(), c: { currentVersion: 0, logVersion: 0 } }; states.set(entityId, state); }
|
|
368
362
|
return state;
|
|
369
363
|
};
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
if (!events.length || events[0]!.version !== job.cursor + 1 || events[events.length - 1]!.version - events[0]!.version !== events.length - 1) {
|
|
383
|
-
throw new Error('Snapshot source contains a gap');
|
|
384
|
-
}
|
|
385
|
-
const selected = this.selection(job);
|
|
386
|
-
const groups = new Map<string, CheckpointEvent[]>();
|
|
387
|
-
for (const event of events) {
|
|
388
|
-
const group = groups.get(event.entityId) ?? [];
|
|
389
|
-
group.push(event); groups.set(event.entityId, group);
|
|
364
|
+
let phase = job.phase as Phase, cursor = job.cursor, versionCursor = job.version_cursor;
|
|
365
|
+
if (phase === 'seed') {
|
|
366
|
+
const layout = this.layout(job.log_id);
|
|
367
|
+
const checkpoint = layout.checkpoint ? JSON.parse(layout.checkpoint) as CheckpointManifest : null;
|
|
368
|
+
if (!checkpoint || checkpoint.throughVersion !== job.cursor) throw new Error('The previous snapshot changed while seeding');
|
|
369
|
+
while (versionCursor < checkpoint.pages.length && Date.now() - started < STEP_MS) {
|
|
370
|
+
const chunk = await this.chunk(job.log_id, versionCursor);
|
|
371
|
+
for (const event of [...chunk.events].sort((a, b) => a.version - b.version)) foldCheckpointEvent(work(event.entityId).s, event, SELECTORS, standardTypes);
|
|
372
|
+
for (const claim of chunk.claims) work(claim.entityId).c = { currentVersion: claim.currentVersion, logVersion: claim.logVersion };
|
|
373
|
+
versionCursor++;
|
|
374
|
+
}
|
|
375
|
+
if (versionCursor >= checkpoint.pages.length) { phase = 'fold'; versionCursor = 0; }
|
|
390
376
|
}
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
377
|
+
if (phase === 'fold') {
|
|
378
|
+
const selected = cursor < job.through_version ? this.selection(job) : () => true;
|
|
379
|
+
while (cursor < job.through_version && Date.now() - started < STEP_MS) {
|
|
380
|
+
const events = await this.readEvents(job.log_id, cursor, job.through_version, FOLD_EVENTS);
|
|
381
|
+
if (!events.length || events[0]!.version !== cursor + 1 || events[events.length - 1]!.version - events[0]!.version !== events.length - 1) {
|
|
382
|
+
throw new Error('Snapshot source contains a gap');
|
|
383
|
+
}
|
|
384
|
+
for (const event of events) {
|
|
385
|
+
const state = work(event.entityId);
|
|
386
|
+
if (selected(event) && !directives.includes(event.type)) foldCheckpointEvent(state.s, event, SELECTORS, standardTypes);
|
|
387
|
+
state.c = { currentVersion: event.entityVersion, logVersion: event.version };
|
|
388
|
+
}
|
|
389
|
+
cursor = events[events.length - 1]!.version;
|
|
390
|
+
}
|
|
391
|
+
if (cursor >= job.through_version) { phase = 'pages'; versionCursor = 0; }
|
|
398
392
|
}
|
|
399
|
-
this.
|
|
400
|
-
this.assertWritable();
|
|
401
|
-
this.saveWork(job.id, states);
|
|
402
|
-
this.query('UPDATE log_maintenance_jobs SET cursor = ?, updated_at = ? WHERE id = ?',
|
|
403
|
-
events[events.length - 1]!.version, new Date().toISOString(), job.id);
|
|
404
|
-
});
|
|
393
|
+
await this.saveState(job, states, { phase, cursor, versionCursor });
|
|
405
394
|
}
|
|
406
395
|
|
|
407
|
-
/**
|
|
408
|
-
private async
|
|
396
|
+
/** Write claim pages and the retained source events in version order, then publish the manifest. */
|
|
397
|
+
private async pages(job: JobRow): Promise<void> {
|
|
398
|
+
const states = await this.loadState(job);
|
|
399
|
+
if (!states) return;
|
|
400
|
+
const started = Date.now();
|
|
409
401
|
const pages = JSON.parse(job.pages) as CheckpointPage[];
|
|
410
|
-
const
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
402
|
+
const pageKey = () => `v1/${this.appId}/${job.log_id}/snapshots/${job.id}/${pages.length}.json`;
|
|
403
|
+
const write = async (chunk: CheckpointChunk) => {
|
|
404
|
+
const value = JSON.stringify(chunk);
|
|
405
|
+
const key = pageKey();
|
|
406
|
+
pages.push({ key, sha256: await this.put(key, value), count: chunk.events.length, bytes: new TextEncoder().encode(value).byteLength });
|
|
407
|
+
};
|
|
408
|
+
if (!pages.length && job.version_cursor === 0) {
|
|
409
|
+
const claims = [...states].filter(([, state]) => state.c.logVersion > 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)
|
|
410
|
+
.map(([entityId, state]) => ({ entityId, currentVersion: state.c.currentVersion, logVersion: state.c.logVersion }));
|
|
411
|
+
for (let offset = 0; offset < claims.length; offset += CLAIMS_PER_PAGE) await write({ events: [], claims: claims.slice(offset, offset + CLAIMS_PER_PAGE) });
|
|
415
412
|
}
|
|
416
|
-
const
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
413
|
+
const retained = new Set<number>();
|
|
414
|
+
for (const state of states.values()) for (const version of checkpointStateVersions(state.s)) retained.add(version);
|
|
415
|
+
const versions = [...retained].filter(version => version > job.version_cursor).sort((a, b) => a - b);
|
|
416
|
+
let versionCursor = job.version_cursor;
|
|
417
|
+
let chunk: CheckpointChunk = { events: [], claims: [] };
|
|
418
|
+
let bytes = 2;
|
|
419
|
+
let offset = 0;
|
|
420
|
+
while (offset < versions.length && Date.now() - started < STEP_MS) {
|
|
421
|
+
const wanted = versions.slice(offset, offset + READ_VERSIONS);
|
|
422
|
+
const events = await this.readVersions(job.log_id, wanted);
|
|
423
|
+
if (events.length !== wanted.length || events.some((event, index) => event.version !== wanted[index])) throw new Error('Snapshot source is missing retained events');
|
|
424
|
+
for (const event of events) {
|
|
425
|
+
const size = new TextEncoder().encode(JSON.stringify(event)).byteLength + 1;
|
|
426
|
+
if (chunk.events.length && bytes + size >= MAX_PAGE_BYTES) {
|
|
427
|
+
await write(chunk);
|
|
428
|
+
versionCursor = chunk.events[chunk.events.length - 1]!.version;
|
|
429
|
+
chunk = { events: [], claims: [] }; bytes = 2;
|
|
430
|
+
}
|
|
431
|
+
chunk.events.push(event); bytes += size;
|
|
432
|
+
}
|
|
433
|
+
offset += wanted.length;
|
|
422
434
|
}
|
|
423
|
-
if (
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
pages.push({ key, sha256: await this.put(key, value), count: 0, bytes: new TextEncoder().encode(value).byteLength });
|
|
435
|
+
if (chunk.events.length) {
|
|
436
|
+
await write(chunk);
|
|
437
|
+
versionCursor = chunk.events[chunk.events.length - 1]!.version;
|
|
427
438
|
}
|
|
439
|
+
this.assertWritable();
|
|
440
|
+
if (offset < versions.length) {
|
|
441
|
+
this.query('UPDATE log_maintenance_jobs SET version_cursor = ?, pages = ?, updated_at = ? WHERE id = ?',
|
|
442
|
+
versionCursor, JSON.stringify(pages), new Date().toISOString(), job.id);
|
|
443
|
+
return;
|
|
444
|
+
}
|
|
445
|
+
const manifest: CheckpointManifest = { format: 1, appId: this.appId, logId: job.log_id, id: job.id,
|
|
446
|
+
throughVersion: job.through_version, createdAt: new Date().toISOString(), pages,
|
|
447
|
+
eventCount: pages.reduce((count, page) => count + page.count, 0), sourceEventCount: job.through_version,
|
|
448
|
+
sha256: await checkpointDigest(JSON.stringify(pages)) };
|
|
428
449
|
this.storage.transactionSync(() => {
|
|
429
450
|
this.assertWritable();
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
451
|
+
this.query(`INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint) VALUES (?, 0, 0, ?)
|
|
452
|
+
ON CONFLICT(log_id) DO UPDATE SET checkpoint = excluded.checkpoint`, job.log_id, JSON.stringify(manifest));
|
|
453
|
+
this.query("UPDATE log_maintenance_jobs SET status = 'cleaning', version_cursor = ?, pages = ?, updated_at = ? WHERE id = ?",
|
|
454
|
+
versionCursor, JSON.stringify(pages), new Date().toISOString(), job.id);
|
|
433
455
|
});
|
|
434
456
|
}
|
|
435
457
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
const rows = this.query<{ version: number }>(`SELECT version FROM log_checkpoint_versions WHERE job_id = ? AND version > ?
|
|
440
|
-
ORDER BY version LIMIT ?`, job.id, job.version_cursor, PAGE);
|
|
441
|
-
if (!rows.length) {
|
|
442
|
-
const manifest: CheckpointManifest = { format: 1, appId: this.appId, logId: job.log_id, id: job.id,
|
|
443
|
-
throughVersion: job.through_version, createdAt: new Date().toISOString(), pages,
|
|
444
|
-
eventCount: pages.reduce((count, page) => count + page.count, 0), sourceEventCount: job.through_version,
|
|
445
|
-
sha256: await checkpointDigest(JSON.stringify(pages)) };
|
|
446
|
-
this.storage.transactionSync(() => {
|
|
447
|
-
this.assertWritable();
|
|
448
|
-
this.query(`INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint) VALUES (?, 0, 0, ?)
|
|
449
|
-
ON CONFLICT(log_id) DO UPDATE SET checkpoint = excluded.checkpoint`, job.log_id, JSON.stringify(manifest));
|
|
450
|
-
this.query("UPDATE log_maintenance_jobs SET status = 'cleaning', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
|
|
451
|
-
});
|
|
452
|
-
return;
|
|
453
|
-
}
|
|
454
|
-
const wanted = rows.map(row => row.version);
|
|
455
|
-
const events = await this.readVersions(job.log_id, wanted);
|
|
456
|
-
if (events.length !== wanted.length || events.some((event, index) => event.version !== wanted[index])) throw new Error('Snapshot source is missing retained events');
|
|
457
|
-
const chunk: CheckpointChunk = { events: [], claims: [] };
|
|
458
|
-
let bytes = 2;
|
|
459
|
-
for (const event of events) {
|
|
460
|
-
const size = new TextEncoder().encode(JSON.stringify(event)).byteLength + 1;
|
|
461
|
-
if (chunk.events.length && bytes + size >= MAX_PAGE_BYTES) break;
|
|
462
|
-
chunk.events.push(event); bytes += size;
|
|
463
|
-
}
|
|
464
|
-
const value = JSON.stringify(chunk);
|
|
465
|
-
const key = `v1/${this.appId}/${job.log_id}/snapshots/${job.id}/${pages.length}.json`;
|
|
466
|
-
pages.push({ key, sha256: await this.put(key, value), count: chunk.events.length, bytes: new TextEncoder().encode(value).byteLength });
|
|
458
|
+
private async clean(job: JobRow): Promise<void> {
|
|
459
|
+
const reference = workReference(job.entity_cursor);
|
|
460
|
+
if (reference) await this.deleteObject(reference.key);
|
|
467
461
|
this.assertWritable();
|
|
468
|
-
this.query(
|
|
469
|
-
chunk.events[chunk.events.length - 1]!.version, JSON.stringify(pages), new Date().toISOString(), job.id);
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
private clean(job: JobRow): void {
|
|
473
|
-
if (this.deleteWork(job.id)) {
|
|
474
|
-
this.query('UPDATE log_maintenance_jobs SET updated_at = ? WHERE id = ?', new Date().toISOString(), job.id);
|
|
475
|
-
return;
|
|
476
|
-
}
|
|
477
|
-
this.query("UPDATE log_maintenance_jobs SET status = 'complete', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
|
|
462
|
+
this.query("UPDATE log_maintenance_jobs SET status = 'complete', entity_cursor = '', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
|
|
478
463
|
}
|
|
479
464
|
|
|
480
465
|
/** R2 rejects an upload whose bytes do not match the digest, so no read-back is needed. */
|
|
@@ -511,6 +496,7 @@ export class LogMaintenanceStore {
|
|
|
511
496
|
this.query(`UPDATE log_events SET archive_key = ?, payload_bytes = COALESCE(payload_bytes, length(CAST(payload AS BLOB))), payload = CASE WHEN event_type IN
|
|
512
497
|
('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') THEN payload ELSE '{}' END
|
|
513
498
|
WHERE log_id = ? AND version >= ? AND version <= ? AND archive_key IS NULL`, key, job.log_id, first, events[events.length - 1]!.version);
|
|
499
|
+
this.query('UPDATE log_storage SET archived_through = MAX(archived_through, ?) WHERE log_id = ?', events[events.length - 1]!.version, job.log_id);
|
|
514
500
|
this.query("UPDATE log_maintenance_jobs SET status = 'archiving', cursor = ?, updated_at = ? WHERE id = ?",
|
|
515
501
|
events[events.length - 1]!.version, new Date().toISOString(), job.id);
|
|
516
502
|
});
|
|
@@ -536,6 +522,11 @@ export class LogMaintenanceStore {
|
|
|
536
522
|
}
|
|
537
523
|
}
|
|
538
524
|
|
|
525
|
+
/** The job's current work-state object, stored as `key#sha256` in the job row. */
|
|
526
|
+
function workReference(value: string): { key: string; sha256: string } | null {
|
|
527
|
+
const separator = value.lastIndexOf('#');
|
|
528
|
+
return separator > 0 ? { key: value.slice(0, separator), sha256: value.slice(separator + 1) } : null;
|
|
529
|
+
}
|
|
539
530
|
function day(): string { return new Date().toISOString().slice(0, 10); }
|
|
540
531
|
/** Start of the UTC day after `dayValue` (YYYY-MM-DD). */
|
|
541
532
|
function nextMidnight(dayValue: string): number { return Date.parse(`${dayValue}T00:00:00.000Z`) + 86_400_000; }
|
package/src/schema.ts
CHANGED
|
@@ -234,6 +234,27 @@ const LEDGER_APP_MIGRATIONS: readonly Migration[] = [
|
|
|
234
234
|
ALTER TABLE log_maintenance_jobs ADD COLUMN paused_status TEXT;
|
|
235
235
|
ALTER TABLE log_maintenance_jobs ADD COLUMN markers TEXT;
|
|
236
236
|
` },
|
|
237
|
+
// Snapshot work state moves to one R2 object per job (SQL writes per step, not per entity), and
|
|
238
|
+
// archiving no longer writes an index entry per event: the archived prefix is tracked per log.
|
|
239
|
+
{ id: 16, sql: `
|
|
240
|
+
ALTER TABLE log_storage ADD COLUMN archived_through INTEGER NOT NULL DEFAULT 0;
|
|
241
|
+
UPDATE log_storage SET archived_through = COALESCE((SELECT MAX(version) FROM log_events
|
|
242
|
+
WHERE log_events.log_id = log_storage.log_id AND archive_key IS NOT NULL), 0);
|
|
243
|
+
INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint, archived_through)
|
|
244
|
+
SELECT log_id, 0, 0, NULL, MAX(version) FROM log_events
|
|
245
|
+
WHERE archive_key IS NOT NULL AND log_id NOT IN (SELECT log_id FROM log_storage) GROUP BY log_id;
|
|
246
|
+
DROP INDEX log_events_archived_cursor;
|
|
247
|
+
DROP TABLE log_checkpoint_work;
|
|
248
|
+
DROP TABLE log_checkpoint_versions;
|
|
249
|
+
-- Builds in progress restart with the new builder. Their counters measured the old one, so a
|
|
250
|
+
-- build that paused on them may continue today.
|
|
251
|
+
UPDATE log_maintenance_jobs SET phase = NULL, entity_cursor = '', version_cursor = 0, pages = '[]', markers = NULL,
|
|
252
|
+
writes = 0, reads = 0, attempts = 0, error = NULL, retry_at = NULL, status = 'building',
|
|
253
|
+
pause_reason = NULL, resume_at = NULL, paused_status = NULL
|
|
254
|
+
WHERE kind = 'snapshot' AND (status = 'building' OR (status = 'paused' AND COALESCE(paused_status, 'building') = 'building'));
|
|
255
|
+
UPDATE log_maintenance_jobs SET entity_cursor = '', status = 'cleaning', pause_reason = NULL, resume_at = NULL, paused_status = NULL
|
|
256
|
+
WHERE kind = 'snapshot' AND (status = 'cleaning' OR (status = 'paused' AND paused_status = 'cleaning'));
|
|
257
|
+
` },
|
|
237
258
|
];
|
|
238
259
|
|
|
239
260
|
const APP_REGISTRY_MIGRATIONS: readonly Migration[] = [
|