@hearthlog/ledger 0.9.0 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -172,3 +172,63 @@ Restore validates the complete export before provisioning, creates an empty
172
172
  target application, replays and verifies every page, rotates the bootstrap
173
173
  credential, and prints the new token. A failed restore revokes the partial
174
174
  target.
175
+
176
+
177
+ ## Snapshot baselines and compaction
178
+
179
+ The existing log history panel has manual **Create snapshot** and **Compact log** controls.
180
+ A snapshot captures a complete accepted boundary. Compaction moves its prefix payloads into a private archive;
181
+ application/log identities, monotonically increasing event versions, entity claims, action boundaries and retry
182
+ receipts stay intact. SQLite retains the compact event identity registry. Historical reads and retries resolve
183
+ original payloads from the archive; compaction does not expire history or delete domain records/media.
184
+
185
+ The generated Ledger configuration binds `LEDGER_ARCHIVE` to `<ledgerWorker>-archive`. Tenant provisioning
186
+ ensures that private R2 bucket. Keep it private, without an expiry lifecycle. Deploy compatible code and ensure
187
+ this binding before starting maintenance. No namespace or deployed Durable Object identity changes.
188
+
189
+ Authenticated endpoints under `/logs/:logId`:
190
+
191
+ - `GET /maintenance`: availability, verified snapshot manifest, storage generation, and current job.
192
+ - `GET /maintenance-preview`: the current snapshot/generation and prefix payload counts/bytes for review.
193
+ - `POST /maintenance`: `{ requestId, kind: "snapshot" | "compact", expectedCheckpointId?, expectedGeneration? }`.
194
+ Compaction requires the checkpoint ID and generation from the reviewed preview; stale previews fail with 409.
195
+ - `POST /maintenance/step`: `{ requestId }`, resumable work using the same request identity.
196
+ - `GET /checkpoint?page=N`: one checksum-verified canonical replay chunk and its entity claims.
197
+
198
+ From the development site, the same operation can run without an open browser:
199
+
200
+ ```bash
201
+ pnpm -C dev ledger:maintain --log <log-id> --kind snapshot
202
+ pnpm -C dev ledger:maintain --log <log-id> --kind compact
203
+ pnpm -C dev ledger:maintain --log <log-id> --resume
204
+ ```
205
+
206
+ Set `LEDGER_BASE_URL` and `LEDGER_AUTH_TOKEN` in the process environment; the CLI never prints credentials.
207
+ Use `--request-id` to retry an explicit operation and `--once` for one bounded work step.
208
+
209
+ Jobs persist every source batch and publish only verified snapshots. Canonical snapshots keep original source
210
+ witnesses for raw field values, explicit nulls, timestamps and lifecycle transitions; dynamic/nonstandard facts
211
+ are retained losslessly. Browser projection uses a separate baseline table plus accepted suffix and pending
212
+ overlay. Fresh/behind devices can install a baseline atomically; historical comparisons or directives that
213
+ reach into the baseline recover original history on demand. Existing HTTP event readers remain compatible
214
+ through transparent archive hydration, and old socket cursors recover through HTTP rather than receiving
215
+ payload placeholders. The SDK's server checkpoint reader lets Stash seed a cold media index from the same base.
216
+
217
+ `MAINTENANCE_DAILY_WRITES` defaults to `10000` per job (bounded to 128–100000), and
218
+ `MAINTENANCE_DAILY_READS` defaults to `1000000` (bounded to 10000–1000000000). Jobs pause at either budget. Budgets use consumed SQL cursor
219
+ counts and can overrun by one bounded step. Structured step logs include SQL rows, R2 operations/bytes, elapsed
220
+ time and replay counts, without payloads or credentials. Resume with the same ID on a later UTC quota day;
221
+ other applications consume account quota too. The browser
222
+ can be closed safely and the operation resumed from the panel. No automatic snapshots or archive expiry run.
223
+ Deleting a whole log uses bounded background cleanup of that log's private archive objects, after normal
224
+ fork/deletion checks. Individual archived-history deletion is a separate future feature.
225
+
226
+ Use the full logical export or identity-preserving application transfer for backup/moves: they expand archived
227
+ payloads into the existing portable format. A SQLite-only export returns `409 archive_bundle_required` when
228
+ external checkpoint/archive objects are required, avoiding a misleading incomplete backup. Rollback after
229
+ compaction needs a compatible reader or a full logical restore first.
230
+
231
+ The first format bounds a single entity's replay witnesses to 8 MB and stores page descriptors in SQLite.
232
+ Exceeding a snapshot work/metadata bound fails the job while source history stays readable. Measure maintenance
233
+ work and storage on representative data; retained identity metadata still grows, and physical SQLite file size
234
+ need not shrink immediately when payload space becomes reusable.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hearthlog/ledger",
3
- "version": "0.9.0",
3
+ "version": "0.10.1",
4
4
  "description": "Hearthlog's Ledger: the event store Worker (Elysia over SQLite Durable Objects), its contracts and tests, and the logical backup tooling",
5
5
  "license": "AGPL-3.0-only",
6
6
  "repository": {
@@ -30,7 +30,7 @@
30
30
  "dependencies": {
31
31
  "@sinclair/typebox": "0.34.52",
32
32
  "elysia": "1.4.30",
33
- "@hearthlog/contracts": "0.9.0"
33
+ "@hearthlog/contracts": "0.10.1"
34
34
  },
35
35
  "peerDependencies": {
36
36
  "@hearthlog/generated": "*"
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env node
2
+ import { randomUUID } from 'node:crypto';
3
+ import { parseArgs } from 'node:util';
4
+ import type { LogMaintenance } from '@hearthlog/contracts/history/checkpoint';
5
+
6
+ const { values } = parseArgs({ options: {
7
+ log: { type: 'string' }, kind: { type: 'string', default: 'snapshot' },
8
+ 'request-id': { type: 'string' }, resume: { type: 'boolean' }, once: { type: 'boolean' }, help: { type: 'boolean' },
9
+ } });
10
+
11
+ async function main(): Promise<void> {
12
+ if (values.help) {
13
+ console.log('ledger maintenance --log <uuid> [--kind snapshot|compact] [--request-id <uuid>] [--resume] [--once]');
14
+ console.log('Uses LEDGER_BASE_URL and LEDGER_AUTH_TOKEN. Compaction retains original events in the private archive.');
15
+ return;
16
+ }
17
+ if (!values.log || !/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(values.log)) throw new Error('--log must be a UUID');
18
+ if (values.kind !== 'snapshot' && values.kind !== 'compact') throw new Error('--kind must be snapshot or compact');
19
+ const base = process.env.LEDGER_BASE_URL;
20
+ const token = process.env.LEDGER_AUTH_TOKEN;
21
+ if (!base || !token) throw new Error('Set LEDGER_BASE_URL and LEDGER_AUTH_TOKEN');
22
+ const origin = new URL(base);
23
+ if (!['http:', 'https:'].includes(origin.protocol) || origin.username || origin.password) throw new Error('Use an HTTP(S) Ledger URL without embedded credentials');
24
+ const path = `/logs/${values.log}/maintenance`;
25
+ async function request(suffix = '', body?: unknown): Promise<LogMaintenance> {
26
+ const response = await fetch(`${origin.href.replace(/\/$/, '')}${path}${suffix}`, {
27
+ method: body ? 'POST' : 'GET', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
28
+ ...(body ? { body: JSON.stringify(body) } : {}),
29
+ });
30
+ if (!response.ok) throw new Error(`Ledger maintenance request failed (HTTP ${response.status})`);
31
+ return response.json() as Promise<LogMaintenance>;
32
+ }
33
+ let state = await request();
34
+ const requestId = values['request-id'] ?? (values.resume ? state.job?.id : randomUUID());
35
+ if (!requestId) throw new Error('No maintenance operation to resume');
36
+ if (!values.resume) {
37
+ state = await request('', { requestId, kind: values.kind,
38
+ expectedCheckpointId: state.checkpoint?.id, expectedGeneration: state.storageGeneration });
39
+ }
40
+ do {
41
+ if (state.job?.status === 'complete') break;
42
+ state = await request('/step', { requestId });
43
+ console.log(JSON.stringify(state.job));
44
+ if (state.job?.status === 'failed') throw new Error(state.job.error ?? 'Maintenance failed');
45
+ } while (!values.once && state.job?.status !== 'complete' && state.job?.status !== 'paused');
46
+ console.log(JSON.stringify({ logId: values.log, requestId, status: state.job?.status,
47
+ checkpointId: state.checkpoint?.id ?? null, throughVersion: state.checkpoint?.throughVersion ?? 0,
48
+ compactedThroughVersion: state.compactedThroughVersion, storageGeneration: state.storageGeneration }));
49
+ }
50
+
51
+ try { await main(); }
52
+ catch (cause) { console.error(cause instanceof Error ? cause.message : 'Maintenance failed'); process.exitCode = 1; }
package/src/http.ts CHANGED
@@ -32,6 +32,7 @@ import {
32
32
  importFinishBody,
33
33
  importRecordsBody,
34
34
  forkLogBody,
35
+ maintenanceBody, maintenanceStepBody, checkpointPageQuery,
35
36
  logVisibilityBody,
36
37
  galleryPhotoReferenceParams,
37
38
  logLineageBody,
@@ -444,6 +445,21 @@ const app = new Elysia({
444
445
  response: { 200: "LogResponse" },
445
446
  },
446
447
  )
448
+ .get('/logs/:logId/maintenance', async ({ access, params, request }) =>
449
+ rpcResponse(await ledger(access).logMaintenance(access, params.logId), requestOrigin(request)),
450
+ { params: logParams })
451
+ .get('/logs/:logId/maintenance-preview', async ({ access, params, request }) =>
452
+ rpcResponse(await ledger(access).compactionPreview(access, params.logId), requestOrigin(request)),
453
+ { params: logParams })
454
+ .post('/logs/:logId/maintenance', async ({ access, params, body, request }) =>
455
+ rpcResponse(await ledger(access).startLogMaintenance(access, params.logId, body.requestId, body.kind, body.expectedCheckpointId, body.expectedGeneration), requestOrigin(request)),
456
+ { params: logParams, body: maintenanceBody })
457
+ .post('/logs/:logId/maintenance/step', async ({ access, params, body, request }) =>
458
+ rpcResponse(await ledger(access).stepLogMaintenance(access, params.logId, body.requestId), requestOrigin(request)),
459
+ { params: logParams, body: maintenanceStepBody })
460
+ .get('/logs/:logId/checkpoint', async ({ access, params, query, request }) =>
461
+ rpcResponse(await ledger(access).checkpointChunk(access, params.logId, query.page), requestOrigin(request)),
462
+ { params: logParams, query: checkpointPageQuery })
447
463
  .get(
448
464
  "/logs/:logId/gallery-photos/:photoId/files/:fileVersionId/reference",
449
465
  async ({ access, params, request }) =>
package/src/ledger-app.ts CHANGED
@@ -1,4 +1,6 @@
1
1
  import { DurableObject } from "cloudflare:workers";
2
+ import { LogMaintenanceStore } from './log-maintenance';
3
+ import type { CheckpointChunk, LogMaintenance, CompactionPreview } from '@hearthlog/contracts/history/checkpoint';
2
4
 
3
5
  import { SOCKET_PROTOCOL } from "./auth";
4
6
  import {
@@ -140,6 +142,67 @@ interface SocketSessionRow extends Record<string, SqlStorageValue> {
140
142
  }
141
143
 
142
144
  export class LedgerApp extends DurableObject<Env> {
145
+ private maintenanceQueue: Promise<unknown> = Promise.resolve();
146
+
147
+ private maintenance(assertWritable: () => void = () => undefined): LogMaintenanceStore {
148
+ const appId = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one().app_id;
149
+ const store = new LogMaintenanceStore(this.ctx.storage, this.env.LEDGER_ARCHIVE, appId,
150
+ async (logId, after, through, limit, entityId) => {
151
+ const rows = store.observeSql(this.ctx.storage.sql.exec<EventRow>(`WITH candidates AS (
152
+ SELECT *, SUM(COALESCE(payload_bytes, length(CAST(payload AS BLOB)))) OVER (ORDER BY version) AS byte_count,
153
+ ROW_NUMBER() OVER (ORDER BY version) AS row_number
154
+ FROM (SELECT * FROM log_events WHERE log_id = ? AND version > ? AND version <= ?
155
+ ${entityId === undefined ? '' : 'AND entity_id = ?'} ORDER BY version LIMIT ?)
156
+ ) SELECT * FROM candidates WHERE byte_count <= 1500000 OR row_number = 1`,
157
+ logId, after, through, ...(entityId === undefined ? [] : [entityId]), limit)).toArray();
158
+ return store.hydrate(logId, rows.map(storedEvent));
159
+ }, assertWritable, Math.max(128, Math.min(100000, Number(this.env.MAINTENANCE_DAILY_WRITES) || 10000)),
160
+ Math.max(10000, Math.min(1_000_000_000, Number(this.env.MAINTENANCE_DAILY_READS) || 1_000_000)));
161
+ return store;
162
+ }
163
+
164
+ async logMaintenance(access: AccessClaims, logId: string): Promise<RpcResult<LogMaintenance>> {
165
+ return rpcCall(() => {
166
+ this.currentAccess(access);
167
+ if (!this.log(parseUuid(logId, 'log id'))) throw validationError('Log not found');
168
+ return rpcSuccess(this.maintenance().state(logId));
169
+ });
170
+ }
171
+
172
+ async compactionPreview(access: AccessClaims, logId: string): Promise<RpcResult<CompactionPreview>> {
173
+ return rpcCall(() => {
174
+ this.currentAccess(access);
175
+ if (!this.log(parseUuid(logId, 'log id'))) throw validationError('Log not found');
176
+ return rpcSuccess(this.maintenance().preview(logId));
177
+ });
178
+ }
179
+
180
+ async startLogMaintenance(access: AccessClaims, logId: string, id: string, kind: 'snapshot' | 'compact', expectedCheckpointId?: string, expectedGeneration?: number): Promise<RpcResult<LogMaintenance>> {
181
+ return rpcCall(() => {
182
+ this.writableAccess(access);
183
+ return rpcSuccess(this.maintenance().start(parseUuid(logId, 'log id'), parseUuid(id, 'request id'), kind, expectedCheckpointId, expectedGeneration));
184
+ });
185
+ }
186
+
187
+ async stepLogMaintenance(access: AccessClaims, logId: string, id: string): Promise<RpcResult<LogMaintenance>> {
188
+ return rpcCall(async () => {
189
+ this.writableAccess(access);
190
+ const run = this.maintenanceQueue.then(() => this.maintenance(() => { this.writableAccess(access); }).step(parseUuid(logId, 'log id'), parseUuid(id, 'request id')));
191
+ this.maintenanceQueue = run.then(() => undefined, () => undefined);
192
+ const result = await run;
193
+ this.currentAccess(access);
194
+ return rpcSuccess(result);
195
+ });
196
+ }
197
+
198
+ async checkpointChunk(access: AccessClaims, logId: string, index: number): Promise<RpcResult<Omit<CheckpointChunk, 'events'> & { events: StoredEvent[] }>> {
199
+ return rpcCall(async () => {
200
+ this.currentAccess(access);
201
+ const result = await this.maintenance().chunk(parseUuid(logId, 'log id'), parseNonnegativeInteger(index, 'page'));
202
+ this.currentAccess(access);
203
+ return rpcSuccess({ ...result, events: result.events.map(event => ({ ...event, payload: event.payload as StoredEvent['payload'] })) });
204
+ });
205
+ }
143
206
  constructor(ctx: DurableObjectState, env: Env) {
144
207
  super(ctx, env);
145
208
  this.ctx.blockConcurrencyWhile(async () => {
@@ -147,6 +210,23 @@ export class LedgerApp extends DurableObject<Env> {
147
210
  });
148
211
  }
149
212
 
213
+ /** Delete storage belonging to an explicitly deleted log, in bounded batches. */
214
+ override async alarm(): Promise<void> {
215
+ const job = this.ctx.storage.sql.exec<{ log_id: string }>('SELECT log_id FROM log_archive_deletions LIMIT 1').toArray()[0];
216
+ if (!job) return;
217
+ try {
218
+ const app = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one();
219
+ const bucket = this.env.LEDGER_ARCHIVE;
220
+ const objects = await bucket.list({ prefix: `v1/${app.app_id}/${job.log_id}/`, limit: 128 });
221
+ if (objects.objects.length) await bucket.delete(objects.objects.map(object => object.key));
222
+ if (!objects.truncated) this.ctx.storage.sql.exec('DELETE FROM log_archive_deletions WHERE log_id = ?', job.log_id);
223
+ } finally {
224
+ if (this.ctx.storage.sql.exec('SELECT 1 FROM log_archive_deletions LIMIT 1').toArray().length) {
225
+ await this.ctx.storage.setAlarm(Date.now() + 60_000);
226
+ }
227
+ }
228
+ }
229
+
150
230
  override async fetch(request: Request): Promise<Response> {
151
231
  const url = new URL(request.url);
152
232
  const pathname = url.pathname;
@@ -222,7 +302,7 @@ export class LedgerApp extends DurableObject<Env> {
222
302
  return rpcSuccess({
223
303
  appId: claims.appId,
224
304
  principalId: claims.principalId,
225
- capabilities: ["action-metadata-v1", "workspace-restore-v1", "event-excision-v1"],
305
+ capabilities: ["snapshot-baseline-v1", "action-metadata-v1", "workspace-restore-v1", "event-excision-v1"],
226
306
  ...(fence ? { fence } : {}),
227
307
  });
228
308
  });
@@ -247,9 +327,19 @@ export class LedgerApp extends DurableObject<Env> {
247
327
  }
248
328
 
249
329
  async administratorSetLogLineage(appId: string, logId: string, payload: unknown): Promise<RpcResult<LogSummary>> {
250
- return rpcCall(() => {
330
+ return rpcCall(async () => {
251
331
  this.administratorApplication(appId);
252
332
  const { sourceLogId, sourceVersion } = decodeRpc(logLineageBody, payload, 'log lineage');
333
+ // Prefix identity is logical, even when the two logs use different
334
+ // physical archive layouts. Never compare '{}' payload placeholders.
335
+ for (let after = 0; after < sourceVersion;) {
336
+ const child = await this.hydrateEvents(logId, this.eventsThrough(logId, after, sourceVersion, 128));
337
+ const parent = await this.hydrateEvents(sourceLogId, this.eventsThrough(sourceLogId, after, sourceVersion, 128));
338
+ if (!child.length || child.length !== parent.length || child.some((event, index) => event.id !== parent[index]!.id
339
+ || JSON.stringify(event.payload) !== JSON.stringify(parent[index]!.payload))) throw validationError('Fork prefix does not match its source');
340
+ after = child[child.length - 1]!.version;
341
+ }
342
+
253
343
  this.ctx.storage.transactionSync(() => {
254
344
  const target = this.log(logId);
255
345
  const source = this.log(sourceLogId);
@@ -272,7 +362,7 @@ export class LedgerApp extends DurableObject<Env> {
272
362
  `SELECT COUNT(*) AS count FROM log_events AS child
273
363
  LEFT JOIN log_events AS parent ON parent.log_id = ? AND parent.version = child.version
274
364
  WHERE child.log_id = ? AND child.version <= ?
275
- AND (parent.event_id IS NULL OR parent.event_id != child.event_id OR parent.payload != child.payload)`,
365
+ AND (parent.event_id IS NULL OR parent.event_id != child.event_id)`,
276
366
  sourceLogId, logId, sourceVersion,
277
367
  ).toArray()[0]?.count ?? 0;
278
368
  if (mismatches) throw validationError('Fork prefix does not match its source');
@@ -342,6 +432,10 @@ export class LedgerApp extends DurableObject<Env> {
342
432
 
343
433
  async administratorExportSqlite(appId: string): Promise<Response> {
344
434
  this.administratorApplication(appId);
435
+ if (this.ctx.storage.sql.exec('SELECT 1 FROM log_events WHERE archive_key IS NOT NULL LIMIT 1').toArray().length
436
+ || this.ctx.storage.sql.exec('SELECT 1 FROM log_storage WHERE checkpoint IS NOT NULL LIMIT 1').toArray().length) {
437
+ return jsonError('archive_bundle_required', 'Use the complete logical export or application transfer; archived payloads are outside SQLite.', 409);
438
+ }
345
439
  return sqliteExport(
346
440
  this.ctx.storage,
347
441
  `ledger-${appId}-${new Date().toISOString().replaceAll(":", "-")}`,
@@ -456,7 +550,7 @@ export class LedgerApp extends DurableObject<Env> {
456
550
  logId: string,
457
551
  query: AdminEventsQuery,
458
552
  ): Promise<RpcResult<AdminEventsResponse>> {
459
- return rpcCall(() => {
553
+ return rpcCall(async () => {
460
554
  this.administratorApplication(appId);
461
555
  const id = parseUuid(logId, "log id");
462
556
  const parsed = decodeRpc(
@@ -492,7 +586,7 @@ export class LedgerApp extends DurableObject<Env> {
492
586
  limit + 1,
493
587
  )
494
588
  .toArray();
495
- const events = rows.slice(0, limit).map(storedEvent);
589
+ const events = await this.hydrateEvents(id, rows.slice(0, limit).map(storedEvent));
496
590
  return rpcSuccess({
497
591
  events,
498
592
  headVersion: log.version,
@@ -610,9 +704,10 @@ export class LedgerApp extends DurableObject<Env> {
610
704
  access: AccessClaims,
611
705
  logId: string,
612
706
  ): Promise<RpcResult<null>> {
613
- return rpcCall(() => {
707
+ return rpcCall(async () => {
614
708
  const claims = this.writableAccess(access);
615
709
  this.deleteLogValue(parseUuid(logId, "logId"), claims);
710
+ await this.ctx.storage.setAlarm(Date.now() + 1000);
616
711
  return rpcSuccess(null, 204);
617
712
  });
618
713
  }
@@ -624,11 +719,14 @@ export class LedgerApp extends DurableObject<Env> {
624
719
  limit: number | undefined,
625
720
  replayVersion = 0,
626
721
  ): Promise<RpcResult<EventBatchResponse>> {
627
- return rpcCall(() => {
722
+ return rpcCall(async () => {
628
723
  this.currentAccess(access);
629
724
  const id = parseUuid(logId, "logId");
630
725
  this.assertReplayCompatible(id, replayVersion);
631
- return rpcSuccess(this.getEventsValue(id, afterVersion, limit));
726
+ const batch = this.getEventsValue(id, afterVersion, limit);
727
+ const events = await this.hydrateEvents(id, batch.events);
728
+ this.currentAccess(access);
729
+ return rpcSuccess({ events });
632
730
  });
633
731
  }
634
732
 
@@ -721,7 +819,8 @@ export class LedgerApp extends DurableObject<Env> {
721
819
  log.id, after, TRANSFER_PAGE,
722
820
  ).toArray();
723
821
  if (rows.length === 0) break;
724
- digest = await chainChecksum(digest, rows.map(checksumLine));
822
+ const hydrated = await this.hydrateEvents(log.id, rows.map(storedEvent));
823
+ digest = await chainChecksum(digest, rows.map((row, index) => checksumLine({ ...row, payload: JSON.stringify(hydrated[index]!.payload) })));
725
824
  count += rows.length;
726
825
  after = rows[rows.length - 1]!.version;
727
826
  }
@@ -758,7 +857,7 @@ export class LedgerApp extends DurableObject<Env> {
758
857
  throw transferError("not_importing", "Only an import in progress can be aborted");
759
858
  }
760
859
  this.ctx.storage.transactionSync(() => {
761
- for (const table of ["gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
860
+ for (const table of ["log_storage", "log_archive_pages", "log_maintenance_jobs", "gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
762
861
  this.ctx.storage.sql.exec(`DELETE FROM ${table}`);
763
862
  }
764
863
  });
@@ -1006,9 +1105,9 @@ export class LedgerApp extends DurableObject<Env> {
1006
1105
  );
1007
1106
  this.ctx.storage.sql.exec(
1008
1107
  `INSERT INTO log_events (log_id, version, event_id, event_type, entity_id, entity_version,
1009
- event_timestamp, payload, action_id, action_index, action_size, actor)
1108
+ event_timestamp, payload, action_id, action_index, action_size, actor, archive_key, payload_bytes)
1010
1109
  SELECT ?, version, event_id, event_type, entity_id, entity_version,
1011
- event_timestamp, payload, action_id, action_index, action_size, actor
1110
+ event_timestamp, payload, action_id, action_index, action_size, actor, archive_key, payload_bytes
1012
1111
  FROM log_events WHERE log_id = ? AND version <= ? ORDER BY version`,
1013
1112
  id, sourceLogId, targetVersion,
1014
1113
  );
@@ -1058,13 +1157,25 @@ export class LedgerApp extends DurableObject<Env> {
1058
1157
  }
1059
1158
  this.ctx.storage.transactionSync(() => {
1060
1159
  this.currentAccess(access);
1160
+ if (this.ctx.storage.sql.exec("SELECT 1 FROM log_maintenance_jobs WHERE log_id = ? AND status NOT IN ('complete', 'failed') LIMIT 1", logId).toArray().length) {
1161
+ throw new LedgerError(409, { error: 'Conflict', code: 'maintenance_in_progress', message: 'Finish log maintenance before deleting this log.' });
1162
+ }
1061
1163
  if (this.ctx.storage.sql.exec('SELECT 1 FROM logs WHERE source_log_id = ? LIMIT 1', logId).toArray().length) {
1062
1164
  throw new LedgerError(409, { error: 'Conflict', code: 'log_has_forks',
1063
1165
  message: 'Remove descendant logs before deleting their source.' });
1064
1166
  }
1167
+ if (this.ctx.storage.sql.exec(`SELECT 1 FROM log_events WHERE log_id != ? AND archive_key IN
1168
+ (SELECT object_key FROM log_archive_pages WHERE log_id = ?) LIMIT 1`, logId, logId).toArray().length) {
1169
+ throw new LedgerError(409, { error: 'Conflict', code: 'log_has_archive_dependents',
1170
+ message: 'Other logs still depend on this archived history.' });
1171
+ }
1065
1172
  for (const query of [
1173
+ "DELETE FROM log_checkpoint_entities WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
1066
1174
  "DELETE FROM log_fork_requests WHERE fork_log_id = ?",
1067
1175
  "DELETE FROM append_requests WHERE log_id = ?",
1176
+ "DELETE FROM log_storage WHERE log_id = ?",
1177
+ "DELETE FROM log_archive_pages WHERE log_id = ?",
1178
+ "DELETE FROM log_maintenance_jobs WHERE log_id = ?",
1068
1179
  "DELETE FROM gallery_photo_references WHERE log_id = ?",
1069
1180
  "DELETE FROM entity_claims WHERE log_id = ?",
1070
1181
  "DELETE FROM log_events WHERE log_id = ?",
@@ -1072,6 +1183,7 @@ export class LedgerApp extends DurableObject<Env> {
1072
1183
  ]) {
1073
1184
  this.ctx.storage.sql.exec(query, logId);
1074
1185
  }
1186
+ this.ctx.storage.sql.exec('INSERT OR IGNORE INTO log_archive_deletions (log_id) VALUES (?)', logId);
1075
1187
  });
1076
1188
  this.closeLogSockets(logId, 1008, "log_deleted");
1077
1189
  }
@@ -1113,6 +1225,18 @@ export class LedgerApp extends DurableObject<Env> {
1113
1225
  validateIncomingEvents(request.events);
1114
1226
  const requestHash = await hashAppendRequest(logId, request);
1115
1227
 
1228
+ // The original response may now live in R2. Resolve it before entering
1229
+ // the synchronous append transaction; accepted requests never append twice.
1230
+ const receipt = this.ctx.storage.sql.exec<{ request_hash: string; first_version: number | null; last_version: number | null }>(
1231
+ 'SELECT request_hash, first_version, last_version FROM append_requests WHERE log_id = ? AND request_id = ?', logId, request.requestId).toArray()[0];
1232
+ if (receipt && receipt.first_version !== null && receipt.last_version !== null) {
1233
+ if (receipt.request_hash !== requestHash) throw new LedgerError(409, { error: 'Conflict', code: 'idempotency_mismatch' });
1234
+ const result = this.replayedAppend(logId, receipt.first_version, receipt.last_version);
1235
+ result.events = await this.hydrateEvents(logId, result.events);
1236
+ this.currentAccess(access);
1237
+ return result;
1238
+ }
1239
+
1116
1240
  const response = this.ctx.storage.transactionSync(() => {
1117
1241
  this.currentAccess(access);
1118
1242
  this.assertReplayCompatible(logId, replayVersion);
@@ -1479,6 +1603,12 @@ export class LedgerApp extends DurableObject<Env> {
1479
1603
  return;
1480
1604
  }
1481
1605
  const head = this.logHead(attachment.logId);
1606
+ const archived = this.ctx.storage.sql.exec<{ version: number | null }>('SELECT MAX(version) AS version FROM log_events WHERE log_id = ? AND archive_key IS NOT NULL', attachment.logId).one().version ?? 0;
1607
+ if (attachment.cursor < archived) {
1608
+ this.sendFrame(ws, { type: 'resync_required', headVersion: head, reason: 'gap_too_large' });
1609
+ ws.close(1008, 'baseline resync required');
1610
+ return;
1611
+ }
1482
1612
  if (attachment.cursor > head) {
1483
1613
  this.sendFrame(ws, {
1484
1614
  type: "resync_required",
@@ -1756,7 +1886,7 @@ export class LedgerApp extends DurableObject<Env> {
1756
1886
  });
1757
1887
  totals.logs += 1;
1758
1888
  for (let after = 0; after < log.version;) {
1759
- const events = this.eventsThrough(log.id, after, log.version, TRANSFER_PAGE);
1889
+ const events = await this.hydrateEvents(log.id, this.eventsThrough(log.id, after, log.version, TRANSFER_PAGE));
1760
1890
  if (events.length === 0) throw new Error(`log ${log.id} export stopped at version ${after}`);
1761
1891
  yield line({ t: "events", logId: log.id, events });
1762
1892
  after = events[events.length - 1]!.version;
@@ -1831,6 +1961,14 @@ export class LedgerApp extends DurableObject<Env> {
1831
1961
  return log.version;
1832
1962
  }
1833
1963
 
1964
+ private async hydrateEvents(logId: string, events: StoredEvent[]): Promise<StoredEvent[]> {
1965
+ if (!events.length) return events;
1966
+ const ordered = [...events].sort((a, b) => a.version - b.version);
1967
+ const hydrated = await this.maintenance().hydrate(logId, ordered);
1968
+ const byId = new Map(hydrated.map(event => [event.id, event]));
1969
+ return events.map(event => ({ ...event, payload: byId.get(event.id)!.payload as StoredEvent['payload'] }));
1970
+ }
1971
+
1834
1972
  private eventsAfter(
1835
1973
  logId: string,
1836
1974
  after: number,
@@ -1899,12 +2037,12 @@ export class LedgerApp extends DurableObject<Env> {
1899
2037
  let cursor = 0;
1900
2038
  let firstEvent = true;
1901
2039
  while (cursor < log.headVersion) {
1902
- const events = this.eventsThrough(
2040
+ const events = await this.hydrateEvents(log.id, this.eventsThrough(
1903
2041
  log.id,
1904
2042
  cursor,
1905
2043
  log.headVersion,
1906
2044
  MAX_CATCH_UP,
1907
- );
2045
+ ));
1908
2046
  if (events.length === 0) {
1909
2047
  throw new Error(
1910
2048
  `log ${log.id} export stopped at version ${cursor}`,
@@ -0,0 +1,345 @@
1
+ import { EVENT_SCHEMAS } from '#generated/event-schemas';
2
+ import { checkpointDigest, checkpointEntityEvents, checkpointSelection, type CheckpointChunk, type CheckpointEvent, type CheckpointManifest,
3
+ type CheckpointPage, type LogMaintenance, type MaintenanceJob, type CompactionPreview } from '@hearthlog/contracts/history/checkpoint';
4
+ import { LedgerError, validationError } from './errors';
5
+
6
+ const PAGE = 128;
7
+ const MAX_PAGE_BYTES = 1_500_000;
8
+ const DEFAULT_DAY_WRITE_BUDGET = 10_000;
9
+ const standardTypes = new Set(Object.keys(EVENT_SCHEMAS));
10
+ const directives = ['EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED'];
11
+ type Row = Record<string, SqlStorageValue>;
12
+ interface JobRow extends Row {
13
+ id: string; log_id: string; kind: MaintenanceJob['kind']; through_version: number;
14
+ status: MaintenanceJob['status']; cursor: number; entity_cursor: string; pages: string;
15
+ error: string | null; updated_at: string; day: string; writes: number; reads: number;
16
+ }
17
+ interface LayoutRow extends Row { log_id: string; compacted_through: number; generation: number; checkpoint: string | null; }
18
+ type ReadEvents = (logId: string, after: number, through: number, limit: number, entityId?: string) => Promise<CheckpointEvent[]>;
19
+
20
+ /** Durable job state lives in SQLite; immutable payloads live in the private bucket. */
21
+ export class LogMaintenanceStore {
22
+ private cursors: Array<{ readonly rowsRead: number; readonly rowsWritten: number }> = [];
23
+ private io = { r2Reads: 0, r2Writes: 0, r2Deletes: 0, r2ReadBytes: 0, r2WrittenBytes: 0 };
24
+
25
+ /** Also meter the bounded source scan provided by LedgerApp. */
26
+ observeSql<T extends { readonly rowsRead: number; readonly rowsWritten: number }>(cursor: T): T {
27
+ this.cursors.push(cursor);
28
+ return cursor;
29
+ }
30
+
31
+ private exec<T extends Record<string, SqlStorageValue> = Record<string, SqlStorageValue>>(query: string, ...bindings: SqlStorageValue[]): SqlStorageCursor<T> {
32
+ return this.observeSql(this.storage.sql.exec<T>(query, ...bindings));
33
+ }
34
+
35
+ private async getObject(key: string): Promise<R2ObjectBody | null> {
36
+ this.io.r2Reads++;
37
+ const object = await this.bucket!.get(key);
38
+ this.io.r2ReadBytes += object?.size ?? 0;
39
+ return object;
40
+ }
41
+
42
+ private async deleteObject(key: string): Promise<void> {
43
+ this.io.r2Deletes++;
44
+ await this.bucket!.delete(key);
45
+ }
46
+
47
+ constructor(private readonly storage: DurableObjectStorage, private readonly bucket: R2Bucket | undefined,
48
+ private readonly appId: string, private readonly readEvents: ReadEvents, private readonly assertWritable: () => void = () => undefined,
49
+ private readonly dayWriteBudget = DEFAULT_DAY_WRITE_BUDGET, private readonly dayReadBudget = 1_000_000) {}
50
+
51
+ state(logId: string, jobId?: string): LogMaintenance {
52
+ const layout = this.layout(logId);
53
+ const job = this.exec<JobRow>(`SELECT * FROM log_maintenance_jobs WHERE log_id = ? ${jobId ? 'AND id = ?' : ''} ORDER BY rowid DESC LIMIT 1`, logId, ...(jobId ? [jobId] : [])).toArray()[0];
54
+ return { compactedThroughVersion: layout.compacted_through, storageGeneration: layout.generation,
55
+ checkpoint: layout.checkpoint ? JSON.parse(layout.checkpoint) : null,
56
+ job: job ? publicJob(job) : null, available: Boolean(this.bucket) };
57
+ }
58
+
59
+ preview(logId: string): CompactionPreview {
60
+ const state = this.state(logId);
61
+ const totals = this.exec<{ count: number; bytes: number }>(`SELECT COUNT(*) AS count,
62
+ COALESCE(SUM(length(CAST(payload AS BLOB))), 0) AS bytes FROM log_events
63
+ WHERE log_id = ? AND version <= ? AND archive_key IS NULL`, logId, state.checkpoint?.throughVersion ?? 0).one();
64
+ return { checkpointId: state.checkpoint?.id ?? null, storageGeneration: state.storageGeneration,
65
+ throughVersion: state.checkpoint?.throughVersion ?? 0, activePayloadEvents: totals.count, activePayloadBytes: totals.bytes,
66
+ snapshotBytes: state.checkpoint?.pages.reduce((bytes, page) => bytes + page.bytes, 0) ?? 0 };
67
+ }
68
+
69
+ start(logId: string, id: string, kind: MaintenanceJob['kind'], expectedCheckpointId?: string, expectedGeneration?: number): LogMaintenance {
70
+ if (!this.bucket) throw new LedgerError(503, { error: 'Unavailable', code: 'archive_unconfigured', message: 'Configure private Ledger archive storage first.' });
71
+ this.storage.transactionSync(() => {
72
+ this.assertWritable();
73
+ const prior = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ?', id).toArray()[0];
74
+ if (prior) {
75
+ if (prior.log_id !== logId || prior.kind !== kind) throw validationError('Maintenance request ID already used');
76
+ return;
77
+ }
78
+ const active = this.exec<JobRow>("SELECT * FROM log_maintenance_jobs WHERE log_id = ? AND status NOT IN ('complete', 'failed') LIMIT 1", logId).toArray()[0];
79
+ if (active) throw new LedgerError(409, { error: 'Conflict', code: 'maintenance_in_progress', message: 'Resume the existing maintenance operation.' });
80
+ const log = this.exec<{ version: number; status: string }>('SELECT version, status FROM logs WHERE id = ?', logId).toArray()[0];
81
+ if (!log || log.status !== 'ready') throw validationError('Choose a ready log');
82
+ const boundary = this.exec<{ action_index: number | null; action_size: number | null }>('SELECT action_index, action_size FROM log_events WHERE log_id = ? AND version = ?', logId, log.version).toArray()[0];
83
+ if (boundary?.action_size && boundary.action_index !== boundary.action_size - 1) throw validationError('Incomplete action boundary');
84
+ const checkpoint = this.state(logId).checkpoint;
85
+ if (kind === 'compact' && (expectedCheckpointId !== checkpoint?.id || expectedGeneration !== this.layout(logId).generation)) {
86
+ throw new LedgerError(409, { error: 'Conflict', code: 'stale_maintenance_preview', message: 'The snapshot changed. Review compaction again.' });
87
+ }
88
+ if (kind === 'compact' && (!checkpoint || checkpoint.throughVersion <= this.layout(logId).compacted_through)) {
89
+ throw new LedgerError(409, { error: 'Conflict', code: 'snapshot_required', message: 'Create a newer verified snapshot before compacting.' });
90
+ }
91
+ this.exec(`INSERT INTO log_maintenance_jobs (id, log_id, kind, through_version, status, cursor, entity_cursor, pages, updated_at, day, writes)
92
+ VALUES (?, ?, ?, ?, ?, 0, '', '[]', ?, ?, 0)`, id, logId, kind,
93
+ kind === 'compact' ? checkpoint!.throughVersion : log.version, kind === 'compact' ? 'archiving' : 'building', new Date().toISOString(), day());
94
+ });
95
+ return this.state(logId, id);
96
+ }
97
+
98
+ async step(logId: string, id: string): Promise<LogMaintenance> {
99
+ this.cursors = [];
100
+ this.io = { r2Reads: 0, r2Writes: 0, r2Deletes: 0, r2ReadBytes: 0, r2WrittenBytes: 0 };
101
+ const started = Date.now();
102
+ let result: LogMaintenance | undefined;
103
+ try {
104
+ const before = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId).toArray()[0];
105
+ result = await this.stepValue(logId, id);
106
+ this.assertWritable();
107
+ this.exec('UPDATE log_maintenance_jobs SET reads = ?, writes = ? WHERE id = ? AND log_id = ?',
108
+ (before?.day === day() ? before.reads : 0) + this.cursors.reduce((count, cursor) => count + cursor.rowsRead, 0),
109
+ (before?.day === day() ? before.writes : 0) + this.cursors.reduce((count, cursor) => count + cursor.rowsWritten, 0) + 1,
110
+ id, logId);
111
+ return result;
112
+ } finally {
113
+ console.info('ledger_maintenance_step', { appId: this.appId, logId, jobId: id,
114
+ status: result?.job?.status, cursor: result?.job?.cursor, elapsedMs: Date.now() - started,
115
+ rowsRead: this.cursors.reduce((count, cursor) => count + cursor.rowsRead, 0),
116
+ rowsWritten: this.cursors.reduce((count, cursor) => count + cursor.rowsWritten, 0), ...this.io,
117
+ snapshotEvents: result?.checkpoint?.eventCount, sourceEvents: result?.checkpoint?.sourceEventCount });
118
+ }
119
+ }
120
+
121
+ private async stepValue(logId: string, id: string): Promise<LogMaintenance> {
122
+ if (!this.bucket) throw new Error('Archive storage unavailable');
123
+ this.assertWritable();
124
+ let job = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId).toArray()[0];
125
+ if (!job) throw validationError('Maintenance operation not found');
126
+ if (job.status === 'complete') return this.state(logId, id);
127
+ if (this.state(logId).job?.id !== id) throw new LedgerError(409, { error: 'Conflict', code: 'stale_maintenance', message: 'A newer maintenance operation replaced this one.' });
128
+ if (job.status === 'failed') {
129
+ this.exec("UPDATE log_maintenance_jobs SET status = ?, error = NULL WHERE id = ?", job.kind === 'snapshot' ? 'building' : 'archiving', id);
130
+ job = { ...job, status: job.kind === 'snapshot' ? 'building' : 'archiving', error: null };
131
+ }
132
+ if (job.day !== day()) {
133
+ this.exec('UPDATE log_maintenance_jobs SET day = ?, writes = 0, reads = 0, status = ? WHERE id = ?', day(), job.kind === 'snapshot' ? 'building' : 'archiving', id);
134
+ job = { ...job, day: day(), writes: 0, reads: 0, status: job.kind === 'snapshot' ? 'building' : 'archiving' };
135
+ }
136
+ if (job.writes >= this.dayWriteBudget || job.reads >= this.dayReadBudget) {
137
+ this.exec("UPDATE log_maintenance_jobs SET status = 'paused' WHERE id = ?", id);
138
+ return this.state(logId, id);
139
+ }
140
+ try {
141
+ if (job.status === 'building') await this.build(job);
142
+ else if (job.status === 'cleaning') await this.clean(job);
143
+ else if (job.status === 'archiving' || job.status === 'reclaiming') await this.archive(job);
144
+ } catch (cause) {
145
+ // Keep source rows and page cursors untouched when verification fails.
146
+ this.assertWritable();
147
+ this.exec("UPDATE log_maintenance_jobs SET status = 'failed', error = ?, updated_at = ? WHERE id = ?",
148
+ cause instanceof Error ? cause.message : 'Maintenance failed', new Date().toISOString(), id);
149
+ }
150
+ return this.state(logId, id);
151
+ }
152
+
153
+ async chunk(logId: string, index: number): Promise<CheckpointChunk> {
154
+ const manifest = this.state(logId).checkpoint;
155
+ const page = manifest?.pages[index];
156
+ if (!page || !this.bucket) throw validationError('Snapshot page not found');
157
+ const object = await this.getObject(page.key);
158
+ if (!object) throw archiveError('Snapshot page is missing');
159
+ const value = await object.text();
160
+ if (await checkpointDigest(value) !== page.sha256) throw archiveError('Snapshot checksum mismatch');
161
+ return JSON.parse(value) as CheckpointChunk;
162
+ }
163
+
164
+ /** Bounded archive pages are verified before any accepted payload is reclaimed. */
165
+ async hydrate(logId: string, events: CheckpointEvent[]): Promise<CheckpointEvent[]> {
166
+ if (!events.length) return events;
167
+ const references = this.exec<{ event_id: string; archive_key: string }>(
168
+ 'SELECT event_id, archive_key FROM log_events WHERE log_id = ? AND event_id IN (SELECT value FROM json_each(?)) AND archive_key IS NOT NULL',
169
+ logId, JSON.stringify(events.map(event => event.id))).toArray();
170
+ if (!references.length) return events;
171
+ if (!this.bucket) throw archiveError('Archive storage unavailable');
172
+ const wanted = new Set(events.map(event => event.id));
173
+ const payloads = new Map<string, CheckpointEvent>();
174
+ for (const key of new Set(references.map(row => row.archive_key))) {
175
+ if (!key.startsWith(`v1/${this.appId}/`)) throw archiveError('Archive scope mismatch');
176
+ const reference = this.exec<{ sha256: string }>('SELECT sha256 FROM log_archive_pages WHERE object_key = ? LIMIT 1', key).toArray()[0];
177
+ if (!reference) throw archiveError('Archive manifest is missing');
178
+ const object = await this.getObject(key);
179
+ if (!object) throw archiveError('Archived history is missing');
180
+ const value = await object.text();
181
+ if (await checkpointDigest(value) !== reference.sha256) throw archiveError('Archive checksum mismatch');
182
+ for (const event of JSON.parse(value) as CheckpointEvent[]) if (wanted.has(event.id)) payloads.set(event.id, event);
183
+ }
184
+ const archivedIds = new Set(references.map(row => row.event_id));
185
+ return events.map(event => {
186
+ if (!archivedIds.has(event.id)) return event;
187
+ const original = payloads.get(event.id);
188
+ if (!original || original.version !== event.version || original.entityId !== event.entityId
189
+ || original.entityVersion !== event.entityVersion || original.type !== event.type || original.timestamp !== event.timestamp) throw archiveError('Archive identity mismatch');
190
+ return original;
191
+ });
192
+ }
193
+
194
+ private layout(logId: string): LayoutRow {
195
+ return this.exec<LayoutRow>('SELECT * FROM log_storage WHERE log_id = ?', logId).toArray()[0]
196
+ ?? { log_id: logId, compacted_through: 0, generation: 0, checkpoint: null };
197
+ }
198
+
199
+ private async build(job: JobRow): Promise<void> {
200
+ if (job.cursor < job.through_version) {
201
+ // Scan the primary log index once. Work records are populated in
202
+ // budgeted batches rather than building an index over old history
203
+ // during a schema migration (which could exhaust free-tier writes).
204
+ const events = await this.readEvents(job.log_id, job.cursor, job.through_version, 16);
205
+ if (!events.length || events[0]!.version !== job.cursor + 1) throw new Error('Snapshot source contains a gap');
206
+ const selected = this.selection(job.log_id, job.through_version);
207
+ const groups = new Map<string, CheckpointEvent[]>();
208
+ for (const event of events) {
209
+ const group = groups.get(event.entityId) ?? [];
210
+ group.push(event); groups.set(event.entityId, group);
211
+ }
212
+ for (const [entityId, source] of groups) {
213
+ const prior = this.exec<{ object_key: string }>('SELECT object_key FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, entityId).toArray()[0];
214
+ const previous = prior ? await this.readWork(prior.object_key) : { events: [], claims: [] };
215
+ const last = source[source.length - 1]!;
216
+ const witnesses = checkpointEntityEvents([...previous.events, ...source.filter(event => selected(event) && !directives.includes(event.type))], [
217
+ event => typeof event.payload.status === 'string' ? `status:${event.payload.status}` : null,
218
+ event => typeof event.payload.thumbnailRef === 'string' ? 'thumbnail:string' : null,
219
+ ], standardTypes);
220
+ const value = JSON.stringify({ events: witnesses, claims: [{ entityId, currentVersion: last.entityVersion, logVersion: last.version }] });
221
+ if (new TextEncoder().encode(value).byteLength > 8_000_000) throw new Error('Entity replay state exceeds the snapshot work budget');
222
+ const key = `v1/${this.appId}/${job.log_id}/work/${job.id}/${last.version}.json`;
223
+ await this.putVerified(key, value);
224
+ this.storage.transactionSync(() => {
225
+ this.assertWritable();
226
+ this.exec(`INSERT INTO log_checkpoint_entities (job_id, entity_id, object_key) VALUES (?, ?, ?)
227
+ ON CONFLICT(job_id, entity_id) DO UPDATE SET object_key = excluded.object_key`, job.id, entityId, key);
228
+ this.exec('UPDATE log_maintenance_jobs SET writes = writes + 2 WHERE id = ?', job.id);
229
+ });
230
+ if (prior && prior.object_key !== key) await this.deleteObject(prior.object_key);
231
+ }
232
+ this.assertWritable();
233
+ this.exec('UPDATE log_maintenance_jobs SET cursor = ?, writes = writes + 1, updated_at = ? WHERE id = ?',
234
+ events[events.length - 1]!.version, new Date().toISOString(), job.id);
235
+ return;
236
+ }
237
+ const pages = JSON.parse(job.pages) as CheckpointPage[];
238
+ const entities = this.exec<{ entity_id: string; object_key: string }>(`SELECT entity_id, object_key FROM log_checkpoint_entities
239
+ WHERE job_id = ? AND entity_id > ? ORDER BY entity_id LIMIT 16`, job.id, job.entity_cursor).toArray();
240
+ if (!entities.length) {
241
+ const manifest: CheckpointManifest = { format: 1, appId: this.appId, logId: job.log_id, id: job.id,
242
+ throughVersion: job.through_version, createdAt: new Date().toISOString(), pages,
243
+ eventCount: pages.reduce((count, page) => count + page.count, 0), sourceEventCount: job.through_version,
244
+ sha256: await checkpointDigest(JSON.stringify(pages)) };
245
+ this.storage.transactionSync(() => {
246
+ this.assertWritable();
247
+ this.exec(`INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint) VALUES (?, 0, 0, ?)
248
+ ON CONFLICT(log_id) DO UPDATE SET checkpoint = excluded.checkpoint`, job.log_id, JSON.stringify(manifest));
249
+ this.exec("UPDATE log_maintenance_jobs SET status = 'cleaning', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
250
+ });
251
+ return;
252
+ }
253
+ const chunk: CheckpointChunk = { events: [], claims: [] };
254
+ let last = job.entity_cursor;
255
+ for (const entity of entities) {
256
+ const part = await this.readWork(entity.object_key);
257
+ chunk.events.push(...part.events); chunk.claims.push(...part.claims);
258
+ last = entity.entity_id;
259
+ if (new TextEncoder().encode(JSON.stringify(chunk)).byteLength >= MAX_PAGE_BYTES) break;
260
+ }
261
+ chunk.events.sort((a, b) => a.version - b.version);
262
+ const value = JSON.stringify(chunk);
263
+ const key = `v1/${this.appId}/${job.log_id}/snapshots/${job.id}/${pages.length}.json`;
264
+ const sha256 = await this.putVerified(key, value);
265
+ pages.push({ key, sha256, count: chunk.events.length, bytes: new TextEncoder().encode(value).byteLength });
266
+ this.assertWritable();
267
+ this.exec('UPDATE log_maintenance_jobs SET entity_cursor = ?, pages = ?, writes = writes + 1, updated_at = ? WHERE id = ?',
268
+ last, JSON.stringify(pages), new Date().toISOString(), job.id);
269
+ }
270
+
271
+ private async readWork(key: string): Promise<CheckpointChunk> {
272
+ const object = await this.getObject(key);
273
+ if (!object) throw new Error('Snapshot work page is missing');
274
+ const encoded = await object.text();
275
+ if (await checkpointDigest(encoded) !== object.customMetadata?.sha256) throw new Error('Snapshot work checksum mismatch');
276
+ return JSON.parse(encoded) as CheckpointChunk;
277
+ }
278
+
279
+ private async clean(job: JobRow): Promise<void> {
280
+ const rows = this.exec<{ entity_id: string; object_key: string }>('SELECT entity_id, object_key FROM log_checkpoint_entities WHERE job_id = ? LIMIT 16', job.id).toArray();
281
+ for (const row of rows) {
282
+ await this.deleteObject(row.object_key);
283
+ this.assertWritable();
284
+ this.exec('DELETE FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, row.entity_id);
285
+ }
286
+ this.exec('UPDATE log_maintenance_jobs SET status = ?, writes = writes + ?, updated_at = ? WHERE id = ?',
287
+ rows.length ? 'cleaning' : 'complete', rows.length + 1, new Date().toISOString(), job.id);
288
+ }
289
+
290
+ private async putVerified(key: string, value: string): Promise<string> {
291
+ const sha256 = await checkpointDigest(value);
292
+ this.io.r2Writes++;
293
+ this.io.r2WrittenBytes += new TextEncoder().encode(value).byteLength;
294
+ await this.bucket!.put(key, value, { customMetadata: { sha256 }, httpMetadata: { contentType: 'application/json' } });
295
+ const check = await this.getObject(key);
296
+ if (!check || await checkpointDigest(await check.text()) !== sha256) throw new Error('Snapshot verification failed');
297
+ return sha256;
298
+ }
299
+
300
+ private async archive(job: JobRow): Promise<void> {
301
+ // Metadata stays as the compact identity/action registry. Only payloads
302
+ // move, so historical boundaries and event/request uniqueness survive.
303
+ const rows = this.exec<{ version: number }>(`SELECT version FROM log_events WHERE log_id = ?
304
+ AND version <= ? AND version > ? AND archive_key IS NULL ORDER BY version LIMIT ?`, job.log_id, job.through_version, job.cursor, PAGE).toArray();
305
+ if (!rows.length) {
306
+ this.storage.transactionSync(() => {
307
+ this.assertWritable();
308
+ this.exec('UPDATE log_storage SET compacted_through = MAX(compacted_through, ?), generation = generation + 1 WHERE log_id = ?', job.through_version, job.log_id);
309
+ this.exec("UPDATE log_maintenance_jobs SET status = 'complete', cursor = ?, updated_at = ? WHERE id = ?", job.through_version, new Date().toISOString(), job.id);
310
+ });
311
+ return;
312
+ }
313
+ const first = rows[0]!.version;
314
+ const events = await this.readEvents(job.log_id, first - 1, rows[rows.length - 1]!.version, PAGE);
315
+ const value = JSON.stringify(events);
316
+ const sha256 = await checkpointDigest(value);
317
+ const key = `v1/${this.appId}/${job.log_id}/archive/${job.id}/${first}.json`;
318
+ await this.putVerified(key, value);
319
+ this.storage.transactionSync(() => {
320
+ this.assertWritable();
321
+ this.exec(`INSERT OR IGNORE INTO log_archive_pages (log_id, first_version, last_version, object_key, sha256) VALUES (?, ?, ?, ?, ?)`,
322
+ job.log_id, first, events[events.length - 1]!.version, key, sha256);
323
+ const reclaimed = this.exec(`UPDATE log_events SET archive_key = ?, payload_bytes = COALESCE(payload_bytes, length(CAST(payload AS BLOB))), payload = CASE WHEN event_type IN
324
+ ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') THEN payload ELSE '{}' END
325
+ WHERE log_id = ? AND version >= ? AND version <= ? AND archive_key IS NULL`, key, job.log_id, first, events[events.length - 1]!.version);
326
+ reclaimed.toArray();
327
+ this.exec("UPDATE log_maintenance_jobs SET status = 'archiving', cursor = ?, writes = writes + ?, updated_at = ? WHERE id = ?",
328
+ events[events.length - 1]!.version, reclaimed.rowsWritten + 2, new Date().toISOString(), job.id);
329
+ });
330
+ }
331
+
332
+ private selection(logId: string, through: number): (event: CheckpointEvent) => boolean {
333
+ const rows = this.exec<{ version: number; event_id: string; event_type: string; payload: string }>(`SELECT version, event_id, event_type, payload FROM log_events
334
+ WHERE log_id = ? AND version <= ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') ORDER BY version DESC`, logId, through).toArray();
335
+ return checkpointSelection(rows.map(event => ({ id: event.event_id, type: event.event_type,
336
+ version: event.version, payload: JSON.parse(event.payload) as Record<string, unknown> })), through);
337
+ }
338
+ }
339
+
340
+ function day(): string { return new Date().toISOString().slice(0, 10); }
341
+ function archiveError(message: string): LedgerError { return new LedgerError(503, { error: 'Unavailable', code: 'archive_unavailable', message }); }
342
+ function publicJob(row: JobRow): MaintenanceJob {
343
+ return { id: row.id, logId: row.log_id, kind: row.kind, throughVersion: row.through_version, status: row.status,
344
+ cursor: row.cursor, error: row.error, updatedAt: row.updated_at };
345
+ }
package/src/schema.ts CHANGED
@@ -187,6 +187,34 @@ const LEDGER_APP_MIGRATIONS: readonly Migration[] = [
187
187
  ALTER TABLE app_metadata ADD COLUMN fence_mode TEXT;
188
188
  ALTER TABLE app_metadata ADD COLUMN fence_destination TEXT;
189
189
  ` },
190
+ { id: 14, sql: `
191
+ ALTER TABLE log_events ADD COLUMN archive_key TEXT;
192
+ ALTER TABLE log_events ADD COLUMN payload_bytes INTEGER;
193
+ CREATE INDEX log_events_snapshot_markers ON log_events(log_id, version)
194
+ WHERE event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED');
195
+ CREATE INDEX log_events_archived_cursor ON log_events(log_id, version) WHERE archive_key IS NOT NULL;
196
+ CREATE TABLE log_storage (
197
+ log_id TEXT PRIMARY KEY, compacted_through INTEGER NOT NULL DEFAULT 0,
198
+ generation INTEGER NOT NULL DEFAULT 0, checkpoint TEXT
199
+ );
200
+ CREATE TABLE log_maintenance_jobs (
201
+ id TEXT PRIMARY KEY, log_id TEXT NOT NULL, kind TEXT NOT NULL,
202
+ through_version INTEGER NOT NULL, status TEXT NOT NULL, cursor INTEGER NOT NULL,
203
+ entity_cursor TEXT NOT NULL, pages TEXT NOT NULL, error TEXT,
204
+ updated_at TEXT NOT NULL, day TEXT NOT NULL, writes INTEGER NOT NULL, reads INTEGER NOT NULL DEFAULT 0
205
+ );
206
+ CREATE INDEX log_maintenance_scope ON log_maintenance_jobs(log_id);
207
+ CREATE TABLE log_checkpoint_entities (
208
+ job_id TEXT NOT NULL, entity_id TEXT NOT NULL, object_key TEXT NOT NULL,
209
+ PRIMARY KEY(job_id, entity_id)
210
+ );
211
+ CREATE TABLE log_archive_deletions (log_id TEXT PRIMARY KEY);
212
+ CREATE TABLE log_archive_pages (
213
+ log_id TEXT NOT NULL, first_version INTEGER NOT NULL, last_version INTEGER NOT NULL,
214
+ object_key TEXT NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY(log_id, first_version)
215
+ );
216
+ CREATE INDEX log_archive_object ON log_archive_pages(object_key);
217
+ ` },
190
218
  ];
191
219
 
192
220
  const APP_REGISTRY_MIGRATIONS: readonly Migration[] = [
package/src/schemas.ts CHANGED
@@ -128,6 +128,13 @@ export const logParams = t.Object(
128
128
  strictObject,
129
129
  );
130
130
 
131
+ export const maintenanceBody = t.Object({
132
+ requestId: uuid('request id'), kind: t.Union([t.Literal('snapshot'), t.Literal('compact')]),
133
+ expectedCheckpointId: t.Optional(uuid('checkpoint id')), expectedGeneration: t.Optional(t.Integer(queryInteger)),
134
+ }, strictObject);
135
+ export const maintenanceStepBody = t.Object({ requestId: uuid('request id') }, strictObject);
136
+ export const checkpointPageQuery = t.Object({ page: t.Numeric(queryInteger) }, strictObject);
137
+
131
138
  export const galleryPhotoReferenceParams = t.Object(
132
139
  {
133
140
  logId: uuid('log id'),