@hearthlog/ledger 0.10.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -192,7 +192,8 @@ Authenticated endpoints under `/logs/:logId`:
192
192
  - `GET /maintenance-preview`: the current snapshot/generation and prefix payload counts/bytes for review.
193
193
  - `POST /maintenance`: `{ requestId, kind: "snapshot" | "compact", expectedCheckpointId?, expectedGeneration? }`.
194
194
  Compaction requires the checkpoint ID and generation from the reviewed preview; stale previews fail with 409.
195
- - `POST /maintenance/step`: `{ requestId }`, resumable work using the same request identity.
195
+ - `POST /maintenance/step`: `{ requestId }`; runs one bounded step now (sharing any step already in flight) and restarts a
196
+ job that stopped after repeated failures. Normally unnecessary: the Ledger advances jobs by itself (see below).
196
197
  - `GET /checkpoint?page=N`: one checksum-verified canonical replay chunk and its entity claims.
197
198
 
198
199
  From the development site, the same operation can run without an open browser:
@@ -206,6 +207,15 @@ pnpm -C dev ledger:maintain --log <log-id> --resume
206
207
  Set `LEDGER_BASE_URL` and `LEDGER_AUTH_TOKEN` in the process environment; the CLI never prints credentials.
207
208
  Use `--request-id` to retry an explicit operation and `--once` for one bounded work step.
208
209
 
210
+ A started job is carried to completion by the application object's alarm, with no caller or open browser. Each step
211
+ folds up to 512 source events into small per-entity replay slots (source versions only, in SQLite), then writes the
212
+ retained events as checksummed pages. A later snapshot starts from the previous one's retained facts instead of re-reading
213
+ archived history, unless a restore/removal marker has been added since. A failing step is retried with backoff; the
214
+ error and attempt count stay visible, and after five attempts the job stops as `failed` until it is explicitly retried
215
+ (the maintenance panel's Retry, `POST /maintenance/step`, or the CLI). A job that uses its daily budget is `paused`
216
+ with `pauseReason` and `resumeAt`, and resumes by itself at the next UTC day. Opening the panel re-arms a job whose
217
+ alarm was lost.
218
+
209
219
  Jobs persist every source batch and publish only verified snapshots. Canonical snapshots keep original source
210
220
  witnesses for raw field values, explicit nulls, timestamps and lifecycle transitions; dynamic/nonstandard facts
211
221
  are retained losslessly. Browser projection uses a separate baseline table plus accepted suffix and pending
@@ -217,9 +227,8 @@ payload placeholders. The SDK's server checkpoint reader lets Stash seed a cold
217
227
  `MAINTENANCE_DAILY_WRITES` defaults to `10000` per job (bounded to 128–100000), and
218
228
  `MAINTENANCE_DAILY_READS` defaults to `1000000` (bounded to 10000–1000000000). Jobs pause at either budget. Budgets use consumed SQL cursor
219
229
  counts and can overrun by one bounded step. Structured step logs include SQL rows, R2 operations/bytes, elapsed
220
- time and replay counts, without payloads or credentials. Resume with the same ID on a later UTC quota day;
221
- other applications consume account quota too. The browser
222
- can be closed safely and the operation resumed from the panel. No automatic snapshots or archive expiry run.
230
+ time and replay counts, without payloads or credentials. Other applications consume account quota too. The browser
231
+ can be closed safely; the job continues on the server. No automatic snapshots or archive expiry run.
223
232
  Deleting a whole log uses bounded background cleanup of that log's private archive objects, after normal
224
233
  fork/deletion checks. Individual archived-history deletion is a separate future feature.
225
234
 
@@ -228,7 +237,7 @@ payloads into the existing portable format. A SQLite-only export returns `409 ar
228
237
  external checkpoint/archive objects are required, avoiding a misleading incomplete backup. Rollback after
229
238
  compaction needs a compatible reader or a full logical restore first.
230
239
 
231
- The first format bounds a single entity's replay witnesses to 8 MB and stores page descriptors in SQLite.
240
+ The first format bounds a single entity's replay state to about 1.5 MB and stores page descriptors in SQLite.
232
241
  Exceeding a snapshot work/metadata bound fails the job while source history stays readable. Measure maintenance
233
242
  work and storage on representative data; retained identity metadata still grows, and physical SQLite file size
234
243
  need not shrink immediately when payload space becomes reusable.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hearthlog/ledger",
3
- "version": "0.10.1",
3
+ "version": "0.11.0",
4
4
  "description": "Hearthlog's Ledger: the event store Worker (Elysia over SQLite Durable Objects), its contracts and tests, and the logical backup tooling",
5
5
  "license": "AGPL-3.0-only",
6
6
  "repository": {
@@ -30,7 +30,7 @@
30
30
  "dependencies": {
31
31
  "@sinclair/typebox": "0.34.52",
32
32
  "elysia": "1.4.30",
33
- "@hearthlog/contracts": "0.10.1"
33
+ "@hearthlog/contracts": "0.11.0"
34
34
  },
35
35
  "peerDependencies": {
36
36
  "@hearthlog/generated": "*"
@@ -41,7 +41,11 @@ async function main(): Promise<void> {
41
41
  if (state.job?.status === 'complete') break;
42
42
  state = await request('/step', { requestId });
43
43
  console.log(JSON.stringify(state.job));
44
+ if (!values.once && state.job && !['complete', 'failed', 'paused'].includes(state.job.status)) {
45
+ await new Promise(resolve => setTimeout(resolve, 500));
46
+ }
44
47
  if (state.job?.status === 'failed') throw new Error(state.job.error ?? 'Maintenance failed');
48
+ if (state.job?.status === 'paused') console.error(`Paused (${state.job.pauseReason ?? 'budget'}); the Ledger resumes at ${state.job.resumeAt ?? 'the next UTC day'}.`);
45
49
  } while (!values.once && state.job?.status !== 'complete' && state.job?.status !== 'paused');
46
50
  console.log(JSON.stringify({ logId: values.log, requestId, status: state.job?.status,
47
51
  checkpointId: state.checkpoint?.id ?? null, throughVersion: state.checkpoint?.throughVersion ?? 0,
package/src/ledger-app.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { MaintenanceSteps } from './maintenance-steps';
1
2
  import { DurableObject } from "cloudflare:workers";
2
3
  import { LogMaintenanceStore } from './log-maintenance';
3
4
  import type { CheckpointChunk, LogMaintenance, CompactionPreview } from '@hearthlog/contracts/history/checkpoint';
@@ -142,19 +143,23 @@ interface SocketSessionRow extends Record<string, SqlStorageValue> {
142
143
  }
143
144
 
144
145
  export class LedgerApp extends DurableObject<Env> {
145
- private maintenanceQueue: Promise<unknown> = Promise.resolve();
146
+ private readonly maintenanceSteps = new MaintenanceSteps<LogMaintenance>(work => this.ctx.waitUntil(work));
146
147
 
147
148
  private maintenance(assertWritable: () => void = () => undefined): LogMaintenanceStore {
148
149
  const appId = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one().app_id;
149
150
  const store = new LogMaintenanceStore(this.ctx.storage, this.env.LEDGER_ARCHIVE, appId,
150
- async (logId, after, through, limit, entityId) => {
151
+ async (logId, after, through, limit) => {
151
152
  const rows = store.observeSql(this.ctx.storage.sql.exec<EventRow>(`WITH candidates AS (
152
153
  SELECT *, SUM(COALESCE(payload_bytes, length(CAST(payload AS BLOB)))) OVER (ORDER BY version) AS byte_count,
153
154
  ROW_NUMBER() OVER (ORDER BY version) AS row_number
154
- FROM (SELECT * FROM log_events WHERE log_id = ? AND version > ? AND version <= ?
155
- ${entityId === undefined ? '' : 'AND entity_id = ?'} ORDER BY version LIMIT ?)
155
+ FROM (SELECT * FROM log_events WHERE log_id = ? AND version > ? AND version <= ? ORDER BY version LIMIT ?)
156
156
  ) SELECT * FROM candidates WHERE byte_count <= 1500000 OR row_number = 1`,
157
- logId, after, through, ...(entityId === undefined ? [] : [entityId]), limit)).toArray();
157
+ logId, after, through, limit)).toArray();
158
+ return store.hydrate(logId, rows.map(storedEvent));
159
+ }, async (logId, versions) => {
160
+ const rows = store.observeSql(this.ctx.storage.sql.exec<EventRow>(
161
+ 'SELECT * FROM log_events WHERE log_id = ? AND version IN (SELECT value FROM json_each(?)) ORDER BY version',
162
+ logId, JSON.stringify(versions))).toArray();
158
163
  return store.hydrate(logId, rows.map(storedEvent));
159
164
  }, assertWritable, Math.max(128, Math.min(100000, Number(this.env.MAINTENANCE_DAILY_WRITES) || 10000)),
160
165
  Math.max(10000, Math.min(1_000_000_000, Number(this.env.MAINTENANCE_DAILY_READS) || 1_000_000)));
@@ -162,10 +167,13 @@ export class LedgerApp extends DurableObject<Env> {
162
167
  }
163
168
 
164
169
  async logMaintenance(access: AccessClaims, logId: string): Promise<RpcResult<LogMaintenance>> {
165
- return rpcCall(() => {
170
+ return rpcCall(async () => {
166
171
  this.currentAccess(access);
167
172
  if (!this.log(parseUuid(logId, 'log id'))) throw validationError('Log not found');
168
- return rpcSuccess(this.maintenance().state(logId));
173
+ const store = this.maintenance();
174
+ // A job begun before alarms existed (or whose alarm was lost) is picked up again here.
175
+ if (store.hasActiveJob() && await this.ctx.storage.getAlarm() === null) await this.scheduleAlarm();
176
+ return rpcSuccess(store.state(logId));
169
177
  });
170
178
  }
171
179
 
@@ -178,19 +186,26 @@ export class LedgerApp extends DurableObject<Env> {
178
186
  }
179
187
 
180
188
  async startLogMaintenance(access: AccessClaims, logId: string, id: string, kind: 'snapshot' | 'compact', expectedCheckpointId?: string, expectedGeneration?: number): Promise<RpcResult<LogMaintenance>> {
181
- return rpcCall(() => {
189
+ return rpcCall(async () => {
182
190
  this.writableAccess(access);
183
- return rpcSuccess(this.maintenance().start(parseUuid(logId, 'log id'), parseUuid(id, 'request id'), kind, expectedCheckpointId, expectedGeneration));
191
+ const state = this.maintenance().start(parseUuid(logId, 'log id'), parseUuid(id, 'request id'), kind, expectedCheckpointId, expectedGeneration);
192
+ // The server carries the job to completion; the caller only needs to watch.
193
+ await this.scheduleAlarm();
194
+ return rpcSuccess(state);
184
195
  });
185
196
  }
186
197
 
187
198
  async stepLogMaintenance(access: AccessClaims, logId: string, id: string): Promise<RpcResult<LogMaintenance>> {
188
199
  return rpcCall(async () => {
189
200
  this.writableAccess(access);
190
- const run = this.maintenanceQueue.then(() => this.maintenance(() => { this.writableAccess(access); }).step(parseUuid(logId, 'log id'), parseUuid(id, 'request id')));
191
- this.maintenanceQueue = run.then(() => undefined, () => undefined);
192
- const result = await run;
201
+ const parsedLog = parseUuid(logId, 'log id');
202
+ const parsedId = parseUuid(id, 'request id');
203
+ const result = await this.maintenanceSteps.advance(`${parsedLog}:${parsedId}`,
204
+ () => this.maintenance(() => { this.writableAccess(access); }).step(parsedLog, parsedId),
205
+ () => this.maintenance().state(parsedLog, parsedId));
193
206
  this.currentAccess(access);
207
+ // An explicit step also restarts a job that stopped after repeated failures.
208
+ await this.scheduleAlarm();
194
209
  return rpcSuccess(result);
195
210
  });
196
211
  }
@@ -210,21 +225,59 @@ export class LedgerApp extends DurableObject<Env> {
210
225
  });
211
226
  }
212
227
 
213
- /** Delete storage belonging to an explicitly deleted log, in bounded batches. */
228
+ /** Advance log maintenance and delete storage of deleted logs, each in bounded batches. */
214
229
  override async alarm(): Promise<void> {
230
+ let failed = false;
231
+ try { await this.deleteArchivedLogs(); }
232
+ catch (cause) { failed = true; console.error('ledger_archive_deletion_failed', normalizeError(cause).message); }
233
+ try { await this.advanceMaintenance(); }
234
+ catch (cause) { failed = true; console.error('ledger_maintenance_alarm_failed', normalizeError(cause).message); }
235
+ await this.scheduleAlarm({ deletionDelayMs: 60_000, failureDelayMs: failed ? 30_000 : undefined });
236
+ }
237
+
238
+ private async deleteArchivedLogs(): Promise<void> {
215
239
  const job = this.ctx.storage.sql.exec<{ log_id: string }>('SELECT log_id FROM log_archive_deletions LIMIT 1').toArray()[0];
216
240
  if (!job) return;
217
- try {
218
- const app = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one();
219
- const bucket = this.env.LEDGER_ARCHIVE;
220
- const objects = await bucket.list({ prefix: `v1/${app.app_id}/${job.log_id}/`, limit: 128 });
221
- if (objects.objects.length) await bucket.delete(objects.objects.map(object => object.key));
222
- if (!objects.truncated) this.ctx.storage.sql.exec('DELETE FROM log_archive_deletions WHERE log_id = ?', job.log_id);
223
- } finally {
224
- if (this.ctx.storage.sql.exec('SELECT 1 FROM log_archive_deletions LIMIT 1').toArray().length) {
225
- await this.ctx.storage.setAlarm(Date.now() + 60_000);
226
- }
241
+ const app = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one();
242
+ const bucket = this.env.LEDGER_ARCHIVE;
243
+ const objects = await bucket.list({ prefix: `v1/${app.app_id}/${job.log_id}/`, limit: 128 });
244
+ if (objects.objects.length) await bucket.delete(objects.objects.map(object => object.key));
245
+ if (!objects.truncated) this.ctx.storage.sql.exec('DELETE FROM log_archive_deletions WHERE log_id = ?', job.log_id);
246
+ }
247
+
248
+ /** Run due maintenance steps for a few seconds, so a job finishes without any caller. */
249
+ private async advanceMaintenance(): Promise<void> {
250
+ if (this.metadataOptional() === undefined) return;
251
+ const started = Date.now();
252
+ const assertWritable = () => this.assertApplicationWritable();
253
+ do {
254
+ const due = this.maintenance(assertWritable).dueJobs()[0];
255
+ if (!due) return;
256
+ await this.maintenanceSteps.run(`${due.logId}:${due.id}`, () => this.maintenance(assertWritable).step(due.logId, due.id));
257
+ } while (Date.now() - started < 5000);
258
+ }
259
+
260
+ /** Wake at the earliest time any pending work (maintenance step, retry, resume, deletion) is due. */
261
+ private async scheduleAlarm(options: { deletionDelayMs?: number; failureDelayMs?: number } = {}): Promise<void> {
262
+ const now = Date.now();
263
+ const wakes: number[] = [];
264
+ if (this.ctx.storage.sql.exec('SELECT 1 FROM log_archive_deletions LIMIT 1').toArray().length) {
265
+ wakes.push(now + (options.deletionDelayMs ?? 1000));
266
+ }
267
+ if (this.metadataOptional() !== undefined) {
268
+ const wake = this.maintenance().nextWake();
269
+ if (wake !== null) wakes.push(Math.max(wake, now + (options.failureDelayMs ?? 50)));
227
270
  }
271
+ if (wakes.length) await this.ctx.storage.setAlarm(Math.min(...wakes));
272
+ }
273
+
274
+ /** Background work has no caller to authorize it, but must still stop for revoked or fenced applications. */
275
+ private assertApplicationWritable(): void {
276
+ if (this.metadata().revoked_at !== null) {
277
+ throw new LedgerError(401, { error: "access token has been rotated or revoked", code: "unauthorized" });
278
+ }
279
+ const fence = this.fence();
280
+ if (fence) throw fenceError(fence);
228
281
  }
229
282
 
230
283
  override async fetch(request: Request): Promise<Response> {
@@ -707,7 +760,7 @@ export class LedgerApp extends DurableObject<Env> {
707
760
  return rpcCall(async () => {
708
761
  const claims = this.writableAccess(access);
709
762
  this.deleteLogValue(parseUuid(logId, "logId"), claims);
710
- await this.ctx.storage.setAlarm(Date.now() + 1000);
763
+ await this.scheduleAlarm();
711
764
  return rpcSuccess(null, 204);
712
765
  });
713
766
  }
@@ -857,7 +910,7 @@ export class LedgerApp extends DurableObject<Env> {
857
910
  throw transferError("not_importing", "Only an import in progress can be aborted");
858
911
  }
859
912
  this.ctx.storage.transactionSync(() => {
860
- for (const table of ["log_storage", "log_archive_pages", "log_maintenance_jobs", "gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
913
+ for (const table of ["log_checkpoint_work", "log_checkpoint_versions", "log_checkpoint_entities", "log_storage", "log_archive_pages", "log_maintenance_jobs", "gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
861
914
  this.ctx.storage.sql.exec(`DELETE FROM ${table}`);
862
915
  }
863
916
  });
@@ -1171,6 +1224,8 @@ export class LedgerApp extends DurableObject<Env> {
1171
1224
  }
1172
1225
  for (const query of [
1173
1226
  "DELETE FROM log_checkpoint_entities WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
1227
+ "DELETE FROM log_checkpoint_work WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
1228
+ "DELETE FROM log_checkpoint_versions WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
1174
1229
  "DELETE FROM log_fork_requests WHERE fork_log_id = ?",
1175
1230
  "DELETE FROM append_requests WHERE log_id = ?",
1176
1231
  "DELETE FROM log_storage WHERE log_id = ?",
@@ -1,21 +1,44 @@
1
1
  import { EVENT_SCHEMAS } from '#generated/event-schemas';
2
- import { checkpointDigest, checkpointEntityEvents, checkpointSelection, type CheckpointChunk, type CheckpointEvent, type CheckpointManifest,
3
- type CheckpointPage, type LogMaintenance, type MaintenanceJob, type CompactionPreview } from '@hearthlog/contracts/history/checkpoint';
2
+ import { checkpointDigest, checkpointSelection, checkpointStateVersions, foldCheckpointEvent, newCheckpointEntityState,
3
+ type CheckpointChunk, type CheckpointEntityState, type CheckpointEvent, type CheckpointManifest, type CheckpointPage,
4
+ type CheckpointSelector, type LogMaintenance, type MaintenanceJob, type CompactionPreview } from '@hearthlog/contracts/history/checkpoint';
4
5
  import { LedgerError, validationError } from './errors';
5
6
 
7
+ /** Events archived per compaction step. */
6
8
  const PAGE = 128;
9
+ /** Source events folded per snapshot step (also bounded by the byte window of the reader). */
10
+ const FOLD_EVENTS = 512;
11
+ /** Entities indexed into snapshot pages per step. */
12
+ const INDEX_ENTITIES = 256;
13
+ const CLEAN_ROWS = 256;
7
14
  const MAX_PAGE_BYTES = 1_500_000;
15
+ const MAX_STATE_BYTES = 1_500_000;
16
+ const MAX_MARKER_BYTES = 1_000_000;
17
+ /** Automatic attempts before a job stops and waits for an explicit retry. */
18
+ export const MAX_ATTEMPTS = 5;
8
19
  const DEFAULT_DAY_WRITE_BUDGET = 10_000;
9
20
  const standardTypes = new Set(Object.keys(EVENT_SCHEMAS));
10
21
  const directives = ['EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED'];
22
+ const SELECTORS: readonly CheckpointSelector[] = [
23
+ event => typeof event.payload.status === 'string' ? `status:${event.payload.status}` : null,
24
+ event => typeof event.payload.thumbnailRef === 'string' ? 'thumbnail:string' : null,
25
+ ];
11
26
  type Row = Record<string, SqlStorageValue>;
27
+ type Phase = 'reset' | 'seed' | 'fold' | 'index' | 'pages';
12
28
  interface JobRow extends Row {
13
29
  id: string; log_id: string; kind: MaintenanceJob['kind']; through_version: number;
14
30
  status: MaintenanceJob['status']; cursor: number; entity_cursor: string; pages: string;
15
31
  error: string | null; updated_at: string; day: string; writes: number; reads: number;
32
+ phase: Phase | null; version_cursor: number; attempts: number; retry_at: string | null;
33
+ resume_at: string | null; pause_reason: 'writes' | 'reads' | null; paused_status: MaintenanceJob['status'] | null;
34
+ markers: string | null;
16
35
  }
17
36
  interface LayoutRow extends Row { log_id: string; compacted_through: number; generation: number; checkpoint: string | null; }
18
- type ReadEvents = (logId: string, after: number, through: number, limit: number, entityId?: string) => Promise<CheckpointEvent[]>;
37
+ /** Replay slots for one entity plus the claim it held at the last source event. */
38
+ interface WorkState { s: CheckpointEntityState; c: { currentVersion: number; logVersion: number }; }
39
+ type Marker = Pick<CheckpointEvent, 'id' | 'type' | 'version' | 'payload'>;
40
+ type ReadEvents = (logId: string, after: number, through: number, limit: number) => Promise<CheckpointEvent[]>;
41
+ type ReadVersions = (logId: string, versions: number[]) => Promise<CheckpointEvent[]>;
19
42
 
20
43
  /** Durable job state lives in SQLite; immutable payloads live in the private bucket. */
21
44
  export class LogMaintenanceStore {
@@ -28,8 +51,10 @@ export class LogMaintenanceStore {
28
51
  return cursor;
29
52
  }
30
53
 
31
- private exec<T extends Record<string, SqlStorageValue> = Record<string, SqlStorageValue>>(query: string, ...bindings: SqlStorageValue[]): SqlStorageCursor<T> {
32
- return this.observeSql(this.storage.sql.exec<T>(query, ...bindings));
54
+ /** Run a statement to completion so its row counts are final, and meter it. */
55
+ private query<T extends Record<string, SqlStorageValue> = Record<string, SqlStorageValue>>(statement: string, ...bindings: SqlStorageValue[]): T[] {
56
+ const cursor = this.observeSql(this.storage.sql.exec<T>(statement, ...bindings));
57
+ return cursor.toArray();
33
58
  }
34
59
 
35
60
  private async getObject(key: string): Promise<R2ObjectBody | null> {
@@ -45,12 +70,16 @@ export class LogMaintenanceStore {
45
70
  }
46
71
 
47
72
  constructor(private readonly storage: DurableObjectStorage, private readonly bucket: R2Bucket | undefined,
48
- private readonly appId: string, private readonly readEvents: ReadEvents, private readonly assertWritable: () => void = () => undefined,
73
+ private readonly appId: string, private readonly readEvents: ReadEvents, private readonly readVersions: ReadVersions,
74
+ private readonly assertWritable: () => void = () => undefined,
49
75
  private readonly dayWriteBudget = DEFAULT_DAY_WRITE_BUDGET, private readonly dayReadBudget = 1_000_000) {}
50
76
 
51
77
  state(logId: string, jobId?: string): LogMaintenance {
52
78
  const layout = this.layout(logId);
53
- const job = this.exec<JobRow>(`SELECT * FROM log_maintenance_jobs WHERE log_id = ? ${jobId ? 'AND id = ?' : ''} ORDER BY rowid DESC LIMIT 1`, logId, ...(jobId ? [jobId] : [])).toArray()[0];
79
+ // Polled often: leave out the large work columns (pages, markers).
80
+ const job = this.query<JobRow>(`SELECT id, log_id, kind, through_version, status, cursor, error, updated_at, attempts, retry_at,
81
+ resume_at, pause_reason FROM log_maintenance_jobs WHERE log_id = ? ${jobId ? 'AND id = ?' : ''} ORDER BY rowid DESC LIMIT 1`,
82
+ logId, ...(jobId ? [jobId] : []))[0];
54
83
  return { compactedThroughVersion: layout.compacted_through, storageGeneration: layout.generation,
55
84
  checkpoint: layout.checkpoint ? JSON.parse(layout.checkpoint) : null,
56
85
  job: job ? publicJob(job) : null, available: Boolean(this.bucket) };
@@ -58,9 +87,9 @@ export class LogMaintenanceStore {
58
87
 
59
88
  preview(logId: string): CompactionPreview {
60
89
  const state = this.state(logId);
61
- const totals = this.exec<{ count: number; bytes: number }>(`SELECT COUNT(*) AS count,
90
+ const totals = this.query<{ count: number; bytes: number }>(`SELECT COUNT(*) AS count,
62
91
  COALESCE(SUM(length(CAST(payload AS BLOB))), 0) AS bytes FROM log_events
63
- WHERE log_id = ? AND version <= ? AND archive_key IS NULL`, logId, state.checkpoint?.throughVersion ?? 0).one();
92
+ WHERE log_id = ? AND version <= ? AND archive_key IS NULL`, logId, state.checkpoint?.throughVersion ?? 0)[0]!;
64
93
  return { checkpointId: state.checkpoint?.id ?? null, storageGeneration: state.storageGeneration,
65
94
  throughVersion: state.checkpoint?.throughVersion ?? 0, activePayloadEvents: totals.count, activePayloadBytes: totals.bytes,
66
95
  snapshotBytes: state.checkpoint?.pages.reduce((bytes, page) => bytes + page.bytes, 0) ?? 0 };
@@ -70,16 +99,16 @@ export class LogMaintenanceStore {
70
99
  if (!this.bucket) throw new LedgerError(503, { error: 'Unavailable', code: 'archive_unconfigured', message: 'Configure private Ledger archive storage first.' });
71
100
  this.storage.transactionSync(() => {
72
101
  this.assertWritable();
73
- const prior = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ?', id).toArray()[0];
102
+ const prior = this.query<JobRow>('SELECT id, log_id, kind FROM log_maintenance_jobs WHERE id = ?', id)[0];
74
103
  if (prior) {
75
104
  if (prior.log_id !== logId || prior.kind !== kind) throw validationError('Maintenance request ID already used');
76
105
  return;
77
106
  }
78
- const active = this.exec<JobRow>("SELECT * FROM log_maintenance_jobs WHERE log_id = ? AND status NOT IN ('complete', 'failed') LIMIT 1", logId).toArray()[0];
107
+ const active = this.query<JobRow>("SELECT id FROM log_maintenance_jobs WHERE log_id = ? AND status NOT IN ('complete', 'failed') LIMIT 1", logId)[0];
79
108
  if (active) throw new LedgerError(409, { error: 'Conflict', code: 'maintenance_in_progress', message: 'Resume the existing maintenance operation.' });
80
- const log = this.exec<{ version: number; status: string }>('SELECT version, status FROM logs WHERE id = ?', logId).toArray()[0];
109
+ const log = this.query<{ version: number; status: string }>('SELECT version, status FROM logs WHERE id = ?', logId)[0];
81
110
  if (!log || log.status !== 'ready') throw validationError('Choose a ready log');
82
- const boundary = this.exec<{ action_index: number | null; action_size: number | null }>('SELECT action_index, action_size FROM log_events WHERE log_id = ? AND version = ?', logId, log.version).toArray()[0];
111
+ const boundary = this.query<{ action_index: number | null; action_size: number | null }>('SELECT action_index, action_size FROM log_events WHERE log_id = ? AND version = ?', logId, log.version)[0];
83
112
  if (boundary?.action_size && boundary.action_index !== boundary.action_size - 1) throw validationError('Incomplete action boundary');
84
113
  const checkpoint = this.state(logId).checkpoint;
85
114
  if (kind === 'compact' && (expectedCheckpointId !== checkpoint?.id || expectedGeneration !== this.layout(logId).generation)) {
@@ -88,9 +117,19 @@ export class LogMaintenanceStore {
88
117
  if (kind === 'compact' && (!checkpoint || checkpoint.throughVersion <= this.layout(logId).compacted_through)) {
89
118
  throw new LedgerError(409, { error: 'Conflict', code: 'snapshot_required', message: 'Create a newer verified snapshot before compacting.' });
90
119
  }
91
- this.exec(`INSERT INTO log_maintenance_jobs (id, log_id, kind, through_version, status, cursor, entity_cursor, pages, updated_at, day, writes)
92
- VALUES (?, ?, ?, ?, ?, 0, '', '[]', ?, ?, 0)`, id, logId, kind,
93
- kind === 'compact' ? checkpoint!.throughVersion : log.version, kind === 'compact' ? 'archiving' : 'building', new Date().toISOString(), day());
120
+ // A new snapshot can start from the previous one's retained facts
121
+ // instead of re-reading the whole (archived) history, unless a
122
+ // replay directive since then could reach back behind it.
123
+ let phase: Phase = 'fold', cursor = 0;
124
+ if (kind === 'snapshot' && checkpoint && checkpoint.throughVersion > 0 && checkpoint.throughVersion <= log.version
125
+ && !this.query(`SELECT 1 FROM log_events WHERE log_id = ? AND version > ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') LIMIT 1`,
126
+ logId, checkpoint.throughVersion).length) {
127
+ phase = 'seed'; cursor = checkpoint.throughVersion;
128
+ }
129
+ this.query(`INSERT INTO log_maintenance_jobs (id, log_id, kind, through_version, status, cursor, entity_cursor, pages, updated_at, day, writes, phase)
130
+ VALUES (?, ?, ?, ?, ?, ?, '', '[]', ?, ?, 0, ?)`, id, logId, kind,
131
+ kind === 'compact' ? checkpoint!.throughVersion : log.version, kind === 'compact' ? 'archiving' : 'building', cursor,
132
+ new Date().toISOString(), day(), kind === 'compact' ? null : phase);
94
133
  });
95
134
  return this.state(logId, id);
96
135
  }
@@ -101,10 +140,10 @@ export class LogMaintenanceStore {
101
140
  const started = Date.now();
102
141
  let result: LogMaintenance | undefined;
103
142
  try {
104
- const before = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId).toArray()[0];
143
+ const before = this.query<JobRow>('SELECT day, reads, writes FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId)[0];
105
144
  result = await this.stepValue(logId, id);
106
145
  this.assertWritable();
107
- this.exec('UPDATE log_maintenance_jobs SET reads = ?, writes = ? WHERE id = ? AND log_id = ?',
146
+ this.query('UPDATE log_maintenance_jobs SET reads = ?, writes = ? WHERE id = ? AND log_id = ?',
108
147
  (before?.day === day() ? before.reads : 0) + this.cursors.reduce((count, cursor) => count + cursor.rowsRead, 0),
109
148
  (before?.day === day() ? before.writes : 0) + this.cursors.reduce((count, cursor) => count + cursor.rowsWritten, 0) + 1,
110
149
  id, logId);
@@ -118,34 +157,81 @@ export class LogMaintenanceStore {
118
157
  }
119
158
  }
120
159
 
160
+ /** Active jobs whose next step is due now, least recently advanced first. */
161
+ dueJobs(now = Date.now()): Array<{ logId: string; id: string }> {
162
+ return this.activeJobs().filter(job => job.wake <= now).sort((a, b) => a.updatedAt.localeCompare(b.updatedAt))
163
+ .map(job => ({ logId: job.logId, id: job.id }));
164
+ }
165
+
166
+ /** Earliest time any active job needs another step, or null when none does. */
167
+ nextWake(): number | null {
168
+ const wakes = this.activeJobs().map(job => job.wake);
169
+ return wakes.length ? Math.min(...wakes) : null;
170
+ }
171
+
172
+ hasActiveJob(): boolean {
173
+ return this.query("SELECT 1 FROM log_maintenance_jobs WHERE status NOT IN ('complete', 'failed') LIMIT 1").length > 0;
174
+ }
175
+
176
+ private activeJobs(): Array<{ id: string; logId: string; wake: number; updatedAt: string }> {
177
+ const now = Date.now();
178
+ return this.query<JobRow>(`SELECT id, log_id, status, day, updated_at, retry_at, resume_at FROM log_maintenance_jobs
179
+ WHERE status NOT IN ('complete', 'failed')`).map(job => {
180
+ let wake = now;
181
+ if (job.status === 'paused') wake = job.resume_at ? Date.parse(job.resume_at) : nextMidnight(job.day);
182
+ else if (job.retry_at) wake = Date.parse(job.retry_at);
183
+ return { id: job.id, logId: job.log_id, wake: Number.isFinite(wake) ? wake : now, updatedAt: job.updated_at };
184
+ });
185
+ }
186
+
121
187
  private async stepValue(logId: string, id: string): Promise<LogMaintenance> {
122
188
  if (!this.bucket) throw new Error('Archive storage unavailable');
123
189
  this.assertWritable();
124
- let job = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId).toArray()[0];
190
+ let job = this.query<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId)[0];
125
191
  if (!job) throw validationError('Maintenance operation not found');
126
192
  if (job.status === 'complete') return this.state(logId, id);
127
193
  if (this.state(logId).job?.id !== id) throw new LedgerError(409, { error: 'Conflict', code: 'stale_maintenance', message: 'A newer maintenance operation replaced this one.' });
194
+ const resumeStatus = (row: JobRow): MaintenanceJob['status'] => row.paused_status && row.paused_status !== 'paused' && row.paused_status !== 'failed'
195
+ ? row.paused_status : row.kind === 'snapshot' ? 'building' : 'archiving';
128
196
  if (job.status === 'failed') {
129
- this.exec("UPDATE log_maintenance_jobs SET status = ?, error = NULL WHERE id = ?", job.kind === 'snapshot' ? 'building' : 'archiving', id);
130
- job = { ...job, status: job.kind === 'snapshot' ? 'building' : 'archiving', error: null };
197
+ // An explicit retry: start counting attempts again from the failed step.
198
+ this.query("UPDATE log_maintenance_jobs SET status = ?, error = NULL, attempts = 0, retry_at = NULL, paused_status = NULL WHERE id = ?", resumeStatus(job), id);
199
+ job = { ...job, status: resumeStatus(job), error: null, attempts: 0, retry_at: null, paused_status: null };
131
200
  }
132
201
  if (job.day !== day()) {
133
- this.exec('UPDATE log_maintenance_jobs SET day = ?, writes = 0, reads = 0, status = ? WHERE id = ?', day(), job.kind === 'snapshot' ? 'building' : 'archiving', id);
134
- job = { ...job, day: day(), writes: 0, reads: 0, status: job.kind === 'snapshot' ? 'building' : 'archiving' };
202
+ const status = job.status === 'paused' ? resumeStatus(job) : job.status;
203
+ this.query('UPDATE log_maintenance_jobs SET day = ?, writes = 0, reads = 0, status = ?, pause_reason = NULL, resume_at = NULL, paused_status = NULL WHERE id = ?', day(), status, id);
204
+ job = { ...job, day: day(), writes: 0, reads: 0, status, pause_reason: null, resume_at: null, paused_status: null };
135
205
  }
136
- if (job.writes >= this.dayWriteBudget || job.reads >= this.dayReadBudget) {
137
- this.exec("UPDATE log_maintenance_jobs SET status = 'paused' WHERE id = ?", id);
206
+ if (job.status === 'paused' || job.writes >= this.dayWriteBudget || job.reads >= this.dayReadBudget) {
207
+ if (job.status !== 'paused') {
208
+ const reason = job.writes >= this.dayWriteBudget ? 'writes' : 'reads';
209
+ this.query("UPDATE log_maintenance_jobs SET status = 'paused', paused_status = ?, pause_reason = ?, resume_at = ?, updated_at = ? WHERE id = ?",
210
+ job.status, reason, new Date(nextMidnight(job.day)).toISOString(), new Date().toISOString(), id);
211
+ }
138
212
  return this.state(logId, id);
139
213
  }
140
214
  try {
141
215
  if (job.status === 'building') await this.build(job);
142
- else if (job.status === 'cleaning') await this.clean(job);
216
+ else if (job.status === 'cleaning') this.clean(job);
143
217
  else if (job.status === 'archiving' || job.status === 'reclaiming') await this.archive(job);
218
+ if (job.attempts > 0 || job.error) {
219
+ this.query('UPDATE log_maintenance_jobs SET attempts = 0, error = NULL, retry_at = NULL WHERE id = ?', id);
220
+ }
144
221
  } catch (cause) {
145
- // Keep source rows and page cursors untouched when verification fails.
222
+ // Access problems are not job failures; source rows and cursors stay untouched.
146
223
  this.assertWritable();
147
- this.exec("UPDATE log_maintenance_jobs SET status = 'failed', error = ?, updated_at = ? WHERE id = ?",
148
- cause instanceof Error ? cause.message : 'Maintenance failed', new Date().toISOString(), id);
224
+ const attempts = job.attempts + 1;
225
+ const message = cause instanceof Error ? cause.message : 'Maintenance failed';
226
+ const now = new Date().toISOString();
227
+ if (attempts >= MAX_ATTEMPTS) {
228
+ this.query("UPDATE log_maintenance_jobs SET status = 'failed', paused_status = ?, error = ?, attempts = ?, retry_at = NULL, updated_at = ? WHERE id = ?",
229
+ job.status, message, attempts, now, id);
230
+ } else {
231
+ // Stay in the current stage and let the alarm retry with backoff; the error stays visible meanwhile.
232
+ this.query('UPDATE log_maintenance_jobs SET error = ?, attempts = ?, retry_at = ?, updated_at = ? WHERE id = ?',
233
+ message, attempts, new Date(Date.now() + Math.min(600_000, 15_000 * 2 ** (attempts - 1))).toISOString(), now, id);
234
+ }
149
235
  }
150
236
  return this.state(logId, id);
151
237
  }
@@ -164,16 +250,16 @@ export class LogMaintenanceStore {
164
250
  /** Bounded archive pages are verified before any accepted payload is reclaimed. */
165
251
  async hydrate(logId: string, events: CheckpointEvent[]): Promise<CheckpointEvent[]> {
166
252
  if (!events.length) return events;
167
- const references = this.exec<{ event_id: string; archive_key: string }>(
253
+ const references = this.query<{ event_id: string; archive_key: string }>(
168
254
  'SELECT event_id, archive_key FROM log_events WHERE log_id = ? AND event_id IN (SELECT value FROM json_each(?)) AND archive_key IS NOT NULL',
169
- logId, JSON.stringify(events.map(event => event.id))).toArray();
255
+ logId, JSON.stringify(events.map(event => event.id)));
170
256
  if (!references.length) return events;
171
257
  if (!this.bucket) throw archiveError('Archive storage unavailable');
172
258
  const wanted = new Set(events.map(event => event.id));
173
259
  const payloads = new Map<string, CheckpointEvent>();
174
260
  for (const key of new Set(references.map(row => row.archive_key))) {
175
261
  if (!key.startsWith(`v1/${this.appId}/`)) throw archiveError('Archive scope mismatch');
176
- const reference = this.exec<{ sha256: string }>('SELECT sha256 FROM log_archive_pages WHERE object_key = ? LIMIT 1', key).toArray()[0];
262
+ const reference = this.query<{ sha256: string }>('SELECT sha256 FROM log_archive_pages WHERE object_key = ? LIMIT 1', key)[0];
177
263
  if (!reference) throw archiveError('Archive manifest is missing');
178
264
  const object = await this.getObject(key);
179
265
  if (!object) throw archiveError('Archived history is missing');
@@ -192,154 +278,270 @@ export class LogMaintenanceStore {
192
278
  }
193
279
 
194
280
  private layout(logId: string): LayoutRow {
195
- return this.exec<LayoutRow>('SELECT * FROM log_storage WHERE log_id = ?', logId).toArray()[0]
281
+ return this.query<LayoutRow>('SELECT * FROM log_storage WHERE log_id = ?', logId)[0]
196
282
  ?? { log_id: logId, compacted_through: 0, generation: 0, checkpoint: null };
197
283
  }
198
284
 
199
285
  private async build(job: JobRow): Promise<void> {
200
- if (job.cursor < job.through_version) {
201
- // Scan the primary log index once. Work records are populated in
202
- // budgeted batches rather than building an index over old history
203
- // during a schema migration (which could exhaust free-tier writes).
204
- const events = await this.readEvents(job.log_id, job.cursor, job.through_version, 16);
205
- if (!events.length || events[0]!.version !== job.cursor + 1) throw new Error('Snapshot source contains a gap');
206
- const selected = this.selection(job.log_id, job.through_version);
207
- const groups = new Map<string, CheckpointEvent[]>();
208
- for (const event of events) {
209
- const group = groups.get(event.entityId) ?? [];
210
- group.push(event); groups.set(event.entityId, group);
211
- }
212
- for (const [entityId, source] of groups) {
213
- const prior = this.exec<{ object_key: string }>('SELECT object_key FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, entityId).toArray()[0];
214
- const previous = prior ? await this.readWork(prior.object_key) : { events: [], claims: [] };
215
- const last = source[source.length - 1]!;
216
- const witnesses = checkpointEntityEvents([...previous.events, ...source.filter(event => selected(event) && !directives.includes(event.type))], [
217
- event => typeof event.payload.status === 'string' ? `status:${event.payload.status}` : null,
218
- event => typeof event.payload.thumbnailRef === 'string' ? 'thumbnail:string' : null,
219
- ], standardTypes);
220
- const value = JSON.stringify({ events: witnesses, claims: [{ entityId, currentVersion: last.entityVersion, logVersion: last.version }] });
221
- if (new TextEncoder().encode(value).byteLength > 8_000_000) throw new Error('Entity replay state exceeds the snapshot work budget');
222
- const key = `v1/${this.appId}/${job.log_id}/work/${job.id}/${last.version}.json`;
223
- await this.putVerified(key, value);
224
- this.storage.transactionSync(() => {
225
- this.assertWritable();
226
- this.exec(`INSERT INTO log_checkpoint_entities (job_id, entity_id, object_key) VALUES (?, ?, ?)
227
- ON CONFLICT(job_id, entity_id) DO UPDATE SET object_key = excluded.object_key`, job.id, entityId, key);
228
- this.exec('UPDATE log_maintenance_jobs SET writes = writes + 2 WHERE id = ?', job.id);
229
- });
230
- if (prior && prior.object_key !== key) await this.deleteObject(prior.object_key);
286
+ switch (job.phase ?? 'reset') {
287
+ case 'reset': return this.reset(job);
288
+ case 'seed': return this.seed(job);
289
+ case 'fold':
290
+ if (job.cursor < job.through_version) return this.fold(job);
291
+ this.setPhase(job, 'index');
292
+ return this.index({ ...job, phase: 'index' });
293
+ case 'index': return this.index(job);
294
+ case 'pages': return this.pages(job);
295
+ }
296
+ }
297
+
298
+ private setPhase(job: JobRow, phase: Phase, extra: { cursor?: number; versionCursor?: number; entityCursor?: string; pages?: CheckpointPage[] } = {}): void {
299
+ this.assertWritable();
300
+ this.query('UPDATE log_maintenance_jobs SET phase = ?, cursor = ?, version_cursor = ?, entity_cursor = ?, pages = ?, updated_at = ? WHERE id = ?',
301
+ phase, extra.cursor ?? job.cursor, extra.versionCursor ?? 0, extra.entityCursor ?? '', JSON.stringify(extra.pages ?? []),
302
+ new Date().toISOString(), job.id);
303
+ }
304
+
305
+ /** Discard partial work (including work files of the former R2-based builder) and restart from the first event. */
306
+ private async reset(job: JobRow): Promise<void> {
307
+ const legacy = this.query<{ entity_id: string; object_key: string }>('SELECT entity_id, object_key FROM log_checkpoint_entities WHERE job_id = ? LIMIT 16', job.id);
308
+ if (legacy.length) {
309
+ for (const row of legacy) {
310
+ if (row.object_key) await this.deleteObject(row.object_key);
311
+ this.assertWritable();
312
+ this.query('DELETE FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, row.entity_id);
231
313
  }
314
+ return;
315
+ }
316
+ if (this.deleteWork(job.id)) return;
317
+ this.setPhase(job, 'fold', { cursor: 0 });
318
+ }
319
+
320
+ /** Delete a bounded batch of work rows; true while any remain. */
321
+ private deleteWork(jobId: string): boolean {
322
+ this.assertWritable();
323
+ const entities = this.query<{ entity_id: string }>('SELECT entity_id FROM log_checkpoint_work WHERE job_id = ? LIMIT ?', jobId, CLEAN_ROWS);
324
+ if (entities.length) {
325
+ this.query('DELETE FROM log_checkpoint_work WHERE job_id = ? AND entity_id IN (SELECT value FROM json_each(?))', jobId, JSON.stringify(entities.map(row => row.entity_id)));
326
+ return true;
327
+ }
328
+ const versions = this.query<{ version: number }>('SELECT version FROM log_checkpoint_versions WHERE job_id = ? LIMIT ?', jobId, CLEAN_ROWS * 2);
329
+ if (versions.length) {
330
+ this.query('DELETE FROM log_checkpoint_versions WHERE job_id = ? AND version IN (SELECT value FROM json_each(?))', jobId, JSON.stringify(versions.map(row => row.version)));
331
+ return true;
332
+ }
333
+ return false;
334
+ }
335
+
336
+ private loadWork(jobId: string, entityIds: string[]): Map<string, WorkState> {
337
+ return new Map(this.query<{ entity_id: string; state: string }>(
338
+ 'SELECT entity_id, state FROM log_checkpoint_work WHERE job_id = ? AND entity_id IN (SELECT value FROM json_each(?))',
339
+ jobId, JSON.stringify(entityIds)).map(row => [row.entity_id, JSON.parse(row.state) as WorkState]));
340
+ }
341
+
342
+ /** Persist updated entity states in one statement. */
343
+ private saveWork(jobId: string, states: Map<string, WorkState>): void {
344
+ const rows: Array<{ id: string; state: WorkState }> = [];
345
+ for (const [id, state] of states) {
346
+ if (JSON.stringify(state).length > MAX_STATE_BYTES) throw new Error('Entity replay state exceeds the snapshot work budget');
347
+ rows.push({ id, state });
348
+ }
349
+ this.query(`INSERT INTO log_checkpoint_work (job_id, entity_id, state)
350
+ SELECT ?, json_extract(value, '$.id'), json_extract(value, '$.state') FROM json_each(?) WHERE true
351
+ ON CONFLICT(job_id, entity_id) DO UPDATE SET state = excluded.state`, jobId, JSON.stringify(rows));
352
+ }
353
+
354
+ /** Start from the previous snapshot's retained facts: one of its pages per step. */
355
+ private async seed(job: JobRow): Promise<void> {
356
+ const checkpoint = this.layout(job.log_id).checkpoint ? JSON.parse(this.layout(job.log_id).checkpoint!) as CheckpointManifest : null;
357
+ if (!checkpoint || checkpoint.throughVersion !== job.cursor) throw new Error('The previous snapshot changed while seeding');
358
+ if (job.version_cursor >= checkpoint.pages.length) {
359
+ this.setPhase(job, 'fold');
360
+ return;
361
+ }
362
+ const chunk = await this.chunk(job.log_id, job.version_cursor);
363
+ const touched = new Set<string>([...chunk.events.map(event => event.entityId), ...chunk.claims.map(claim => claim.entityId)]);
364
+ const states = this.loadWork(job.id, [...touched]);
365
+ const work = (entityId: string): WorkState => {
366
+ let state = states.get(entityId);
367
+ if (!state) { state = { s: newCheckpointEntityState(), c: { currentVersion: 0, logVersion: 0 } }; states.set(entityId, state); }
368
+ return state;
369
+ };
370
+ for (const event of [...chunk.events].sort((a, b) => a.version - b.version)) foldCheckpointEvent(work(event.entityId).s, event, SELECTORS, standardTypes);
371
+ for (const claim of chunk.claims) work(claim.entityId).c = { currentVersion: claim.currentVersion, logVersion: claim.logVersion };
372
+ this.storage.transactionSync(() => {
373
+ this.assertWritable();
374
+ this.saveWork(job.id, states);
375
+ this.query('UPDATE log_maintenance_jobs SET version_cursor = ?, updated_at = ? WHERE id = ?', job.version_cursor + 1, new Date().toISOString(), job.id);
376
+ });
377
+ }
378
+
379
+ /** Fold the next window of source events into per-entity replay slots. */
380
+ private async fold(job: JobRow): Promise<void> {
381
+ const events = await this.readEvents(job.log_id, job.cursor, job.through_version, FOLD_EVENTS);
382
+ if (!events.length || events[0]!.version !== job.cursor + 1 || events[events.length - 1]!.version - events[0]!.version !== events.length - 1) {
383
+ throw new Error('Snapshot source contains a gap');
384
+ }
385
+ const selected = this.selection(job);
386
+ const groups = new Map<string, CheckpointEvent[]>();
387
+ for (const event of events) {
388
+ const group = groups.get(event.entityId) ?? [];
389
+ group.push(event); groups.set(event.entityId, group);
390
+ }
391
+ const states = this.loadWork(job.id, [...groups.keys()]);
392
+ for (const [entityId, source] of groups) {
393
+ const state = states.get(entityId) ?? { s: newCheckpointEntityState(), c: { currentVersion: 0, logVersion: 0 } };
394
+ for (const event of source) if (selected(event) && !directives.includes(event.type)) foldCheckpointEvent(state.s, event, SELECTORS, standardTypes);
395
+ const last = source[source.length - 1]!;
396
+ state.c = { currentVersion: last.entityVersion, logVersion: last.version };
397
+ states.set(entityId, state);
398
+ }
399
+ this.storage.transactionSync(() => {
232
400
  this.assertWritable();
233
- this.exec('UPDATE log_maintenance_jobs SET cursor = ?, writes = writes + 1, updated_at = ? WHERE id = ?',
401
+ this.saveWork(job.id, states);
402
+ this.query('UPDATE log_maintenance_jobs SET cursor = ?, updated_at = ? WHERE id = ?',
234
403
  events[events.length - 1]!.version, new Date().toISOString(), job.id);
404
+ });
405
+ }
406
+
407
+ /** Turn entity slots into claim pages and the list of source versions to retain. */
408
+ private async index(job: JobRow): Promise<void> {
409
+ const pages = JSON.parse(job.pages) as CheckpointPage[];
410
+ const rows = this.query<{ entity_id: string; state: string }>(`SELECT entity_id, state FROM log_checkpoint_work
411
+ WHERE job_id = ? AND entity_id > ? ORDER BY entity_id LIMIT ?`, job.id, job.entity_cursor, INDEX_ENTITIES);
412
+ if (!rows.length) {
413
+ this.setPhase(job, 'pages', { pages });
235
414
  return;
236
415
  }
416
+ const versions = new Set<number>();
417
+ const claims: CheckpointChunk['claims'] = [];
418
+ for (const row of rows) {
419
+ const work = JSON.parse(row.state) as WorkState;
420
+ for (const version of checkpointStateVersions(work.s)) versions.add(version);
421
+ if (work.c.logVersion > 0) claims.push({ entityId: row.entity_id, currentVersion: work.c.currentVersion, logVersion: work.c.logVersion });
422
+ }
423
+ if (claims.length) {
424
+ const value = JSON.stringify({ events: [], claims } satisfies CheckpointChunk);
425
+ const key = `v1/${this.appId}/${job.log_id}/snapshots/${job.id}/${pages.length}.json`;
426
+ pages.push({ key, sha256: await this.put(key, value), count: 0, bytes: new TextEncoder().encode(value).byteLength });
427
+ }
428
+ this.storage.transactionSync(() => {
429
+ this.assertWritable();
430
+ if (versions.size) this.query('INSERT OR IGNORE INTO log_checkpoint_versions (job_id, version) SELECT ?, value FROM json_each(?)', job.id, JSON.stringify([...versions]));
431
+ this.query('UPDATE log_maintenance_jobs SET entity_cursor = ?, pages = ?, updated_at = ? WHERE id = ?',
432
+ rows[rows.length - 1]!.entity_id, JSON.stringify(pages), new Date().toISOString(), job.id);
433
+ });
434
+ }
435
+
436
+ /** Write retained source events in version order, then publish the manifest. */
437
+ private async pages(job: JobRow): Promise<void> {
237
438
  const pages = JSON.parse(job.pages) as CheckpointPage[];
238
- const entities = this.exec<{ entity_id: string; object_key: string }>(`SELECT entity_id, object_key FROM log_checkpoint_entities
239
- WHERE job_id = ? AND entity_id > ? ORDER BY entity_id LIMIT 16`, job.id, job.entity_cursor).toArray();
240
- if (!entities.length) {
439
+ const rows = this.query<{ version: number }>(`SELECT version FROM log_checkpoint_versions WHERE job_id = ? AND version > ?
440
+ ORDER BY version LIMIT ?`, job.id, job.version_cursor, PAGE);
441
+ if (!rows.length) {
241
442
  const manifest: CheckpointManifest = { format: 1, appId: this.appId, logId: job.log_id, id: job.id,
242
443
  throughVersion: job.through_version, createdAt: new Date().toISOString(), pages,
243
444
  eventCount: pages.reduce((count, page) => count + page.count, 0), sourceEventCount: job.through_version,
244
445
  sha256: await checkpointDigest(JSON.stringify(pages)) };
245
446
  this.storage.transactionSync(() => {
246
447
  this.assertWritable();
247
- this.exec(`INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint) VALUES (?, 0, 0, ?)
448
+ this.query(`INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint) VALUES (?, 0, 0, ?)
248
449
  ON CONFLICT(log_id) DO UPDATE SET checkpoint = excluded.checkpoint`, job.log_id, JSON.stringify(manifest));
249
- this.exec("UPDATE log_maintenance_jobs SET status = 'cleaning', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
450
+ this.query("UPDATE log_maintenance_jobs SET status = 'cleaning', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
250
451
  });
251
452
  return;
252
453
  }
454
+ const wanted = rows.map(row => row.version);
455
+ const events = await this.readVersions(job.log_id, wanted);
456
+ if (events.length !== wanted.length || events.some((event, index) => event.version !== wanted[index])) throw new Error('Snapshot source is missing retained events');
253
457
  const chunk: CheckpointChunk = { events: [], claims: [] };
254
- let last = job.entity_cursor;
255
- for (const entity of entities) {
256
- const part = await this.readWork(entity.object_key);
257
- chunk.events.push(...part.events); chunk.claims.push(...part.claims);
258
- last = entity.entity_id;
259
- if (new TextEncoder().encode(JSON.stringify(chunk)).byteLength >= MAX_PAGE_BYTES) break;
458
+ let bytes = 2;
459
+ for (const event of events) {
460
+ const size = new TextEncoder().encode(JSON.stringify(event)).byteLength + 1;
461
+ if (chunk.events.length && bytes + size >= MAX_PAGE_BYTES) break;
462
+ chunk.events.push(event); bytes += size;
260
463
  }
261
- chunk.events.sort((a, b) => a.version - b.version);
262
464
  const value = JSON.stringify(chunk);
263
465
  const key = `v1/${this.appId}/${job.log_id}/snapshots/${job.id}/${pages.length}.json`;
264
- const sha256 = await this.putVerified(key, value);
265
- pages.push({ key, sha256, count: chunk.events.length, bytes: new TextEncoder().encode(value).byteLength });
466
+ pages.push({ key, sha256: await this.put(key, value), count: chunk.events.length, bytes: new TextEncoder().encode(value).byteLength });
266
467
  this.assertWritable();
267
- this.exec('UPDATE log_maintenance_jobs SET entity_cursor = ?, pages = ?, writes = writes + 1, updated_at = ? WHERE id = ?',
268
- last, JSON.stringify(pages), new Date().toISOString(), job.id);
269
- }
270
-
271
- private async readWork(key: string): Promise<CheckpointChunk> {
272
- const object = await this.getObject(key);
273
- if (!object) throw new Error('Snapshot work page is missing');
274
- const encoded = await object.text();
275
- if (await checkpointDigest(encoded) !== object.customMetadata?.sha256) throw new Error('Snapshot work checksum mismatch');
276
- return JSON.parse(encoded) as CheckpointChunk;
468
+ this.query('UPDATE log_maintenance_jobs SET version_cursor = ?, pages = ?, updated_at = ? WHERE id = ?',
469
+ chunk.events[chunk.events.length - 1]!.version, JSON.stringify(pages), new Date().toISOString(), job.id);
277
470
  }
278
471
 
279
- private async clean(job: JobRow): Promise<void> {
280
- const rows = this.exec<{ entity_id: string; object_key: string }>('SELECT entity_id, object_key FROM log_checkpoint_entities WHERE job_id = ? LIMIT 16', job.id).toArray();
281
- for (const row of rows) {
282
- await this.deleteObject(row.object_key);
283
- this.assertWritable();
284
- this.exec('DELETE FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, row.entity_id);
472
+ private clean(job: JobRow): void {
473
+ if (this.deleteWork(job.id)) {
474
+ this.query('UPDATE log_maintenance_jobs SET updated_at = ? WHERE id = ?', new Date().toISOString(), job.id);
475
+ return;
285
476
  }
286
- this.exec('UPDATE log_maintenance_jobs SET status = ?, writes = writes + ?, updated_at = ? WHERE id = ?',
287
- rows.length ? 'cleaning' : 'complete', rows.length + 1, new Date().toISOString(), job.id);
477
+ this.query("UPDATE log_maintenance_jobs SET status = 'complete', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
288
478
  }
289
479
 
290
- private async putVerified(key: string, value: string): Promise<string> {
480
+ /** R2 rejects an upload whose bytes do not match the digest, so no read-back is needed. */
481
+ private async put(key: string, value: string): Promise<string> {
291
482
  const sha256 = await checkpointDigest(value);
292
483
  this.io.r2Writes++;
293
484
  this.io.r2WrittenBytes += new TextEncoder().encode(value).byteLength;
294
- await this.bucket!.put(key, value, { customMetadata: { sha256 }, httpMetadata: { contentType: 'application/json' } });
295
- const check = await this.getObject(key);
296
- if (!check || await checkpointDigest(await check.text()) !== sha256) throw new Error('Snapshot verification failed');
485
+ await this.bucket!.put(key, value, { sha256, customMetadata: { sha256 }, httpMetadata: { contentType: 'application/json' } });
297
486
  return sha256;
298
487
  }
299
488
 
300
489
  private async archive(job: JobRow): Promise<void> {
301
490
  // Metadata stays as the compact identity/action registry. Only payloads
302
491
  // move, so historical boundaries and event/request uniqueness survive.
303
- const rows = this.exec<{ version: number }>(`SELECT version FROM log_events WHERE log_id = ?
304
- AND version <= ? AND version > ? AND archive_key IS NULL ORDER BY version LIMIT ?`, job.log_id, job.through_version, job.cursor, PAGE).toArray();
492
+ const rows = this.query<{ version: number }>(`SELECT version FROM log_events WHERE log_id = ?
493
+ AND version <= ? AND version > ? AND archive_key IS NULL ORDER BY version LIMIT ?`, job.log_id, job.through_version, job.cursor, PAGE);
305
494
  if (!rows.length) {
306
495
  this.storage.transactionSync(() => {
307
- this.assertWritable();
308
- this.exec('UPDATE log_storage SET compacted_through = MAX(compacted_through, ?), generation = generation + 1 WHERE log_id = ?', job.through_version, job.log_id);
309
- this.exec("UPDATE log_maintenance_jobs SET status = 'complete', cursor = ?, updated_at = ? WHERE id = ?", job.through_version, new Date().toISOString(), job.id);
496
+ this.assertWritable();
497
+ this.query('UPDATE log_storage SET compacted_through = MAX(compacted_through, ?), generation = generation + 1 WHERE log_id = ?', job.through_version, job.log_id);
498
+ this.query("UPDATE log_maintenance_jobs SET status = 'complete', cursor = ?, updated_at = ? WHERE id = ?", job.through_version, new Date().toISOString(), job.id);
310
499
  });
311
500
  return;
312
501
  }
313
502
  const first = rows[0]!.version;
314
503
  const events = await this.readEvents(job.log_id, first - 1, rows[rows.length - 1]!.version, PAGE);
315
504
  const value = JSON.stringify(events);
316
- const sha256 = await checkpointDigest(value);
317
505
  const key = `v1/${this.appId}/${job.log_id}/archive/${job.id}/${first}.json`;
318
- await this.putVerified(key, value);
506
+ const sha256 = await this.put(key, value);
319
507
  this.storage.transactionSync(() => {
320
508
  this.assertWritable();
321
- this.exec(`INSERT OR IGNORE INTO log_archive_pages (log_id, first_version, last_version, object_key, sha256) VALUES (?, ?, ?, ?, ?)`,
509
+ this.query(`INSERT OR IGNORE INTO log_archive_pages (log_id, first_version, last_version, object_key, sha256) VALUES (?, ?, ?, ?, ?)`,
322
510
  job.log_id, first, events[events.length - 1]!.version, key, sha256);
323
- const reclaimed = this.exec(`UPDATE log_events SET archive_key = ?, payload_bytes = COALESCE(payload_bytes, length(CAST(payload AS BLOB))), payload = CASE WHEN event_type IN
511
+ this.query(`UPDATE log_events SET archive_key = ?, payload_bytes = COALESCE(payload_bytes, length(CAST(payload AS BLOB))), payload = CASE WHEN event_type IN
324
512
  ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') THEN payload ELSE '{}' END
325
513
  WHERE log_id = ? AND version >= ? AND version <= ? AND archive_key IS NULL`, key, job.log_id, first, events[events.length - 1]!.version);
326
- reclaimed.toArray();
327
- this.exec("UPDATE log_maintenance_jobs SET status = 'archiving', cursor = ?, writes = writes + ?, updated_at = ? WHERE id = ?",
328
- events[events.length - 1]!.version, reclaimed.rowsWritten + 2, new Date().toISOString(), job.id);
514
+ this.query("UPDATE log_maintenance_jobs SET status = 'archiving', cursor = ?, updated_at = ? WHERE id = ?",
515
+ events[events.length - 1]!.version, new Date().toISOString(), job.id);
329
516
  });
330
517
  }
331
518
 
332
- private selection(logId: string, through: number): (event: CheckpointEvent) => boolean {
333
- const rows = this.exec<{ version: number; event_id: string; event_type: string; payload: string }>(`SELECT version, event_id, event_type, payload FROM log_events
334
- WHERE log_id = ? AND version <= ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') ORDER BY version DESC`, logId, through).toArray();
335
- return checkpointSelection(rows.map(event => ({ id: event.event_id, type: event.event_type,
336
- version: event.version, payload: JSON.parse(event.payload) as Record<string, unknown> })), through);
519
+ /** Replay directives up to the snapshot boundary, resolved once per job. */
520
+ private selection(job: JobRow): (event: CheckpointEvent) => boolean {
521
+ let markers: Marker[];
522
+ if (job.markers) {
523
+ markers = JSON.parse(job.markers) as Marker[];
524
+ } else {
525
+ markers = this.query<{ version: number; event_id: string; event_type: string; payload: string }>(`SELECT version, event_id, event_type, payload FROM log_events
526
+ WHERE log_id = ? AND version <= ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') ORDER BY version DESC`,
527
+ job.log_id, job.through_version).map(event => ({ id: event.event_id, type: event.event_type,
528
+ version: event.version, payload: JSON.parse(event.payload) as Record<string, unknown> }));
529
+ const encoded = JSON.stringify(markers);
530
+ if (encoded.length < MAX_MARKER_BYTES) {
531
+ this.assertWritable();
532
+ this.query('UPDATE log_maintenance_jobs SET markers = ? WHERE id = ?', encoded, job.id);
533
+ }
534
+ }
535
+ return checkpointSelection(markers, job.through_version);
337
536
  }
338
537
  }
339
538
 
340
539
  function day(): string { return new Date().toISOString().slice(0, 10); }
540
+ /** Start of the UTC day after `dayValue` (YYYY-MM-DD). */
541
+ function nextMidnight(dayValue: string): number { return Date.parse(`${dayValue}T00:00:00.000Z`) + 86_400_000; }
341
542
  function archiveError(message: string): LedgerError { return new LedgerError(503, { error: 'Unavailable', code: 'archive_unavailable', message }); }
342
543
  function publicJob(row: JobRow): MaintenanceJob {
343
544
  return { id: row.id, logId: row.log_id, kind: row.kind, throughVersion: row.through_version, status: row.status,
344
- cursor: row.cursor, error: row.error, updatedAt: row.updated_at };
545
+ cursor: row.cursor, error: row.error, updatedAt: row.updated_at, attempts: row.attempts, retryAt: row.retry_at,
546
+ pauseReason: row.pause_reason, resumeAt: row.resume_at };
345
547
  }
@@ -0,0 +1,38 @@
1
+ /** One in-flight step per operation (alarm and callers share it), with a bounded wait for HTTP/RPC callers.
2
+ * Losing a response must not enqueue another copy of the same running step.
3
+ */
4
+ export class MaintenanceSteps<T> {
5
+ private queue: Promise<unknown> = Promise.resolve();
6
+ private readonly running = new Map<string, Promise<T>>();
7
+
8
+ constructor(private readonly keepAlive: (work: Promise<unknown>) => void, private readonly responseWaitMs = 1000) {}
9
+
10
+ /** Queue one step behind earlier ones (or join the identical step already in flight) and resolve with its real result. */
11
+ run(key: string, work: () => Promise<T>): Promise<T> {
12
+ let run = this.running.get(key);
13
+ if (!run) {
14
+ run = this.queue.then(work);
15
+ this.running.set(key, run);
16
+ const settled = run.then(() => undefined, () => undefined).then(() => {
17
+ this.running.delete(key);
18
+ });
19
+ this.queue = settled;
20
+ this.keepAlive(settled);
21
+ }
22
+ return run;
23
+ }
24
+
25
+ async advance(key: string, work: () => Promise<T>, current: () => T): Promise<T> {
26
+ const run = this.run(key, work);
27
+ let timer: ReturnType<typeof setTimeout> | undefined;
28
+ try {
29
+ return await Promise.race([run, new Promise<T>((resolve, reject) => {
30
+ timer = setTimeout(() => {
31
+ try { resolve(current()); } catch (cause) { reject(cause); }
32
+ }, this.responseWaitMs);
33
+ })]);
34
+ } finally {
35
+ clearTimeout(timer);
36
+ }
37
+ }
38
+ }
package/src/schema.ts CHANGED
@@ -215,6 +215,25 @@ const LEDGER_APP_MIGRATIONS: readonly Migration[] = [
215
215
  );
216
216
  CREATE INDEX log_archive_object ON log_archive_pages(object_key);
217
217
  ` },
218
+ // Snapshot work state now lives in SQLite as per-entity replay slots (no R2
219
+ // work files). Jobs are advanced by an alarm and record retry/pause details.
220
+ { id: 15, sql: `
221
+ CREATE TABLE log_checkpoint_work (
222
+ job_id TEXT NOT NULL, entity_id TEXT NOT NULL, state TEXT NOT NULL,
223
+ PRIMARY KEY(job_id, entity_id)
224
+ );
225
+ CREATE TABLE log_checkpoint_versions (
226
+ job_id TEXT NOT NULL, version INTEGER NOT NULL, PRIMARY KEY(job_id, version)
227
+ );
228
+ ALTER TABLE log_maintenance_jobs ADD COLUMN phase TEXT;
229
+ ALTER TABLE log_maintenance_jobs ADD COLUMN version_cursor INTEGER NOT NULL DEFAULT 0;
230
+ ALTER TABLE log_maintenance_jobs ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0;
231
+ ALTER TABLE log_maintenance_jobs ADD COLUMN retry_at TEXT;
232
+ ALTER TABLE log_maintenance_jobs ADD COLUMN resume_at TEXT;
233
+ ALTER TABLE log_maintenance_jobs ADD COLUMN pause_reason TEXT;
234
+ ALTER TABLE log_maintenance_jobs ADD COLUMN paused_status TEXT;
235
+ ALTER TABLE log_maintenance_jobs ADD COLUMN markers TEXT;
236
+ ` },
218
237
  ];
219
238
 
220
239
  const APP_REGISTRY_MIGRATIONS: readonly Migration[] = [