@hearthlog/ledger 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -5
- package/package.json +2 -2
- package/scripts/maintenance.ts +1 -0
- package/src/ledger-app.ts +73 -21
- package/src/log-maintenance.ts +321 -119
- package/src/maintenance-steps.ts +8 -2
- package/src/schema.ts +19 -0
package/README.md
CHANGED
|
@@ -192,7 +192,8 @@ Authenticated endpoints under `/logs/:logId`:
|
|
|
192
192
|
- `GET /maintenance-preview`: the current snapshot/generation and prefix payload counts/bytes for review.
|
|
193
193
|
- `POST /maintenance`: `{ requestId, kind: "snapshot" | "compact", expectedCheckpointId?, expectedGeneration? }`.
|
|
194
194
|
Compaction requires the checkpoint ID and generation from the reviewed preview; stale previews fail with 409.
|
|
195
|
-
- `POST /maintenance/step`: `{ requestId }
|
|
195
|
+
- `POST /maintenance/step`: `{ requestId }`; runs one bounded step now (sharing any step already in flight) and restarts a
|
|
196
|
+
job that stopped after repeated failures. Normally unnecessary: the Ledger advances jobs by itself (see below).
|
|
196
197
|
- `GET /checkpoint?page=N`: one checksum-verified canonical replay chunk and its entity claims.
|
|
197
198
|
|
|
198
199
|
From the development site, the same operation can run without an open browser:
|
|
@@ -206,6 +207,15 @@ pnpm -C dev ledger:maintain --log <log-id> --resume
|
|
|
206
207
|
Set `LEDGER_BASE_URL` and `LEDGER_AUTH_TOKEN` in the process environment; the CLI never prints credentials.
|
|
207
208
|
Use `--request-id` to retry an explicit operation and `--once` for one bounded work step.
|
|
208
209
|
|
|
210
|
+
A started job is carried to completion by the application object's alarm, with no caller or open browser. Each step
|
|
211
|
+
folds up to 512 source events into small per-entity replay slots (source versions only, in SQLite), then writes the
|
|
212
|
+
retained events as checksummed pages. A later snapshot starts from the previous one's retained facts instead of re-reading
|
|
213
|
+
archived history, unless a restore/removal marker has been added since. A failing step is retried with backoff; the
|
|
214
|
+
error and attempt count stay visible, and after five attempts the job stops as `failed` until it is explicitly retried
|
|
215
|
+
(the maintenance panel's Retry, `POST /maintenance/step`, or the CLI). A job that uses its daily budget is `paused`
|
|
216
|
+
with `pauseReason` and `resumeAt`, and resumes by itself at the next UTC day. Opening the panel re-arms a job whose
|
|
217
|
+
alarm was lost.
|
|
218
|
+
|
|
209
219
|
Jobs persist every source batch and publish only verified snapshots. Canonical snapshots keep original source
|
|
210
220
|
witnesses for raw field values, explicit nulls, timestamps and lifecycle transitions; dynamic/nonstandard facts
|
|
211
221
|
are retained losslessly. Browser projection uses a separate baseline table plus accepted suffix and pending
|
|
@@ -217,9 +227,8 @@ payload placeholders. The SDK's server checkpoint reader lets Stash seed a cold
|
|
|
217
227
|
`MAINTENANCE_DAILY_WRITES` defaults to `10000` per job (bounded to 128–100000), and
|
|
218
228
|
`MAINTENANCE_DAILY_READS` defaults to `1000000` (bounded to 10000–1000000000). Jobs pause at either budget. Budgets use consumed SQL cursor
|
|
219
229
|
counts and can overrun by one bounded step. Structured step logs include SQL rows, R2 operations/bytes, elapsed
|
|
220
|
-
time and replay counts, without payloads or credentials.
|
|
221
|
-
|
|
222
|
-
can be closed safely and the operation resumed from the panel. No automatic snapshots or archive expiry run.
|
|
230
|
+
time and replay counts, without payloads or credentials. Other applications consume account quota too. The browser
|
|
231
|
+
can be closed safely; the job continues on the server. No automatic snapshots or archive expiry run.
|
|
223
232
|
Deleting a whole log uses bounded background cleanup of that log's private archive objects, after normal
|
|
224
233
|
fork/deletion checks. Individual archived-history deletion is a separate future feature.
|
|
225
234
|
|
|
@@ -228,7 +237,7 @@ payloads into the existing portable format. A SQLite-only export returns `409 ar
|
|
|
228
237
|
external checkpoint/archive objects are required, avoiding a misleading incomplete backup. Rollback after
|
|
229
238
|
compaction needs a compatible reader or a full logical restore first.
|
|
230
239
|
|
|
231
|
-
The first format bounds a single entity's replay
|
|
240
|
+
The first format bounds a single entity's replay state to about 1.5 MB and stores page descriptors in SQLite.
|
|
232
241
|
Exceeding a snapshot work/metadata bound fails the job while source history stays readable. Measure maintenance
|
|
233
242
|
work and storage on representative data; retained identity metadata still grows, and physical SQLite file size
|
|
234
243
|
need not shrink immediately when payload space becomes reusable.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hearthlog/ledger",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Hearthlog's Ledger: the event store Worker (Elysia over SQLite Durable Objects), its contracts and tests, and the logical backup tooling",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"repository": {
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"dependencies": {
|
|
31
31
|
"@sinclair/typebox": "0.34.52",
|
|
32
32
|
"elysia": "1.4.30",
|
|
33
|
-
"@hearthlog/contracts": "0.
|
|
33
|
+
"@hearthlog/contracts": "0.11.0"
|
|
34
34
|
},
|
|
35
35
|
"peerDependencies": {
|
|
36
36
|
"@hearthlog/generated": "*"
|
package/scripts/maintenance.ts
CHANGED
|
@@ -45,6 +45,7 @@ async function main(): Promise<void> {
|
|
|
45
45
|
await new Promise(resolve => setTimeout(resolve, 500));
|
|
46
46
|
}
|
|
47
47
|
if (state.job?.status === 'failed') throw new Error(state.job.error ?? 'Maintenance failed');
|
|
48
|
+
if (state.job?.status === 'paused') console.error(`Paused (${state.job.pauseReason ?? 'budget'}); the Ledger resumes at ${state.job.resumeAt ?? 'the next UTC day'}.`);
|
|
48
49
|
} while (!values.once && state.job?.status !== 'complete' && state.job?.status !== 'paused');
|
|
49
50
|
console.log(JSON.stringify({ logId: values.log, requestId, status: state.job?.status,
|
|
50
51
|
checkpointId: state.checkpoint?.id ?? null, throughVersion: state.checkpoint?.throughVersion ?? 0,
|
package/src/ledger-app.ts
CHANGED
|
@@ -148,14 +148,18 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
148
148
|
private maintenance(assertWritable: () => void = () => undefined): LogMaintenanceStore {
|
|
149
149
|
const appId = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one().app_id;
|
|
150
150
|
const store = new LogMaintenanceStore(this.ctx.storage, this.env.LEDGER_ARCHIVE, appId,
|
|
151
|
-
async (logId, after, through, limit
|
|
151
|
+
async (logId, after, through, limit) => {
|
|
152
152
|
const rows = store.observeSql(this.ctx.storage.sql.exec<EventRow>(`WITH candidates AS (
|
|
153
153
|
SELECT *, SUM(COALESCE(payload_bytes, length(CAST(payload AS BLOB)))) OVER (ORDER BY version) AS byte_count,
|
|
154
154
|
ROW_NUMBER() OVER (ORDER BY version) AS row_number
|
|
155
|
-
FROM (SELECT * FROM log_events WHERE log_id = ? AND version > ? AND version <= ?
|
|
156
|
-
${entityId === undefined ? '' : 'AND entity_id = ?'} ORDER BY version LIMIT ?)
|
|
155
|
+
FROM (SELECT * FROM log_events WHERE log_id = ? AND version > ? AND version <= ? ORDER BY version LIMIT ?)
|
|
157
156
|
) SELECT * FROM candidates WHERE byte_count <= 1500000 OR row_number = 1`,
|
|
158
|
-
logId, after, through,
|
|
157
|
+
logId, after, through, limit)).toArray();
|
|
158
|
+
return store.hydrate(logId, rows.map(storedEvent));
|
|
159
|
+
}, async (logId, versions) => {
|
|
160
|
+
const rows = store.observeSql(this.ctx.storage.sql.exec<EventRow>(
|
|
161
|
+
'SELECT * FROM log_events WHERE log_id = ? AND version IN (SELECT value FROM json_each(?)) ORDER BY version',
|
|
162
|
+
logId, JSON.stringify(versions))).toArray();
|
|
159
163
|
return store.hydrate(logId, rows.map(storedEvent));
|
|
160
164
|
}, assertWritable, Math.max(128, Math.min(100000, Number(this.env.MAINTENANCE_DAILY_WRITES) || 10000)),
|
|
161
165
|
Math.max(10000, Math.min(1_000_000_000, Number(this.env.MAINTENANCE_DAILY_READS) || 1_000_000)));
|
|
@@ -163,10 +167,13 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
163
167
|
}
|
|
164
168
|
|
|
165
169
|
async logMaintenance(access: AccessClaims, logId: string): Promise<RpcResult<LogMaintenance>> {
|
|
166
|
-
return rpcCall(() => {
|
|
170
|
+
return rpcCall(async () => {
|
|
167
171
|
this.currentAccess(access);
|
|
168
172
|
if (!this.log(parseUuid(logId, 'log id'))) throw validationError('Log not found');
|
|
169
|
-
|
|
173
|
+
const store = this.maintenance();
|
|
174
|
+
// A job begun before alarms existed (or whose alarm was lost) is picked up again here.
|
|
175
|
+
if (store.hasActiveJob() && await this.ctx.storage.getAlarm() === null) await this.scheduleAlarm();
|
|
176
|
+
return rpcSuccess(store.state(logId));
|
|
170
177
|
});
|
|
171
178
|
}
|
|
172
179
|
|
|
@@ -179,9 +186,12 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
179
186
|
}
|
|
180
187
|
|
|
181
188
|
async startLogMaintenance(access: AccessClaims, logId: string, id: string, kind: 'snapshot' | 'compact', expectedCheckpointId?: string, expectedGeneration?: number): Promise<RpcResult<LogMaintenance>> {
|
|
182
|
-
return rpcCall(() => {
|
|
189
|
+
return rpcCall(async () => {
|
|
183
190
|
this.writableAccess(access);
|
|
184
|
-
|
|
191
|
+
const state = this.maintenance().start(parseUuid(logId, 'log id'), parseUuid(id, 'request id'), kind, expectedCheckpointId, expectedGeneration);
|
|
192
|
+
// The server carries the job to completion; the caller only needs to watch.
|
|
193
|
+
await this.scheduleAlarm();
|
|
194
|
+
return rpcSuccess(state);
|
|
185
195
|
});
|
|
186
196
|
}
|
|
187
197
|
|
|
@@ -194,6 +204,8 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
194
204
|
() => this.maintenance(() => { this.writableAccess(access); }).step(parsedLog, parsedId),
|
|
195
205
|
() => this.maintenance().state(parsedLog, parsedId));
|
|
196
206
|
this.currentAccess(access);
|
|
207
|
+
// An explicit step also restarts a job that stopped after repeated failures.
|
|
208
|
+
await this.scheduleAlarm();
|
|
197
209
|
return rpcSuccess(result);
|
|
198
210
|
});
|
|
199
211
|
}
|
|
@@ -213,21 +225,59 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
213
225
|
});
|
|
214
226
|
}
|
|
215
227
|
|
|
216
|
-
/**
|
|
228
|
+
/** Advance log maintenance and delete storage of deleted logs, each in bounded batches. */
|
|
217
229
|
override async alarm(): Promise<void> {
|
|
230
|
+
let failed = false;
|
|
231
|
+
try { await this.deleteArchivedLogs(); }
|
|
232
|
+
catch (cause) { failed = true; console.error('ledger_archive_deletion_failed', normalizeError(cause).message); }
|
|
233
|
+
try { await this.advanceMaintenance(); }
|
|
234
|
+
catch (cause) { failed = true; console.error('ledger_maintenance_alarm_failed', normalizeError(cause).message); }
|
|
235
|
+
await this.scheduleAlarm({ deletionDelayMs: 60_000, failureDelayMs: failed ? 30_000 : undefined });
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
private async deleteArchivedLogs(): Promise<void> {
|
|
218
239
|
const job = this.ctx.storage.sql.exec<{ log_id: string }>('SELECT log_id FROM log_archive_deletions LIMIT 1').toArray()[0];
|
|
219
240
|
if (!job) return;
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
241
|
+
const app = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one();
|
|
242
|
+
const bucket = this.env.LEDGER_ARCHIVE;
|
|
243
|
+
const objects = await bucket.list({ prefix: `v1/${app.app_id}/${job.log_id}/`, limit: 128 });
|
|
244
|
+
if (objects.objects.length) await bucket.delete(objects.objects.map(object => object.key));
|
|
245
|
+
if (!objects.truncated) this.ctx.storage.sql.exec('DELETE FROM log_archive_deletions WHERE log_id = ?', job.log_id);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Run due maintenance steps for a few seconds, so a job finishes without any caller. */
|
|
249
|
+
private async advanceMaintenance(): Promise<void> {
|
|
250
|
+
if (this.metadataOptional() === undefined) return;
|
|
251
|
+
const started = Date.now();
|
|
252
|
+
const assertWritable = () => this.assertApplicationWritable();
|
|
253
|
+
do {
|
|
254
|
+
const due = this.maintenance(assertWritable).dueJobs()[0];
|
|
255
|
+
if (!due) return;
|
|
256
|
+
await this.maintenanceSteps.run(`${due.logId}:${due.id}`, () => this.maintenance(assertWritable).step(due.logId, due.id));
|
|
257
|
+
} while (Date.now() - started < 5000);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** Wake at the earliest time any pending work (maintenance step, retry, resume, deletion) is due. */
|
|
261
|
+
private async scheduleAlarm(options: { deletionDelayMs?: number; failureDelayMs?: number } = {}): Promise<void> {
|
|
262
|
+
const now = Date.now();
|
|
263
|
+
const wakes: number[] = [];
|
|
264
|
+
if (this.ctx.storage.sql.exec('SELECT 1 FROM log_archive_deletions LIMIT 1').toArray().length) {
|
|
265
|
+
wakes.push(now + (options.deletionDelayMs ?? 1000));
|
|
266
|
+
}
|
|
267
|
+
if (this.metadataOptional() !== undefined) {
|
|
268
|
+
const wake = this.maintenance().nextWake();
|
|
269
|
+
if (wake !== null) wakes.push(Math.max(wake, now + (options.failureDelayMs ?? 50)));
|
|
230
270
|
}
|
|
271
|
+
if (wakes.length) await this.ctx.storage.setAlarm(Math.min(...wakes));
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** Background work has no caller to authorize it, but must still stop for revoked or fenced applications. */
|
|
275
|
+
private assertApplicationWritable(): void {
|
|
276
|
+
if (this.metadata().revoked_at !== null) {
|
|
277
|
+
throw new LedgerError(401, { error: "access token has been rotated or revoked", code: "unauthorized" });
|
|
278
|
+
}
|
|
279
|
+
const fence = this.fence();
|
|
280
|
+
if (fence) throw fenceError(fence);
|
|
231
281
|
}
|
|
232
282
|
|
|
233
283
|
override async fetch(request: Request): Promise<Response> {
|
|
@@ -710,7 +760,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
710
760
|
return rpcCall(async () => {
|
|
711
761
|
const claims = this.writableAccess(access);
|
|
712
762
|
this.deleteLogValue(parseUuid(logId, "logId"), claims);
|
|
713
|
-
await this.
|
|
763
|
+
await this.scheduleAlarm();
|
|
714
764
|
return rpcSuccess(null, 204);
|
|
715
765
|
});
|
|
716
766
|
}
|
|
@@ -860,7 +910,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
860
910
|
throw transferError("not_importing", "Only an import in progress can be aborted");
|
|
861
911
|
}
|
|
862
912
|
this.ctx.storage.transactionSync(() => {
|
|
863
|
-
for (const table of ["log_storage", "log_archive_pages", "log_maintenance_jobs", "gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
|
|
913
|
+
for (const table of ["log_checkpoint_work", "log_checkpoint_versions", "log_checkpoint_entities", "log_storage", "log_archive_pages", "log_maintenance_jobs", "gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
|
|
864
914
|
this.ctx.storage.sql.exec(`DELETE FROM ${table}`);
|
|
865
915
|
}
|
|
866
916
|
});
|
|
@@ -1174,6 +1224,8 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1174
1224
|
}
|
|
1175
1225
|
for (const query of [
|
|
1176
1226
|
"DELETE FROM log_checkpoint_entities WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
|
|
1227
|
+
"DELETE FROM log_checkpoint_work WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
|
|
1228
|
+
"DELETE FROM log_checkpoint_versions WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
|
|
1177
1229
|
"DELETE FROM log_fork_requests WHERE fork_log_id = ?",
|
|
1178
1230
|
"DELETE FROM append_requests WHERE log_id = ?",
|
|
1179
1231
|
"DELETE FROM log_storage WHERE log_id = ?",
|
package/src/log-maintenance.ts
CHANGED
|
@@ -1,21 +1,44 @@
|
|
|
1
1
|
import { EVENT_SCHEMAS } from '#generated/event-schemas';
|
|
2
|
-
import { checkpointDigest,
|
|
3
|
-
type
|
|
2
|
+
import { checkpointDigest, checkpointSelection, checkpointStateVersions, foldCheckpointEvent, newCheckpointEntityState,
|
|
3
|
+
type CheckpointChunk, type CheckpointEntityState, type CheckpointEvent, type CheckpointManifest, type CheckpointPage,
|
|
4
|
+
type CheckpointSelector, type LogMaintenance, type MaintenanceJob, type CompactionPreview } from '@hearthlog/contracts/history/checkpoint';
|
|
4
5
|
import { LedgerError, validationError } from './errors';
|
|
5
6
|
|
|
7
|
+
/** Events archived per compaction step. */
|
|
6
8
|
const PAGE = 128;
|
|
9
|
+
/** Source events folded per snapshot step (also bounded by the byte window of the reader). */
|
|
10
|
+
const FOLD_EVENTS = 512;
|
|
11
|
+
/** Entities indexed into snapshot pages per step. */
|
|
12
|
+
const INDEX_ENTITIES = 256;
|
|
13
|
+
const CLEAN_ROWS = 256;
|
|
7
14
|
const MAX_PAGE_BYTES = 1_500_000;
|
|
15
|
+
const MAX_STATE_BYTES = 1_500_000;
|
|
16
|
+
const MAX_MARKER_BYTES = 1_000_000;
|
|
17
|
+
/** Automatic attempts before a job stops and waits for an explicit retry. */
|
|
18
|
+
export const MAX_ATTEMPTS = 5;
|
|
8
19
|
const DEFAULT_DAY_WRITE_BUDGET = 10_000;
|
|
9
20
|
const standardTypes = new Set(Object.keys(EVENT_SCHEMAS));
|
|
10
21
|
const directives = ['EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED'];
|
|
22
|
+
const SELECTORS: readonly CheckpointSelector[] = [
|
|
23
|
+
event => typeof event.payload.status === 'string' ? `status:${event.payload.status}` : null,
|
|
24
|
+
event => typeof event.payload.thumbnailRef === 'string' ? 'thumbnail:string' : null,
|
|
25
|
+
];
|
|
11
26
|
type Row = Record<string, SqlStorageValue>;
|
|
27
|
+
type Phase = 'reset' | 'seed' | 'fold' | 'index' | 'pages';
|
|
12
28
|
interface JobRow extends Row {
|
|
13
29
|
id: string; log_id: string; kind: MaintenanceJob['kind']; through_version: number;
|
|
14
30
|
status: MaintenanceJob['status']; cursor: number; entity_cursor: string; pages: string;
|
|
15
31
|
error: string | null; updated_at: string; day: string; writes: number; reads: number;
|
|
32
|
+
phase: Phase | null; version_cursor: number; attempts: number; retry_at: string | null;
|
|
33
|
+
resume_at: string | null; pause_reason: 'writes' | 'reads' | null; paused_status: MaintenanceJob['status'] | null;
|
|
34
|
+
markers: string | null;
|
|
16
35
|
}
|
|
17
36
|
interface LayoutRow extends Row { log_id: string; compacted_through: number; generation: number; checkpoint: string | null; }
|
|
18
|
-
|
|
37
|
+
/** Replay slots for one entity plus the claim it held at the last source event. */
|
|
38
|
+
interface WorkState { s: CheckpointEntityState; c: { currentVersion: number; logVersion: number }; }
|
|
39
|
+
type Marker = Pick<CheckpointEvent, 'id' | 'type' | 'version' | 'payload'>;
|
|
40
|
+
type ReadEvents = (logId: string, after: number, through: number, limit: number) => Promise<CheckpointEvent[]>;
|
|
41
|
+
type ReadVersions = (logId: string, versions: number[]) => Promise<CheckpointEvent[]>;
|
|
19
42
|
|
|
20
43
|
/** Durable job state lives in SQLite; immutable payloads live in the private bucket. */
|
|
21
44
|
export class LogMaintenanceStore {
|
|
@@ -28,8 +51,10 @@ export class LogMaintenanceStore {
|
|
|
28
51
|
return cursor;
|
|
29
52
|
}
|
|
30
53
|
|
|
31
|
-
|
|
32
|
-
|
|
54
|
+
/** Run a statement to completion so its row counts are final, and meter it. */
|
|
55
|
+
private query<T extends Record<string, SqlStorageValue> = Record<string, SqlStorageValue>>(statement: string, ...bindings: SqlStorageValue[]): T[] {
|
|
56
|
+
const cursor = this.observeSql(this.storage.sql.exec<T>(statement, ...bindings));
|
|
57
|
+
return cursor.toArray();
|
|
33
58
|
}
|
|
34
59
|
|
|
35
60
|
private async getObject(key: string): Promise<R2ObjectBody | null> {
|
|
@@ -45,12 +70,16 @@ export class LogMaintenanceStore {
|
|
|
45
70
|
}
|
|
46
71
|
|
|
47
72
|
constructor(private readonly storage: DurableObjectStorage, private readonly bucket: R2Bucket | undefined,
|
|
48
|
-
private readonly appId: string, private readonly readEvents: ReadEvents, private readonly
|
|
73
|
+
private readonly appId: string, private readonly readEvents: ReadEvents, private readonly readVersions: ReadVersions,
|
|
74
|
+
private readonly assertWritable: () => void = () => undefined,
|
|
49
75
|
private readonly dayWriteBudget = DEFAULT_DAY_WRITE_BUDGET, private readonly dayReadBudget = 1_000_000) {}
|
|
50
76
|
|
|
51
77
|
state(logId: string, jobId?: string): LogMaintenance {
|
|
52
78
|
const layout = this.layout(logId);
|
|
53
|
-
|
|
79
|
+
// Polled often: leave out the large work columns (pages, markers).
|
|
80
|
+
const job = this.query<JobRow>(`SELECT id, log_id, kind, through_version, status, cursor, error, updated_at, attempts, retry_at,
|
|
81
|
+
resume_at, pause_reason FROM log_maintenance_jobs WHERE log_id = ? ${jobId ? 'AND id = ?' : ''} ORDER BY rowid DESC LIMIT 1`,
|
|
82
|
+
logId, ...(jobId ? [jobId] : []))[0];
|
|
54
83
|
return { compactedThroughVersion: layout.compacted_through, storageGeneration: layout.generation,
|
|
55
84
|
checkpoint: layout.checkpoint ? JSON.parse(layout.checkpoint) : null,
|
|
56
85
|
job: job ? publicJob(job) : null, available: Boolean(this.bucket) };
|
|
@@ -58,9 +87,9 @@ export class LogMaintenanceStore {
|
|
|
58
87
|
|
|
59
88
|
preview(logId: string): CompactionPreview {
|
|
60
89
|
const state = this.state(logId);
|
|
61
|
-
const totals = this.
|
|
90
|
+
const totals = this.query<{ count: number; bytes: number }>(`SELECT COUNT(*) AS count,
|
|
62
91
|
COALESCE(SUM(length(CAST(payload AS BLOB))), 0) AS bytes FROM log_events
|
|
63
|
-
WHERE log_id = ? AND version <= ? AND archive_key IS NULL`, logId, state.checkpoint?.throughVersion ?? 0)
|
|
92
|
+
WHERE log_id = ? AND version <= ? AND archive_key IS NULL`, logId, state.checkpoint?.throughVersion ?? 0)[0]!;
|
|
64
93
|
return { checkpointId: state.checkpoint?.id ?? null, storageGeneration: state.storageGeneration,
|
|
65
94
|
throughVersion: state.checkpoint?.throughVersion ?? 0, activePayloadEvents: totals.count, activePayloadBytes: totals.bytes,
|
|
66
95
|
snapshotBytes: state.checkpoint?.pages.reduce((bytes, page) => bytes + page.bytes, 0) ?? 0 };
|
|
@@ -70,16 +99,16 @@ export class LogMaintenanceStore {
|
|
|
70
99
|
if (!this.bucket) throw new LedgerError(503, { error: 'Unavailable', code: 'archive_unconfigured', message: 'Configure private Ledger archive storage first.' });
|
|
71
100
|
this.storage.transactionSync(() => {
|
|
72
101
|
this.assertWritable();
|
|
73
|
-
const prior = this.
|
|
102
|
+
const prior = this.query<JobRow>('SELECT id, log_id, kind FROM log_maintenance_jobs WHERE id = ?', id)[0];
|
|
74
103
|
if (prior) {
|
|
75
104
|
if (prior.log_id !== logId || prior.kind !== kind) throw validationError('Maintenance request ID already used');
|
|
76
105
|
return;
|
|
77
106
|
}
|
|
78
|
-
const active = this.
|
|
107
|
+
const active = this.query<JobRow>("SELECT id FROM log_maintenance_jobs WHERE log_id = ? AND status NOT IN ('complete', 'failed') LIMIT 1", logId)[0];
|
|
79
108
|
if (active) throw new LedgerError(409, { error: 'Conflict', code: 'maintenance_in_progress', message: 'Resume the existing maintenance operation.' });
|
|
80
|
-
const log = this.
|
|
109
|
+
const log = this.query<{ version: number; status: string }>('SELECT version, status FROM logs WHERE id = ?', logId)[0];
|
|
81
110
|
if (!log || log.status !== 'ready') throw validationError('Choose a ready log');
|
|
82
|
-
const boundary = this.
|
|
111
|
+
const boundary = this.query<{ action_index: number | null; action_size: number | null }>('SELECT action_index, action_size FROM log_events WHERE log_id = ? AND version = ?', logId, log.version)[0];
|
|
83
112
|
if (boundary?.action_size && boundary.action_index !== boundary.action_size - 1) throw validationError('Incomplete action boundary');
|
|
84
113
|
const checkpoint = this.state(logId).checkpoint;
|
|
85
114
|
if (kind === 'compact' && (expectedCheckpointId !== checkpoint?.id || expectedGeneration !== this.layout(logId).generation)) {
|
|
@@ -88,9 +117,19 @@ export class LogMaintenanceStore {
|
|
|
88
117
|
if (kind === 'compact' && (!checkpoint || checkpoint.throughVersion <= this.layout(logId).compacted_through)) {
|
|
89
118
|
throw new LedgerError(409, { error: 'Conflict', code: 'snapshot_required', message: 'Create a newer verified snapshot before compacting.' });
|
|
90
119
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
120
|
+
// A new snapshot can start from the previous one's retained facts
|
|
121
|
+
// instead of re-reading the whole (archived) history, unless a
|
|
122
|
+
// replay directive since then could reach back behind it.
|
|
123
|
+
let phase: Phase = 'fold', cursor = 0;
|
|
124
|
+
if (kind === 'snapshot' && checkpoint && checkpoint.throughVersion > 0 && checkpoint.throughVersion <= log.version
|
|
125
|
+
&& !this.query(`SELECT 1 FROM log_events WHERE log_id = ? AND version > ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') LIMIT 1`,
|
|
126
|
+
logId, checkpoint.throughVersion).length) {
|
|
127
|
+
phase = 'seed'; cursor = checkpoint.throughVersion;
|
|
128
|
+
}
|
|
129
|
+
this.query(`INSERT INTO log_maintenance_jobs (id, log_id, kind, through_version, status, cursor, entity_cursor, pages, updated_at, day, writes, phase)
|
|
130
|
+
VALUES (?, ?, ?, ?, ?, ?, '', '[]', ?, ?, 0, ?)`, id, logId, kind,
|
|
131
|
+
kind === 'compact' ? checkpoint!.throughVersion : log.version, kind === 'compact' ? 'archiving' : 'building', cursor,
|
|
132
|
+
new Date().toISOString(), day(), kind === 'compact' ? null : phase);
|
|
94
133
|
});
|
|
95
134
|
return this.state(logId, id);
|
|
96
135
|
}
|
|
@@ -101,10 +140,10 @@ export class LogMaintenanceStore {
|
|
|
101
140
|
const started = Date.now();
|
|
102
141
|
let result: LogMaintenance | undefined;
|
|
103
142
|
try {
|
|
104
|
-
const before = this.
|
|
143
|
+
const before = this.query<JobRow>('SELECT day, reads, writes FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId)[0];
|
|
105
144
|
result = await this.stepValue(logId, id);
|
|
106
145
|
this.assertWritable();
|
|
107
|
-
this.
|
|
146
|
+
this.query('UPDATE log_maintenance_jobs SET reads = ?, writes = ? WHERE id = ? AND log_id = ?',
|
|
108
147
|
(before?.day === day() ? before.reads : 0) + this.cursors.reduce((count, cursor) => count + cursor.rowsRead, 0),
|
|
109
148
|
(before?.day === day() ? before.writes : 0) + this.cursors.reduce((count, cursor) => count + cursor.rowsWritten, 0) + 1,
|
|
110
149
|
id, logId);
|
|
@@ -118,34 +157,81 @@ export class LogMaintenanceStore {
|
|
|
118
157
|
}
|
|
119
158
|
}
|
|
120
159
|
|
|
160
|
+
/** Active jobs whose next step is due now, least recently advanced first. */
|
|
161
|
+
dueJobs(now = Date.now()): Array<{ logId: string; id: string }> {
|
|
162
|
+
return this.activeJobs().filter(job => job.wake <= now).sort((a, b) => a.updatedAt.localeCompare(b.updatedAt))
|
|
163
|
+
.map(job => ({ logId: job.logId, id: job.id }));
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Earliest time any active job needs another step, or null when none does. */
|
|
167
|
+
nextWake(): number | null {
|
|
168
|
+
const wakes = this.activeJobs().map(job => job.wake);
|
|
169
|
+
return wakes.length ? Math.min(...wakes) : null;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
hasActiveJob(): boolean {
|
|
173
|
+
return this.query("SELECT 1 FROM log_maintenance_jobs WHERE status NOT IN ('complete', 'failed') LIMIT 1").length > 0;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
private activeJobs(): Array<{ id: string; logId: string; wake: number; updatedAt: string }> {
|
|
177
|
+
const now = Date.now();
|
|
178
|
+
return this.query<JobRow>(`SELECT id, log_id, status, day, updated_at, retry_at, resume_at FROM log_maintenance_jobs
|
|
179
|
+
WHERE status NOT IN ('complete', 'failed')`).map(job => {
|
|
180
|
+
let wake = now;
|
|
181
|
+
if (job.status === 'paused') wake = job.resume_at ? Date.parse(job.resume_at) : nextMidnight(job.day);
|
|
182
|
+
else if (job.retry_at) wake = Date.parse(job.retry_at);
|
|
183
|
+
return { id: job.id, logId: job.log_id, wake: Number.isFinite(wake) ? wake : now, updatedAt: job.updated_at };
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
121
187
|
private async stepValue(logId: string, id: string): Promise<LogMaintenance> {
|
|
122
188
|
if (!this.bucket) throw new Error('Archive storage unavailable');
|
|
123
189
|
this.assertWritable();
|
|
124
|
-
let job = this.
|
|
190
|
+
let job = this.query<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId)[0];
|
|
125
191
|
if (!job) throw validationError('Maintenance operation not found');
|
|
126
192
|
if (job.status === 'complete') return this.state(logId, id);
|
|
127
193
|
if (this.state(logId).job?.id !== id) throw new LedgerError(409, { error: 'Conflict', code: 'stale_maintenance', message: 'A newer maintenance operation replaced this one.' });
|
|
194
|
+
const resumeStatus = (row: JobRow): MaintenanceJob['status'] => row.paused_status && row.paused_status !== 'paused' && row.paused_status !== 'failed'
|
|
195
|
+
? row.paused_status : row.kind === 'snapshot' ? 'building' : 'archiving';
|
|
128
196
|
if (job.status === 'failed') {
|
|
129
|
-
|
|
130
|
-
|
|
197
|
+
// An explicit retry: start counting attempts again from the failed step.
|
|
198
|
+
this.query("UPDATE log_maintenance_jobs SET status = ?, error = NULL, attempts = 0, retry_at = NULL, paused_status = NULL WHERE id = ?", resumeStatus(job), id);
|
|
199
|
+
job = { ...job, status: resumeStatus(job), error: null, attempts: 0, retry_at: null, paused_status: null };
|
|
131
200
|
}
|
|
132
201
|
if (job.day !== day()) {
|
|
133
|
-
|
|
134
|
-
|
|
202
|
+
const status = job.status === 'paused' ? resumeStatus(job) : job.status;
|
|
203
|
+
this.query('UPDATE log_maintenance_jobs SET day = ?, writes = 0, reads = 0, status = ?, pause_reason = NULL, resume_at = NULL, paused_status = NULL WHERE id = ?', day(), status, id);
|
|
204
|
+
job = { ...job, day: day(), writes: 0, reads: 0, status, pause_reason: null, resume_at: null, paused_status: null };
|
|
135
205
|
}
|
|
136
|
-
if (job.writes >= this.dayWriteBudget || job.reads >= this.dayReadBudget) {
|
|
137
|
-
|
|
206
|
+
if (job.status === 'paused' || job.writes >= this.dayWriteBudget || job.reads >= this.dayReadBudget) {
|
|
207
|
+
if (job.status !== 'paused') {
|
|
208
|
+
const reason = job.writes >= this.dayWriteBudget ? 'writes' : 'reads';
|
|
209
|
+
this.query("UPDATE log_maintenance_jobs SET status = 'paused', paused_status = ?, pause_reason = ?, resume_at = ?, updated_at = ? WHERE id = ?",
|
|
210
|
+
job.status, reason, new Date(nextMidnight(job.day)).toISOString(), new Date().toISOString(), id);
|
|
211
|
+
}
|
|
138
212
|
return this.state(logId, id);
|
|
139
213
|
}
|
|
140
214
|
try {
|
|
141
215
|
if (job.status === 'building') await this.build(job);
|
|
142
|
-
else if (job.status === 'cleaning')
|
|
216
|
+
else if (job.status === 'cleaning') this.clean(job);
|
|
143
217
|
else if (job.status === 'archiving' || job.status === 'reclaiming') await this.archive(job);
|
|
218
|
+
if (job.attempts > 0 || job.error) {
|
|
219
|
+
this.query('UPDATE log_maintenance_jobs SET attempts = 0, error = NULL, retry_at = NULL WHERE id = ?', id);
|
|
220
|
+
}
|
|
144
221
|
} catch (cause) {
|
|
145
|
-
//
|
|
222
|
+
// Access problems are not job failures; source rows and cursors stay untouched.
|
|
146
223
|
this.assertWritable();
|
|
147
|
-
|
|
148
|
-
|
|
224
|
+
const attempts = job.attempts + 1;
|
|
225
|
+
const message = cause instanceof Error ? cause.message : 'Maintenance failed';
|
|
226
|
+
const now = new Date().toISOString();
|
|
227
|
+
if (attempts >= MAX_ATTEMPTS) {
|
|
228
|
+
this.query("UPDATE log_maintenance_jobs SET status = 'failed', paused_status = ?, error = ?, attempts = ?, retry_at = NULL, updated_at = ? WHERE id = ?",
|
|
229
|
+
job.status, message, attempts, now, id);
|
|
230
|
+
} else {
|
|
231
|
+
// Stay in the current stage and let the alarm retry with backoff; the error stays visible meanwhile.
|
|
232
|
+
this.query('UPDATE log_maintenance_jobs SET error = ?, attempts = ?, retry_at = ?, updated_at = ? WHERE id = ?',
|
|
233
|
+
message, attempts, new Date(Date.now() + Math.min(600_000, 15_000 * 2 ** (attempts - 1))).toISOString(), now, id);
|
|
234
|
+
}
|
|
149
235
|
}
|
|
150
236
|
return this.state(logId, id);
|
|
151
237
|
}
|
|
@@ -164,16 +250,16 @@ export class LogMaintenanceStore {
|
|
|
164
250
|
/** Bounded archive pages are verified before any accepted payload is reclaimed. */
|
|
165
251
|
async hydrate(logId: string, events: CheckpointEvent[]): Promise<CheckpointEvent[]> {
|
|
166
252
|
if (!events.length) return events;
|
|
167
|
-
const references = this.
|
|
253
|
+
const references = this.query<{ event_id: string; archive_key: string }>(
|
|
168
254
|
'SELECT event_id, archive_key FROM log_events WHERE log_id = ? AND event_id IN (SELECT value FROM json_each(?)) AND archive_key IS NOT NULL',
|
|
169
|
-
logId, JSON.stringify(events.map(event => event.id)))
|
|
255
|
+
logId, JSON.stringify(events.map(event => event.id)));
|
|
170
256
|
if (!references.length) return events;
|
|
171
257
|
if (!this.bucket) throw archiveError('Archive storage unavailable');
|
|
172
258
|
const wanted = new Set(events.map(event => event.id));
|
|
173
259
|
const payloads = new Map<string, CheckpointEvent>();
|
|
174
260
|
for (const key of new Set(references.map(row => row.archive_key))) {
|
|
175
261
|
if (!key.startsWith(`v1/${this.appId}/`)) throw archiveError('Archive scope mismatch');
|
|
176
|
-
const reference = this.
|
|
262
|
+
const reference = this.query<{ sha256: string }>('SELECT sha256 FROM log_archive_pages WHERE object_key = ? LIMIT 1', key)[0];
|
|
177
263
|
if (!reference) throw archiveError('Archive manifest is missing');
|
|
178
264
|
const object = await this.getObject(key);
|
|
179
265
|
if (!object) throw archiveError('Archived history is missing');
|
|
@@ -192,154 +278,270 @@ export class LogMaintenanceStore {
|
|
|
192
278
|
}
|
|
193
279
|
|
|
194
280
|
private layout(logId: string): LayoutRow {
|
|
195
|
-
return this.
|
|
281
|
+
return this.query<LayoutRow>('SELECT * FROM log_storage WHERE log_id = ?', logId)[0]
|
|
196
282
|
?? { log_id: logId, compacted_through: 0, generation: 0, checkpoint: null };
|
|
197
283
|
}
|
|
198
284
|
|
|
199
285
|
private async build(job: JobRow): Promise<void> {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
this.
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
ON CONFLICT(job_id, entity_id) DO UPDATE SET object_key = excluded.object_key`, job.id, entityId, key);
|
|
228
|
-
this.exec('UPDATE log_maintenance_jobs SET writes = writes + 2 WHERE id = ?', job.id);
|
|
229
|
-
});
|
|
230
|
-
if (prior && prior.object_key !== key) await this.deleteObject(prior.object_key);
|
|
286
|
+
switch (job.phase ?? 'reset') {
|
|
287
|
+
case 'reset': return this.reset(job);
|
|
288
|
+
case 'seed': return this.seed(job);
|
|
289
|
+
case 'fold':
|
|
290
|
+
if (job.cursor < job.through_version) return this.fold(job);
|
|
291
|
+
this.setPhase(job, 'index');
|
|
292
|
+
return this.index({ ...job, phase: 'index' });
|
|
293
|
+
case 'index': return this.index(job);
|
|
294
|
+
case 'pages': return this.pages(job);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
private setPhase(job: JobRow, phase: Phase, extra: { cursor?: number; versionCursor?: number; entityCursor?: string; pages?: CheckpointPage[] } = {}): void {
|
|
299
|
+
this.assertWritable();
|
|
300
|
+
this.query('UPDATE log_maintenance_jobs SET phase = ?, cursor = ?, version_cursor = ?, entity_cursor = ?, pages = ?, updated_at = ? WHERE id = ?',
|
|
301
|
+
phase, extra.cursor ?? job.cursor, extra.versionCursor ?? 0, extra.entityCursor ?? '', JSON.stringify(extra.pages ?? []),
|
|
302
|
+
new Date().toISOString(), job.id);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Discard partial work (including work files of the former R2-based builder) and restart from the first event. */
|
|
306
|
+
private async reset(job: JobRow): Promise<void> {
|
|
307
|
+
const legacy = this.query<{ entity_id: string; object_key: string }>('SELECT entity_id, object_key FROM log_checkpoint_entities WHERE job_id = ? LIMIT 16', job.id);
|
|
308
|
+
if (legacy.length) {
|
|
309
|
+
for (const row of legacy) {
|
|
310
|
+
if (row.object_key) await this.deleteObject(row.object_key);
|
|
311
|
+
this.assertWritable();
|
|
312
|
+
this.query('DELETE FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, row.entity_id);
|
|
231
313
|
}
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
if (this.deleteWork(job.id)) return;
|
|
317
|
+
this.setPhase(job, 'fold', { cursor: 0 });
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** Delete a bounded batch of work rows; true while any remain. */
|
|
321
|
+
private deleteWork(jobId: string): boolean {
|
|
322
|
+
this.assertWritable();
|
|
323
|
+
const entities = this.query<{ entity_id: string }>('SELECT entity_id FROM log_checkpoint_work WHERE job_id = ? LIMIT ?', jobId, CLEAN_ROWS);
|
|
324
|
+
if (entities.length) {
|
|
325
|
+
this.query('DELETE FROM log_checkpoint_work WHERE job_id = ? AND entity_id IN (SELECT value FROM json_each(?))', jobId, JSON.stringify(entities.map(row => row.entity_id)));
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
328
|
+
const versions = this.query<{ version: number }>('SELECT version FROM log_checkpoint_versions WHERE job_id = ? LIMIT ?', jobId, CLEAN_ROWS * 2);
|
|
329
|
+
if (versions.length) {
|
|
330
|
+
this.query('DELETE FROM log_checkpoint_versions WHERE job_id = ? AND version IN (SELECT value FROM json_each(?))', jobId, JSON.stringify(versions.map(row => row.version)));
|
|
331
|
+
return true;
|
|
332
|
+
}
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
private loadWork(jobId: string, entityIds: string[]): Map<string, WorkState> {
|
|
337
|
+
return new Map(this.query<{ entity_id: string; state: string }>(
|
|
338
|
+
'SELECT entity_id, state FROM log_checkpoint_work WHERE job_id = ? AND entity_id IN (SELECT value FROM json_each(?))',
|
|
339
|
+
jobId, JSON.stringify(entityIds)).map(row => [row.entity_id, JSON.parse(row.state) as WorkState]));
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/** Persist updated entity states in one statement. */
|
|
343
|
+
private saveWork(jobId: string, states: Map<string, WorkState>): void {
|
|
344
|
+
const rows: Array<{ id: string; state: WorkState }> = [];
|
|
345
|
+
for (const [id, state] of states) {
|
|
346
|
+
if (JSON.stringify(state).length > MAX_STATE_BYTES) throw new Error('Entity replay state exceeds the snapshot work budget');
|
|
347
|
+
rows.push({ id, state });
|
|
348
|
+
}
|
|
349
|
+
this.query(`INSERT INTO log_checkpoint_work (job_id, entity_id, state)
|
|
350
|
+
SELECT ?, json_extract(value, '$.id'), json_extract(value, '$.state') FROM json_each(?) WHERE true
|
|
351
|
+
ON CONFLICT(job_id, entity_id) DO UPDATE SET state = excluded.state`, jobId, JSON.stringify(rows));
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Start from the previous snapshot's retained facts: one of its pages per step. */
|
|
355
|
+
private async seed(job: JobRow): Promise<void> {
|
|
356
|
+
const checkpoint = this.layout(job.log_id).checkpoint ? JSON.parse(this.layout(job.log_id).checkpoint!) as CheckpointManifest : null;
|
|
357
|
+
if (!checkpoint || checkpoint.throughVersion !== job.cursor) throw new Error('The previous snapshot changed while seeding');
|
|
358
|
+
if (job.version_cursor >= checkpoint.pages.length) {
|
|
359
|
+
this.setPhase(job, 'fold');
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
const chunk = await this.chunk(job.log_id, job.version_cursor);
|
|
363
|
+
const touched = new Set<string>([...chunk.events.map(event => event.entityId), ...chunk.claims.map(claim => claim.entityId)]);
|
|
364
|
+
const states = this.loadWork(job.id, [...touched]);
|
|
365
|
+
const work = (entityId: string): WorkState => {
|
|
366
|
+
let state = states.get(entityId);
|
|
367
|
+
if (!state) { state = { s: newCheckpointEntityState(), c: { currentVersion: 0, logVersion: 0 } }; states.set(entityId, state); }
|
|
368
|
+
return state;
|
|
369
|
+
};
|
|
370
|
+
for (const event of [...chunk.events].sort((a, b) => a.version - b.version)) foldCheckpointEvent(work(event.entityId).s, event, SELECTORS, standardTypes);
|
|
371
|
+
for (const claim of chunk.claims) work(claim.entityId).c = { currentVersion: claim.currentVersion, logVersion: claim.logVersion };
|
|
372
|
+
this.storage.transactionSync(() => {
|
|
373
|
+
this.assertWritable();
|
|
374
|
+
this.saveWork(job.id, states);
|
|
375
|
+
this.query('UPDATE log_maintenance_jobs SET version_cursor = ?, updated_at = ? WHERE id = ?', job.version_cursor + 1, new Date().toISOString(), job.id);
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/** Fold the next window of source events into per-entity replay slots. */
|
|
380
|
+
private async fold(job: JobRow): Promise<void> {
|
|
381
|
+
const events = await this.readEvents(job.log_id, job.cursor, job.through_version, FOLD_EVENTS);
|
|
382
|
+
if (!events.length || events[0]!.version !== job.cursor + 1 || events[events.length - 1]!.version - events[0]!.version !== events.length - 1) {
|
|
383
|
+
throw new Error('Snapshot source contains a gap');
|
|
384
|
+
}
|
|
385
|
+
const selected = this.selection(job);
|
|
386
|
+
const groups = new Map<string, CheckpointEvent[]>();
|
|
387
|
+
for (const event of events) {
|
|
388
|
+
const group = groups.get(event.entityId) ?? [];
|
|
389
|
+
group.push(event); groups.set(event.entityId, group);
|
|
390
|
+
}
|
|
391
|
+
const states = this.loadWork(job.id, [...groups.keys()]);
|
|
392
|
+
for (const [entityId, source] of groups) {
|
|
393
|
+
const state = states.get(entityId) ?? { s: newCheckpointEntityState(), c: { currentVersion: 0, logVersion: 0 } };
|
|
394
|
+
for (const event of source) if (selected(event) && !directives.includes(event.type)) foldCheckpointEvent(state.s, event, SELECTORS, standardTypes);
|
|
395
|
+
const last = source[source.length - 1]!;
|
|
396
|
+
state.c = { currentVersion: last.entityVersion, logVersion: last.version };
|
|
397
|
+
states.set(entityId, state);
|
|
398
|
+
}
|
|
399
|
+
this.storage.transactionSync(() => {
|
|
232
400
|
this.assertWritable();
|
|
233
|
-
this.
|
|
401
|
+
this.saveWork(job.id, states);
|
|
402
|
+
this.query('UPDATE log_maintenance_jobs SET cursor = ?, updated_at = ? WHERE id = ?',
|
|
234
403
|
events[events.length - 1]!.version, new Date().toISOString(), job.id);
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** Turn entity slots into claim pages and the list of source versions to retain. */
|
|
408
|
+
private async index(job: JobRow): Promise<void> {
|
|
409
|
+
const pages = JSON.parse(job.pages) as CheckpointPage[];
|
|
410
|
+
const rows = this.query<{ entity_id: string; state: string }>(`SELECT entity_id, state FROM log_checkpoint_work
|
|
411
|
+
WHERE job_id = ? AND entity_id > ? ORDER BY entity_id LIMIT ?`, job.id, job.entity_cursor, INDEX_ENTITIES);
|
|
412
|
+
if (!rows.length) {
|
|
413
|
+
this.setPhase(job, 'pages', { pages });
|
|
235
414
|
return;
|
|
236
415
|
}
|
|
416
|
+
const versions = new Set<number>();
|
|
417
|
+
const claims: CheckpointChunk['claims'] = [];
|
|
418
|
+
for (const row of rows) {
|
|
419
|
+
const work = JSON.parse(row.state) as WorkState;
|
|
420
|
+
for (const version of checkpointStateVersions(work.s)) versions.add(version);
|
|
421
|
+
if (work.c.logVersion > 0) claims.push({ entityId: row.entity_id, currentVersion: work.c.currentVersion, logVersion: work.c.logVersion });
|
|
422
|
+
}
|
|
423
|
+
if (claims.length) {
|
|
424
|
+
const value = JSON.stringify({ events: [], claims } satisfies CheckpointChunk);
|
|
425
|
+
const key = `v1/${this.appId}/${job.log_id}/snapshots/${job.id}/${pages.length}.json`;
|
|
426
|
+
pages.push({ key, sha256: await this.put(key, value), count: 0, bytes: new TextEncoder().encode(value).byteLength });
|
|
427
|
+
}
|
|
428
|
+
this.storage.transactionSync(() => {
|
|
429
|
+
this.assertWritable();
|
|
430
|
+
if (versions.size) this.query('INSERT OR IGNORE INTO log_checkpoint_versions (job_id, version) SELECT ?, value FROM json_each(?)', job.id, JSON.stringify([...versions]));
|
|
431
|
+
this.query('UPDATE log_maintenance_jobs SET entity_cursor = ?, pages = ?, updated_at = ? WHERE id = ?',
|
|
432
|
+
rows[rows.length - 1]!.entity_id, JSON.stringify(pages), new Date().toISOString(), job.id);
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/** Write retained source events in version order, then publish the manifest. */
|
|
437
|
+
private async pages(job: JobRow): Promise<void> {
|
|
237
438
|
const pages = JSON.parse(job.pages) as CheckpointPage[];
|
|
238
|
-
const
|
|
239
|
-
|
|
240
|
-
if (!
|
|
439
|
+
const rows = this.query<{ version: number }>(`SELECT version FROM log_checkpoint_versions WHERE job_id = ? AND version > ?
|
|
440
|
+
ORDER BY version LIMIT ?`, job.id, job.version_cursor, PAGE);
|
|
441
|
+
if (!rows.length) {
|
|
241
442
|
const manifest: CheckpointManifest = { format: 1, appId: this.appId, logId: job.log_id, id: job.id,
|
|
242
443
|
throughVersion: job.through_version, createdAt: new Date().toISOString(), pages,
|
|
243
444
|
eventCount: pages.reduce((count, page) => count + page.count, 0), sourceEventCount: job.through_version,
|
|
244
445
|
sha256: await checkpointDigest(JSON.stringify(pages)) };
|
|
245
446
|
this.storage.transactionSync(() => {
|
|
246
447
|
this.assertWritable();
|
|
247
|
-
this.
|
|
448
|
+
this.query(`INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint) VALUES (?, 0, 0, ?)
|
|
248
449
|
ON CONFLICT(log_id) DO UPDATE SET checkpoint = excluded.checkpoint`, job.log_id, JSON.stringify(manifest));
|
|
249
|
-
this.
|
|
450
|
+
this.query("UPDATE log_maintenance_jobs SET status = 'cleaning', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
|
|
250
451
|
});
|
|
251
452
|
return;
|
|
252
453
|
}
|
|
454
|
+
const wanted = rows.map(row => row.version);
|
|
455
|
+
const events = await this.readVersions(job.log_id, wanted);
|
|
456
|
+
if (events.length !== wanted.length || events.some((event, index) => event.version !== wanted[index])) throw new Error('Snapshot source is missing retained events');
|
|
253
457
|
const chunk: CheckpointChunk = { events: [], claims: [] };
|
|
254
|
-
let
|
|
255
|
-
for (const
|
|
256
|
-
const
|
|
257
|
-
chunk.events.
|
|
258
|
-
|
|
259
|
-
if (new TextEncoder().encode(JSON.stringify(chunk)).byteLength >= MAX_PAGE_BYTES) break;
|
|
458
|
+
let bytes = 2;
|
|
459
|
+
for (const event of events) {
|
|
460
|
+
const size = new TextEncoder().encode(JSON.stringify(event)).byteLength + 1;
|
|
461
|
+
if (chunk.events.length && bytes + size >= MAX_PAGE_BYTES) break;
|
|
462
|
+
chunk.events.push(event); bytes += size;
|
|
260
463
|
}
|
|
261
|
-
chunk.events.sort((a, b) => a.version - b.version);
|
|
262
464
|
const value = JSON.stringify(chunk);
|
|
263
465
|
const key = `v1/${this.appId}/${job.log_id}/snapshots/${job.id}/${pages.length}.json`;
|
|
264
|
-
|
|
265
|
-
pages.push({ key, sha256, count: chunk.events.length, bytes: new TextEncoder().encode(value).byteLength });
|
|
466
|
+
pages.push({ key, sha256: await this.put(key, value), count: chunk.events.length, bytes: new TextEncoder().encode(value).byteLength });
|
|
266
467
|
this.assertWritable();
|
|
267
|
-
this.
|
|
268
|
-
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
private async readWork(key: string): Promise<CheckpointChunk> {
|
|
272
|
-
const object = await this.getObject(key);
|
|
273
|
-
if (!object) throw new Error('Snapshot work page is missing');
|
|
274
|
-
const encoded = await object.text();
|
|
275
|
-
if (await checkpointDigest(encoded) !== object.customMetadata?.sha256) throw new Error('Snapshot work checksum mismatch');
|
|
276
|
-
return JSON.parse(encoded) as CheckpointChunk;
|
|
468
|
+
this.query('UPDATE log_maintenance_jobs SET version_cursor = ?, pages = ?, updated_at = ? WHERE id = ?',
|
|
469
|
+
chunk.events[chunk.events.length - 1]!.version, JSON.stringify(pages), new Date().toISOString(), job.id);
|
|
277
470
|
}
|
|
278
471
|
|
|
279
|
-
private
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
this.assertWritable();
|
|
284
|
-
this.exec('DELETE FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, row.entity_id);
|
|
472
|
+
private clean(job: JobRow): void {
|
|
473
|
+
if (this.deleteWork(job.id)) {
|
|
474
|
+
this.query('UPDATE log_maintenance_jobs SET updated_at = ? WHERE id = ?', new Date().toISOString(), job.id);
|
|
475
|
+
return;
|
|
285
476
|
}
|
|
286
|
-
this.
|
|
287
|
-
rows.length ? 'cleaning' : 'complete', rows.length + 1, new Date().toISOString(), job.id);
|
|
477
|
+
this.query("UPDATE log_maintenance_jobs SET status = 'complete', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
|
|
288
478
|
}
|
|
289
479
|
|
|
290
|
-
|
|
480
|
+
/** R2 rejects an upload whose bytes do not match the digest, so no read-back is needed. */
|
|
481
|
+
private async put(key: string, value: string): Promise<string> {
|
|
291
482
|
const sha256 = await checkpointDigest(value);
|
|
292
483
|
this.io.r2Writes++;
|
|
293
484
|
this.io.r2WrittenBytes += new TextEncoder().encode(value).byteLength;
|
|
294
|
-
await this.bucket!.put(key, value, { customMetadata: { sha256 }, httpMetadata: { contentType: 'application/json' } });
|
|
295
|
-
const check = await this.getObject(key);
|
|
296
|
-
if (!check || await checkpointDigest(await check.text()) !== sha256) throw new Error('Snapshot verification failed');
|
|
485
|
+
await this.bucket!.put(key, value, { sha256, customMetadata: { sha256 }, httpMetadata: { contentType: 'application/json' } });
|
|
297
486
|
return sha256;
|
|
298
487
|
}
|
|
299
488
|
|
|
300
489
|
private async archive(job: JobRow): Promise<void> {
|
|
301
490
|
// Metadata stays as the compact identity/action registry. Only payloads
|
|
302
491
|
// move, so historical boundaries and event/request uniqueness survive.
|
|
303
|
-
const rows = this.
|
|
304
|
-
AND version <= ? AND version > ? AND archive_key IS NULL ORDER BY version LIMIT ?`, job.log_id, job.through_version, job.cursor, PAGE)
|
|
492
|
+
const rows = this.query<{ version: number }>(`SELECT version FROM log_events WHERE log_id = ?
|
|
493
|
+
AND version <= ? AND version > ? AND archive_key IS NULL ORDER BY version LIMIT ?`, job.log_id, job.through_version, job.cursor, PAGE);
|
|
305
494
|
if (!rows.length) {
|
|
306
495
|
this.storage.transactionSync(() => {
|
|
307
|
-
|
|
308
|
-
this.
|
|
309
|
-
this.
|
|
496
|
+
this.assertWritable();
|
|
497
|
+
this.query('UPDATE log_storage SET compacted_through = MAX(compacted_through, ?), generation = generation + 1 WHERE log_id = ?', job.through_version, job.log_id);
|
|
498
|
+
this.query("UPDATE log_maintenance_jobs SET status = 'complete', cursor = ?, updated_at = ? WHERE id = ?", job.through_version, new Date().toISOString(), job.id);
|
|
310
499
|
});
|
|
311
500
|
return;
|
|
312
501
|
}
|
|
313
502
|
const first = rows[0]!.version;
|
|
314
503
|
const events = await this.readEvents(job.log_id, first - 1, rows[rows.length - 1]!.version, PAGE);
|
|
315
504
|
const value = JSON.stringify(events);
|
|
316
|
-
const sha256 = await checkpointDigest(value);
|
|
317
505
|
const key = `v1/${this.appId}/${job.log_id}/archive/${job.id}/${first}.json`;
|
|
318
|
-
await this.
|
|
506
|
+
const sha256 = await this.put(key, value);
|
|
319
507
|
this.storage.transactionSync(() => {
|
|
320
508
|
this.assertWritable();
|
|
321
|
-
this.
|
|
509
|
+
this.query(`INSERT OR IGNORE INTO log_archive_pages (log_id, first_version, last_version, object_key, sha256) VALUES (?, ?, ?, ?, ?)`,
|
|
322
510
|
job.log_id, first, events[events.length - 1]!.version, key, sha256);
|
|
323
|
-
|
|
511
|
+
this.query(`UPDATE log_events SET archive_key = ?, payload_bytes = COALESCE(payload_bytes, length(CAST(payload AS BLOB))), payload = CASE WHEN event_type IN
|
|
324
512
|
('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') THEN payload ELSE '{}' END
|
|
325
513
|
WHERE log_id = ? AND version >= ? AND version <= ? AND archive_key IS NULL`, key, job.log_id, first, events[events.length - 1]!.version);
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
events[events.length - 1]!.version, reclaimed.rowsWritten + 2, new Date().toISOString(), job.id);
|
|
514
|
+
this.query("UPDATE log_maintenance_jobs SET status = 'archiving', cursor = ?, updated_at = ? WHERE id = ?",
|
|
515
|
+
events[events.length - 1]!.version, new Date().toISOString(), job.id);
|
|
329
516
|
});
|
|
330
517
|
}
|
|
331
518
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
519
|
+
/** Replay directives up to the snapshot boundary, resolved once per job. */
|
|
520
|
+
private selection(job: JobRow): (event: CheckpointEvent) => boolean {
|
|
521
|
+
let markers: Marker[];
|
|
522
|
+
if (job.markers) {
|
|
523
|
+
markers = JSON.parse(job.markers) as Marker[];
|
|
524
|
+
} else {
|
|
525
|
+
markers = this.query<{ version: number; event_id: string; event_type: string; payload: string }>(`SELECT version, event_id, event_type, payload FROM log_events
|
|
526
|
+
WHERE log_id = ? AND version <= ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') ORDER BY version DESC`,
|
|
527
|
+
job.log_id, job.through_version).map(event => ({ id: event.event_id, type: event.event_type,
|
|
528
|
+
version: event.version, payload: JSON.parse(event.payload) as Record<string, unknown> }));
|
|
529
|
+
const encoded = JSON.stringify(markers);
|
|
530
|
+
if (encoded.length < MAX_MARKER_BYTES) {
|
|
531
|
+
this.assertWritable();
|
|
532
|
+
this.query('UPDATE log_maintenance_jobs SET markers = ? WHERE id = ?', encoded, job.id);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
return checkpointSelection(markers, job.through_version);
|
|
337
536
|
}
|
|
338
537
|
}
|
|
339
538
|
|
|
340
539
|
function day(): string { return new Date().toISOString().slice(0, 10); }
|
|
540
|
+
/** Start of the UTC day after `dayValue` (YYYY-MM-DD). */
|
|
541
|
+
function nextMidnight(dayValue: string): number { return Date.parse(`${dayValue}T00:00:00.000Z`) + 86_400_000; }
|
|
341
542
|
function archiveError(message: string): LedgerError { return new LedgerError(503, { error: 'Unavailable', code: 'archive_unavailable', message }); }
|
|
342
543
|
function publicJob(row: JobRow): MaintenanceJob {
|
|
343
544
|
return { id: row.id, logId: row.log_id, kind: row.kind, throughVersion: row.through_version, status: row.status,
|
|
344
|
-
cursor: row.cursor, error: row.error, updatedAt: row.updated_at
|
|
545
|
+
cursor: row.cursor, error: row.error, updatedAt: row.updated_at, attempts: row.attempts, retryAt: row.retry_at,
|
|
546
|
+
pauseReason: row.pause_reason, resumeAt: row.resume_at };
|
|
345
547
|
}
|
package/src/maintenance-steps.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/** One in-flight step per operation, with a bounded wait for HTTP/RPC callers.
|
|
1
|
+
/** One in-flight step per operation (alarm and callers share it), with a bounded wait for HTTP/RPC callers.
|
|
2
2
|
* Losing a response must not enqueue another copy of the same running step.
|
|
3
3
|
*/
|
|
4
4
|
export class MaintenanceSteps<T> {
|
|
@@ -7,7 +7,8 @@ export class MaintenanceSteps<T> {
|
|
|
7
7
|
|
|
8
8
|
constructor(private readonly keepAlive: (work: Promise<unknown>) => void, private readonly responseWaitMs = 1000) {}
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
/** Queue one step behind earlier ones (or join the identical step already in flight) and resolve with its real result. */
|
|
11
|
+
run(key: string, work: () => Promise<T>): Promise<T> {
|
|
11
12
|
let run = this.running.get(key);
|
|
12
13
|
if (!run) {
|
|
13
14
|
run = this.queue.then(work);
|
|
@@ -18,6 +19,11 @@ export class MaintenanceSteps<T> {
|
|
|
18
19
|
this.queue = settled;
|
|
19
20
|
this.keepAlive(settled);
|
|
20
21
|
}
|
|
22
|
+
return run;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async advance(key: string, work: () => Promise<T>, current: () => T): Promise<T> {
|
|
26
|
+
const run = this.run(key, work);
|
|
21
27
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
22
28
|
try {
|
|
23
29
|
return await Promise.race([run, new Promise<T>((resolve, reject) => {
|
package/src/schema.ts
CHANGED
|
@@ -215,6 +215,25 @@ const LEDGER_APP_MIGRATIONS: readonly Migration[] = [
|
|
|
215
215
|
);
|
|
216
216
|
CREATE INDEX log_archive_object ON log_archive_pages(object_key);
|
|
217
217
|
` },
|
|
218
|
+
// Snapshot work state now lives in SQLite as per-entity replay slots (no R2
|
|
219
|
+
// work files). Jobs are advanced by an alarm and record retry/pause details.
|
|
220
|
+
{ id: 15, sql: `
|
|
221
|
+
CREATE TABLE log_checkpoint_work (
|
|
222
|
+
job_id TEXT NOT NULL, entity_id TEXT NOT NULL, state TEXT NOT NULL,
|
|
223
|
+
PRIMARY KEY(job_id, entity_id)
|
|
224
|
+
);
|
|
225
|
+
CREATE TABLE log_checkpoint_versions (
|
|
226
|
+
job_id TEXT NOT NULL, version INTEGER NOT NULL, PRIMARY KEY(job_id, version)
|
|
227
|
+
);
|
|
228
|
+
ALTER TABLE log_maintenance_jobs ADD COLUMN phase TEXT;
|
|
229
|
+
ALTER TABLE log_maintenance_jobs ADD COLUMN version_cursor INTEGER NOT NULL DEFAULT 0;
|
|
230
|
+
ALTER TABLE log_maintenance_jobs ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0;
|
|
231
|
+
ALTER TABLE log_maintenance_jobs ADD COLUMN retry_at TEXT;
|
|
232
|
+
ALTER TABLE log_maintenance_jobs ADD COLUMN resume_at TEXT;
|
|
233
|
+
ALTER TABLE log_maintenance_jobs ADD COLUMN pause_reason TEXT;
|
|
234
|
+
ALTER TABLE log_maintenance_jobs ADD COLUMN paused_status TEXT;
|
|
235
|
+
ALTER TABLE log_maintenance_jobs ADD COLUMN markers TEXT;
|
|
236
|
+
` },
|
|
218
237
|
];
|
|
219
238
|
|
|
220
239
|
const APP_REGISTRY_MIGRATIONS: readonly Migration[] = [
|