@hearthlog/ledger 0.9.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -0
- package/package.json +2 -2
- package/scripts/maintenance.ts +55 -0
- package/src/http.ts +16 -0
- package/src/ledger-app.ts +156 -15
- package/src/log-maintenance.ts +345 -0
- package/src/maintenance-steps.ts +32 -0
- package/src/schema.ts +28 -0
- package/src/schemas.ts +7 -0
package/README.md
CHANGED
|
@@ -172,3 +172,63 @@ Restore validates the complete export before provisioning, creates an empty
|
|
|
172
172
|
target application, replays and verifies every page, rotates the bootstrap
|
|
173
173
|
credential, and prints the new token. A failed restore revokes the partial
|
|
174
174
|
target.
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
## Snapshot baselines and compaction
|
|
178
|
+
|
|
179
|
+
The existing log history panel has manual **Create snapshot** and **Compact log** controls.
|
|
180
|
+
A snapshot captures a complete accepted boundary. Compaction moves its prefix payloads into a private archive;
|
|
181
|
+
application/log identities, monotonically increasing event versions, entity claims, action boundaries and retry
|
|
182
|
+
receipts stay intact. SQLite retains the compact event identity registry. Historical reads and retries resolve
|
|
183
|
+
original payloads from the archive; compaction does not expire history or delete domain records/media.
|
|
184
|
+
|
|
185
|
+
The generated Ledger configuration binds `LEDGER_ARCHIVE` to `<ledgerWorker>-archive`. Tenant provisioning
|
|
186
|
+
ensures that private R2 bucket. Keep it private, without an expiry lifecycle. Deploy compatible code and ensure
|
|
187
|
+
this binding before starting maintenance. No namespace or deployed Durable Object identity changes.
|
|
188
|
+
|
|
189
|
+
Authenticated endpoints under `/logs/:logId`:
|
|
190
|
+
|
|
191
|
+
- `GET /maintenance`: availability, verified snapshot manifest, storage generation, and current job.
|
|
192
|
+
- `GET /maintenance-preview`: the current snapshot/generation and prefix payload counts/bytes for review.
|
|
193
|
+
- `POST /maintenance`: `{ requestId, kind: "snapshot" | "compact", expectedCheckpointId?, expectedGeneration? }`.
|
|
194
|
+
Compaction requires the checkpoint ID and generation from the reviewed preview; stale previews fail with 409.
|
|
195
|
+
- `POST /maintenance/step`: `{ requestId }`, resumable work using the same request identity.
|
|
196
|
+
- `GET /checkpoint?page=N`: one checksum-verified canonical replay chunk and its entity claims.
|
|
197
|
+
|
|
198
|
+
From the development site, the same operation can run without an open browser:
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
pnpm -C dev ledger:maintain --log <log-id> --kind snapshot
|
|
202
|
+
pnpm -C dev ledger:maintain --log <log-id> --kind compact
|
|
203
|
+
pnpm -C dev ledger:maintain --log <log-id> --resume
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Set `LEDGER_BASE_URL` and `LEDGER_AUTH_TOKEN` in the process environment; the CLI never prints credentials.
|
|
207
|
+
Use `--request-id` to retry an explicit operation and `--once` for one bounded work step.
|
|
208
|
+
|
|
209
|
+
Jobs persist every source batch and publish only verified snapshots. Canonical snapshots keep original source
|
|
210
|
+
witnesses for raw field values, explicit nulls, timestamps and lifecycle transitions; dynamic/nonstandard facts
|
|
211
|
+
are retained losslessly. Browser projection uses a separate baseline table plus accepted suffix and pending
|
|
212
|
+
overlay. Fresh/behind devices can install a baseline atomically; historical comparisons or directives that
|
|
213
|
+
reach into the baseline recover original history on demand. Existing HTTP event readers remain compatible
|
|
214
|
+
through transparent archive hydration, and old socket cursors recover through HTTP rather than receiving
|
|
215
|
+
payload placeholders. The SDK's server checkpoint reader lets Stash seed a cold media index from the same base.
|
|
216
|
+
|
|
217
|
+
`MAINTENANCE_DAILY_WRITES` defaults to `10000` per job (bounded to 128–100000), and
|
|
218
|
+
`MAINTENANCE_DAILY_READS` defaults to `1000000` (bounded to 10000–1000000000). Jobs pause at either budget. Budgets use consumed SQL cursor
|
|
219
|
+
counts and can overrun by one bounded step. Structured step logs include SQL rows, R2 operations/bytes, elapsed
|
|
220
|
+
time and replay counts, without payloads or credentials. Resume with the same ID on a later UTC quota day;
|
|
221
|
+
other applications consume account quota too. The browser
|
|
222
|
+
can be closed safely and the operation resumed from the panel. No automatic snapshots or archive expiry run.
|
|
223
|
+
Deleting a whole log uses bounded background cleanup of that log's private archive objects, after normal
|
|
224
|
+
fork/deletion checks. Individual archived-history deletion is a separate future feature.
|
|
225
|
+
|
|
226
|
+
Use the full logical export or identity-preserving application transfer for backup/moves: they expand archived
|
|
227
|
+
payloads into the existing portable format. A SQLite-only export returns `409 archive_bundle_required` when
|
|
228
|
+
external checkpoint/archive objects are required, avoiding a misleading incomplete backup. Rollback after
|
|
229
|
+
compaction needs a compatible reader or a full logical restore first.
|
|
230
|
+
|
|
231
|
+
The first format bounds a single entity's replay witnesses to 8 MB and stores page descriptors in SQLite.
|
|
232
|
+
Exceeding a snapshot work/metadata bound fails the job while source history stays readable. Measure maintenance
|
|
233
|
+
work and storage on representative data; retained identity metadata still grows, and physical SQLite file size
|
|
234
|
+
need not shrink immediately when payload space becomes reusable.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hearthlog/ledger",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.2",
|
|
4
4
|
"description": "Hearthlog's Ledger: the event store Worker (Elysia over SQLite Durable Objects), its contracts and tests, and the logical backup tooling",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"repository": {
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"dependencies": {
|
|
31
31
|
"@sinclair/typebox": "0.34.52",
|
|
32
32
|
"elysia": "1.4.30",
|
|
33
|
-
"@hearthlog/contracts": "0.
|
|
33
|
+
"@hearthlog/contracts": "0.10.2"
|
|
34
34
|
},
|
|
35
35
|
"peerDependencies": {
|
|
36
36
|
"@hearthlog/generated": "*"
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { parseArgs } from 'node:util';
|
|
4
|
+
import type { LogMaintenance } from '@hearthlog/contracts/history/checkpoint';
|
|
5
|
+
|
|
6
|
+
const { values } = parseArgs({ options: {
|
|
7
|
+
log: { type: 'string' }, kind: { type: 'string', default: 'snapshot' },
|
|
8
|
+
'request-id': { type: 'string' }, resume: { type: 'boolean' }, once: { type: 'boolean' }, help: { type: 'boolean' },
|
|
9
|
+
} });
|
|
10
|
+
|
|
11
|
+
async function main(): Promise<void> {
|
|
12
|
+
if (values.help) {
|
|
13
|
+
console.log('ledger maintenance --log <uuid> [--kind snapshot|compact] [--request-id <uuid>] [--resume] [--once]');
|
|
14
|
+
console.log('Uses LEDGER_BASE_URL and LEDGER_AUTH_TOKEN. Compaction retains original events in the private archive.');
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
if (!values.log || !/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(values.log)) throw new Error('--log must be a UUID');
|
|
18
|
+
if (values.kind !== 'snapshot' && values.kind !== 'compact') throw new Error('--kind must be snapshot or compact');
|
|
19
|
+
const base = process.env.LEDGER_BASE_URL;
|
|
20
|
+
const token = process.env.LEDGER_AUTH_TOKEN;
|
|
21
|
+
if (!base || !token) throw new Error('Set LEDGER_BASE_URL and LEDGER_AUTH_TOKEN');
|
|
22
|
+
const origin = new URL(base);
|
|
23
|
+
if (!['http:', 'https:'].includes(origin.protocol) || origin.username || origin.password) throw new Error('Use an HTTP(S) Ledger URL without embedded credentials');
|
|
24
|
+
const path = `/logs/${values.log}/maintenance`;
|
|
25
|
+
async function request(suffix = '', body?: unknown): Promise<LogMaintenance> {
|
|
26
|
+
const response = await fetch(`${origin.href.replace(/\/$/, '')}${path}${suffix}`, {
|
|
27
|
+
method: body ? 'POST' : 'GET', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
|
|
28
|
+
...(body ? { body: JSON.stringify(body) } : {}),
|
|
29
|
+
});
|
|
30
|
+
if (!response.ok) throw new Error(`Ledger maintenance request failed (HTTP ${response.status})`);
|
|
31
|
+
return response.json() as Promise<LogMaintenance>;
|
|
32
|
+
}
|
|
33
|
+
let state = await request();
|
|
34
|
+
const requestId = values['request-id'] ?? (values.resume ? state.job?.id : randomUUID());
|
|
35
|
+
if (!requestId) throw new Error('No maintenance operation to resume');
|
|
36
|
+
if (!values.resume) {
|
|
37
|
+
state = await request('', { requestId, kind: values.kind,
|
|
38
|
+
expectedCheckpointId: state.checkpoint?.id, expectedGeneration: state.storageGeneration });
|
|
39
|
+
}
|
|
40
|
+
do {
|
|
41
|
+
if (state.job?.status === 'complete') break;
|
|
42
|
+
state = await request('/step', { requestId });
|
|
43
|
+
console.log(JSON.stringify(state.job));
|
|
44
|
+
if (!values.once && state.job && !['complete', 'failed', 'paused'].includes(state.job.status)) {
|
|
45
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
46
|
+
}
|
|
47
|
+
if (state.job?.status === 'failed') throw new Error(state.job.error ?? 'Maintenance failed');
|
|
48
|
+
} while (!values.once && state.job?.status !== 'complete' && state.job?.status !== 'paused');
|
|
49
|
+
console.log(JSON.stringify({ logId: values.log, requestId, status: state.job?.status,
|
|
50
|
+
checkpointId: state.checkpoint?.id ?? null, throughVersion: state.checkpoint?.throughVersion ?? 0,
|
|
51
|
+
compactedThroughVersion: state.compactedThroughVersion, storageGeneration: state.storageGeneration }));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
try { await main(); }
|
|
55
|
+
catch (cause) { console.error(cause instanceof Error ? cause.message : 'Maintenance failed'); process.exitCode = 1; }
|
package/src/http.ts
CHANGED
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
importFinishBody,
|
|
33
33
|
importRecordsBody,
|
|
34
34
|
forkLogBody,
|
|
35
|
+
maintenanceBody, maintenanceStepBody, checkpointPageQuery,
|
|
35
36
|
logVisibilityBody,
|
|
36
37
|
galleryPhotoReferenceParams,
|
|
37
38
|
logLineageBody,
|
|
@@ -444,6 +445,21 @@ const app = new Elysia({
|
|
|
444
445
|
response: { 200: "LogResponse" },
|
|
445
446
|
},
|
|
446
447
|
)
|
|
448
|
+
.get('/logs/:logId/maintenance', async ({ access, params, request }) =>
|
|
449
|
+
rpcResponse(await ledger(access).logMaintenance(access, params.logId), requestOrigin(request)),
|
|
450
|
+
{ params: logParams })
|
|
451
|
+
.get('/logs/:logId/maintenance-preview', async ({ access, params, request }) =>
|
|
452
|
+
rpcResponse(await ledger(access).compactionPreview(access, params.logId), requestOrigin(request)),
|
|
453
|
+
{ params: logParams })
|
|
454
|
+
.post('/logs/:logId/maintenance', async ({ access, params, body, request }) =>
|
|
455
|
+
rpcResponse(await ledger(access).startLogMaintenance(access, params.logId, body.requestId, body.kind, body.expectedCheckpointId, body.expectedGeneration), requestOrigin(request)),
|
|
456
|
+
{ params: logParams, body: maintenanceBody })
|
|
457
|
+
.post('/logs/:logId/maintenance/step', async ({ access, params, body, request }) =>
|
|
458
|
+
rpcResponse(await ledger(access).stepLogMaintenance(access, params.logId, body.requestId), requestOrigin(request)),
|
|
459
|
+
{ params: logParams, body: maintenanceStepBody })
|
|
460
|
+
.get('/logs/:logId/checkpoint', async ({ access, params, query, request }) =>
|
|
461
|
+
rpcResponse(await ledger(access).checkpointChunk(access, params.logId, query.page), requestOrigin(request)),
|
|
462
|
+
{ params: logParams, query: checkpointPageQuery })
|
|
447
463
|
.get(
|
|
448
464
|
"/logs/:logId/gallery-photos/:photoId/files/:fileVersionId/reference",
|
|
449
465
|
async ({ access, params, request }) =>
|
package/src/ledger-app.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import { MaintenanceSteps } from './maintenance-steps';
|
|
1
2
|
import { DurableObject } from "cloudflare:workers";
|
|
3
|
+
import { LogMaintenanceStore } from './log-maintenance';
|
|
4
|
+
import type { CheckpointChunk, LogMaintenance, CompactionPreview } from '@hearthlog/contracts/history/checkpoint';
|
|
2
5
|
|
|
3
6
|
import { SOCKET_PROTOCOL } from "./auth";
|
|
4
7
|
import {
|
|
@@ -140,6 +143,69 @@ interface SocketSessionRow extends Record<string, SqlStorageValue> {
|
|
|
140
143
|
}
|
|
141
144
|
|
|
142
145
|
export class LedgerApp extends DurableObject<Env> {
|
|
146
|
+
private readonly maintenanceSteps = new MaintenanceSteps<LogMaintenance>(work => this.ctx.waitUntil(work));
|
|
147
|
+
|
|
148
|
+
private maintenance(assertWritable: () => void = () => undefined): LogMaintenanceStore {
|
|
149
|
+
const appId = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one().app_id;
|
|
150
|
+
const store = new LogMaintenanceStore(this.ctx.storage, this.env.LEDGER_ARCHIVE, appId,
|
|
151
|
+
async (logId, after, through, limit, entityId) => {
|
|
152
|
+
const rows = store.observeSql(this.ctx.storage.sql.exec<EventRow>(`WITH candidates AS (
|
|
153
|
+
SELECT *, SUM(COALESCE(payload_bytes, length(CAST(payload AS BLOB)))) OVER (ORDER BY version) AS byte_count,
|
|
154
|
+
ROW_NUMBER() OVER (ORDER BY version) AS row_number
|
|
155
|
+
FROM (SELECT * FROM log_events WHERE log_id = ? AND version > ? AND version <= ?
|
|
156
|
+
${entityId === undefined ? '' : 'AND entity_id = ?'} ORDER BY version LIMIT ?)
|
|
157
|
+
) SELECT * FROM candidates WHERE byte_count <= 1500000 OR row_number = 1`,
|
|
158
|
+
logId, after, through, ...(entityId === undefined ? [] : [entityId]), limit)).toArray();
|
|
159
|
+
return store.hydrate(logId, rows.map(storedEvent));
|
|
160
|
+
}, assertWritable, Math.max(128, Math.min(100000, Number(this.env.MAINTENANCE_DAILY_WRITES) || 10000)),
|
|
161
|
+
Math.max(10000, Math.min(1_000_000_000, Number(this.env.MAINTENANCE_DAILY_READS) || 1_000_000)));
|
|
162
|
+
return store;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async logMaintenance(access: AccessClaims, logId: string): Promise<RpcResult<LogMaintenance>> {
|
|
166
|
+
return rpcCall(() => {
|
|
167
|
+
this.currentAccess(access);
|
|
168
|
+
if (!this.log(parseUuid(logId, 'log id'))) throw validationError('Log not found');
|
|
169
|
+
return rpcSuccess(this.maintenance().state(logId));
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async compactionPreview(access: AccessClaims, logId: string): Promise<RpcResult<CompactionPreview>> {
|
|
174
|
+
return rpcCall(() => {
|
|
175
|
+
this.currentAccess(access);
|
|
176
|
+
if (!this.log(parseUuid(logId, 'log id'))) throw validationError('Log not found');
|
|
177
|
+
return rpcSuccess(this.maintenance().preview(logId));
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async startLogMaintenance(access: AccessClaims, logId: string, id: string, kind: 'snapshot' | 'compact', expectedCheckpointId?: string, expectedGeneration?: number): Promise<RpcResult<LogMaintenance>> {
|
|
182
|
+
return rpcCall(() => {
|
|
183
|
+
this.writableAccess(access);
|
|
184
|
+
return rpcSuccess(this.maintenance().start(parseUuid(logId, 'log id'), parseUuid(id, 'request id'), kind, expectedCheckpointId, expectedGeneration));
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
async stepLogMaintenance(access: AccessClaims, logId: string, id: string): Promise<RpcResult<LogMaintenance>> {
|
|
189
|
+
return rpcCall(async () => {
|
|
190
|
+
this.writableAccess(access);
|
|
191
|
+
const parsedLog = parseUuid(logId, 'log id');
|
|
192
|
+
const parsedId = parseUuid(id, 'request id');
|
|
193
|
+
const result = await this.maintenanceSteps.advance(`${parsedLog}:${parsedId}`,
|
|
194
|
+
() => this.maintenance(() => { this.writableAccess(access); }).step(parsedLog, parsedId),
|
|
195
|
+
() => this.maintenance().state(parsedLog, parsedId));
|
|
196
|
+
this.currentAccess(access);
|
|
197
|
+
return rpcSuccess(result);
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async checkpointChunk(access: AccessClaims, logId: string, index: number): Promise<RpcResult<Omit<CheckpointChunk, 'events'> & { events: StoredEvent[] }>> {
|
|
202
|
+
return rpcCall(async () => {
|
|
203
|
+
this.currentAccess(access);
|
|
204
|
+
const result = await this.maintenance().chunk(parseUuid(logId, 'log id'), parseNonnegativeInteger(index, 'page'));
|
|
205
|
+
this.currentAccess(access);
|
|
206
|
+
return rpcSuccess({ ...result, events: result.events.map(event => ({ ...event, payload: event.payload as StoredEvent['payload'] })) });
|
|
207
|
+
});
|
|
208
|
+
}
|
|
143
209
|
constructor(ctx: DurableObjectState, env: Env) {
|
|
144
210
|
super(ctx, env);
|
|
145
211
|
this.ctx.blockConcurrencyWhile(async () => {
|
|
@@ -147,6 +213,23 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
147
213
|
});
|
|
148
214
|
}
|
|
149
215
|
|
|
216
|
+
/** Delete storage belonging to an explicitly deleted log, in bounded batches. */
|
|
217
|
+
override async alarm(): Promise<void> {
|
|
218
|
+
const job = this.ctx.storage.sql.exec<{ log_id: string }>('SELECT log_id FROM log_archive_deletions LIMIT 1').toArray()[0];
|
|
219
|
+
if (!job) return;
|
|
220
|
+
try {
|
|
221
|
+
const app = this.ctx.storage.sql.exec<{ app_id: string }>('SELECT app_id FROM app_metadata WHERE singleton = 1').one();
|
|
222
|
+
const bucket = this.env.LEDGER_ARCHIVE;
|
|
223
|
+
const objects = await bucket.list({ prefix: `v1/${app.app_id}/${job.log_id}/`, limit: 128 });
|
|
224
|
+
if (objects.objects.length) await bucket.delete(objects.objects.map(object => object.key));
|
|
225
|
+
if (!objects.truncated) this.ctx.storage.sql.exec('DELETE FROM log_archive_deletions WHERE log_id = ?', job.log_id);
|
|
226
|
+
} finally {
|
|
227
|
+
if (this.ctx.storage.sql.exec('SELECT 1 FROM log_archive_deletions LIMIT 1').toArray().length) {
|
|
228
|
+
await this.ctx.storage.setAlarm(Date.now() + 60_000);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
150
233
|
override async fetch(request: Request): Promise<Response> {
|
|
151
234
|
const url = new URL(request.url);
|
|
152
235
|
const pathname = url.pathname;
|
|
@@ -222,7 +305,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
222
305
|
return rpcSuccess({
|
|
223
306
|
appId: claims.appId,
|
|
224
307
|
principalId: claims.principalId,
|
|
225
|
-
capabilities: ["action-metadata-v1", "workspace-restore-v1", "event-excision-v1"],
|
|
308
|
+
capabilities: ["snapshot-baseline-v1", "action-metadata-v1", "workspace-restore-v1", "event-excision-v1"],
|
|
226
309
|
...(fence ? { fence } : {}),
|
|
227
310
|
});
|
|
228
311
|
});
|
|
@@ -247,9 +330,19 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
247
330
|
}
|
|
248
331
|
|
|
249
332
|
async administratorSetLogLineage(appId: string, logId: string, payload: unknown): Promise<RpcResult<LogSummary>> {
|
|
250
|
-
return rpcCall(() => {
|
|
333
|
+
return rpcCall(async () => {
|
|
251
334
|
this.administratorApplication(appId);
|
|
252
335
|
const { sourceLogId, sourceVersion } = decodeRpc(logLineageBody, payload, 'log lineage');
|
|
336
|
+
// Prefix identity is logical, even when the two logs use different
|
|
337
|
+
// physical archive layouts. Never compare '{}' payload placeholders.
|
|
338
|
+
for (let after = 0; after < sourceVersion;) {
|
|
339
|
+
const child = await this.hydrateEvents(logId, this.eventsThrough(logId, after, sourceVersion, 128));
|
|
340
|
+
const parent = await this.hydrateEvents(sourceLogId, this.eventsThrough(sourceLogId, after, sourceVersion, 128));
|
|
341
|
+
if (!child.length || child.length !== parent.length || child.some((event, index) => event.id !== parent[index]!.id
|
|
342
|
+
|| JSON.stringify(event.payload) !== JSON.stringify(parent[index]!.payload))) throw validationError('Fork prefix does not match its source');
|
|
343
|
+
after = child[child.length - 1]!.version;
|
|
344
|
+
}
|
|
345
|
+
|
|
253
346
|
this.ctx.storage.transactionSync(() => {
|
|
254
347
|
const target = this.log(logId);
|
|
255
348
|
const source = this.log(sourceLogId);
|
|
@@ -272,7 +365,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
272
365
|
`SELECT COUNT(*) AS count FROM log_events AS child
|
|
273
366
|
LEFT JOIN log_events AS parent ON parent.log_id = ? AND parent.version = child.version
|
|
274
367
|
WHERE child.log_id = ? AND child.version <= ?
|
|
275
|
-
AND (parent.event_id IS NULL OR parent.event_id != child.event_id
|
|
368
|
+
AND (parent.event_id IS NULL OR parent.event_id != child.event_id)`,
|
|
276
369
|
sourceLogId, logId, sourceVersion,
|
|
277
370
|
).toArray()[0]?.count ?? 0;
|
|
278
371
|
if (mismatches) throw validationError('Fork prefix does not match its source');
|
|
@@ -342,6 +435,10 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
342
435
|
|
|
343
436
|
async administratorExportSqlite(appId: string): Promise<Response> {
|
|
344
437
|
this.administratorApplication(appId);
|
|
438
|
+
if (this.ctx.storage.sql.exec('SELECT 1 FROM log_events WHERE archive_key IS NOT NULL LIMIT 1').toArray().length
|
|
439
|
+
|| this.ctx.storage.sql.exec('SELECT 1 FROM log_storage WHERE checkpoint IS NOT NULL LIMIT 1').toArray().length) {
|
|
440
|
+
return jsonError('archive_bundle_required', 'Use the complete logical export or application transfer; archived payloads are outside SQLite.', 409);
|
|
441
|
+
}
|
|
345
442
|
return sqliteExport(
|
|
346
443
|
this.ctx.storage,
|
|
347
444
|
`ledger-${appId}-${new Date().toISOString().replaceAll(":", "-")}`,
|
|
@@ -456,7 +553,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
456
553
|
logId: string,
|
|
457
554
|
query: AdminEventsQuery,
|
|
458
555
|
): Promise<RpcResult<AdminEventsResponse>> {
|
|
459
|
-
return rpcCall(() => {
|
|
556
|
+
return rpcCall(async () => {
|
|
460
557
|
this.administratorApplication(appId);
|
|
461
558
|
const id = parseUuid(logId, "log id");
|
|
462
559
|
const parsed = decodeRpc(
|
|
@@ -492,7 +589,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
492
589
|
limit + 1,
|
|
493
590
|
)
|
|
494
591
|
.toArray();
|
|
495
|
-
const events = rows.slice(0, limit).map(storedEvent);
|
|
592
|
+
const events = await this.hydrateEvents(id, rows.slice(0, limit).map(storedEvent));
|
|
496
593
|
return rpcSuccess({
|
|
497
594
|
events,
|
|
498
595
|
headVersion: log.version,
|
|
@@ -610,9 +707,10 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
610
707
|
access: AccessClaims,
|
|
611
708
|
logId: string,
|
|
612
709
|
): Promise<RpcResult<null>> {
|
|
613
|
-
return rpcCall(() => {
|
|
710
|
+
return rpcCall(async () => {
|
|
614
711
|
const claims = this.writableAccess(access);
|
|
615
712
|
this.deleteLogValue(parseUuid(logId, "logId"), claims);
|
|
713
|
+
await this.ctx.storage.setAlarm(Date.now() + 1000);
|
|
616
714
|
return rpcSuccess(null, 204);
|
|
617
715
|
});
|
|
618
716
|
}
|
|
@@ -624,11 +722,14 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
624
722
|
limit: number | undefined,
|
|
625
723
|
replayVersion = 0,
|
|
626
724
|
): Promise<RpcResult<EventBatchResponse>> {
|
|
627
|
-
return rpcCall(() => {
|
|
725
|
+
return rpcCall(async () => {
|
|
628
726
|
this.currentAccess(access);
|
|
629
727
|
const id = parseUuid(logId, "logId");
|
|
630
728
|
this.assertReplayCompatible(id, replayVersion);
|
|
631
|
-
|
|
729
|
+
const batch = this.getEventsValue(id, afterVersion, limit);
|
|
730
|
+
const events = await this.hydrateEvents(id, batch.events);
|
|
731
|
+
this.currentAccess(access);
|
|
732
|
+
return rpcSuccess({ events });
|
|
632
733
|
});
|
|
633
734
|
}
|
|
634
735
|
|
|
@@ -721,7 +822,8 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
721
822
|
log.id, after, TRANSFER_PAGE,
|
|
722
823
|
).toArray();
|
|
723
824
|
if (rows.length === 0) break;
|
|
724
|
-
|
|
825
|
+
const hydrated = await this.hydrateEvents(log.id, rows.map(storedEvent));
|
|
826
|
+
digest = await chainChecksum(digest, rows.map((row, index) => checksumLine({ ...row, payload: JSON.stringify(hydrated[index]!.payload) })));
|
|
725
827
|
count += rows.length;
|
|
726
828
|
after = rows[rows.length - 1]!.version;
|
|
727
829
|
}
|
|
@@ -758,7 +860,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
758
860
|
throw transferError("not_importing", "Only an import in progress can be aborted");
|
|
759
861
|
}
|
|
760
862
|
this.ctx.storage.transactionSync(() => {
|
|
761
|
-
for (const table of ["gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
|
|
863
|
+
for (const table of ["log_storage", "log_archive_pages", "log_maintenance_jobs", "gallery_photo_references", "log_fork_requests", "append_requests", "entity_claims", "log_events", "logs", "app_metadata"]) {
|
|
762
864
|
this.ctx.storage.sql.exec(`DELETE FROM ${table}`);
|
|
763
865
|
}
|
|
764
866
|
});
|
|
@@ -1006,9 +1108,9 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1006
1108
|
);
|
|
1007
1109
|
this.ctx.storage.sql.exec(
|
|
1008
1110
|
`INSERT INTO log_events (log_id, version, event_id, event_type, entity_id, entity_version,
|
|
1009
|
-
event_timestamp, payload, action_id, action_index, action_size, actor)
|
|
1111
|
+
event_timestamp, payload, action_id, action_index, action_size, actor, archive_key, payload_bytes)
|
|
1010
1112
|
SELECT ?, version, event_id, event_type, entity_id, entity_version,
|
|
1011
|
-
event_timestamp, payload, action_id, action_index, action_size, actor
|
|
1113
|
+
event_timestamp, payload, action_id, action_index, action_size, actor, archive_key, payload_bytes
|
|
1012
1114
|
FROM log_events WHERE log_id = ? AND version <= ? ORDER BY version`,
|
|
1013
1115
|
id, sourceLogId, targetVersion,
|
|
1014
1116
|
);
|
|
@@ -1058,13 +1160,25 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1058
1160
|
}
|
|
1059
1161
|
this.ctx.storage.transactionSync(() => {
|
|
1060
1162
|
this.currentAccess(access);
|
|
1163
|
+
if (this.ctx.storage.sql.exec("SELECT 1 FROM log_maintenance_jobs WHERE log_id = ? AND status NOT IN ('complete', 'failed') LIMIT 1", logId).toArray().length) {
|
|
1164
|
+
throw new LedgerError(409, { error: 'Conflict', code: 'maintenance_in_progress', message: 'Finish log maintenance before deleting this log.' });
|
|
1165
|
+
}
|
|
1061
1166
|
if (this.ctx.storage.sql.exec('SELECT 1 FROM logs WHERE source_log_id = ? LIMIT 1', logId).toArray().length) {
|
|
1062
1167
|
throw new LedgerError(409, { error: 'Conflict', code: 'log_has_forks',
|
|
1063
1168
|
message: 'Remove descendant logs before deleting their source.' });
|
|
1064
1169
|
}
|
|
1170
|
+
if (this.ctx.storage.sql.exec(`SELECT 1 FROM log_events WHERE log_id != ? AND archive_key IN
|
|
1171
|
+
(SELECT object_key FROM log_archive_pages WHERE log_id = ?) LIMIT 1`, logId, logId).toArray().length) {
|
|
1172
|
+
throw new LedgerError(409, { error: 'Conflict', code: 'log_has_archive_dependents',
|
|
1173
|
+
message: 'Other logs still depend on this archived history.' });
|
|
1174
|
+
}
|
|
1065
1175
|
for (const query of [
|
|
1176
|
+
"DELETE FROM log_checkpoint_entities WHERE job_id IN (SELECT id FROM log_maintenance_jobs WHERE log_id = ?)",
|
|
1066
1177
|
"DELETE FROM log_fork_requests WHERE fork_log_id = ?",
|
|
1067
1178
|
"DELETE FROM append_requests WHERE log_id = ?",
|
|
1179
|
+
"DELETE FROM log_storage WHERE log_id = ?",
|
|
1180
|
+
"DELETE FROM log_archive_pages WHERE log_id = ?",
|
|
1181
|
+
"DELETE FROM log_maintenance_jobs WHERE log_id = ?",
|
|
1068
1182
|
"DELETE FROM gallery_photo_references WHERE log_id = ?",
|
|
1069
1183
|
"DELETE FROM entity_claims WHERE log_id = ?",
|
|
1070
1184
|
"DELETE FROM log_events WHERE log_id = ?",
|
|
@@ -1072,6 +1186,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1072
1186
|
]) {
|
|
1073
1187
|
this.ctx.storage.sql.exec(query, logId);
|
|
1074
1188
|
}
|
|
1189
|
+
this.ctx.storage.sql.exec('INSERT OR IGNORE INTO log_archive_deletions (log_id) VALUES (?)', logId);
|
|
1075
1190
|
});
|
|
1076
1191
|
this.closeLogSockets(logId, 1008, "log_deleted");
|
|
1077
1192
|
}
|
|
@@ -1113,6 +1228,18 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1113
1228
|
validateIncomingEvents(request.events);
|
|
1114
1229
|
const requestHash = await hashAppendRequest(logId, request);
|
|
1115
1230
|
|
|
1231
|
+
// The original response may now live in R2. Resolve it before entering
|
|
1232
|
+
// the synchronous append transaction; accepted requests never append twice.
|
|
1233
|
+
const receipt = this.ctx.storage.sql.exec<{ request_hash: string; first_version: number | null; last_version: number | null }>(
|
|
1234
|
+
'SELECT request_hash, first_version, last_version FROM append_requests WHERE log_id = ? AND request_id = ?', logId, request.requestId).toArray()[0];
|
|
1235
|
+
if (receipt && receipt.first_version !== null && receipt.last_version !== null) {
|
|
1236
|
+
if (receipt.request_hash !== requestHash) throw new LedgerError(409, { error: 'Conflict', code: 'idempotency_mismatch' });
|
|
1237
|
+
const result = this.replayedAppend(logId, receipt.first_version, receipt.last_version);
|
|
1238
|
+
result.events = await this.hydrateEvents(logId, result.events);
|
|
1239
|
+
this.currentAccess(access);
|
|
1240
|
+
return result;
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1116
1243
|
const response = this.ctx.storage.transactionSync(() => {
|
|
1117
1244
|
this.currentAccess(access);
|
|
1118
1245
|
this.assertReplayCompatible(logId, replayVersion);
|
|
@@ -1479,6 +1606,12 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1479
1606
|
return;
|
|
1480
1607
|
}
|
|
1481
1608
|
const head = this.logHead(attachment.logId);
|
|
1609
|
+
const archived = this.ctx.storage.sql.exec<{ version: number | null }>('SELECT MAX(version) AS version FROM log_events WHERE log_id = ? AND archive_key IS NOT NULL', attachment.logId).one().version ?? 0;
|
|
1610
|
+
if (attachment.cursor < archived) {
|
|
1611
|
+
this.sendFrame(ws, { type: 'resync_required', headVersion: head, reason: 'gap_too_large' });
|
|
1612
|
+
ws.close(1008, 'baseline resync required');
|
|
1613
|
+
return;
|
|
1614
|
+
}
|
|
1482
1615
|
if (attachment.cursor > head) {
|
|
1483
1616
|
this.sendFrame(ws, {
|
|
1484
1617
|
type: "resync_required",
|
|
@@ -1756,7 +1889,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1756
1889
|
});
|
|
1757
1890
|
totals.logs += 1;
|
|
1758
1891
|
for (let after = 0; after < log.version;) {
|
|
1759
|
-
const events = this.eventsThrough(log.id, after, log.version, TRANSFER_PAGE);
|
|
1892
|
+
const events = await this.hydrateEvents(log.id, this.eventsThrough(log.id, after, log.version, TRANSFER_PAGE));
|
|
1760
1893
|
if (events.length === 0) throw new Error(`log ${log.id} export stopped at version ${after}`);
|
|
1761
1894
|
yield line({ t: "events", logId: log.id, events });
|
|
1762
1895
|
after = events[events.length - 1]!.version;
|
|
@@ -1831,6 +1964,14 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1831
1964
|
return log.version;
|
|
1832
1965
|
}
|
|
1833
1966
|
|
|
1967
|
+
private async hydrateEvents(logId: string, events: StoredEvent[]): Promise<StoredEvent[]> {
|
|
1968
|
+
if (!events.length) return events;
|
|
1969
|
+
const ordered = [...events].sort((a, b) => a.version - b.version);
|
|
1970
|
+
const hydrated = await this.maintenance().hydrate(logId, ordered);
|
|
1971
|
+
const byId = new Map(hydrated.map(event => [event.id, event]));
|
|
1972
|
+
return events.map(event => ({ ...event, payload: byId.get(event.id)!.payload as StoredEvent['payload'] }));
|
|
1973
|
+
}
|
|
1974
|
+
|
|
1834
1975
|
private eventsAfter(
|
|
1835
1976
|
logId: string,
|
|
1836
1977
|
after: number,
|
|
@@ -1899,12 +2040,12 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1899
2040
|
let cursor = 0;
|
|
1900
2041
|
let firstEvent = true;
|
|
1901
2042
|
while (cursor < log.headVersion) {
|
|
1902
|
-
const events = this.eventsThrough(
|
|
2043
|
+
const events = await this.hydrateEvents(log.id, this.eventsThrough(
|
|
1903
2044
|
log.id,
|
|
1904
2045
|
cursor,
|
|
1905
2046
|
log.headVersion,
|
|
1906
2047
|
MAX_CATCH_UP,
|
|
1907
|
-
);
|
|
2048
|
+
));
|
|
1908
2049
|
if (events.length === 0) {
|
|
1909
2050
|
throw new Error(
|
|
1910
2051
|
`log ${log.id} export stopped at version ${cursor}`,
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
import { EVENT_SCHEMAS } from '#generated/event-schemas';
|
|
2
|
+
import { checkpointDigest, checkpointEntityEvents, checkpointSelection, type CheckpointChunk, type CheckpointEvent, type CheckpointManifest,
|
|
3
|
+
type CheckpointPage, type LogMaintenance, type MaintenanceJob, type CompactionPreview } from '@hearthlog/contracts/history/checkpoint';
|
|
4
|
+
import { LedgerError, validationError } from './errors';
|
|
5
|
+
|
|
6
|
+
const PAGE = 128;
|
|
7
|
+
const MAX_PAGE_BYTES = 1_500_000;
|
|
8
|
+
const DEFAULT_DAY_WRITE_BUDGET = 10_000;
|
|
9
|
+
const standardTypes = new Set(Object.keys(EVENT_SCHEMAS));
|
|
10
|
+
const directives = ['EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED'];
|
|
11
|
+
type Row = Record<string, SqlStorageValue>;
|
|
12
|
+
interface JobRow extends Row {
|
|
13
|
+
id: string; log_id: string; kind: MaintenanceJob['kind']; through_version: number;
|
|
14
|
+
status: MaintenanceJob['status']; cursor: number; entity_cursor: string; pages: string;
|
|
15
|
+
error: string | null; updated_at: string; day: string; writes: number; reads: number;
|
|
16
|
+
}
|
|
17
|
+
interface LayoutRow extends Row { log_id: string; compacted_through: number; generation: number; checkpoint: string | null; }
|
|
18
|
+
type ReadEvents = (logId: string, after: number, through: number, limit: number, entityId?: string) => Promise<CheckpointEvent[]>;
|
|
19
|
+
|
|
20
|
+
/** Durable job state lives in SQLite; immutable payloads live in the private bucket. */
|
|
21
|
+
export class LogMaintenanceStore {
|
|
22
|
+
private cursors: Array<{ readonly rowsRead: number; readonly rowsWritten: number }> = [];
|
|
23
|
+
private io = { r2Reads: 0, r2Writes: 0, r2Deletes: 0, r2ReadBytes: 0, r2WrittenBytes: 0 };
|
|
24
|
+
|
|
25
|
+
/** Also meter the bounded source scan provided by LedgerApp. */
|
|
26
|
+
observeSql<T extends { readonly rowsRead: number; readonly rowsWritten: number }>(cursor: T): T {
|
|
27
|
+
this.cursors.push(cursor);
|
|
28
|
+
return cursor;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
private exec<T extends Record<string, SqlStorageValue> = Record<string, SqlStorageValue>>(query: string, ...bindings: SqlStorageValue[]): SqlStorageCursor<T> {
|
|
32
|
+
return this.observeSql(this.storage.sql.exec<T>(query, ...bindings));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
private async getObject(key: string): Promise<R2ObjectBody | null> {
|
|
36
|
+
this.io.r2Reads++;
|
|
37
|
+
const object = await this.bucket!.get(key);
|
|
38
|
+
this.io.r2ReadBytes += object?.size ?? 0;
|
|
39
|
+
return object;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
private async deleteObject(key: string): Promise<void> {
|
|
43
|
+
this.io.r2Deletes++;
|
|
44
|
+
await this.bucket!.delete(key);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
constructor(private readonly storage: DurableObjectStorage, private readonly bucket: R2Bucket | undefined,
|
|
48
|
+
private readonly appId: string, private readonly readEvents: ReadEvents, private readonly assertWritable: () => void = () => undefined,
|
|
49
|
+
private readonly dayWriteBudget = DEFAULT_DAY_WRITE_BUDGET, private readonly dayReadBudget = 1_000_000) {}
|
|
50
|
+
|
|
51
|
+
state(logId: string, jobId?: string): LogMaintenance {
|
|
52
|
+
const layout = this.layout(logId);
|
|
53
|
+
const job = this.exec<JobRow>(`SELECT * FROM log_maintenance_jobs WHERE log_id = ? ${jobId ? 'AND id = ?' : ''} ORDER BY rowid DESC LIMIT 1`, logId, ...(jobId ? [jobId] : [])).toArray()[0];
|
|
54
|
+
return { compactedThroughVersion: layout.compacted_through, storageGeneration: layout.generation,
|
|
55
|
+
checkpoint: layout.checkpoint ? JSON.parse(layout.checkpoint) : null,
|
|
56
|
+
job: job ? publicJob(job) : null, available: Boolean(this.bucket) };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
preview(logId: string): CompactionPreview {
|
|
60
|
+
const state = this.state(logId);
|
|
61
|
+
const totals = this.exec<{ count: number; bytes: number }>(`SELECT COUNT(*) AS count,
|
|
62
|
+
COALESCE(SUM(length(CAST(payload AS BLOB))), 0) AS bytes FROM log_events
|
|
63
|
+
WHERE log_id = ? AND version <= ? AND archive_key IS NULL`, logId, state.checkpoint?.throughVersion ?? 0).one();
|
|
64
|
+
return { checkpointId: state.checkpoint?.id ?? null, storageGeneration: state.storageGeneration,
|
|
65
|
+
throughVersion: state.checkpoint?.throughVersion ?? 0, activePayloadEvents: totals.count, activePayloadBytes: totals.bytes,
|
|
66
|
+
snapshotBytes: state.checkpoint?.pages.reduce((bytes, page) => bytes + page.bytes, 0) ?? 0 };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
start(logId: string, id: string, kind: MaintenanceJob['kind'], expectedCheckpointId?: string, expectedGeneration?: number): LogMaintenance {
|
|
70
|
+
if (!this.bucket) throw new LedgerError(503, { error: 'Unavailable', code: 'archive_unconfigured', message: 'Configure private Ledger archive storage first.' });
|
|
71
|
+
this.storage.transactionSync(() => {
|
|
72
|
+
this.assertWritable();
|
|
73
|
+
const prior = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ?', id).toArray()[0];
|
|
74
|
+
if (prior) {
|
|
75
|
+
if (prior.log_id !== logId || prior.kind !== kind) throw validationError('Maintenance request ID already used');
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const active = this.exec<JobRow>("SELECT * FROM log_maintenance_jobs WHERE log_id = ? AND status NOT IN ('complete', 'failed') LIMIT 1", logId).toArray()[0];
|
|
79
|
+
if (active) throw new LedgerError(409, { error: 'Conflict', code: 'maintenance_in_progress', message: 'Resume the existing maintenance operation.' });
|
|
80
|
+
const log = this.exec<{ version: number; status: string }>('SELECT version, status FROM logs WHERE id = ?', logId).toArray()[0];
|
|
81
|
+
if (!log || log.status !== 'ready') throw validationError('Choose a ready log');
|
|
82
|
+
const boundary = this.exec<{ action_index: number | null; action_size: number | null }>('SELECT action_index, action_size FROM log_events WHERE log_id = ? AND version = ?', logId, log.version).toArray()[0];
|
|
83
|
+
if (boundary?.action_size && boundary.action_index !== boundary.action_size - 1) throw validationError('Incomplete action boundary');
|
|
84
|
+
const checkpoint = this.state(logId).checkpoint;
|
|
85
|
+
if (kind === 'compact' && (expectedCheckpointId !== checkpoint?.id || expectedGeneration !== this.layout(logId).generation)) {
|
|
86
|
+
throw new LedgerError(409, { error: 'Conflict', code: 'stale_maintenance_preview', message: 'The snapshot changed. Review compaction again.' });
|
|
87
|
+
}
|
|
88
|
+
if (kind === 'compact' && (!checkpoint || checkpoint.throughVersion <= this.layout(logId).compacted_through)) {
|
|
89
|
+
throw new LedgerError(409, { error: 'Conflict', code: 'snapshot_required', message: 'Create a newer verified snapshot before compacting.' });
|
|
90
|
+
}
|
|
91
|
+
this.exec(`INSERT INTO log_maintenance_jobs (id, log_id, kind, through_version, status, cursor, entity_cursor, pages, updated_at, day, writes)
|
|
92
|
+
VALUES (?, ?, ?, ?, ?, 0, '', '[]', ?, ?, 0)`, id, logId, kind,
|
|
93
|
+
kind === 'compact' ? checkpoint!.throughVersion : log.version, kind === 'compact' ? 'archiving' : 'building', new Date().toISOString(), day());
|
|
94
|
+
});
|
|
95
|
+
return this.state(logId, id);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async step(logId: string, id: string): Promise<LogMaintenance> {
|
|
99
|
+
this.cursors = [];
|
|
100
|
+
this.io = { r2Reads: 0, r2Writes: 0, r2Deletes: 0, r2ReadBytes: 0, r2WrittenBytes: 0 };
|
|
101
|
+
const started = Date.now();
|
|
102
|
+
let result: LogMaintenance | undefined;
|
|
103
|
+
try {
|
|
104
|
+
const before = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId).toArray()[0];
|
|
105
|
+
result = await this.stepValue(logId, id);
|
|
106
|
+
this.assertWritable();
|
|
107
|
+
this.exec('UPDATE log_maintenance_jobs SET reads = ?, writes = ? WHERE id = ? AND log_id = ?',
|
|
108
|
+
(before?.day === day() ? before.reads : 0) + this.cursors.reduce((count, cursor) => count + cursor.rowsRead, 0),
|
|
109
|
+
(before?.day === day() ? before.writes : 0) + this.cursors.reduce((count, cursor) => count + cursor.rowsWritten, 0) + 1,
|
|
110
|
+
id, logId);
|
|
111
|
+
return result;
|
|
112
|
+
} finally {
|
|
113
|
+
console.info('ledger_maintenance_step', { appId: this.appId, logId, jobId: id,
|
|
114
|
+
status: result?.job?.status, cursor: result?.job?.cursor, elapsedMs: Date.now() - started,
|
|
115
|
+
rowsRead: this.cursors.reduce((count, cursor) => count + cursor.rowsRead, 0),
|
|
116
|
+
rowsWritten: this.cursors.reduce((count, cursor) => count + cursor.rowsWritten, 0), ...this.io,
|
|
117
|
+
snapshotEvents: result?.checkpoint?.eventCount, sourceEvents: result?.checkpoint?.sourceEventCount });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
private async stepValue(logId: string, id: string): Promise<LogMaintenance> {
|
|
122
|
+
if (!this.bucket) throw new Error('Archive storage unavailable');
|
|
123
|
+
this.assertWritable();
|
|
124
|
+
let job = this.exec<JobRow>('SELECT * FROM log_maintenance_jobs WHERE id = ? AND log_id = ?', id, logId).toArray()[0];
|
|
125
|
+
if (!job) throw validationError('Maintenance operation not found');
|
|
126
|
+
if (job.status === 'complete') return this.state(logId, id);
|
|
127
|
+
if (this.state(logId).job?.id !== id) throw new LedgerError(409, { error: 'Conflict', code: 'stale_maintenance', message: 'A newer maintenance operation replaced this one.' });
|
|
128
|
+
if (job.status === 'failed') {
|
|
129
|
+
this.exec("UPDATE log_maintenance_jobs SET status = ?, error = NULL WHERE id = ?", job.kind === 'snapshot' ? 'building' : 'archiving', id);
|
|
130
|
+
job = { ...job, status: job.kind === 'snapshot' ? 'building' : 'archiving', error: null };
|
|
131
|
+
}
|
|
132
|
+
if (job.day !== day()) {
|
|
133
|
+
this.exec('UPDATE log_maintenance_jobs SET day = ?, writes = 0, reads = 0, status = ? WHERE id = ?', day(), job.kind === 'snapshot' ? 'building' : 'archiving', id);
|
|
134
|
+
job = { ...job, day: day(), writes: 0, reads: 0, status: job.kind === 'snapshot' ? 'building' : 'archiving' };
|
|
135
|
+
}
|
|
136
|
+
if (job.writes >= this.dayWriteBudget || job.reads >= this.dayReadBudget) {
|
|
137
|
+
this.exec("UPDATE log_maintenance_jobs SET status = 'paused' WHERE id = ?", id);
|
|
138
|
+
return this.state(logId, id);
|
|
139
|
+
}
|
|
140
|
+
try {
|
|
141
|
+
if (job.status === 'building') await this.build(job);
|
|
142
|
+
else if (job.status === 'cleaning') await this.clean(job);
|
|
143
|
+
else if (job.status === 'archiving' || job.status === 'reclaiming') await this.archive(job);
|
|
144
|
+
} catch (cause) {
|
|
145
|
+
// Keep source rows and page cursors untouched when verification fails.
|
|
146
|
+
this.assertWritable();
|
|
147
|
+
this.exec("UPDATE log_maintenance_jobs SET status = 'failed', error = ?, updated_at = ? WHERE id = ?",
|
|
148
|
+
cause instanceof Error ? cause.message : 'Maintenance failed', new Date().toISOString(), id);
|
|
149
|
+
}
|
|
150
|
+
return this.state(logId, id);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
async chunk(logId: string, index: number): Promise<CheckpointChunk> {
|
|
154
|
+
const manifest = this.state(logId).checkpoint;
|
|
155
|
+
const page = manifest?.pages[index];
|
|
156
|
+
if (!page || !this.bucket) throw validationError('Snapshot page not found');
|
|
157
|
+
const object = await this.getObject(page.key);
|
|
158
|
+
if (!object) throw archiveError('Snapshot page is missing');
|
|
159
|
+
const value = await object.text();
|
|
160
|
+
if (await checkpointDigest(value) !== page.sha256) throw archiveError('Snapshot checksum mismatch');
|
|
161
|
+
return JSON.parse(value) as CheckpointChunk;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Bounded archive pages are verified before any accepted payload is reclaimed. */
|
|
165
|
+
async hydrate(logId: string, events: CheckpointEvent[]): Promise<CheckpointEvent[]> {
|
|
166
|
+
if (!events.length) return events;
|
|
167
|
+
const references = this.exec<{ event_id: string; archive_key: string }>(
|
|
168
|
+
'SELECT event_id, archive_key FROM log_events WHERE log_id = ? AND event_id IN (SELECT value FROM json_each(?)) AND archive_key IS NOT NULL',
|
|
169
|
+
logId, JSON.stringify(events.map(event => event.id))).toArray();
|
|
170
|
+
if (!references.length) return events;
|
|
171
|
+
if (!this.bucket) throw archiveError('Archive storage unavailable');
|
|
172
|
+
const wanted = new Set(events.map(event => event.id));
|
|
173
|
+
const payloads = new Map<string, CheckpointEvent>();
|
|
174
|
+
for (const key of new Set(references.map(row => row.archive_key))) {
|
|
175
|
+
if (!key.startsWith(`v1/${this.appId}/`)) throw archiveError('Archive scope mismatch');
|
|
176
|
+
const reference = this.exec<{ sha256: string }>('SELECT sha256 FROM log_archive_pages WHERE object_key = ? LIMIT 1', key).toArray()[0];
|
|
177
|
+
if (!reference) throw archiveError('Archive manifest is missing');
|
|
178
|
+
const object = await this.getObject(key);
|
|
179
|
+
if (!object) throw archiveError('Archived history is missing');
|
|
180
|
+
const value = await object.text();
|
|
181
|
+
if (await checkpointDigest(value) !== reference.sha256) throw archiveError('Archive checksum mismatch');
|
|
182
|
+
for (const event of JSON.parse(value) as CheckpointEvent[]) if (wanted.has(event.id)) payloads.set(event.id, event);
|
|
183
|
+
}
|
|
184
|
+
const archivedIds = new Set(references.map(row => row.event_id));
|
|
185
|
+
return events.map(event => {
|
|
186
|
+
if (!archivedIds.has(event.id)) return event;
|
|
187
|
+
const original = payloads.get(event.id);
|
|
188
|
+
if (!original || original.version !== event.version || original.entityId !== event.entityId
|
|
189
|
+
|| original.entityVersion !== event.entityVersion || original.type !== event.type || original.timestamp !== event.timestamp) throw archiveError('Archive identity mismatch');
|
|
190
|
+
return original;
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
private layout(logId: string): LayoutRow {
|
|
195
|
+
return this.exec<LayoutRow>('SELECT * FROM log_storage WHERE log_id = ?', logId).toArray()[0]
|
|
196
|
+
?? { log_id: logId, compacted_through: 0, generation: 0, checkpoint: null };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
private async build(job: JobRow): Promise<void> {
|
|
200
|
+
if (job.cursor < job.through_version) {
|
|
201
|
+
// Scan the primary log index once. Work records are populated in
|
|
202
|
+
// budgeted batches rather than building an index over old history
|
|
203
|
+
// during a schema migration (which could exhaust free-tier writes).
|
|
204
|
+
const events = await this.readEvents(job.log_id, job.cursor, job.through_version, 16);
|
|
205
|
+
if (!events.length || events[0]!.version !== job.cursor + 1) throw new Error('Snapshot source contains a gap');
|
|
206
|
+
const selected = this.selection(job.log_id, job.through_version);
|
|
207
|
+
const groups = new Map<string, CheckpointEvent[]>();
|
|
208
|
+
for (const event of events) {
|
|
209
|
+
const group = groups.get(event.entityId) ?? [];
|
|
210
|
+
group.push(event); groups.set(event.entityId, group);
|
|
211
|
+
}
|
|
212
|
+
for (const [entityId, source] of groups) {
|
|
213
|
+
const prior = this.exec<{ object_key: string }>('SELECT object_key FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, entityId).toArray()[0];
|
|
214
|
+
const previous = prior ? await this.readWork(prior.object_key) : { events: [], claims: [] };
|
|
215
|
+
const last = source[source.length - 1]!;
|
|
216
|
+
const witnesses = checkpointEntityEvents([...previous.events, ...source.filter(event => selected(event) && !directives.includes(event.type))], [
|
|
217
|
+
event => typeof event.payload.status === 'string' ? `status:${event.payload.status}` : null,
|
|
218
|
+
event => typeof event.payload.thumbnailRef === 'string' ? 'thumbnail:string' : null,
|
|
219
|
+
], standardTypes);
|
|
220
|
+
const value = JSON.stringify({ events: witnesses, claims: [{ entityId, currentVersion: last.entityVersion, logVersion: last.version }] });
|
|
221
|
+
if (new TextEncoder().encode(value).byteLength > 8_000_000) throw new Error('Entity replay state exceeds the snapshot work budget');
|
|
222
|
+
const key = `v1/${this.appId}/${job.log_id}/work/${job.id}/${last.version}.json`;
|
|
223
|
+
await this.putVerified(key, value);
|
|
224
|
+
this.storage.transactionSync(() => {
|
|
225
|
+
this.assertWritable();
|
|
226
|
+
this.exec(`INSERT INTO log_checkpoint_entities (job_id, entity_id, object_key) VALUES (?, ?, ?)
|
|
227
|
+
ON CONFLICT(job_id, entity_id) DO UPDATE SET object_key = excluded.object_key`, job.id, entityId, key);
|
|
228
|
+
this.exec('UPDATE log_maintenance_jobs SET writes = writes + 2 WHERE id = ?', job.id);
|
|
229
|
+
});
|
|
230
|
+
if (prior && prior.object_key !== key) await this.deleteObject(prior.object_key);
|
|
231
|
+
}
|
|
232
|
+
this.assertWritable();
|
|
233
|
+
this.exec('UPDATE log_maintenance_jobs SET cursor = ?, writes = writes + 1, updated_at = ? WHERE id = ?',
|
|
234
|
+
events[events.length - 1]!.version, new Date().toISOString(), job.id);
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
const pages = JSON.parse(job.pages) as CheckpointPage[];
|
|
238
|
+
const entities = this.exec<{ entity_id: string; object_key: string }>(`SELECT entity_id, object_key FROM log_checkpoint_entities
|
|
239
|
+
WHERE job_id = ? AND entity_id > ? ORDER BY entity_id LIMIT 16`, job.id, job.entity_cursor).toArray();
|
|
240
|
+
if (!entities.length) {
|
|
241
|
+
const manifest: CheckpointManifest = { format: 1, appId: this.appId, logId: job.log_id, id: job.id,
|
|
242
|
+
throughVersion: job.through_version, createdAt: new Date().toISOString(), pages,
|
|
243
|
+
eventCount: pages.reduce((count, page) => count + page.count, 0), sourceEventCount: job.through_version,
|
|
244
|
+
sha256: await checkpointDigest(JSON.stringify(pages)) };
|
|
245
|
+
this.storage.transactionSync(() => {
|
|
246
|
+
this.assertWritable();
|
|
247
|
+
this.exec(`INSERT INTO log_storage (log_id, compacted_through, generation, checkpoint) VALUES (?, 0, 0, ?)
|
|
248
|
+
ON CONFLICT(log_id) DO UPDATE SET checkpoint = excluded.checkpoint`, job.log_id, JSON.stringify(manifest));
|
|
249
|
+
this.exec("UPDATE log_maintenance_jobs SET status = 'cleaning', updated_at = ? WHERE id = ?", new Date().toISOString(), job.id);
|
|
250
|
+
});
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
const chunk: CheckpointChunk = { events: [], claims: [] };
|
|
254
|
+
let last = job.entity_cursor;
|
|
255
|
+
for (const entity of entities) {
|
|
256
|
+
const part = await this.readWork(entity.object_key);
|
|
257
|
+
chunk.events.push(...part.events); chunk.claims.push(...part.claims);
|
|
258
|
+
last = entity.entity_id;
|
|
259
|
+
if (new TextEncoder().encode(JSON.stringify(chunk)).byteLength >= MAX_PAGE_BYTES) break;
|
|
260
|
+
}
|
|
261
|
+
chunk.events.sort((a, b) => a.version - b.version);
|
|
262
|
+
const value = JSON.stringify(chunk);
|
|
263
|
+
const key = `v1/${this.appId}/${job.log_id}/snapshots/${job.id}/${pages.length}.json`;
|
|
264
|
+
const sha256 = await this.putVerified(key, value);
|
|
265
|
+
pages.push({ key, sha256, count: chunk.events.length, bytes: new TextEncoder().encode(value).byteLength });
|
|
266
|
+
this.assertWritable();
|
|
267
|
+
this.exec('UPDATE log_maintenance_jobs SET entity_cursor = ?, pages = ?, writes = writes + 1, updated_at = ? WHERE id = ?',
|
|
268
|
+
last, JSON.stringify(pages), new Date().toISOString(), job.id);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
private async readWork(key: string): Promise<CheckpointChunk> {
|
|
272
|
+
const object = await this.getObject(key);
|
|
273
|
+
if (!object) throw new Error('Snapshot work page is missing');
|
|
274
|
+
const encoded = await object.text();
|
|
275
|
+
if (await checkpointDigest(encoded) !== object.customMetadata?.sha256) throw new Error('Snapshot work checksum mismatch');
|
|
276
|
+
return JSON.parse(encoded) as CheckpointChunk;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
private async clean(job: JobRow): Promise<void> {
|
|
280
|
+
const rows = this.exec<{ entity_id: string; object_key: string }>('SELECT entity_id, object_key FROM log_checkpoint_entities WHERE job_id = ? LIMIT 16', job.id).toArray();
|
|
281
|
+
for (const row of rows) {
|
|
282
|
+
await this.deleteObject(row.object_key);
|
|
283
|
+
this.assertWritable();
|
|
284
|
+
this.exec('DELETE FROM log_checkpoint_entities WHERE job_id = ? AND entity_id = ?', job.id, row.entity_id);
|
|
285
|
+
}
|
|
286
|
+
this.exec('UPDATE log_maintenance_jobs SET status = ?, writes = writes + ?, updated_at = ? WHERE id = ?',
|
|
287
|
+
rows.length ? 'cleaning' : 'complete', rows.length + 1, new Date().toISOString(), job.id);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
private async putVerified(key: string, value: string): Promise<string> {
|
|
291
|
+
const sha256 = await checkpointDigest(value);
|
|
292
|
+
this.io.r2Writes++;
|
|
293
|
+
this.io.r2WrittenBytes += new TextEncoder().encode(value).byteLength;
|
|
294
|
+
await this.bucket!.put(key, value, { customMetadata: { sha256 }, httpMetadata: { contentType: 'application/json' } });
|
|
295
|
+
const check = await this.getObject(key);
|
|
296
|
+
if (!check || await checkpointDigest(await check.text()) !== sha256) throw new Error('Snapshot verification failed');
|
|
297
|
+
return sha256;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
private async archive(job: JobRow): Promise<void> {
|
|
301
|
+
// Metadata stays as the compact identity/action registry. Only payloads
|
|
302
|
+
// move, so historical boundaries and event/request uniqueness survive.
|
|
303
|
+
const rows = this.exec<{ version: number }>(`SELECT version FROM log_events WHERE log_id = ?
|
|
304
|
+
AND version <= ? AND version > ? AND archive_key IS NULL ORDER BY version LIMIT ?`, job.log_id, job.through_version, job.cursor, PAGE).toArray();
|
|
305
|
+
if (!rows.length) {
|
|
306
|
+
this.storage.transactionSync(() => {
|
|
307
|
+
this.assertWritable();
|
|
308
|
+
this.exec('UPDATE log_storage SET compacted_through = MAX(compacted_through, ?), generation = generation + 1 WHERE log_id = ?', job.through_version, job.log_id);
|
|
309
|
+
this.exec("UPDATE log_maintenance_jobs SET status = 'complete', cursor = ?, updated_at = ? WHERE id = ?", job.through_version, new Date().toISOString(), job.id);
|
|
310
|
+
});
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
const first = rows[0]!.version;
|
|
314
|
+
const events = await this.readEvents(job.log_id, first - 1, rows[rows.length - 1]!.version, PAGE);
|
|
315
|
+
const value = JSON.stringify(events);
|
|
316
|
+
const sha256 = await checkpointDigest(value);
|
|
317
|
+
const key = `v1/${this.appId}/${job.log_id}/archive/${job.id}/${first}.json`;
|
|
318
|
+
await this.putVerified(key, value);
|
|
319
|
+
this.storage.transactionSync(() => {
|
|
320
|
+
this.assertWritable();
|
|
321
|
+
this.exec(`INSERT OR IGNORE INTO log_archive_pages (log_id, first_version, last_version, object_key, sha256) VALUES (?, ?, ?, ?, ?)`,
|
|
322
|
+
job.log_id, first, events[events.length - 1]!.version, key, sha256);
|
|
323
|
+
const reclaimed = this.exec(`UPDATE log_events SET archive_key = ?, payload_bytes = COALESCE(payload_bytes, length(CAST(payload AS BLOB))), payload = CASE WHEN event_type IN
|
|
324
|
+
('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') THEN payload ELSE '{}' END
|
|
325
|
+
WHERE log_id = ? AND version >= ? AND version <= ? AND archive_key IS NULL`, key, job.log_id, first, events[events.length - 1]!.version);
|
|
326
|
+
reclaimed.toArray();
|
|
327
|
+
this.exec("UPDATE log_maintenance_jobs SET status = 'archiving', cursor = ?, writes = writes + ?, updated_at = ? WHERE id = ?",
|
|
328
|
+
events[events.length - 1]!.version, reclaimed.rowsWritten + 2, new Date().toISOString(), job.id);
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
private selection(logId: string, through: number): (event: CheckpointEvent) => boolean {
|
|
333
|
+
const rows = this.exec<{ version: number; event_id: string; event_type: string; payload: string }>(`SELECT version, event_id, event_type, payload FROM log_events
|
|
334
|
+
WHERE log_id = ? AND version <= ? AND event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED') ORDER BY version DESC`, logId, through).toArray();
|
|
335
|
+
return checkpointSelection(rows.map(event => ({ id: event.event_id, type: event.event_type,
|
|
336
|
+
version: event.version, payload: JSON.parse(event.payload) as Record<string, unknown> })), through);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
function day(): string { return new Date().toISOString().slice(0, 10); }
|
|
341
|
+
function archiveError(message: string): LedgerError { return new LedgerError(503, { error: 'Unavailable', code: 'archive_unavailable', message }); }
|
|
342
|
+
function publicJob(row: JobRow): MaintenanceJob {
|
|
343
|
+
return { id: row.id, logId: row.log_id, kind: row.kind, throughVersion: row.through_version, status: row.status,
|
|
344
|
+
cursor: row.cursor, error: row.error, updatedAt: row.updated_at };
|
|
345
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/** One in-flight step per operation, with a bounded wait for HTTP/RPC callers.
|
|
2
|
+
* Losing a response must not enqueue another copy of the same running step.
|
|
3
|
+
*/
|
|
4
|
+
export class MaintenanceSteps<T> {
|
|
5
|
+
private queue: Promise<unknown> = Promise.resolve();
|
|
6
|
+
private readonly running = new Map<string, Promise<T>>();
|
|
7
|
+
|
|
8
|
+
constructor(private readonly keepAlive: (work: Promise<unknown>) => void, private readonly responseWaitMs = 1000) {}
|
|
9
|
+
|
|
10
|
+
async advance(key: string, work: () => Promise<T>, current: () => T): Promise<T> {
|
|
11
|
+
let run = this.running.get(key);
|
|
12
|
+
if (!run) {
|
|
13
|
+
run = this.queue.then(work);
|
|
14
|
+
this.running.set(key, run);
|
|
15
|
+
const settled = run.then(() => undefined, () => undefined).then(() => {
|
|
16
|
+
this.running.delete(key);
|
|
17
|
+
});
|
|
18
|
+
this.queue = settled;
|
|
19
|
+
this.keepAlive(settled);
|
|
20
|
+
}
|
|
21
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
22
|
+
try {
|
|
23
|
+
return await Promise.race([run, new Promise<T>((resolve, reject) => {
|
|
24
|
+
timer = setTimeout(() => {
|
|
25
|
+
try { resolve(current()); } catch (cause) { reject(cause); }
|
|
26
|
+
}, this.responseWaitMs);
|
|
27
|
+
})]);
|
|
28
|
+
} finally {
|
|
29
|
+
clearTimeout(timer);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
package/src/schema.ts
CHANGED
|
@@ -187,6 +187,34 @@ const LEDGER_APP_MIGRATIONS: readonly Migration[] = [
|
|
|
187
187
|
ALTER TABLE app_metadata ADD COLUMN fence_mode TEXT;
|
|
188
188
|
ALTER TABLE app_metadata ADD COLUMN fence_destination TEXT;
|
|
189
189
|
` },
|
|
190
|
+
{ id: 14, sql: `
|
|
191
|
+
ALTER TABLE log_events ADD COLUMN archive_key TEXT;
|
|
192
|
+
ALTER TABLE log_events ADD COLUMN payload_bytes INTEGER;
|
|
193
|
+
CREATE INDEX log_events_snapshot_markers ON log_events(log_id, version)
|
|
194
|
+
WHERE event_type IN ('EVENT_TOMBSTONED', 'WORKSPACE_RESTORED', 'EVENTS_EXCISED');
|
|
195
|
+
CREATE INDEX log_events_archived_cursor ON log_events(log_id, version) WHERE archive_key IS NOT NULL;
|
|
196
|
+
CREATE TABLE log_storage (
|
|
197
|
+
log_id TEXT PRIMARY KEY, compacted_through INTEGER NOT NULL DEFAULT 0,
|
|
198
|
+
generation INTEGER NOT NULL DEFAULT 0, checkpoint TEXT
|
|
199
|
+
);
|
|
200
|
+
CREATE TABLE log_maintenance_jobs (
|
|
201
|
+
id TEXT PRIMARY KEY, log_id TEXT NOT NULL, kind TEXT NOT NULL,
|
|
202
|
+
through_version INTEGER NOT NULL, status TEXT NOT NULL, cursor INTEGER NOT NULL,
|
|
203
|
+
entity_cursor TEXT NOT NULL, pages TEXT NOT NULL, error TEXT,
|
|
204
|
+
updated_at TEXT NOT NULL, day TEXT NOT NULL, writes INTEGER NOT NULL, reads INTEGER NOT NULL DEFAULT 0
|
|
205
|
+
);
|
|
206
|
+
CREATE INDEX log_maintenance_scope ON log_maintenance_jobs(log_id);
|
|
207
|
+
CREATE TABLE log_checkpoint_entities (
|
|
208
|
+
job_id TEXT NOT NULL, entity_id TEXT NOT NULL, object_key TEXT NOT NULL,
|
|
209
|
+
PRIMARY KEY(job_id, entity_id)
|
|
210
|
+
);
|
|
211
|
+
CREATE TABLE log_archive_deletions (log_id TEXT PRIMARY KEY);
|
|
212
|
+
CREATE TABLE log_archive_pages (
|
|
213
|
+
log_id TEXT NOT NULL, first_version INTEGER NOT NULL, last_version INTEGER NOT NULL,
|
|
214
|
+
object_key TEXT NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY(log_id, first_version)
|
|
215
|
+
);
|
|
216
|
+
CREATE INDEX log_archive_object ON log_archive_pages(object_key);
|
|
217
|
+
` },
|
|
190
218
|
];
|
|
191
219
|
|
|
192
220
|
const APP_REGISTRY_MIGRATIONS: readonly Migration[] = [
|
package/src/schemas.ts
CHANGED
|
@@ -128,6 +128,13 @@ export const logParams = t.Object(
|
|
|
128
128
|
strictObject,
|
|
129
129
|
);
|
|
130
130
|
|
|
131
|
+
export const maintenanceBody = t.Object({
|
|
132
|
+
requestId: uuid('request id'), kind: t.Union([t.Literal('snapshot'), t.Literal('compact')]),
|
|
133
|
+
expectedCheckpointId: t.Optional(uuid('checkpoint id')), expectedGeneration: t.Optional(t.Integer(queryInteger)),
|
|
134
|
+
}, strictObject);
|
|
135
|
+
export const maintenanceStepBody = t.Object({ requestId: uuid('request id') }, strictObject);
|
|
136
|
+
export const checkpointPageQuery = t.Object({ page: t.Numeric(queryInteger) }, strictObject);
|
|
137
|
+
|
|
131
138
|
export const galleryPhotoReferenceParams = t.Object(
|
|
132
139
|
{
|
|
133
140
|
logId: uuid('log id'),
|