@hearthlog/ledger 0.13.7 → 0.13.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -2
- package/docs/database-structure.md +5 -5
- package/package.json +2 -2
- package/src/event-validation.ts +3 -0
- package/src/http.ts +27 -2
- package/src/ledger-app.ts +167 -28
- package/src/schemas.ts +10 -0
package/README.md
CHANGED
|
@@ -82,7 +82,8 @@ Each application also carries an **event policy** (`GET|PUT /admin/v1/apps/<id>/
|
|
|
82
82
|
`contracts` lists the modules whose event types it accepts (the core's always are;
|
|
83
83
|
`EVENT_OWNERS` in `event-schemas.generated.ts` says which module defines each type),
|
|
84
84
|
`dynamicEvents` says whether runtime-defined entity types are allowed, and
|
|
85
|
-
`validation` (`report` or `enforce`) overrides the deployment's `EVENT_VALIDATION
|
|
85
|
+
`validation` (`report` or `enforce`) overrides the deployment's `EVENT_VALIDATION`, and
|
|
86
|
+
`autoSnapshots` (optional, off when absent) lets Ledger snapshot growing logs by itself (see below).
|
|
86
87
|
An application created without a policy gets `{ contracts: null, dynamicEvents: false,
|
|
87
88
|
validation: "enforce" }`; `pnpm -C dev ensure:tenant` creates a site's application with the
|
|
88
89
|
modules the site ships. An application that predates policies has none and behaves
|
|
@@ -133,6 +134,16 @@ tokens, set the old key and ID as `TOKEN_SIGNING_KEY_PREVIOUS` and
|
|
|
133
134
|
`TOKEN_SIGNING_PREVIOUS_KEY_ID`, deploy the new current key, rotate app tokens,
|
|
134
135
|
then remove the previous key after the coordinated window.
|
|
135
136
|
|
|
137
|
+
## Batched appends
|
|
138
|
+
|
|
139
|
+
`POST /logs/{logId}/appends` takes `{ requests: [...] }`, up to 64 ordinary append
|
|
140
|
+
bodies, and applies them in order as if each had been sent alone: each keeps its own
|
|
141
|
+
`requestId` receipt and version checks. It stops at the first request that fails and
|
|
142
|
+
answers `{ results: [{ status, body }] }` with one entry per request it attempted, so a
|
|
143
|
+
resend replays the same results. Sockets receive the accepted events once, after the
|
|
144
|
+
batch. Sessions advertise it with the `append-batch-v1` capability; the frontend sends
|
|
145
|
+
a queue of saves this way and falls back to single appends without it.
|
|
146
|
+
|
|
136
147
|
## Realtime protocol
|
|
137
148
|
|
|
138
149
|
The browser opens
|
|
@@ -230,7 +241,14 @@ payload placeholders. The SDK's server checkpoint reader lets Stash seed a cold
|
|
|
230
241
|
`MAINTENANCE_DAILY_READS` defaults to `1000000` (bounded to 10000–1000000000). Jobs pause at either budget. Budgets use consumed SQL cursor
|
|
231
242
|
counts and can overrun by one bounded step. Structured step logs include SQL rows, R2 operations/bytes, elapsed
|
|
232
243
|
time and replay counts, without payloads or credentials. Other applications consume account quota too. The browser
|
|
233
|
-
can be closed safely; the job continues on the server. No
|
|
244
|
+
can be closed safely; the job continues on the server. No archive expiry runs, and compaction is always manual.
|
|
245
|
+
|
|
246
|
+
Snapshots can start by themselves: when an application's policy has `autoSnapshots: true`, an append that leaves the
|
|
247
|
+
log `AUTO_SNAPSHOT_EVENTS` (default `5000`, bounded to 500–1000000) or more past its last verified snapshot starts a
|
|
248
|
+
snapshot job, unless one is already running or the last failed less than a day ago. It runs under the same budgets
|
|
249
|
+
and pauses as one started by hand. The switch is off by default; set `"autoSnapshots": true` on a site in
|
|
250
|
+
`tenants.json` and apply it with `pnpm -C dev ensure:tenant -- --site <id> --apply-policy`, after one manual snapshot
|
|
251
|
+
on that site's data has finished cleanly.
|
|
234
252
|
Deleting a whole log uses bounded background cleanup of that log's private archive objects, after normal
|
|
235
253
|
fork/deletion checks. Individual archived-history deletion is a separate future feature.
|
|
236
254
|
|
|
@@ -126,11 +126,11 @@ removing a photo event does not immediately delete recoverable original bytes.
|
|
|
126
126
|
### `append_requests`
|
|
127
127
|
|
|
128
128
|
Stores `(request_id, log_id)`, a canonical request hash, the first and last log
|
|
129
|
-
version the request accepted,
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
129
|
+
version the request accepted, and creation time. Exact retries rebuild the response
|
|
130
|
+
from `log_events` through that version range, so they report the original author.
|
|
131
|
+
Rows written before the range columns existed (migration 9) replay the validated
|
|
132
|
+
copy of the response they stored in `result`; newer rows store `{}` there, since a
|
|
133
|
+
full copy of every accepted event only doubled the log's size.
|
|
134
134
|
Reusing a request ID with different content returns
|
|
135
135
|
`409 idempotency_mismatch`. Because the response is one SQLite row, an append
|
|
136
136
|
whose accepted events serialize to more than 1,900,000 bytes is rejected with
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hearthlog/ledger",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.8",
|
|
4
4
|
"description": "Hearthlog's Ledger: the event store Worker (Elysia over SQLite Durable Objects), its contracts and tests, and the logical backup tooling",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"repository": {
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"dependencies": {
|
|
31
31
|
"@sinclair/typebox": "0.34.52",
|
|
32
32
|
"elysia": "1.4.30",
|
|
33
|
-
"@hearthlog/contracts": "0.13.
|
|
33
|
+
"@hearthlog/contracts": "0.13.8"
|
|
34
34
|
},
|
|
35
35
|
"peerDependencies": {
|
|
36
36
|
"@hearthlog/generated": "*"
|
package/src/event-validation.ts
CHANGED
|
@@ -44,6 +44,8 @@ export interface EventPolicy {
|
|
|
44
44
|
dynamicEvents: boolean;
|
|
45
45
|
/** Null follows the deployment's `EVENT_VALIDATION`. */
|
|
46
46
|
validation: EventValidationMode | null;
|
|
47
|
+
/** Start a snapshot job by itself once the log grows `AUTO_SNAPSHOT_EVENTS` past the last snapshot. Off when absent. */
|
|
48
|
+
autoSnapshots?: boolean;
|
|
47
49
|
}
|
|
48
50
|
|
|
49
51
|
/** What an application created without an explicit policy gets: modules unrestricted, nothing undeclared, violations rejected. */
|
|
@@ -57,6 +59,7 @@ export function parseEventPolicy(value: unknown): EventPolicy | null {
|
|
|
57
59
|
contracts: Array.isArray(parsed.contracts) ? parsed.contracts.filter((id): id is string => typeof id === "string") : null,
|
|
58
60
|
dynamicEvents: parsed.dynamicEvents !== false,
|
|
59
61
|
validation: parsed.validation === "enforce" || parsed.validation === "report" ? parsed.validation : null,
|
|
62
|
+
...(parsed.autoSnapshots === true ? { autoSnapshots: true } : {}),
|
|
60
63
|
};
|
|
61
64
|
} catch {
|
|
62
65
|
return null;
|
package/src/http.ts
CHANGED
|
@@ -23,6 +23,7 @@ import type { RpcResult } from "./rpc";
|
|
|
23
23
|
import {
|
|
24
24
|
apiModels,
|
|
25
25
|
appendEventsBody,
|
|
26
|
+
appendBatchBody,
|
|
26
27
|
applicationParams,
|
|
27
28
|
createApplicationBody,
|
|
28
29
|
createLogBody,
|
|
@@ -516,8 +517,8 @@ const app = new Elysia({
|
|
|
516
517
|
.get(
|
|
517
518
|
"/logs/:logId/events",
|
|
518
519
|
async ({ access, params, query, request }) =>
|
|
519
|
-
|
|
520
|
-
await ledger(access).
|
|
520
|
+
textResponse(
|
|
521
|
+
await ledger(access).listEventsText(
|
|
521
522
|
access,
|
|
522
523
|
params.logId,
|
|
523
524
|
query.afterVersion,
|
|
@@ -532,6 +533,21 @@ const app = new Elysia({
|
|
|
532
533
|
response: { 200: "EventBatchResponse" },
|
|
533
534
|
},
|
|
534
535
|
)
|
|
536
|
+
.post(
|
|
537
|
+
"/logs/:logId/appends",
|
|
538
|
+
async ({ access, body, params, request }) =>
|
|
539
|
+
rpcResponse(
|
|
540
|
+
await ledger(access).appendEventBatches(
|
|
541
|
+
access,
|
|
542
|
+
params.logId,
|
|
543
|
+
body,
|
|
544
|
+
Number(request.headers.get("X-Ledger-Replay-Version") ?? 0),
|
|
545
|
+
request.headers.get("X-Ledger-Actor"),
|
|
546
|
+
),
|
|
547
|
+
requestOrigin(request),
|
|
548
|
+
),
|
|
549
|
+
{ body: appendBatchBody, params: logParams },
|
|
550
|
+
)
|
|
535
551
|
.post(
|
|
536
552
|
"/logs/:logId/events",
|
|
537
553
|
async ({ access, body, params, request }) =>
|
|
@@ -642,6 +658,15 @@ function rpcResponse<T>(result: RpcResult<T>, origin: string | null): Response {
|
|
|
642
658
|
return withCors(response, origin);
|
|
643
659
|
}
|
|
644
660
|
|
|
661
|
+
/** A successful body that is already JSON text (an event page) goes out as it is. */
|
|
662
|
+
function textResponse(result: RpcResult<string>, origin: string | null): Response {
|
|
663
|
+
if (!result.ok) return rpcResponse(result, origin);
|
|
664
|
+
return withCors(new Response(result.body, {
|
|
665
|
+
status: result.status,
|
|
666
|
+
headers: { "Content-Type": "application/json" },
|
|
667
|
+
}), origin);
|
|
668
|
+
}
|
|
669
|
+
|
|
645
670
|
function preflight(origin: string | null): Response {
|
|
646
671
|
const headers = corsHeaders(origin);
|
|
647
672
|
headers.set("Access-Control-Max-Age", "86400");
|
package/src/ledger-app.ts
CHANGED
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
LedgerError,
|
|
38
38
|
normalizeError,
|
|
39
39
|
validationError,
|
|
40
|
+
type ErrorBody,
|
|
40
41
|
} from "./errors";
|
|
41
42
|
import { type RpcResult, rpcCall, rpcSuccess } from "./rpc";
|
|
42
43
|
import { fenceError, parseFence, validateFenceRequest, type Fence, type FenceRequest } from "./fence";
|
|
@@ -55,6 +56,7 @@ import {
|
|
|
55
56
|
} from "./transfer";
|
|
56
57
|
import { sqliteExport } from "./sqlite-export";
|
|
57
58
|
import {
|
|
59
|
+
appendBatchBody,
|
|
58
60
|
applicationSummary,
|
|
59
61
|
appendEventsBody,
|
|
60
62
|
type ApplicationSummary,
|
|
@@ -112,6 +114,19 @@ interface LogRow extends Record<string, SqlStorageValue> {
|
|
|
112
114
|
hidden_at: string | null;
|
|
113
115
|
}
|
|
114
116
|
|
|
117
|
+
/** One request's outcome in a batched append: its accepted response, or the error it would have answered alone. */
|
|
118
|
+
export interface AppendBatchResult {
|
|
119
|
+
status: number;
|
|
120
|
+
body: AppendEventsResponse | ErrorBody;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** What one delivery round shares between the sockets of a log. */
|
|
124
|
+
interface DeliveryCache {
|
|
125
|
+
head?: number;
|
|
126
|
+
archived?: number;
|
|
127
|
+
pages: Map<number, Array<{ version: number; text: string }>>;
|
|
128
|
+
}
|
|
129
|
+
|
|
115
130
|
interface EventRow extends Record<string, SqlStorageValue> {
|
|
116
131
|
version: number;
|
|
117
132
|
event_id: string;
|
|
@@ -245,6 +260,34 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
245
260
|
if (!objects.truncated) this.ctx.storage.sql.exec('DELETE FROM log_archive_deletions WHERE log_id = ?', job.log_id);
|
|
246
261
|
}
|
|
247
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Start a snapshot job by itself when the application's policy allows it and the log has grown
|
|
265
|
+
* `AUTO_SNAPSHOT_EVENTS` (default 5,000) past its last verified snapshot. The job runs under the usual budgets and
|
|
266
|
+
* pauses; compaction stays manual. Best effort: an append never fails because of it. A job that failed is not
|
|
267
|
+
* retried for a day, so a log that cannot be snapshotted does not start one on every append.
|
|
268
|
+
*/
|
|
269
|
+
private async considerAutoSnapshot(logId: string): Promise<void> {
|
|
270
|
+
try {
|
|
271
|
+
if (!this.env.LEDGER_ARCHIVE || this.eventPolicy()?.autoSnapshots !== true) return;
|
|
272
|
+
const threshold = Math.max(500, Math.min(1_000_000, Number(this.env.AUTO_SNAPSHOT_EVENTS) || 5000));
|
|
273
|
+
const head = this.log(logId)?.version ?? 0;
|
|
274
|
+
const store = this.maintenance(() => this.assertApplicationWritable());
|
|
275
|
+
const state = store.state(logId);
|
|
276
|
+
if (head - (state.checkpoint?.throughVersion ?? 0) < threshold) return;
|
|
277
|
+
const job = state.job;
|
|
278
|
+
if (job && job.status !== 'complete') {
|
|
279
|
+
if (job.status !== 'failed' || Date.now() - Date.parse(job.updatedAt) < 24 * 60 * 60 * 1000) return;
|
|
280
|
+
}
|
|
281
|
+
store.start(logId, crypto.randomUUID(), 'snapshot');
|
|
282
|
+
console.log(JSON.stringify({ event: 'ledger_auto_snapshot_started', logId, head, throughVersion: state.checkpoint?.throughVersion ?? 0 }));
|
|
283
|
+
await this.scheduleAlarm();
|
|
284
|
+
} catch (cause) {
|
|
285
|
+
// An unfinished action at the head, a job on another log of the same request, or a fence: try after a later append.
|
|
286
|
+
const error = normalizeError(cause);
|
|
287
|
+
if (!(error instanceof LedgerError)) console.error('ledger_auto_snapshot_failed', error.message);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
248
291
|
/** Run due maintenance steps for a few seconds, so a job finishes without any caller. */
|
|
249
292
|
private async advanceMaintenance(): Promise<void> {
|
|
250
293
|
if (this.metadataOptional() === undefined) return;
|
|
@@ -355,7 +398,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
355
398
|
return rpcSuccess({
|
|
356
399
|
appId: claims.appId,
|
|
357
400
|
principalId: claims.principalId,
|
|
358
|
-
capabilities: ["snapshot-baseline-v1", "action-metadata-v1", "workspace-restore-v1", "event-excision-v1"],
|
|
401
|
+
capabilities: ["snapshot-baseline-v1", "action-metadata-v1", "workspace-restore-v1", "event-excision-v1", "append-batch-v1"],
|
|
359
402
|
...(fence ? { fence } : {}),
|
|
360
403
|
});
|
|
361
404
|
});
|
|
@@ -783,6 +826,69 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
783
826
|
});
|
|
784
827
|
}
|
|
785
828
|
|
|
829
|
+
/**
|
|
830
|
+
* Apply several append requests in order, each exactly as appendEvents would (its own request id, hash, receipt
|
|
831
|
+
* and transaction), stopping at the first that fails. One result per attempted request; the rest are not tried,
|
|
832
|
+
* so the caller can resend them. Sockets get one delivery for the whole batch.
|
|
833
|
+
*/
|
|
834
|
+
async appendEventBatches(
|
|
835
|
+
access: AccessClaims,
|
|
836
|
+
logId: string,
|
|
837
|
+
payload: unknown,
|
|
838
|
+
replayVersion = 0,
|
|
839
|
+
actor: string | null = null,
|
|
840
|
+
): Promise<RpcResult<{ results: AppendBatchResult[] }>> {
|
|
841
|
+
return rpcCall(async () => {
|
|
842
|
+
const claims = this.writableAccess(access);
|
|
843
|
+
const id = parseUuid(logId, "logId");
|
|
844
|
+
const { requests } = decodeRpc(appendBatchBody, payload, "append batch request");
|
|
845
|
+
const results: AppendBatchResult[] = [];
|
|
846
|
+
try {
|
|
847
|
+
for (const request of requests) {
|
|
848
|
+
const outcome = await rpcCall(async () => rpcSuccess(
|
|
849
|
+
await this.append(id, request, claims, replayVersion, normalizeActor(actor), { deliver: false })));
|
|
850
|
+
results.push({ status: outcome.status, body: outcome.body });
|
|
851
|
+
if (!outcome.ok) break;
|
|
852
|
+
}
|
|
853
|
+
} finally {
|
|
854
|
+
if (results.some((result) => result.status < 300)) {
|
|
855
|
+
try {
|
|
856
|
+
this.deliver(id);
|
|
857
|
+
} catch (cause) {
|
|
858
|
+
console.error(JSON.stringify({ message: "Ledger socket delivery failed", error: normalizeError(cause).message }));
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
return rpcSuccess({ results });
|
|
863
|
+
});
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
/**
|
|
867
|
+
* The same page as listEvents, as the response's JSON text. Stored payloads are already JSON, so they are spliced
|
|
868
|
+
* in as they are instead of being parsed here, cloned through RPC and serialized again by the Worker. A page that
|
|
869
|
+
* reaches archived rows takes the object path, which reads their payloads from the archive.
|
|
870
|
+
*/
|
|
871
|
+
async listEventsText(
|
|
872
|
+
access: AccessClaims,
|
|
873
|
+
logId: string,
|
|
874
|
+
afterVersion: number | undefined,
|
|
875
|
+
limit: number | undefined,
|
|
876
|
+
replayVersion = 0,
|
|
877
|
+
): Promise<RpcResult<string>> {
|
|
878
|
+
return rpcCall(async () => {
|
|
879
|
+
this.currentAccess(access);
|
|
880
|
+
const id = parseUuid(logId, "logId");
|
|
881
|
+
this.assertReplayCompatible(id, replayVersion);
|
|
882
|
+
this.assertReadableLog(id);
|
|
883
|
+
const rows = this.eventRowsAfter(id, Math.max(0, afterVersion ?? 0), clamp(limit ?? MAX_CATCH_UP, 1, MAX_CATCH_UP));
|
|
884
|
+
const text = rows.some((row) => row.archive_key !== null)
|
|
885
|
+
? JSON.stringify({ events: await this.hydrateEvents(id, rows.map(storedEvent)) })
|
|
886
|
+
: `{"events":[${rows.map(storedEventText).join(",")}]}`;
|
|
887
|
+
this.currentAccess(access);
|
|
888
|
+
return rpcSuccess(text);
|
|
889
|
+
});
|
|
890
|
+
}
|
|
891
|
+
|
|
786
892
|
async appendEvents(
|
|
787
893
|
access: AccessClaims,
|
|
788
894
|
logId: string,
|
|
@@ -1247,11 +1353,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1247
1353
|
this.closeLogSockets(logId, 1008, "log_deleted");
|
|
1248
1354
|
}
|
|
1249
1355
|
|
|
1250
|
-
private
|
|
1251
|
-
logId: string,
|
|
1252
|
-
afterVersion: number | undefined,
|
|
1253
|
-
limit: number | undefined,
|
|
1254
|
-
): { events: StoredEvent[] } {
|
|
1356
|
+
private assertReadableLog(logId: string): void {
|
|
1255
1357
|
const log = this.log(logId);
|
|
1256
1358
|
if (!log) {
|
|
1257
1359
|
throw new LedgerError(404, {
|
|
@@ -1262,6 +1364,14 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1262
1364
|
if (log.status !== 'ready') throw new LedgerError(409, {
|
|
1263
1365
|
error: 'Fork preparing', code: 'fork_preparing',
|
|
1264
1366
|
});
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
private getEventsValue(
|
|
1370
|
+
logId: string,
|
|
1371
|
+
afterVersion: number | undefined,
|
|
1372
|
+
limit: number | undefined,
|
|
1373
|
+
): { events: StoredEvent[] } {
|
|
1374
|
+
this.assertReadableLog(logId);
|
|
1265
1375
|
return {
|
|
1266
1376
|
events: this.eventsAfter(
|
|
1267
1377
|
logId,
|
|
@@ -1277,6 +1387,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1277
1387
|
access: AccessClaims,
|
|
1278
1388
|
replayVersion: number,
|
|
1279
1389
|
actor: string | null,
|
|
1390
|
+
options: { deliver?: boolean } = {},
|
|
1280
1391
|
): Promise<AppendEventsResponse> {
|
|
1281
1392
|
if (request.events.length === 0) {
|
|
1282
1393
|
throw validationError("events must not be empty");
|
|
@@ -1309,7 +1420,9 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1309
1420
|
});
|
|
1310
1421
|
// The append is committed. Realtime delivery is best effort: sockets
|
|
1311
1422
|
// recover through HTTP replay, and a delivery fault must never turn an
|
|
1312
|
-
// accepted append into an error response.
|
|
1423
|
+
// accepted append into an error response. A batch delivers once at its end.
|
|
1424
|
+
await this.considerAutoSnapshot(logId);
|
|
1425
|
+
if (options.deliver === false) return response;
|
|
1313
1426
|
try {
|
|
1314
1427
|
this.deliver(logId);
|
|
1315
1428
|
} catch (cause) {
|
|
@@ -1465,11 +1578,9 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1465
1578
|
stored.push(event);
|
|
1466
1579
|
}
|
|
1467
1580
|
|
|
1468
|
-
//
|
|
1469
|
-
//
|
|
1581
|
+
// One append's accepted events must fit one response (and, for Workers before migration 9, one stored
|
|
1582
|
+
// receipt row); reject what never could before writing anything.
|
|
1470
1583
|
const response = { newVersion: logVersion, events: stored };
|
|
1471
|
-
// The stored copy keeps the pre-authorship shape: a rolled-back Worker
|
|
1472
|
-
// validates it strictly, and replays read log_events anyway.
|
|
1473
1584
|
const result = JSON.stringify({
|
|
1474
1585
|
newVersion: logVersion,
|
|
1475
1586
|
events: stored.map(({ actor: _actor, ...event }) => event),
|
|
@@ -1546,9 +1657,9 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1546
1657
|
request.requestId,
|
|
1547
1658
|
logId,
|
|
1548
1659
|
requestHash,
|
|
1549
|
-
//
|
|
1550
|
-
//
|
|
1551
|
-
|
|
1660
|
+
// Replays read the version range (stored since migration 9, which every deployable Worker has); a copy
|
|
1661
|
+
// of every accepted event here only doubled the log's size.
|
|
1662
|
+
"{}",
|
|
1552
1663
|
new Date().toISOString(),
|
|
1553
1664
|
stored[0]!.version,
|
|
1554
1665
|
logVersion,
|
|
@@ -1654,6 +1765,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1654
1765
|
ws: WebSocket,
|
|
1655
1766
|
attachment: SocketAttachment,
|
|
1656
1767
|
sendReady: boolean,
|
|
1768
|
+
shared?: DeliveryCache,
|
|
1657
1769
|
): void {
|
|
1658
1770
|
try { this.assertReplayCompatible(attachment.logId, attachment.replayVersion ?? 0); }
|
|
1659
1771
|
catch {
|
|
@@ -1661,8 +1773,9 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1661
1773
|
ws.close(1008, "client upgrade required");
|
|
1662
1774
|
return;
|
|
1663
1775
|
}
|
|
1664
|
-
const head = this.logHead(attachment.logId);
|
|
1665
|
-
const archived = this.ctx.storage.sql.exec<{ archived_through: number }>('SELECT archived_through FROM log_storage WHERE log_id = ?', attachment.logId).toArray()[0]?.archived_through ?? 0;
|
|
1776
|
+
const head = shared?.head ?? this.logHead(attachment.logId);
|
|
1777
|
+
const archived = shared?.archived ?? this.ctx.storage.sql.exec<{ archived_through: number }>('SELECT archived_through FROM log_storage WHERE log_id = ?', attachment.logId).toArray()[0]?.archived_through ?? 0;
|
|
1778
|
+
if (shared) { shared.head = head; shared.archived = archived; }
|
|
1666
1779
|
if (attachment.cursor < archived) {
|
|
1667
1780
|
this.sendFrame(ws, { type: 'resync_required', headVersion: head, reason: 'gap_too_large' });
|
|
1668
1781
|
ws.close(1008, 'baseline resync required');
|
|
@@ -1686,18 +1799,19 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1686
1799
|
ws.close(1008, "resync required");
|
|
1687
1800
|
return;
|
|
1688
1801
|
}
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
MAX_CATCH_UP
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
ws.
|
|
1802
|
+
// Sockets at the same cursor share one read and one serialization of their frames.
|
|
1803
|
+
let frames = shared?.pages.get(attachment.cursor);
|
|
1804
|
+
if (!frames) {
|
|
1805
|
+
frames = this.eventsAfter(attachment.logId, attachment.cursor, MAX_CATCH_UP)
|
|
1806
|
+
.map((event) => ({ version: event.version, text: JSON.stringify({ type: "event", event } satisfies SocketFrame) }));
|
|
1807
|
+
shared?.pages.set(attachment.cursor, frames);
|
|
1808
|
+
}
|
|
1809
|
+
for (const frame of frames) {
|
|
1810
|
+
ws.send(frame.text);
|
|
1811
|
+
attachment.cursor = frame.version;
|
|
1698
1812
|
}
|
|
1699
|
-
if (sendReady) {
|
|
1700
|
-
|
|
1813
|
+
if (sendReady) this.sendFrame(ws, { type: "ready", headVersion: head });
|
|
1814
|
+
if (frames.length > 0 || sendReady) {
|
|
1701
1815
|
attachment.lastSentAt = new Date().toISOString();
|
|
1702
1816
|
ws.serializeAttachment(attachment);
|
|
1703
1817
|
}
|
|
@@ -1705,6 +1819,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1705
1819
|
|
|
1706
1820
|
private deliver(logId: string): void {
|
|
1707
1821
|
const metadata = this.metadata();
|
|
1822
|
+
const shared: DeliveryCache = { pages: new Map() };
|
|
1708
1823
|
for (const ws of this.ctx.getWebSockets(socketTag(logId))) {
|
|
1709
1824
|
let attachment: SocketAttachment;
|
|
1710
1825
|
try {
|
|
@@ -1722,7 +1837,7 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
1722
1837
|
continue;
|
|
1723
1838
|
}
|
|
1724
1839
|
try {
|
|
1725
|
-
this.sendCatchUp(ws, attachment, false);
|
|
1840
|
+
this.sendCatchUp(ws, attachment, false, shared);
|
|
1726
1841
|
} catch (cause) {
|
|
1727
1842
|
console.error(
|
|
1728
1843
|
JSON.stringify({
|
|
@@ -2028,6 +2143,22 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
2028
2143
|
return events.map(event => ({ ...event, payload: byId.get(event.id)!.payload as StoredEvent['payload'] }));
|
|
2029
2144
|
}
|
|
2030
2145
|
|
|
2146
|
+
private eventRowsAfter(logId: string, after: number, limit: number): Array<EventRow & { archive_key: string | null }> {
|
|
2147
|
+
return this.ctx.storage.sql
|
|
2148
|
+
.exec<EventRow & { archive_key: string | null }>(
|
|
2149
|
+
`SELECT version, event_id, event_type, entity_id, entity_version,
|
|
2150
|
+
event_timestamp, payload, action_id, action_index, action_size, actor, archive_key
|
|
2151
|
+
FROM log_events
|
|
2152
|
+
WHERE log_id = ? AND version > ?
|
|
2153
|
+
ORDER BY version
|
|
2154
|
+
LIMIT ?`,
|
|
2155
|
+
logId,
|
|
2156
|
+
Math.max(0, after),
|
|
2157
|
+
clamp(limit, 1, MAX_CATCH_UP),
|
|
2158
|
+
)
|
|
2159
|
+
.toArray();
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2031
2162
|
private eventsAfter(
|
|
2032
2163
|
logId: string,
|
|
2033
2164
|
after: number,
|
|
@@ -2136,6 +2267,14 @@ export class LedgerApp extends DurableObject<Env> {
|
|
|
2136
2267
|
}
|
|
2137
2268
|
}
|
|
2138
2269
|
|
|
2270
|
+
/** A stored event as JSON text, its payload spliced in from storage unparsed (it was written by JSON.stringify). */
|
|
2271
|
+
function storedEventText(row: EventRow): string {
|
|
2272
|
+
if (!row.payload.startsWith("{")) throw new Error("invalid stored event payload");
|
|
2273
|
+
const { payload: _payload, ...envelope } = storedEvent({ ...row, payload: "{}" });
|
|
2274
|
+
const head = JSON.stringify(envelope);
|
|
2275
|
+
return `${head.slice(0, -1)},"payload":${row.payload}}`;
|
|
2276
|
+
}
|
|
2277
|
+
|
|
2139
2278
|
function storedEvent(row: EventRow): StoredEvent {
|
|
2140
2279
|
const payload: unknown = JSON.parse(row.payload);
|
|
2141
2280
|
if (
|
package/src/schemas.ts
CHANGED
|
@@ -43,6 +43,7 @@ export const eventPolicyBody = t.Object(
|
|
|
43
43
|
contracts: t.Union([t.Array(t.String({ pattern: "^(core|[a-z0-9]+(\\.[a-z0-9-]+)+)$" }), { maxItems: 64 }), t.Null()]),
|
|
44
44
|
dynamicEvents: t.Boolean(),
|
|
45
45
|
validation: t.Union([t.Literal("report"), t.Literal("enforce"), t.Null()]),
|
|
46
|
+
autoSnapshots: t.Optional(t.Boolean()),
|
|
46
47
|
},
|
|
47
48
|
strictObject,
|
|
48
49
|
);
|
|
@@ -274,6 +275,15 @@ export const appendEventsBody = t.Object(
|
|
|
274
275
|
strictObject,
|
|
275
276
|
);
|
|
276
277
|
|
|
278
|
+
/** Several append requests in one round trip, applied in order; each keeps its own request id and receipt. */
|
|
279
|
+
export const MAX_APPEND_BATCH_REQUESTS = 64;
|
|
280
|
+
export const appendBatchBody = t.Object(
|
|
281
|
+
{
|
|
282
|
+
requests: t.Array(appendEventsBody, { minItems: 1, maxItems: MAX_APPEND_BATCH_REQUESTS }),
|
|
283
|
+
},
|
|
284
|
+
strictObject,
|
|
285
|
+
);
|
|
286
|
+
|
|
277
287
|
export const storedEvent = t.Object(
|
|
278
288
|
{
|
|
279
289
|
action: t.Optional(t.Object({
|