durability 3.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/log.js ADDED
@@ -0,0 +1,705 @@
1
+ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
2
+ if (kind === "m") throw new TypeError("Private method is not writable");
3
+ if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
4
+ if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
5
+ return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
6
+ };
7
+ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
8
+ if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
9
+ if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
10
+ return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
11
+ };
12
+ var _DurabilityLog_instances, _DurabilityLog_engine, _DurabilityLog_routing, _DurabilityLog_retention, _DurabilityLog_cold, _DurabilityLog_leases, _DurabilityLog_maxParallelism, _DurabilityLog_leaseMs, _DurabilityLog_maxBatchSize, _DurabilityLog_encoder, _DurabilityLog_rehydrate, _DurabilityLog_record, _DurabilityLog_settle, _DurabilityLog_advance, _DurabilityLog_between, _DurabilityLog_readFrom, _DurabilityLog_requireLeases;
13
+ import { z } from 'zod';
14
+ import { LogAppendError, LogLeaseError, LogTruncatedError, reportFailure, serializeError, } from './errors.js';
15
+ import { RoutingLoad } from './load.js';
16
+ import { durabilityLogLeaseMigrations, durabilityLogMigrations, migrate, } from './migrations.js';
17
+ import { assertPositiveInteger } from './policy.js';
18
+ import { engineFor, } from './scheduler.js';
19
+ import { deserialize, serialize } from './serialization.js';
20
+ const logRecordRowSchema = z.object({
21
+ offset: z.number().int().nonnegative(),
22
+ dedup_key: z.string(),
23
+ payload: z.string(),
24
+ bytes: z.number().int().nonnegative(),
25
+ appended_at: z.number().int().nonnegative(),
26
+ });
27
+ const logCursorRowSchema = z.object({
28
+ consumer: z.string(),
29
+ offset: z.number().int().nonnegative(),
30
+ committed_at: z.number().int().nonnegative(),
31
+ });
32
+ /** One contiguous run of records moved out of the object into cold storage. */
33
+ const logSegmentRowSchema = z.object({
34
+ first_offset: z.number().int().nonnegative(),
35
+ last_offset: z.number().int().nonnegative(),
36
+ locator: z.string(),
37
+ bytes: z.number().int().nonnegative(),
38
+ flushed_at: z.number().int().nonnegative(),
39
+ });
40
+ /**
41
+ * One in-flight or settled range of offsets held by a consumer. A range is
42
+ * redelivered under a fresh `batch_id`, so an ack from a previous holder is
43
+ * rejected rather than advancing a range someone else now owns.
44
+ */
45
+ const logLeaseRowSchema = z.object({
46
+ batch_id: z.string(),
47
+ consumer: z.string(),
48
+ first_offset: z.number().int().nonnegative(),
49
+ last_offset: z.number().int().nonnegative(),
50
+ state: z.enum(['active', 'pending', 'acked', 'skipped']),
51
+ attempt: z.number().int().positive(),
52
+ expires_at: z.number().int().nonnegative(),
53
+ created_at: z.number().int().nonnegative(),
54
+ });
55
+ const allocationRowSchema = z.object({
56
+ allocated_offset: z.number().int().nonnegative(),
57
+ });
58
+ const offsetRowSchema = z.object({ offset: z.number().int().nonnegative() });
59
+ const seqRowSchema = z.object({ seq: z.number().int().nonnegative() });
60
+ const countRowSchema = z.object({ count: z.number().int().nonnegative() });
61
+ const totalBytesRowSchema = z.object({
62
+ total_bytes: z.number().int().nonnegative(),
63
+ });
64
+ const boundsRowSchema = z.object({
65
+ hot: z.number().int().nonnegative().nullable(),
66
+ cold: z.number().int().nonnegative().nullable(),
67
+ next_offset: z.number().int().nonnegative(),
68
+ });
69
+ /**
70
+ * Inserts the batch and returns each record's offset in input order. A record
71
+ * whose deduplication key is already present keeps its original offset and
72
+ * consumes no new one, so a repeated append is idempotent while that record is
73
+ * still held in the object. Runs inside the caller's transaction.
74
+ */
75
+ const appendRecords = (sql, records, appendedAt) => {
76
+ const assigned = new Map();
77
+ for (const record of records) {
78
+ const existing = offsetRowSchema.optional().parse(sql
79
+ .exec(`SELECT "offset" FROM durability_log_records
80
+ WHERE dedup_key = ? LIMIT 1`, record.dedup_key)
81
+ .toArray()[0]);
82
+ if (existing !== undefined) {
83
+ assigned.set(record.dedup_key, existing.offset);
84
+ }
85
+ }
86
+ const fresh = records.filter((record) => !assigned.has(record.dedup_key));
87
+ const distinct = [...new Map(fresh.map((row) => [row.dedup_key, row]))];
88
+ if (distinct.length === 0) {
89
+ return records.map((record) => assigned.get(record.dedup_key));
90
+ }
91
+ const first = seqRowSchema.parse(sql
92
+ .exec(`UPDATE durability_log_seq
93
+ SET next_offset = next_offset + ?
94
+ WHERE id = 1
95
+ RETURNING next_offset - ? AS seq`, distinct.length, distinct.length)
96
+ .toArray()[0]).seq;
97
+ let added = 0;
98
+ distinct.forEach(([dedupKey, row], index) => {
99
+ const offset = first + index;
100
+ assigned.set(dedupKey, offset);
101
+ added += row.bytes;
102
+ sql.exec(`INSERT INTO durability_log_records
103
+ ("offset", dedup_key, payload, bytes, appended_at)
104
+ VALUES (?, ?, ?, ?, ?)
105
+ ON CONFLICT(dedup_key) DO NOTHING`, offset, dedupKey, row.payload, row.bytes, appendedAt);
106
+ });
107
+ sql.exec(`UPDATE durability_log_seq SET total_bytes = total_bytes + ?
108
+ WHERE id = 1`, added);
109
+ return records.map((record) => assigned.get(record.dedup_key));
110
+ };
111
+ const readRecords = (sql, from, limit) => logRecordRowSchema.array().parse(sql
112
+ .exec(`SELECT * FROM durability_log_records
113
+ WHERE "offset" >= ?
114
+ ORDER BY "offset" ASC
115
+ LIMIT ?`, from, limit)
116
+ .toArray());
117
+ const readBounds = (sql) => {
118
+ const row = boundsRowSchema.parse(sql
119
+ .exec(`SELECT
120
+ (SELECT MIN("offset") FROM durability_log_records) AS hot,
121
+ (SELECT MIN(first_offset) FROM durability_log_segments) AS cold,
122
+ next_offset
123
+ FROM durability_log_seq WHERE id = 1`)
124
+ .toArray()[0]);
125
+ const hotOffset = row.hot ?? row.next_offset;
126
+ return {
127
+ oldestOffset: Math.min(row.cold ?? hotOffset, hotOffset),
128
+ hotOffset,
129
+ nextOffset: row.next_offset,
130
+ };
131
+ };
132
+ const countRecords = (sql) => countRowSchema.parse(sql
133
+ .exec('SELECT COUNT(*) AS count FROM durability_log_records')
134
+ .toArray()[0]).count;
135
+ /** Bytes of record payloads held in the object, excluding flushed segments. */
136
+ const totalBytes = (sql) => totalBytesRowSchema.parse(sql
137
+ .exec('SELECT total_bytes FROM durability_log_seq WHERE id = 1')
138
+ .toArray()[0]).total_bytes;
139
+ /** Oldest retained records in offset order, for flush-or-delete decisions. */
140
+ const listOldest = (sql, limit) => logRecordRowSchema.array().parse(sql
141
+ .exec(`SELECT * FROM durability_log_records
142
+ ORDER BY "offset" ASC
143
+ LIMIT ?`, limit)
144
+ .toArray());
145
+ /**
146
+ * Deletes at most `limit` records below `through`, oldest first, and returns
147
+ * how many were removed. Runs inside the caller's transaction.
148
+ */
149
+ const trimThrough = (sql, through, limit) => {
150
+ const removed = logRecordRowSchema.array().parse(sql
151
+ .exec(`DELETE FROM durability_log_records
152
+ WHERE "offset" IN (
153
+ SELECT "offset" FROM durability_log_records
154
+ WHERE "offset" < ?
155
+ ORDER BY "offset" ASC
156
+ LIMIT ?
157
+ )
158
+ RETURNING *`, through, limit)
159
+ .toArray());
160
+ if (removed.length === 0) {
161
+ return 0;
162
+ }
163
+ sql.exec(`UPDATE durability_log_seq SET total_bytes = MAX(0, total_bytes - ?)
164
+ WHERE id = 1`, removed.reduce((total, row) => total + row.bytes, 0));
165
+ return removed.length;
166
+ };
167
+ /** Records where a flushed run now lives, so a read can rehydrate it. */
168
+ const insertSegment = (sql, row) => {
169
+ sql.exec(`INSERT INTO durability_log_segments
170
+ (first_offset, last_offset, locator, bytes, flushed_at)
171
+ VALUES (?, ?, ?, ?, ?)
172
+ ON CONFLICT(first_offset) DO UPDATE SET
173
+ last_offset = excluded.last_offset,
174
+ locator = excluded.locator,
175
+ bytes = excluded.bytes,
176
+ flushed_at = excluded.flushed_at`, row.first_offset, row.last_offset, row.locator, row.bytes, row.flushed_at);
177
+ };
178
+ /** The segment holding an offset, if that offset was flushed. */
179
+ const findSegment = (sql, offset) => logSegmentRowSchema.optional().parse(sql
180
+ .exec(`SELECT * FROM durability_log_segments
181
+ WHERE first_offset <= ? AND last_offset >= ?
182
+ LIMIT 1`, offset, offset)
183
+ .toArray()[0]);
184
+ const readCursors = (sql) => logCursorRowSchema
185
+ .array()
186
+ .parse(sql
187
+ .exec('SELECT * FROM durability_log_cursors ORDER BY consumer')
188
+ .toArray());
189
+ const readCursor = (sql, consumer) => offsetRowSchema.optional().parse(sql
190
+ .exec(`SELECT "offset" FROM durability_log_cursors
191
+ WHERE consumer = ? LIMIT 1`, consumer)
192
+ .toArray()[0])?.offset;
193
+ /** Advances a cursor, ignoring regressions so a late commit cannot rewind it. */
194
+ const commitCursor = (sql, consumer, offset, committedAt) => {
195
+ sql.exec(`INSERT INTO durability_log_cursors (consumer, "offset", committed_at)
196
+ VALUES (?, ?, ?)
197
+ ON CONFLICT(consumer) DO UPDATE
198
+ SET "offset" = excluded."offset", committed_at = excluded.committed_at
199
+ WHERE excluded."offset" > durability_log_cursors."offset"`, consumer, offset, committedAt);
200
+ };
201
+ /** Highest offset handed to a lease, which runs ahead of the commit. */
202
+ const readAllocation = (sql, consumer) => allocationRowSchema.optional().parse(sql
203
+ .exec(`SELECT allocated_offset FROM durability_log_allocations
204
+ WHERE consumer = ? LIMIT 1`, consumer)
205
+ .toArray()[0])?.allocated_offset;
206
+ const setAllocation = (sql, consumer, offset) => {
207
+ sql.exec(`INSERT INTO durability_log_allocations (consumer, allocated_offset)
208
+ VALUES (?, ?)
209
+ ON CONFLICT(consumer) DO UPDATE SET allocated_offset = excluded.allocated_offset`, consumer, offset);
210
+ };
211
+ const insertLease = (sql, row) => {
212
+ sql.exec(`INSERT INTO durability_log_leases
213
+ (batch_id, consumer, first_offset, last_offset, state, attempt,
214
+ expires_at, created_at)
215
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, row.batch_id, row.consumer, row.first_offset, row.last_offset, row.state, row.attempt, row.expires_at, row.created_at);
216
+ };
217
+ /**
218
+ * Oldest range owed redelivery: explicitly nacked, or held past its expiry.
219
+ * Claiming rewrites its `batch_id`, fencing the previous holder.
220
+ */
221
+ const claimExpired = (sql, consumer, now, batchId, expiresAt) => logLeaseRowSchema.optional().parse(sql
222
+ .exec(`UPDATE durability_log_leases
223
+ SET batch_id = ?, state = 'active', attempt = attempt + 1,
224
+ expires_at = ?
225
+ WHERE batch_id = (
226
+ SELECT batch_id FROM durability_log_leases
227
+ WHERE consumer = ?
228
+ AND (state = 'pending'
229
+ OR (state = 'active' AND expires_at <= ?))
230
+ ORDER BY first_offset ASC
231
+ LIMIT 1
232
+ )
233
+ RETURNING *`, batchId, expiresAt, consumer, now)
234
+ .toArray()[0]);
235
+ /** Ranges still held, so parallelism can be bounded per consumer. */
236
+ const countHeld = (sql, consumer, now) => countRowSchema.parse(sql
237
+ .exec(`SELECT COUNT(*) AS count FROM durability_log_leases
238
+ WHERE consumer = ?
239
+ AND (state = 'pending' OR (state = 'active' AND expires_at > ?))`, consumer, now)
240
+ .toArray()[0]).count;
241
+ /** Settles one range, failing when the batch was fenced or already settled. */
242
+ const settleLease = (sql, batchId, state, expiresAt) => sql
243
+ .exec(`UPDATE durability_log_leases
244
+ SET state = ?, expires_at = ?
245
+ WHERE batch_id = ? AND state = 'active'
246
+ RETURNING batch_id`, state, expiresAt, batchId)
247
+ .toArray().length > 0;
248
+ /**
249
+ * Ranges from `from` in offset order, so a commit can advance over the
250
+ * unbroken run of settled ones and no further.
251
+ */
252
+ const leasesFrom = (sql, consumer, from, limit) => logLeaseRowSchema.array().parse(sql
253
+ .exec(`SELECT * FROM durability_log_leases
254
+ WHERE consumer = ? AND first_offset >= ?
255
+ ORDER BY first_offset ASC
256
+ LIMIT ?`, consumer, from, limit)
257
+ .toArray());
258
+ const deleteLeases = (sql, batchIds) => {
259
+ if (batchIds.length === 0) {
260
+ return;
261
+ }
262
+ sql.exec(`DELETE FROM durability_log_leases
263
+ WHERE batch_id IN (${batchIds.map(() => '?').join(', ')})`, ...batchIds);
264
+ };
265
+ /**
266
+ * A single object holds a hot window, not an unbounded log, so the byte
267
+ * budget is capped well below the platform's per-object limit.
268
+ */
269
+ export const maxLogBytes = 1073741824;
270
+ /**
271
+ * A composable DO capability: an append-only record log addressed by offset,
272
+ * read by pulling consumers that own their cursors. Share a scheduler with
273
+ * operations, alarms, and fanout on the same object.
274
+ *
275
+ * Unlike fanout, records are retained rather than deleted on acknowledgement,
276
+ * so a consumer can rewind or a new one can start from the beginning. Records
277
+ * leave only through `trim`, and a consumer that falls behind retention reads a
278
+ * `LogTruncatedError` rather than silently skipping records.
279
+ */
280
+ export class DurabilityLog {
281
+ constructor(config) {
282
+ _DurabilityLog_instances.add(this);
283
+ _DurabilityLog_engine.set(this, void 0);
284
+ _DurabilityLog_routing.set(this, void 0);
285
+ _DurabilityLog_retention.set(this, void 0);
286
+ _DurabilityLog_cold.set(this, void 0);
287
+ _DurabilityLog_leases.set(this, void 0);
288
+ _DurabilityLog_maxParallelism.set(this, 1);
289
+ _DurabilityLog_leaseMs.set(this, 30000);
290
+ _DurabilityLog_maxBatchSize.set(this, void 0);
291
+ _DurabilityLog_encoder.set(this, new TextEncoder());
292
+ __classPrivateFieldSet(this, _DurabilityLog_maxBatchSize, config.maxBatchSize ?? 100, "f");
293
+ assertPositiveInteger('maxBatchSize', __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
294
+ if (config.retention?.maxAgeMs !== undefined) {
295
+ assertPositiveInteger('retention.maxAgeMs', config.retention.maxAgeMs);
296
+ }
297
+ if (config.retention?.maxRecords !== undefined) {
298
+ assertPositiveInteger('retention.maxRecords', config.retention.maxRecords);
299
+ }
300
+ if (config.retention?.maxBytes !== undefined) {
301
+ assertPositiveInteger('retention.maxBytes', config.retention.maxBytes);
302
+ if (config.retention.maxBytes > maxLogBytes) {
303
+ throw new RangeError(`retention.maxBytes must not exceed ${maxLogBytes} bytes`);
304
+ }
305
+ }
306
+ __classPrivateFieldSet(this, _DurabilityLog_engine, engineFor(config), "f");
307
+ __classPrivateFieldSet(this, _DurabilityLog_routing, config.routing ?? new RoutingLoad(), "f");
308
+ __classPrivateFieldSet(this, _DurabilityLog_retention, config.retention, "f");
309
+ __classPrivateFieldSet(this, _DurabilityLog_cold, config.cold, "f");
310
+ __classPrivateFieldSet(this, _DurabilityLog_leases, config.leases, "f");
311
+ if (__classPrivateFieldGet(this, _DurabilityLog_leases, "f")) {
312
+ __classPrivateFieldSet(this, _DurabilityLog_maxParallelism, __classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxParallelism ?? 1, "f");
313
+ assertPositiveInteger('leases.maxParallelism', __classPrivateFieldGet(this, _DurabilityLog_maxParallelism, "f"));
314
+ __classPrivateFieldSet(this, _DurabilityLog_leaseMs, __classPrivateFieldGet(this, _DurabilityLog_leases, "f").leaseMs ?? 30000, "f");
315
+ assertPositiveInteger('leases.leaseMs', __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
316
+ if (__classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxAttempts !== undefined) {
317
+ assertPositiveInteger('leases.maxAttempts', __classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxAttempts);
318
+ }
319
+ __classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('logLeases', durabilityLogLeaseMigrations);
320
+ }
321
+ __classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('log', durabilityLogMigrations);
322
+ }
323
+ static migrate(context, target) {
324
+ return migrate(context.storage, 'log', durabilityLogMigrations, target);
325
+ }
326
+ /**
327
+ * Appends the batch atomically. Definite input rejections return
328
+ * success:false; storage and commit uncertainty throws, so a caller never
329
+ * reads a transport failure as a definite rejection.
330
+ */
331
+ async append(records) {
332
+ const finish = __classPrivateFieldGet(this, _DurabilityLog_routing, "f").begin('inbound');
333
+ try {
334
+ const inputs = Array.isArray(records) ? records : [records];
335
+ let rows;
336
+ try {
337
+ if (inputs.length === 0) {
338
+ throw new LogAppendError('Append requires at least one record');
339
+ }
340
+ if (inputs.length > __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f")) {
341
+ throw new LogAppendError(`Append of ${inputs.length} records exceeds maxBatchSize ${__classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f")}`);
342
+ }
343
+ rows = inputs.map((input) => {
344
+ if (input.deduplicationKey === '') {
345
+ throw new LogAppendError('Append deduplication keys must be non-empty');
346
+ }
347
+ const payload = serialize(input.body);
348
+ return {
349
+ dedup_key: input.deduplicationKey ?? crypto.randomUUID(),
350
+ payload,
351
+ bytes: __classPrivateFieldGet(this, _DurabilityLog_encoder, "f").encode(payload).byteLength,
352
+ };
353
+ });
354
+ }
355
+ catch (error) {
356
+ if (!(error instanceof LogAppendError)) {
357
+ throw error;
358
+ }
359
+ return {
360
+ success: false,
361
+ error: serializeError(error),
362
+ load: await this.load(),
363
+ };
364
+ }
365
+ const appendedAt = Date.now();
366
+ const offsets = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => appendRecords(sql, rows, appendedAt));
367
+ return { success: true, value: offsets, load: await this.load() };
368
+ }
369
+ finally {
370
+ finish();
371
+ }
372
+ }
373
+ /**
374
+ * Reads forward from an offset. Offsets already flushed are rehydrated from
375
+ * cold storage, so a lagging consumer keeps reading instead of failing; only
376
+ * offsets below everything readable raise `LogTruncatedError`.
377
+ */
378
+ async read(options) {
379
+ const limit = options.limit ?? __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f");
380
+ assertPositiveInteger('limit', limit);
381
+ if (!Number.isInteger(options.from) || options.from < 0) {
382
+ throw new LogAppendError('Read offset must be a non-negative integer');
383
+ }
384
+ const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
385
+ if (options.from < bounds.oldestOffset) {
386
+ throw new LogTruncatedError(options.from, bounds.oldestOffset);
387
+ }
388
+ const capped = Math.min(limit, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
389
+ const records = options.from < bounds.hotOffset
390
+ ? await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, options.from, capped, bounds)
391
+ : readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, options.from, capped).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
392
+ const nextOffset = records.length === 0
393
+ ? Math.max(options.from, bounds.nextOffset)
394
+ : records[records.length - 1].offset + 1;
395
+ return {
396
+ records,
397
+ nextOffset,
398
+ lag: Math.max(0, bounds.nextOffset - nextOffset),
399
+ };
400
+ }
401
+ /** The retained window. `oldestOffset` equals `nextOffset` when empty. */
402
+ async bounds() {
403
+ return readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
404
+ }
405
+ /** Undefined until the consumer commits, so callers choose their own start. */
406
+ async cursor(consumer) {
407
+ return readCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, consumer);
408
+ }
409
+ /** Records progress. Regressions are ignored, so a late commit cannot rewind. */
410
+ async commit(consumer, offset) {
411
+ if (!Number.isInteger(offset) || offset < 0) {
412
+ throw new LogAppendError('Commit offset must be a non-negative integer');
413
+ }
414
+ commitCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, consumer, offset, Date.now());
415
+ }
416
+ async cursors() {
417
+ return Object.fromEntries(readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql).map((row) => [row.consumer, row.offset]));
418
+ }
419
+ /**
420
+ * Enforces retention on the oldest records: with cold storage they are
421
+ * flushed and stay readable through rehydration, without it they are
422
+ * deleted. A cold write failure propagates with the records still in the
423
+ * object, so configured storage cannot be skipped.
424
+ *
425
+ * Retention ignores cursors, as a log should: a consumer slower than
426
+ * everything readable learns through `LogTruncatedError`.
427
+ */
428
+ async trim() {
429
+ if (!__classPrivateFieldGet(this, _DurabilityLog_retention, "f")) {
430
+ return 0;
431
+ }
432
+ const { maxAgeMs, maxRecords, maxBytes } = __classPrivateFieldGet(this, _DurabilityLog_retention, "f");
433
+ const cutoff = maxAgeMs === undefined ? undefined : Date.now() - maxAgeMs;
434
+ let removed = 0;
435
+ for (;;) {
436
+ const oldest = listOldest(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
437
+ if (oldest.length === 0) {
438
+ return removed;
439
+ }
440
+ const excess = maxRecords === undefined
441
+ ? 0
442
+ : Math.max(0, countRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql) - maxRecords);
443
+ const heldBytes = maxBytes === undefined ? 0 : totalBytes(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
444
+ let overBytes = Math.max(0, heldBytes - (maxBytes ?? 0));
445
+ const expiring = takeWhile(oldest, (row, index) => {
446
+ const reclaiming = overBytes > 0;
447
+ overBytes -= row.bytes;
448
+ return (index < excess ||
449
+ reclaiming ||
450
+ (cutoff !== undefined && row.appended_at < cutoff));
451
+ });
452
+ if (expiring.length === 0) {
453
+ return removed;
454
+ }
455
+ const first = expiring[0];
456
+ const last = expiring[expiring.length - 1];
457
+ if (__classPrivateFieldGet(this, _DurabilityLog_cold, "f")) {
458
+ const records = expiring.map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
459
+ // eslint-disable-next-line no-await-in-loop
460
+ const locator = await __classPrivateFieldGet(this, _DurabilityLog_cold, "f").write({
461
+ firstOffset: first.offset,
462
+ lastOffset: last.offset,
463
+ records,
464
+ });
465
+ if (typeof locator !== 'string' || locator.length === 0) {
466
+ throw new TypeError('Cold storage must return a non-empty locator');
467
+ }
468
+ // Indexed before deletion: a crash in between leaves the segment
469
+ // written and readable, never records that point nowhere.
470
+ insertSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, {
471
+ first_offset: first.offset,
472
+ last_offset: last.offset,
473
+ locator,
474
+ bytes: expiring.reduce((total, row) => total + row.bytes, 0),
475
+ flushed_at: Date.now(),
476
+ });
477
+ }
478
+ const deleted = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => trimThrough(sql, last.offset + 1, expiring.length));
479
+ removed += deleted;
480
+ // Records are flushed before deletion, so stopping on partial progress
481
+ // keeps the next pass from writing the remainder a second time.
482
+ if (deleted < expiring.length) {
483
+ return removed;
484
+ }
485
+ }
486
+ }
487
+ /**
488
+ * Forgets flushed segments below an offset, matching whatever lifecycle rule
489
+ * expires them externally. Reads below the new floor then truncate.
490
+ */
491
+ async forgetColdBefore(offset) {
492
+ if (!Number.isInteger(offset) || offset < 0) {
493
+ throw new LogAppendError('Offset must be a non-negative integer');
494
+ }
495
+ return __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql
496
+ .exec(`DELETE FROM durability_log_segments
497
+ WHERE last_offset < ?
498
+ RETURNING first_offset`, offset)
499
+ .toArray().length;
500
+ }
501
+ /**
502
+ * Claims the next unleased range for a consumer, or redelivers one whose
503
+ * holder nacked or let its lease expire. Returns undefined when nothing is
504
+ * available, either because the consumer is at `maxParallelism` or because
505
+ * the log has no unleased records.
506
+ *
507
+ * Two workers sharing a consumer name never receive the same offsets: the
508
+ * range is recorded durably before the records are returned, and a
509
+ * redelivery rewrites the batch id so the previous holder can no longer
510
+ * settle it.
511
+ */
512
+ async lease(consumer, limit) {
513
+ const leases = __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
514
+ const size = Math.min(limit ?? __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"), __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
515
+ assertPositiveInteger('limit', size);
516
+ assertConsumer(consumer);
517
+ const now = Date.now();
518
+ const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
519
+ const reclaimed = claimExpired(sql, consumer, now, crypto.randomUUID(), now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
520
+ if (reclaimed) {
521
+ const poisoned = leases.maxAttempts !== undefined &&
522
+ leases.onPoison !== undefined &&
523
+ reclaimed.attempt > leases.maxAttempts;
524
+ const lease = {
525
+ batchId: reclaimed.batch_id,
526
+ consumer,
527
+ records: await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_between).call(this, reclaimed.first_offset, reclaimed.last_offset),
528
+ firstOffset: reclaimed.first_offset,
529
+ lastOffset: reclaimed.last_offset,
530
+ attempt: reclaimed.attempt,
531
+ expiresAt: reclaimed.expires_at,
532
+ };
533
+ if (!poisoned) {
534
+ return lease;
535
+ }
536
+ // Terminal intent is durable before the callback runs, so a failing
537
+ // handler cannot resurrect a range that already exhausted its attempts.
538
+ __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_settle).call(this, reclaimed.batch_id, 'skipped');
539
+ try {
540
+ await leases.onPoison(lease);
541
+ }
542
+ catch (error) {
543
+ reportFailure('durability.log.poison_handler.failed', { consumer }, error);
544
+ }
545
+ return undefined;
546
+ }
547
+ if (countHeld(sql, consumer, now) >= __classPrivateFieldGet(this, _DurabilityLog_maxParallelism, "f")) {
548
+ return undefined;
549
+ }
550
+ const bounds = readBounds(sql);
551
+ const committed = readCursor(sql, consumer);
552
+ const allocated = readAllocation(sql, consumer);
553
+ const from = Math.max(bounds.oldestOffset, committed ?? 0, allocated ?? 0);
554
+ const records = await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_readFrom).call(this, from, size, bounds);
555
+ if (records.length === 0) {
556
+ return undefined;
557
+ }
558
+ const first = records[0].offset;
559
+ const last = records[records.length - 1].offset;
560
+ const batchId = crypto.randomUUID();
561
+ __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((transaction) => {
562
+ insertLease(transaction, {
563
+ batch_id: batchId,
564
+ consumer,
565
+ first_offset: first,
566
+ last_offset: last,
567
+ state: 'active',
568
+ attempt: 1,
569
+ expires_at: now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"),
570
+ created_at: now,
571
+ });
572
+ setAllocation(transaction, consumer, last + 1);
573
+ });
574
+ return {
575
+ batchId,
576
+ consumer,
577
+ records,
578
+ firstOffset: first,
579
+ lastOffset: last,
580
+ attempt: 1,
581
+ expiresAt: now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"),
582
+ };
583
+ }
584
+ /**
585
+ * Accepts a leased range and advances the cursor over every settled range
586
+ * that forms an unbroken run from it, so parallel consumers commit without
587
+ * skipping offsets another worker is still holding.
588
+ */
589
+ async ack(consumer, batchId) {
590
+ __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
591
+ __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync(() => {
592
+ __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_settle).call(this, batchId, 'acked');
593
+ __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_advance).call(this, consumer);
594
+ });
595
+ }
596
+ /** Returns a leased range for redelivery under a fresh batch id. */
597
+ async nack(consumer, batchId) {
598
+ __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
599
+ __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_settle).call(this, batchId, 'pending');
600
+ }
601
+ /** In-flight and settled-but-uncommitted ranges, for operators and tests. */
602
+ async leases(consumer) {
603
+ __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
604
+ return logLeaseRowSchema.array().parse(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql
605
+ .exec(`SELECT * FROM durability_log_leases
606
+ WHERE consumer = ?
607
+ ORDER BY first_offset ASC`, consumer)
608
+ .toArray());
609
+ }
610
+ /**
611
+ * Backlog is the furthest-behind committed cursor, so routing can weigh
612
+ * partitions by consumer lag. Without cursors there is no backlog to report.
613
+ */
614
+ async load() {
615
+ const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
616
+ const slowest = readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql).reduce((lowest, row) => lowest === undefined ? row.offset : Math.min(lowest, row.offset), undefined);
617
+ return __classPrivateFieldGet(this, _DurabilityLog_routing, "f").snapshot(slowest === undefined ? 0 : Math.max(0, bounds.nextOffset - slowest));
618
+ }
619
+ }
620
+ _DurabilityLog_engine = new WeakMap(), _DurabilityLog_routing = new WeakMap(), _DurabilityLog_retention = new WeakMap(), _DurabilityLog_cold = new WeakMap(), _DurabilityLog_leases = new WeakMap(), _DurabilityLog_maxParallelism = new WeakMap(), _DurabilityLog_leaseMs = new WeakMap(), _DurabilityLog_maxBatchSize = new WeakMap(), _DurabilityLog_encoder = new WeakMap(), _DurabilityLog_instances = new WeakSet(), _DurabilityLog_rehydrate =
621
+ /**
622
+ * Cold reads cost a round trip to external storage and open input gates, so
623
+ * they serve only consumers that fell behind the hot window.
624
+ */
625
+ async function _DurabilityLog_rehydrate(from, limit, bounds) {
626
+ const segment = findSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, from);
627
+ const records = segment
628
+ ? await __classPrivateFieldGet(this, _DurabilityLog_cold, "f")?.read(segment.locator)
629
+ : undefined;
630
+ if (!records) {
631
+ // The index outlived the data, so the offset is unreadable after all.
632
+ throw new LogTruncatedError(from, Math.max(from + 1, bounds.hotOffset));
633
+ }
634
+ return records
635
+ .filter((record) => record.offset >= from)
636
+ .sort((left, right) => left.offset - right.offset)
637
+ .slice(0, limit);
638
+ }, _DurabilityLog_record = function _DurabilityLog_record(row) {
639
+ return {
640
+ offset: row.offset,
641
+ deduplicationKey: row.dedup_key,
642
+ body: deserialize(row.payload),
643
+ appendedAt: row.appended_at,
644
+ };
645
+ }, _DurabilityLog_settle = function _DurabilityLog_settle(batchId, state) {
646
+ // Pending ranges are immediately claimable, so their expiry is in the past.
647
+ const settled = settleLease(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, batchId, state, state === 'pending' ? 0 : Date.now());
648
+ if (settled) {
649
+ return;
650
+ }
651
+ // Either the batch never existed, was fenced by a redelivery, or was
652
+ // already settled; all three are the caller holding a stale handle.
653
+ throw new LogLeaseError(batchId);
654
+ }, _DurabilityLog_advance = function _DurabilityLog_advance(consumer) {
655
+ const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
656
+ const from = readCursor(sql, consumer) ?? 0;
657
+ const candidates = leasesFrom(sql, consumer, from, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
658
+ let cursor = from;
659
+ const settled = [];
660
+ for (const row of candidates) {
661
+ const contiguous = row.first_offset <= cursor;
662
+ const done = row.state === 'acked' || row.state === 'skipped';
663
+ if (!contiguous || !done) {
664
+ break;
665
+ }
666
+ cursor = Math.max(cursor, row.last_offset + 1);
667
+ settled.push(row.batch_id);
668
+ }
669
+ if (settled.length === 0) {
670
+ return;
671
+ }
672
+ commitCursor(sql, consumer, cursor, Date.now());
673
+ deleteLeases(sql, settled);
674
+ }, _DurabilityLog_between =
675
+ /** Reads a stored range, rehydrating it when retention already flushed it. */
676
+ async function _DurabilityLog_between(first, last) {
677
+ const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
678
+ const records = await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_readFrom).call(this, first, last - first + 1, bounds);
679
+ return records.filter((record) => record.offset <= last);
680
+ }, _DurabilityLog_readFrom = async function _DurabilityLog_readFrom(from, limit, bounds) {
681
+ if (from < bounds.hotOffset && from < bounds.nextOffset) {
682
+ return __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, from, limit, bounds);
683
+ }
684
+ return readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, from, limit).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
685
+ }, _DurabilityLog_requireLeases = function _DurabilityLog_requireLeases() {
686
+ if (!__classPrivateFieldGet(this, _DurabilityLog_leases, "f")) {
687
+ throw new TypeError('Leases require the leases option');
688
+ }
689
+ return __classPrivateFieldGet(this, _DurabilityLog_leases, "f");
690
+ };
691
+ const assertConsumer = (consumer) => {
692
+ if (typeof consumer !== 'string' || consumer.length === 0) {
693
+ throw new LogAppendError('Consumer must be a non-empty string');
694
+ }
695
+ };
696
+ const takeWhile = (items, predicate) => {
697
+ const taken = [];
698
+ for (const [index, item] of items.entries()) {
699
+ if (!predicate(item, index)) {
700
+ break;
701
+ }
702
+ taken.push(item);
703
+ }
704
+ return taken;
705
+ };