durability 3.0.1 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +400 -25
- package/dist/client.d.ts +94 -0
- package/dist/client.js +445 -0
- package/dist/durability-alarms.d.ts +14 -1
- package/dist/durability-alarms.js +121 -22
- package/dist/durability.d.ts +33 -2
- package/dist/durability.js +126 -63
- package/dist/errors.d.ts +26 -0
- package/dist/errors.js +38 -0
- package/dist/experimental.d.ts +8 -0
- package/dist/experimental.js +7 -0
- package/dist/fanout.d.ts +25 -2
- package/dist/fanout.js +273 -93
- package/dist/index.d.ts +8 -7
- package/dist/index.js +3 -2
- package/dist/lifecycle.d.ts +8 -0
- package/dist/lifecycle.js +22 -0
- package/dist/log.d.ts +201 -0
- package/dist/log.js +705 -0
- package/dist/migrations.d.ts +28 -1
- package/dist/migrations.js +100 -0
- package/dist/scheduler.d.ts +35 -20
- package/dist/scheduler.js +112 -90
- package/dist/serialization.d.ts +5 -0
- package/dist/serialization.js +16 -0
- package/dist/socket-protocol.d.ts +95 -0
- package/dist/socket-protocol.js +71 -0
- package/dist/socket.d.ts +92 -0
- package/dist/socket.js +273 -0
- package/dist/storage.d.ts +39 -128
- package/dist/storage.js +64 -68
- package/dist/tracked-source.d.ts +87 -0
- package/dist/tracked-source.js +7 -0
- package/dist/tracker.d.ts +96 -0
- package/dist/tracker.js +171 -0
- package/package.json +19 -5
- package/dist/kv-storage.d.ts +0 -2
- package/dist/kv-storage.js +0 -421
- package/dist/sqlite-storage.d.ts +0 -2
- package/dist/sqlite-storage.js +0 -251
package/dist/log.js
ADDED
|
@@ -0,0 +1,705 @@
|
|
|
1
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
2
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
3
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
4
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
5
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
6
|
+
};
|
|
7
|
+
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
8
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
9
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
10
|
+
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
|
+
};
|
|
12
|
+
var _DurabilityLog_instances, _DurabilityLog_engine, _DurabilityLog_routing, _DurabilityLog_retention, _DurabilityLog_cold, _DurabilityLog_leases, _DurabilityLog_maxParallelism, _DurabilityLog_leaseMs, _DurabilityLog_maxBatchSize, _DurabilityLog_encoder, _DurabilityLog_rehydrate, _DurabilityLog_record, _DurabilityLog_settle, _DurabilityLog_advance, _DurabilityLog_between, _DurabilityLog_readFrom, _DurabilityLog_requireLeases;
|
|
13
|
+
import { z } from 'zod';
|
|
14
|
+
import { LogAppendError, LogLeaseError, LogTruncatedError, reportFailure, serializeError, } from './errors.js';
|
|
15
|
+
import { RoutingLoad } from './load.js';
|
|
16
|
+
import { durabilityLogLeaseMigrations, durabilityLogMigrations, migrate, } from './migrations.js';
|
|
17
|
+
import { assertPositiveInteger } from './policy.js';
|
|
18
|
+
import { engineFor, } from './scheduler.js';
|
|
19
|
+
import { deserialize, serialize } from './serialization.js';
|
|
20
|
+
const logRecordRowSchema = z.object({
|
|
21
|
+
offset: z.number().int().nonnegative(),
|
|
22
|
+
dedup_key: z.string(),
|
|
23
|
+
payload: z.string(),
|
|
24
|
+
bytes: z.number().int().nonnegative(),
|
|
25
|
+
appended_at: z.number().int().nonnegative(),
|
|
26
|
+
});
|
|
27
|
+
const logCursorRowSchema = z.object({
|
|
28
|
+
consumer: z.string(),
|
|
29
|
+
offset: z.number().int().nonnegative(),
|
|
30
|
+
committed_at: z.number().int().nonnegative(),
|
|
31
|
+
});
|
|
32
|
+
/** One contiguous run of records moved out of the object into cold storage. */
|
|
33
|
+
const logSegmentRowSchema = z.object({
|
|
34
|
+
first_offset: z.number().int().nonnegative(),
|
|
35
|
+
last_offset: z.number().int().nonnegative(),
|
|
36
|
+
locator: z.string(),
|
|
37
|
+
bytes: z.number().int().nonnegative(),
|
|
38
|
+
flushed_at: z.number().int().nonnegative(),
|
|
39
|
+
});
|
|
40
|
+
/**
|
|
41
|
+
* One in-flight or settled range of offsets held by a consumer. A range is
|
|
42
|
+
* redelivered under a fresh `batch_id`, so an ack from a previous holder is
|
|
43
|
+
* rejected rather than advancing a range someone else now owns.
|
|
44
|
+
*/
|
|
45
|
+
const logLeaseRowSchema = z.object({
|
|
46
|
+
batch_id: z.string(),
|
|
47
|
+
consumer: z.string(),
|
|
48
|
+
first_offset: z.number().int().nonnegative(),
|
|
49
|
+
last_offset: z.number().int().nonnegative(),
|
|
50
|
+
state: z.enum(['active', 'pending', 'acked', 'skipped']),
|
|
51
|
+
attempt: z.number().int().positive(),
|
|
52
|
+
expires_at: z.number().int().nonnegative(),
|
|
53
|
+
created_at: z.number().int().nonnegative(),
|
|
54
|
+
});
|
|
55
|
+
const allocationRowSchema = z.object({
|
|
56
|
+
allocated_offset: z.number().int().nonnegative(),
|
|
57
|
+
});
|
|
58
|
+
const offsetRowSchema = z.object({ offset: z.number().int().nonnegative() });
|
|
59
|
+
const seqRowSchema = z.object({ seq: z.number().int().nonnegative() });
|
|
60
|
+
const countRowSchema = z.object({ count: z.number().int().nonnegative() });
|
|
61
|
+
const totalBytesRowSchema = z.object({
|
|
62
|
+
total_bytes: z.number().int().nonnegative(),
|
|
63
|
+
});
|
|
64
|
+
const boundsRowSchema = z.object({
|
|
65
|
+
hot: z.number().int().nonnegative().nullable(),
|
|
66
|
+
cold: z.number().int().nonnegative().nullable(),
|
|
67
|
+
next_offset: z.number().int().nonnegative(),
|
|
68
|
+
});
|
|
69
|
+
/**
|
|
70
|
+
* Inserts the batch and returns each record's offset in input order. A record
|
|
71
|
+
* whose deduplication key is already present keeps its original offset and
|
|
72
|
+
* consumes no new one, so a repeated append is idempotent while that record is
|
|
73
|
+
* still held in the object. Runs inside the caller's transaction.
|
|
74
|
+
*/
|
|
75
|
+
const appendRecords = (sql, records, appendedAt) => {
|
|
76
|
+
const assigned = new Map();
|
|
77
|
+
for (const record of records) {
|
|
78
|
+
const existing = offsetRowSchema.optional().parse(sql
|
|
79
|
+
.exec(`SELECT "offset" FROM durability_log_records
|
|
80
|
+
WHERE dedup_key = ? LIMIT 1`, record.dedup_key)
|
|
81
|
+
.toArray()[0]);
|
|
82
|
+
if (existing !== undefined) {
|
|
83
|
+
assigned.set(record.dedup_key, existing.offset);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const fresh = records.filter((record) => !assigned.has(record.dedup_key));
|
|
87
|
+
const distinct = [...new Map(fresh.map((row) => [row.dedup_key, row]))];
|
|
88
|
+
if (distinct.length === 0) {
|
|
89
|
+
return records.map((record) => assigned.get(record.dedup_key));
|
|
90
|
+
}
|
|
91
|
+
const first = seqRowSchema.parse(sql
|
|
92
|
+
.exec(`UPDATE durability_log_seq
|
|
93
|
+
SET next_offset = next_offset + ?
|
|
94
|
+
WHERE id = 1
|
|
95
|
+
RETURNING next_offset - ? AS seq`, distinct.length, distinct.length)
|
|
96
|
+
.toArray()[0]).seq;
|
|
97
|
+
let added = 0;
|
|
98
|
+
distinct.forEach(([dedupKey, row], index) => {
|
|
99
|
+
const offset = first + index;
|
|
100
|
+
assigned.set(dedupKey, offset);
|
|
101
|
+
added += row.bytes;
|
|
102
|
+
sql.exec(`INSERT INTO durability_log_records
|
|
103
|
+
("offset", dedup_key, payload, bytes, appended_at)
|
|
104
|
+
VALUES (?, ?, ?, ?, ?)
|
|
105
|
+
ON CONFLICT(dedup_key) DO NOTHING`, offset, dedupKey, row.payload, row.bytes, appendedAt);
|
|
106
|
+
});
|
|
107
|
+
sql.exec(`UPDATE durability_log_seq SET total_bytes = total_bytes + ?
|
|
108
|
+
WHERE id = 1`, added);
|
|
109
|
+
return records.map((record) => assigned.get(record.dedup_key));
|
|
110
|
+
};
|
|
111
|
+
const readRecords = (sql, from, limit) => logRecordRowSchema.array().parse(sql
|
|
112
|
+
.exec(`SELECT * FROM durability_log_records
|
|
113
|
+
WHERE "offset" >= ?
|
|
114
|
+
ORDER BY "offset" ASC
|
|
115
|
+
LIMIT ?`, from, limit)
|
|
116
|
+
.toArray());
|
|
117
|
+
const readBounds = (sql) => {
|
|
118
|
+
const row = boundsRowSchema.parse(sql
|
|
119
|
+
.exec(`SELECT
|
|
120
|
+
(SELECT MIN("offset") FROM durability_log_records) AS hot,
|
|
121
|
+
(SELECT MIN(first_offset) FROM durability_log_segments) AS cold,
|
|
122
|
+
next_offset
|
|
123
|
+
FROM durability_log_seq WHERE id = 1`)
|
|
124
|
+
.toArray()[0]);
|
|
125
|
+
const hotOffset = row.hot ?? row.next_offset;
|
|
126
|
+
return {
|
|
127
|
+
oldestOffset: Math.min(row.cold ?? hotOffset, hotOffset),
|
|
128
|
+
hotOffset,
|
|
129
|
+
nextOffset: row.next_offset,
|
|
130
|
+
};
|
|
131
|
+
};
|
|
132
|
+
const countRecords = (sql) => countRowSchema.parse(sql
|
|
133
|
+
.exec('SELECT COUNT(*) AS count FROM durability_log_records')
|
|
134
|
+
.toArray()[0]).count;
|
|
135
|
+
/** Bytes of record payloads held in the object, excluding flushed segments. */
|
|
136
|
+
const totalBytes = (sql) => totalBytesRowSchema.parse(sql
|
|
137
|
+
.exec('SELECT total_bytes FROM durability_log_seq WHERE id = 1')
|
|
138
|
+
.toArray()[0]).total_bytes;
|
|
139
|
+
/** Oldest retained records in offset order, for flush-or-delete decisions. */
|
|
140
|
+
const listOldest = (sql, limit) => logRecordRowSchema.array().parse(sql
|
|
141
|
+
.exec(`SELECT * FROM durability_log_records
|
|
142
|
+
ORDER BY "offset" ASC
|
|
143
|
+
LIMIT ?`, limit)
|
|
144
|
+
.toArray());
|
|
145
|
+
/**
|
|
146
|
+
* Deletes at most `limit` records below `through`, oldest first, and returns
|
|
147
|
+
* how many were removed. Runs inside the caller's transaction.
|
|
148
|
+
*/
|
|
149
|
+
const trimThrough = (sql, through, limit) => {
|
|
150
|
+
const removed = logRecordRowSchema.array().parse(sql
|
|
151
|
+
.exec(`DELETE FROM durability_log_records
|
|
152
|
+
WHERE "offset" IN (
|
|
153
|
+
SELECT "offset" FROM durability_log_records
|
|
154
|
+
WHERE "offset" < ?
|
|
155
|
+
ORDER BY "offset" ASC
|
|
156
|
+
LIMIT ?
|
|
157
|
+
)
|
|
158
|
+
RETURNING *`, through, limit)
|
|
159
|
+
.toArray());
|
|
160
|
+
if (removed.length === 0) {
|
|
161
|
+
return 0;
|
|
162
|
+
}
|
|
163
|
+
sql.exec(`UPDATE durability_log_seq SET total_bytes = MAX(0, total_bytes - ?)
|
|
164
|
+
WHERE id = 1`, removed.reduce((total, row) => total + row.bytes, 0));
|
|
165
|
+
return removed.length;
|
|
166
|
+
};
|
|
167
|
+
/** Records where a flushed run now lives, so a read can rehydrate it. */
|
|
168
|
+
const insertSegment = (sql, row) => {
|
|
169
|
+
sql.exec(`INSERT INTO durability_log_segments
|
|
170
|
+
(first_offset, last_offset, locator, bytes, flushed_at)
|
|
171
|
+
VALUES (?, ?, ?, ?, ?)
|
|
172
|
+
ON CONFLICT(first_offset) DO UPDATE SET
|
|
173
|
+
last_offset = excluded.last_offset,
|
|
174
|
+
locator = excluded.locator,
|
|
175
|
+
bytes = excluded.bytes,
|
|
176
|
+
flushed_at = excluded.flushed_at`, row.first_offset, row.last_offset, row.locator, row.bytes, row.flushed_at);
|
|
177
|
+
};
|
|
178
|
+
/** The segment holding an offset, if that offset was flushed. */
|
|
179
|
+
const findSegment = (sql, offset) => logSegmentRowSchema.optional().parse(sql
|
|
180
|
+
.exec(`SELECT * FROM durability_log_segments
|
|
181
|
+
WHERE first_offset <= ? AND last_offset >= ?
|
|
182
|
+
LIMIT 1`, offset, offset)
|
|
183
|
+
.toArray()[0]);
|
|
184
|
+
const readCursors = (sql) => logCursorRowSchema
|
|
185
|
+
.array()
|
|
186
|
+
.parse(sql
|
|
187
|
+
.exec('SELECT * FROM durability_log_cursors ORDER BY consumer')
|
|
188
|
+
.toArray());
|
|
189
|
+
const readCursor = (sql, consumer) => offsetRowSchema.optional().parse(sql
|
|
190
|
+
.exec(`SELECT "offset" FROM durability_log_cursors
|
|
191
|
+
WHERE consumer = ? LIMIT 1`, consumer)
|
|
192
|
+
.toArray()[0])?.offset;
|
|
193
|
+
/** Advances a cursor, ignoring regressions so a late commit cannot rewind it. */
|
|
194
|
+
const commitCursor = (sql, consumer, offset, committedAt) => {
|
|
195
|
+
sql.exec(`INSERT INTO durability_log_cursors (consumer, "offset", committed_at)
|
|
196
|
+
VALUES (?, ?, ?)
|
|
197
|
+
ON CONFLICT(consumer) DO UPDATE
|
|
198
|
+
SET "offset" = excluded."offset", committed_at = excluded.committed_at
|
|
199
|
+
WHERE excluded."offset" > durability_log_cursors."offset"`, consumer, offset, committedAt);
|
|
200
|
+
};
|
|
201
|
+
/** Highest offset handed to a lease, which runs ahead of the commit. */
|
|
202
|
+
const readAllocation = (sql, consumer) => allocationRowSchema.optional().parse(sql
|
|
203
|
+
.exec(`SELECT allocated_offset FROM durability_log_allocations
|
|
204
|
+
WHERE consumer = ? LIMIT 1`, consumer)
|
|
205
|
+
.toArray()[0])?.allocated_offset;
|
|
206
|
+
const setAllocation = (sql, consumer, offset) => {
|
|
207
|
+
sql.exec(`INSERT INTO durability_log_allocations (consumer, allocated_offset)
|
|
208
|
+
VALUES (?, ?)
|
|
209
|
+
ON CONFLICT(consumer) DO UPDATE SET allocated_offset = excluded.allocated_offset`, consumer, offset);
|
|
210
|
+
};
|
|
211
|
+
const insertLease = (sql, row) => {
|
|
212
|
+
sql.exec(`INSERT INTO durability_log_leases
|
|
213
|
+
(batch_id, consumer, first_offset, last_offset, state, attempt,
|
|
214
|
+
expires_at, created_at)
|
|
215
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, row.batch_id, row.consumer, row.first_offset, row.last_offset, row.state, row.attempt, row.expires_at, row.created_at);
|
|
216
|
+
};
|
|
217
|
+
/**
|
|
218
|
+
* Oldest range owed redelivery: explicitly nacked, or held past its expiry.
|
|
219
|
+
* Claiming rewrites its `batch_id`, fencing the previous holder.
|
|
220
|
+
*/
|
|
221
|
+
const claimExpired = (sql, consumer, now, batchId, expiresAt) => logLeaseRowSchema.optional().parse(sql
|
|
222
|
+
.exec(`UPDATE durability_log_leases
|
|
223
|
+
SET batch_id = ?, state = 'active', attempt = attempt + 1,
|
|
224
|
+
expires_at = ?
|
|
225
|
+
WHERE batch_id = (
|
|
226
|
+
SELECT batch_id FROM durability_log_leases
|
|
227
|
+
WHERE consumer = ?
|
|
228
|
+
AND (state = 'pending'
|
|
229
|
+
OR (state = 'active' AND expires_at <= ?))
|
|
230
|
+
ORDER BY first_offset ASC
|
|
231
|
+
LIMIT 1
|
|
232
|
+
)
|
|
233
|
+
RETURNING *`, batchId, expiresAt, consumer, now)
|
|
234
|
+
.toArray()[0]);
|
|
235
|
+
/** Ranges still held, so parallelism can be bounded per consumer. */
|
|
236
|
+
const countHeld = (sql, consumer, now) => countRowSchema.parse(sql
|
|
237
|
+
.exec(`SELECT COUNT(*) AS count FROM durability_log_leases
|
|
238
|
+
WHERE consumer = ?
|
|
239
|
+
AND (state = 'pending' OR (state = 'active' AND expires_at > ?))`, consumer, now)
|
|
240
|
+
.toArray()[0]).count;
|
|
241
|
+
/** Settles one range, failing when the batch was fenced or already settled. */
|
|
242
|
+
const settleLease = (sql, batchId, state, expiresAt) => sql
|
|
243
|
+
.exec(`UPDATE durability_log_leases
|
|
244
|
+
SET state = ?, expires_at = ?
|
|
245
|
+
WHERE batch_id = ? AND state = 'active'
|
|
246
|
+
RETURNING batch_id`, state, expiresAt, batchId)
|
|
247
|
+
.toArray().length > 0;
|
|
248
|
+
/**
|
|
249
|
+
* Ranges from `from` in offset order, so a commit can advance over the
|
|
250
|
+
* unbroken run of settled ones and no further.
|
|
251
|
+
*/
|
|
252
|
+
const leasesFrom = (sql, consumer, from, limit) => logLeaseRowSchema.array().parse(sql
|
|
253
|
+
.exec(`SELECT * FROM durability_log_leases
|
|
254
|
+
WHERE consumer = ? AND first_offset >= ?
|
|
255
|
+
ORDER BY first_offset ASC
|
|
256
|
+
LIMIT ?`, consumer, from, limit)
|
|
257
|
+
.toArray());
|
|
258
|
+
const deleteLeases = (sql, batchIds) => {
|
|
259
|
+
if (batchIds.length === 0) {
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
sql.exec(`DELETE FROM durability_log_leases
|
|
263
|
+
WHERE batch_id IN (${batchIds.map(() => '?').join(', ')})`, ...batchIds);
|
|
264
|
+
};
|
|
265
|
+
/**
|
|
266
|
+
* A single object holds a hot window, not an unbounded log, so the byte
|
|
267
|
+
* budget is capped well below the platform's per-object limit.
|
|
268
|
+
*/
|
|
269
|
+
export const maxLogBytes = 1073741824;
|
|
270
|
+
/**
|
|
271
|
+
* A composable DO capability: an append-only record log addressed by offset,
|
|
272
|
+
* read by pulling consumers that own their cursors. Share a scheduler with
|
|
273
|
+
* operations, alarms, and fanout on the same object.
|
|
274
|
+
*
|
|
275
|
+
* Unlike fanout, records are retained rather than deleted on acknowledgement,
|
|
276
|
+
* so a consumer can rewind or a new one can start from the beginning. Records
|
|
277
|
+
* leave only through `trim`, and a consumer that falls behind retention reads a
|
|
278
|
+
* `LogTruncatedError` rather than silently skipping records.
|
|
279
|
+
*/
|
|
280
|
+
export class DurabilityLog {
|
|
281
|
+
constructor(config) {
|
|
282
|
+
_DurabilityLog_instances.add(this);
|
|
283
|
+
_DurabilityLog_engine.set(this, void 0);
|
|
284
|
+
_DurabilityLog_routing.set(this, void 0);
|
|
285
|
+
_DurabilityLog_retention.set(this, void 0);
|
|
286
|
+
_DurabilityLog_cold.set(this, void 0);
|
|
287
|
+
_DurabilityLog_leases.set(this, void 0);
|
|
288
|
+
_DurabilityLog_maxParallelism.set(this, 1);
|
|
289
|
+
_DurabilityLog_leaseMs.set(this, 30000);
|
|
290
|
+
_DurabilityLog_maxBatchSize.set(this, void 0);
|
|
291
|
+
_DurabilityLog_encoder.set(this, new TextEncoder());
|
|
292
|
+
__classPrivateFieldSet(this, _DurabilityLog_maxBatchSize, config.maxBatchSize ?? 100, "f");
|
|
293
|
+
assertPositiveInteger('maxBatchSize', __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
294
|
+
if (config.retention?.maxAgeMs !== undefined) {
|
|
295
|
+
assertPositiveInteger('retention.maxAgeMs', config.retention.maxAgeMs);
|
|
296
|
+
}
|
|
297
|
+
if (config.retention?.maxRecords !== undefined) {
|
|
298
|
+
assertPositiveInteger('retention.maxRecords', config.retention.maxRecords);
|
|
299
|
+
}
|
|
300
|
+
if (config.retention?.maxBytes !== undefined) {
|
|
301
|
+
assertPositiveInteger('retention.maxBytes', config.retention.maxBytes);
|
|
302
|
+
if (config.retention.maxBytes > maxLogBytes) {
|
|
303
|
+
throw new RangeError(`retention.maxBytes must not exceed ${maxLogBytes} bytes`);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
__classPrivateFieldSet(this, _DurabilityLog_engine, engineFor(config), "f");
|
|
307
|
+
__classPrivateFieldSet(this, _DurabilityLog_routing, config.routing ?? new RoutingLoad(), "f");
|
|
308
|
+
__classPrivateFieldSet(this, _DurabilityLog_retention, config.retention, "f");
|
|
309
|
+
__classPrivateFieldSet(this, _DurabilityLog_cold, config.cold, "f");
|
|
310
|
+
__classPrivateFieldSet(this, _DurabilityLog_leases, config.leases, "f");
|
|
311
|
+
if (__classPrivateFieldGet(this, _DurabilityLog_leases, "f")) {
|
|
312
|
+
__classPrivateFieldSet(this, _DurabilityLog_maxParallelism, __classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxParallelism ?? 1, "f");
|
|
313
|
+
assertPositiveInteger('leases.maxParallelism', __classPrivateFieldGet(this, _DurabilityLog_maxParallelism, "f"));
|
|
314
|
+
__classPrivateFieldSet(this, _DurabilityLog_leaseMs, __classPrivateFieldGet(this, _DurabilityLog_leases, "f").leaseMs ?? 30000, "f");
|
|
315
|
+
assertPositiveInteger('leases.leaseMs', __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
|
|
316
|
+
if (__classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxAttempts !== undefined) {
|
|
317
|
+
assertPositiveInteger('leases.maxAttempts', __classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxAttempts);
|
|
318
|
+
}
|
|
319
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('logLeases', durabilityLogLeaseMigrations);
|
|
320
|
+
}
|
|
321
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('log', durabilityLogMigrations);
|
|
322
|
+
}
|
|
323
|
+
static migrate(context, target) {
|
|
324
|
+
return migrate(context.storage, 'log', durabilityLogMigrations, target);
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Appends the batch atomically. Definite input rejections return
|
|
328
|
+
* success:false; storage and commit uncertainty throws, so a caller never
|
|
329
|
+
* reads a transport failure as a definite rejection.
|
|
330
|
+
*/
|
|
331
|
+
async append(records) {
|
|
332
|
+
const finish = __classPrivateFieldGet(this, _DurabilityLog_routing, "f").begin('inbound');
|
|
333
|
+
try {
|
|
334
|
+
const inputs = Array.isArray(records) ? records : [records];
|
|
335
|
+
let rows;
|
|
336
|
+
try {
|
|
337
|
+
if (inputs.length === 0) {
|
|
338
|
+
throw new LogAppendError('Append requires at least one record');
|
|
339
|
+
}
|
|
340
|
+
if (inputs.length > __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f")) {
|
|
341
|
+
throw new LogAppendError(`Append of ${inputs.length} records exceeds maxBatchSize ${__classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f")}`);
|
|
342
|
+
}
|
|
343
|
+
rows = inputs.map((input) => {
|
|
344
|
+
if (input.deduplicationKey === '') {
|
|
345
|
+
throw new LogAppendError('Append deduplication keys must be non-empty');
|
|
346
|
+
}
|
|
347
|
+
const payload = serialize(input.body);
|
|
348
|
+
return {
|
|
349
|
+
dedup_key: input.deduplicationKey ?? crypto.randomUUID(),
|
|
350
|
+
payload,
|
|
351
|
+
bytes: __classPrivateFieldGet(this, _DurabilityLog_encoder, "f").encode(payload).byteLength,
|
|
352
|
+
};
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
catch (error) {
|
|
356
|
+
if (!(error instanceof LogAppendError)) {
|
|
357
|
+
throw error;
|
|
358
|
+
}
|
|
359
|
+
return {
|
|
360
|
+
success: false,
|
|
361
|
+
error: serializeError(error),
|
|
362
|
+
load: await this.load(),
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
const appendedAt = Date.now();
|
|
366
|
+
const offsets = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => appendRecords(sql, rows, appendedAt));
|
|
367
|
+
return { success: true, value: offsets, load: await this.load() };
|
|
368
|
+
}
|
|
369
|
+
finally {
|
|
370
|
+
finish();
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Reads forward from an offset. Offsets already flushed are rehydrated from
|
|
375
|
+
* cold storage, so a lagging consumer keeps reading instead of failing; only
|
|
376
|
+
* offsets below everything readable raise `LogTruncatedError`.
|
|
377
|
+
*/
|
|
378
|
+
async read(options) {
|
|
379
|
+
const limit = options.limit ?? __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f");
|
|
380
|
+
assertPositiveInteger('limit', limit);
|
|
381
|
+
if (!Number.isInteger(options.from) || options.from < 0) {
|
|
382
|
+
throw new LogAppendError('Read offset must be a non-negative integer');
|
|
383
|
+
}
|
|
384
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
385
|
+
if (options.from < bounds.oldestOffset) {
|
|
386
|
+
throw new LogTruncatedError(options.from, bounds.oldestOffset);
|
|
387
|
+
}
|
|
388
|
+
const capped = Math.min(limit, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
389
|
+
const records = options.from < bounds.hotOffset
|
|
390
|
+
? await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, options.from, capped, bounds)
|
|
391
|
+
: readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, options.from, capped).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
392
|
+
const nextOffset = records.length === 0
|
|
393
|
+
? Math.max(options.from, bounds.nextOffset)
|
|
394
|
+
: records[records.length - 1].offset + 1;
|
|
395
|
+
return {
|
|
396
|
+
records,
|
|
397
|
+
nextOffset,
|
|
398
|
+
lag: Math.max(0, bounds.nextOffset - nextOffset),
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
/** The retained window. `oldestOffset` equals `nextOffset` when empty. */
|
|
402
|
+
async bounds() {
|
|
403
|
+
return readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
404
|
+
}
|
|
405
|
+
/** Undefined until the consumer commits, so callers choose their own start. */
|
|
406
|
+
async cursor(consumer) {
|
|
407
|
+
return readCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, consumer);
|
|
408
|
+
}
|
|
409
|
+
/** Records progress. Regressions are ignored, so a late commit cannot rewind. */
|
|
410
|
+
async commit(consumer, offset) {
|
|
411
|
+
if (!Number.isInteger(offset) || offset < 0) {
|
|
412
|
+
throw new LogAppendError('Commit offset must be a non-negative integer');
|
|
413
|
+
}
|
|
414
|
+
commitCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, consumer, offset, Date.now());
|
|
415
|
+
}
|
|
416
|
+
async cursors() {
|
|
417
|
+
return Object.fromEntries(readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql).map((row) => [row.consumer, row.offset]));
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Enforces retention on the oldest records: with cold storage they are
|
|
421
|
+
* flushed and stay readable through rehydration, without it they are
|
|
422
|
+
* deleted. A cold write failure propagates with the records still in the
|
|
423
|
+
* object, so configured storage cannot be skipped.
|
|
424
|
+
*
|
|
425
|
+
* Retention ignores cursors, as a log should: a consumer slower than
|
|
426
|
+
* everything readable learns through `LogTruncatedError`.
|
|
427
|
+
*/
|
|
428
|
+
async trim() {
|
|
429
|
+
if (!__classPrivateFieldGet(this, _DurabilityLog_retention, "f")) {
|
|
430
|
+
return 0;
|
|
431
|
+
}
|
|
432
|
+
const { maxAgeMs, maxRecords, maxBytes } = __classPrivateFieldGet(this, _DurabilityLog_retention, "f");
|
|
433
|
+
const cutoff = maxAgeMs === undefined ? undefined : Date.now() - maxAgeMs;
|
|
434
|
+
let removed = 0;
|
|
435
|
+
for (;;) {
|
|
436
|
+
const oldest = listOldest(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
437
|
+
if (oldest.length === 0) {
|
|
438
|
+
return removed;
|
|
439
|
+
}
|
|
440
|
+
const excess = maxRecords === undefined
|
|
441
|
+
? 0
|
|
442
|
+
: Math.max(0, countRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql) - maxRecords);
|
|
443
|
+
const heldBytes = maxBytes === undefined ? 0 : totalBytes(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
444
|
+
let overBytes = Math.max(0, heldBytes - (maxBytes ?? 0));
|
|
445
|
+
const expiring = takeWhile(oldest, (row, index) => {
|
|
446
|
+
const reclaiming = overBytes > 0;
|
|
447
|
+
overBytes -= row.bytes;
|
|
448
|
+
return (index < excess ||
|
|
449
|
+
reclaiming ||
|
|
450
|
+
(cutoff !== undefined && row.appended_at < cutoff));
|
|
451
|
+
});
|
|
452
|
+
if (expiring.length === 0) {
|
|
453
|
+
return removed;
|
|
454
|
+
}
|
|
455
|
+
const first = expiring[0];
|
|
456
|
+
const last = expiring[expiring.length - 1];
|
|
457
|
+
if (__classPrivateFieldGet(this, _DurabilityLog_cold, "f")) {
|
|
458
|
+
const records = expiring.map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
459
|
+
// eslint-disable-next-line no-await-in-loop
|
|
460
|
+
const locator = await __classPrivateFieldGet(this, _DurabilityLog_cold, "f").write({
|
|
461
|
+
firstOffset: first.offset,
|
|
462
|
+
lastOffset: last.offset,
|
|
463
|
+
records,
|
|
464
|
+
});
|
|
465
|
+
if (typeof locator !== 'string' || locator.length === 0) {
|
|
466
|
+
throw new TypeError('Cold storage must return a non-empty locator');
|
|
467
|
+
}
|
|
468
|
+
// Indexed before deletion: a crash in between leaves the segment
|
|
469
|
+
// written and readable, never records that point nowhere.
|
|
470
|
+
insertSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, {
|
|
471
|
+
first_offset: first.offset,
|
|
472
|
+
last_offset: last.offset,
|
|
473
|
+
locator,
|
|
474
|
+
bytes: expiring.reduce((total, row) => total + row.bytes, 0),
|
|
475
|
+
flushed_at: Date.now(),
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
const deleted = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => trimThrough(sql, last.offset + 1, expiring.length));
|
|
479
|
+
removed += deleted;
|
|
480
|
+
// Records are flushed before deletion, so stopping on partial progress
|
|
481
|
+
// keeps the next pass from writing the remainder a second time.
|
|
482
|
+
if (deleted < expiring.length) {
|
|
483
|
+
return removed;
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Forgets flushed segments below an offset, matching whatever lifecycle rule
|
|
489
|
+
* expires them externally. Reads below the new floor then truncate.
|
|
490
|
+
*/
|
|
491
|
+
async forgetColdBefore(offset) {
|
|
492
|
+
if (!Number.isInteger(offset) || offset < 0) {
|
|
493
|
+
throw new LogAppendError('Offset must be a non-negative integer');
|
|
494
|
+
}
|
|
495
|
+
return __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql
|
|
496
|
+
.exec(`DELETE FROM durability_log_segments
|
|
497
|
+
WHERE last_offset < ?
|
|
498
|
+
RETURNING first_offset`, offset)
|
|
499
|
+
.toArray().length;
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Claims the next unleased range for a consumer, or redelivers one whose
|
|
503
|
+
* holder nacked or let its lease expire. Returns undefined when nothing is
|
|
504
|
+
* available, either because the consumer is at `maxParallelism` or because
|
|
505
|
+
* the log has no unleased records.
|
|
506
|
+
*
|
|
507
|
+
* Two workers sharing a consumer name never receive the same offsets: the
|
|
508
|
+
* range is recorded durably before the records are returned, and a
|
|
509
|
+
* redelivery rewrites the batch id so the previous holder can no longer
|
|
510
|
+
* settle it.
|
|
511
|
+
*/
|
|
512
|
+
async lease(consumer, limit) {
|
|
513
|
+
const leases = __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
|
|
514
|
+
const size = Math.min(limit ?? __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"), __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
515
|
+
assertPositiveInteger('limit', size);
|
|
516
|
+
assertConsumer(consumer);
|
|
517
|
+
const now = Date.now();
|
|
518
|
+
const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
|
|
519
|
+
const reclaimed = claimExpired(sql, consumer, now, crypto.randomUUID(), now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
|
|
520
|
+
if (reclaimed) {
|
|
521
|
+
const poisoned = leases.maxAttempts !== undefined &&
|
|
522
|
+
leases.onPoison !== undefined &&
|
|
523
|
+
reclaimed.attempt > leases.maxAttempts;
|
|
524
|
+
const lease = {
|
|
525
|
+
batchId: reclaimed.batch_id,
|
|
526
|
+
consumer,
|
|
527
|
+
records: await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_between).call(this, reclaimed.first_offset, reclaimed.last_offset),
|
|
528
|
+
firstOffset: reclaimed.first_offset,
|
|
529
|
+
lastOffset: reclaimed.last_offset,
|
|
530
|
+
attempt: reclaimed.attempt,
|
|
531
|
+
expiresAt: reclaimed.expires_at,
|
|
532
|
+
};
|
|
533
|
+
if (!poisoned) {
|
|
534
|
+
return lease;
|
|
535
|
+
}
|
|
536
|
+
// Terminal intent is durable before the callback runs, so a failing
|
|
537
|
+
// handler cannot resurrect a range that already exhausted its attempts.
|
|
538
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_settle).call(this, reclaimed.batch_id, 'skipped');
|
|
539
|
+
try {
|
|
540
|
+
await leases.onPoison(lease);
|
|
541
|
+
}
|
|
542
|
+
catch (error) {
|
|
543
|
+
reportFailure('durability.log.poison_handler.failed', { consumer }, error);
|
|
544
|
+
}
|
|
545
|
+
return undefined;
|
|
546
|
+
}
|
|
547
|
+
if (countHeld(sql, consumer, now) >= __classPrivateFieldGet(this, _DurabilityLog_maxParallelism, "f")) {
|
|
548
|
+
return undefined;
|
|
549
|
+
}
|
|
550
|
+
const bounds = readBounds(sql);
|
|
551
|
+
const committed = readCursor(sql, consumer);
|
|
552
|
+
const allocated = readAllocation(sql, consumer);
|
|
553
|
+
const from = Math.max(bounds.oldestOffset, committed ?? 0, allocated ?? 0);
|
|
554
|
+
const records = await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_readFrom).call(this, from, size, bounds);
|
|
555
|
+
if (records.length === 0) {
|
|
556
|
+
return undefined;
|
|
557
|
+
}
|
|
558
|
+
const first = records[0].offset;
|
|
559
|
+
const last = records[records.length - 1].offset;
|
|
560
|
+
const batchId = crypto.randomUUID();
|
|
561
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((transaction) => {
|
|
562
|
+
insertLease(transaction, {
|
|
563
|
+
batch_id: batchId,
|
|
564
|
+
consumer,
|
|
565
|
+
first_offset: first,
|
|
566
|
+
last_offset: last,
|
|
567
|
+
state: 'active',
|
|
568
|
+
attempt: 1,
|
|
569
|
+
expires_at: now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"),
|
|
570
|
+
created_at: now,
|
|
571
|
+
});
|
|
572
|
+
setAllocation(transaction, consumer, last + 1);
|
|
573
|
+
});
|
|
574
|
+
return {
|
|
575
|
+
batchId,
|
|
576
|
+
consumer,
|
|
577
|
+
records,
|
|
578
|
+
firstOffset: first,
|
|
579
|
+
lastOffset: last,
|
|
580
|
+
attempt: 1,
|
|
581
|
+
expiresAt: now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"),
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* Accepts a leased range and advances the cursor over every settled range
|
|
586
|
+
* that forms an unbroken run from it, so parallel consumers commit without
|
|
587
|
+
* skipping offsets another worker is still holding.
|
|
588
|
+
*/
|
|
589
|
+
async ack(consumer, batchId) {
|
|
590
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
|
|
591
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync(() => {
|
|
592
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_settle).call(this, batchId, 'acked');
|
|
593
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_advance).call(this, consumer);
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
/** Returns a leased range for redelivery under a fresh batch id. */
|
|
597
|
+
async nack(consumer, batchId) {
|
|
598
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
|
|
599
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_settle).call(this, batchId, 'pending');
|
|
600
|
+
}
|
|
601
|
+
/** In-flight and settled-but-uncommitted ranges, for operators and tests. */
|
|
602
|
+
async leases(consumer) {
|
|
603
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
|
|
604
|
+
return logLeaseRowSchema.array().parse(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql
|
|
605
|
+
.exec(`SELECT * FROM durability_log_leases
|
|
606
|
+
WHERE consumer = ?
|
|
607
|
+
ORDER BY first_offset ASC`, consumer)
|
|
608
|
+
.toArray());
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Backlog is the furthest-behind committed cursor, so routing can weigh
|
|
612
|
+
* partitions by consumer lag. Without cursors there is no backlog to report.
|
|
613
|
+
*/
|
|
614
|
+
async load() {
|
|
615
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
616
|
+
const slowest = readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql).reduce((lowest, row) => lowest === undefined ? row.offset : Math.min(lowest, row.offset), undefined);
|
|
617
|
+
return __classPrivateFieldGet(this, _DurabilityLog_routing, "f").snapshot(slowest === undefined ? 0 : Math.max(0, bounds.nextOffset - slowest));
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
_DurabilityLog_engine = new WeakMap(), _DurabilityLog_routing = new WeakMap(), _DurabilityLog_retention = new WeakMap(), _DurabilityLog_cold = new WeakMap(), _DurabilityLog_leases = new WeakMap(), _DurabilityLog_maxParallelism = new WeakMap(), _DurabilityLog_leaseMs = new WeakMap(), _DurabilityLog_maxBatchSize = new WeakMap(), _DurabilityLog_encoder = new WeakMap(), _DurabilityLog_instances = new WeakSet(), _DurabilityLog_rehydrate =
|
|
621
|
+
/**
|
|
622
|
+
* Cold reads cost a round trip to external storage and open input gates, so
|
|
623
|
+
* they serve only consumers that fell behind the hot window.
|
|
624
|
+
*/
|
|
625
|
+
async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
626
|
+
const segment = findSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, from);
|
|
627
|
+
const records = segment
|
|
628
|
+
? await __classPrivateFieldGet(this, _DurabilityLog_cold, "f")?.read(segment.locator)
|
|
629
|
+
: undefined;
|
|
630
|
+
if (!records) {
|
|
631
|
+
// The index outlived the data, so the offset is unreadable after all.
|
|
632
|
+
throw new LogTruncatedError(from, Math.max(from + 1, bounds.hotOffset));
|
|
633
|
+
}
|
|
634
|
+
return records
|
|
635
|
+
.filter((record) => record.offset >= from)
|
|
636
|
+
.sort((left, right) => left.offset - right.offset)
|
|
637
|
+
.slice(0, limit);
|
|
638
|
+
}, _DurabilityLog_record = function _DurabilityLog_record(row) {
|
|
639
|
+
return {
|
|
640
|
+
offset: row.offset,
|
|
641
|
+
deduplicationKey: row.dedup_key,
|
|
642
|
+
body: deserialize(row.payload),
|
|
643
|
+
appendedAt: row.appended_at,
|
|
644
|
+
};
|
|
645
|
+
}, _DurabilityLog_settle = function _DurabilityLog_settle(batchId, state) {
|
|
646
|
+
// Pending ranges are immediately claimable, so their expiry is in the past.
|
|
647
|
+
const settled = settleLease(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, batchId, state, state === 'pending' ? 0 : Date.now());
|
|
648
|
+
if (settled) {
|
|
649
|
+
return;
|
|
650
|
+
}
|
|
651
|
+
// Either the batch never existed, was fenced by a redelivery, or was
|
|
652
|
+
// already settled; all three are the caller holding a stale handle.
|
|
653
|
+
throw new LogLeaseError(batchId);
|
|
654
|
+
}, _DurabilityLog_advance = function _DurabilityLog_advance(consumer) {
|
|
655
|
+
const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
|
|
656
|
+
const from = readCursor(sql, consumer) ?? 0;
|
|
657
|
+
const candidates = leasesFrom(sql, consumer, from, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
658
|
+
let cursor = from;
|
|
659
|
+
const settled = [];
|
|
660
|
+
for (const row of candidates) {
|
|
661
|
+
const contiguous = row.first_offset <= cursor;
|
|
662
|
+
const done = row.state === 'acked' || row.state === 'skipped';
|
|
663
|
+
if (!contiguous || !done) {
|
|
664
|
+
break;
|
|
665
|
+
}
|
|
666
|
+
cursor = Math.max(cursor, row.last_offset + 1);
|
|
667
|
+
settled.push(row.batch_id);
|
|
668
|
+
}
|
|
669
|
+
if (settled.length === 0) {
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
commitCursor(sql, consumer, cursor, Date.now());
|
|
673
|
+
deleteLeases(sql, settled);
|
|
674
|
+
}, _DurabilityLog_between =
|
|
675
|
+
/** Reads a stored range, rehydrating it when retention already flushed it. */
|
|
676
|
+
async function _DurabilityLog_between(first, last) {
|
|
677
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
678
|
+
const records = await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_readFrom).call(this, first, last - first + 1, bounds);
|
|
679
|
+
return records.filter((record) => record.offset <= last);
|
|
680
|
+
}, _DurabilityLog_readFrom = async function _DurabilityLog_readFrom(from, limit, bounds) {
|
|
681
|
+
if (from < bounds.hotOffset && from < bounds.nextOffset) {
|
|
682
|
+
return __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, from, limit, bounds);
|
|
683
|
+
}
|
|
684
|
+
return readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, from, limit).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
685
|
+
}, _DurabilityLog_requireLeases = function _DurabilityLog_requireLeases() {
|
|
686
|
+
if (!__classPrivateFieldGet(this, _DurabilityLog_leases, "f")) {
|
|
687
|
+
throw new TypeError('Leases require the leases option');
|
|
688
|
+
}
|
|
689
|
+
return __classPrivateFieldGet(this, _DurabilityLog_leases, "f");
|
|
690
|
+
};
|
|
691
|
+
const assertConsumer = (consumer) => {
|
|
692
|
+
if (typeof consumer !== 'string' || consumer.length === 0) {
|
|
693
|
+
throw new LogAppendError('Consumer must be a non-empty string');
|
|
694
|
+
}
|
|
695
|
+
};
|
|
696
|
+
const takeWhile = (items, predicate) => {
|
|
697
|
+
const taken = [];
|
|
698
|
+
for (const [index, item] of items.entries()) {
|
|
699
|
+
if (!predicate(item, index)) {
|
|
700
|
+
break;
|
|
701
|
+
}
|
|
702
|
+
taken.push(item);
|
|
703
|
+
}
|
|
704
|
+
return taken;
|
|
705
|
+
};
|