durability 5.1.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +145 -12
- package/dist/{durability-alarms.d.ts → alarms/index.d.ts} +13 -5
- package/dist/{durability-alarms.js → alarms/index.js} +69 -55
- package/dist/alarms/migrations.d.ts +9 -0
- package/dist/alarms/migrations.js +133 -0
- package/dist/experimental.d.ts +10 -10
- package/dist/experimental.js +7 -7
- package/dist/{fanout.d.ts → fanout/index.d.ts} +13 -6
- package/dist/{fanout.js → fanout/index.js} +54 -46
- package/dist/fanout/migrations.d.ts +9 -0
- package/dist/fanout/migrations.js +181 -0
- package/dist/{stored-message.d.ts → fanout/stored-message.d.ts} +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.js +13 -8
- package/dist/{log.d.ts → log/index.d.ts} +15 -3
- package/dist/{log.js → log/index.js} +152 -124
- package/dist/log/migrations.d.ts +18 -0
- package/dist/log/migrations.js +297 -0
- package/dist/{memoize.d.ts → memoize/index.d.ts} +1 -1
- package/dist/{memoize.js → memoize/index.js} +14 -11
- package/dist/memoize/migrations.d.ts +10 -0
- package/dist/memoize/migrations.js +21 -0
- package/dist/migrations.d.ts +0 -111
- package/dist/migrations.js +0 -612
- package/dist/{durability.d.ts → operations/index.d.ts} +13 -5
- package/dist/{durability.js → operations/index.js} +59 -47
- package/dist/operations/migrations.d.ts +10 -0
- package/dist/operations/migrations.js +150 -0
- package/dist/{outbox.d.ts → outbox/index.d.ts} +4 -4
- package/dist/{outbox.js → outbox/index.js} +16 -60
- package/dist/outbox/migrations.d.ts +9 -0
- package/dist/outbox/migrations.js +43 -0
- package/dist/{routing-client.js → routing/client.js} +1 -1
- package/dist/{routing.d.ts → routing/index.d.ts} +3 -3
- package/dist/{routing.js → routing/index.js} +3 -3
- package/dist/{load.d.ts → routing/load.d.ts} +1 -1
- package/dist/{schedule.d.ts → schedule/index.d.ts} +16 -8
- package/dist/{schedule.js → schedule/index.js} +65 -49
- package/dist/schedule/migrations.d.ts +9 -0
- package/dist/schedule/migrations.js +118 -0
- package/dist/{scheduler.d.ts → scheduler/index.d.ts} +75 -25
- package/dist/{scheduler.js → scheduler/index.js} +300 -63
- package/dist/{semaphore-client.d.ts → semaphore/client.d.ts} +1 -1
- package/dist/{semaphore-client.js → semaphore/client.js} +2 -2
- package/dist/{semaphore.d.ts → semaphore/index.d.ts} +10 -10
- package/dist/{semaphore.js → semaphore/index.js} +7 -7
- package/dist/{semaphore-leases.d.ts → semaphore/leases.d.ts} +17 -6
- package/dist/{semaphore-leases.js → semaphore/leases.js} +28 -12
- package/dist/semaphore/migrations.d.ts +18 -0
- package/dist/semaphore/migrations.js +444 -0
- package/dist/{semaphore-protocol.d.ts → semaphore/protocol.d.ts} +16 -8
- package/dist/{semaphore-protocol.js → semaphore/protocol.js} +37 -14
- package/dist/{semaphore-queue-store.d.ts → semaphore/queue-store.d.ts} +12 -6
- package/dist/{semaphore-queue-store.js → semaphore/queue-store.js} +69 -61
- package/dist/{semaphore-queue.d.ts → semaphore/queue.d.ts} +15 -5
- package/dist/{semaphore-queue.js → semaphore/queue.js} +105 -46
- package/dist/{semaphore-server.d.ts → semaphore/server.d.ts} +12 -4
- package/dist/{semaphore-server.js → semaphore/server.js} +51 -24
- package/dist/{semaphore-store.d.ts → semaphore/store.d.ts} +8 -3
- package/dist/{semaphore-store.js → semaphore/store.js} +35 -33
- package/dist/{client.d.ts → socket/client.d.ts} +3 -3
- package/dist/{client.js → socket/client.js} +4 -4
- package/dist/{socket.d.ts → socket/index.d.ts} +15 -4
- package/dist/{socket.js → socket/index.js} +66 -20
- package/dist/socket/migrations.d.ts +9 -0
- package/dist/socket/migrations.js +57 -0
- package/dist/{socket-protocol.d.ts → socket/protocol.d.ts} +1 -1
- package/dist/storage.d.ts +4 -0
- package/dist/storage.js +13 -6
- package/dist/{topology-client.d.ts → topology/client.d.ts} +2 -2
- package/dist/{topology-client.js → topology/client.js} +3 -3
- package/dist/{topology.d.ts → topology/index.d.ts} +17 -7
- package/dist/{topology.js → topology/index.js} +92 -58
- package/dist/topology/migrations.d.ts +10 -0
- package/dist/topology/migrations.js +138 -0
- package/dist/{tracker.d.ts → tracker/index.d.ts} +3 -3
- package/dist/{tracker.js → tracker/index.js} +2 -2
- package/package.json +11 -11
- /package/dist/{stored-message.js → fanout/stored-message.js} +0 -0
- /package/dist/{routing-client.d.ts → routing/client.d.ts} +0 -0
- /package/dist/{load.js → routing/load.js} +0 -0
- /package/dist/{socket-protocol.js → socket/protocol.js} +0 -0
- /package/dist/{topology-snapshot.d.ts → topology/snapshot.d.ts} +0 -0
- /package/dist/{topology-snapshot.js → topology/snapshot.js} +0 -0
- /package/dist/{tracked-source.d.ts → tracker/source.d.ts} +0 -0
- /package/dist/{tracked-source.js → tracker/source.js} +0 -0
|
@@ -9,15 +9,16 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
9
9
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
10
10
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
11
|
};
|
|
12
|
-
var _DurabilityLog_instances, _DurabilityLog_engine, _DurabilityLog_routing, _DurabilityLog_retention, _DurabilityLog_cold, _DurabilityLog_leases, _DurabilityLog_maxParallelism, _DurabilityLog_leaseMs, _DurabilityLog_maxBatchSize, _DurabilityLog_encoder, _DurabilityLog_rehydrate, _DurabilityLog_record, _DurabilityLog_settle, _DurabilityLog_dropTrimmed, _DurabilityLog_advance, _DurabilityLog_between, _DurabilityLog_readFrom, _DurabilityLog_requireLeases;
|
|
12
|
+
var _DurabilityLog_instances, _DurabilityLog_engine, _DurabilityLog_channel, _DurabilityLog_routing, _DurabilityLog_retention, _DurabilityLog_cold, _DurabilityLog_leases, _DurabilityLog_maxParallelism, _DurabilityLog_leaseMs, _DurabilityLog_maxBatchSize, _DurabilityLog_encoder, _DurabilityLog_rehydrate, _DurabilityLog_record, _DurabilityLog_settle, _DurabilityLog_dropTrimmed, _DurabilityLog_advance, _DurabilityLog_between, _DurabilityLog_readFrom, _DurabilityLog_requireLeases;
|
|
13
13
|
import { z } from 'zod';
|
|
14
|
-
import { traceMethods } from '
|
|
15
|
-
import { LogAppendError, LogLeaseError, LogTruncatedError, reportFailure, serializeError, } from '
|
|
16
|
-
import { RoutingLoad } from '
|
|
17
|
-
import { durabilityLogLeaseMigrations, durabilityLogMigrations,
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
14
|
+
import { traceMethods } from '../dev.js';
|
|
15
|
+
import { LogAppendError, LogLeaseError, LogTruncatedError, reportFailure, serializeError, } from '../errors.js';
|
|
16
|
+
import { RoutingLoad, } from '../routing/load.js';
|
|
17
|
+
import { durabilityLogLeaseMigrations, durabilityLogMigrations, } from './migrations.js';
|
|
18
|
+
import { migrateCapability } from '../migrations.js';
|
|
19
|
+
import { assertPositiveInteger } from '../policy.js';
|
|
20
|
+
import { assertChannel, defaultChannel, engineFor, } from '../scheduler/index.js';
|
|
21
|
+
import { deserialize, serialize } from '../serialization.js';
|
|
21
22
|
const logRecordRowSchema = z.object({
|
|
22
23
|
offset: z.number().int().nonnegative(),
|
|
23
24
|
dedup_key: z.string(),
|
|
@@ -73,12 +74,12 @@ const boundsRowSchema = z.object({
|
|
|
73
74
|
* consumes no new one, so a repeated append is idempotent while that record is
|
|
74
75
|
* still held in the object. Runs inside the caller's transaction.
|
|
75
76
|
*/
|
|
76
|
-
const appendRecords = (sql, records, appendedAt) => {
|
|
77
|
+
const appendRecords = (sql, channel, records, appendedAt) => {
|
|
77
78
|
const assigned = new Map();
|
|
78
79
|
for (const record of records) {
|
|
79
80
|
const existing = offsetRowSchema.optional().parse(sql
|
|
80
81
|
.exec(`SELECT "offset" FROM durability_log_records
|
|
81
|
-
WHERE dedup_key = ? LIMIT 1`, record.dedup_key)
|
|
82
|
+
WHERE channel = ? AND dedup_key = ? LIMIT 1`, channel, record.dedup_key)
|
|
82
83
|
.toArray()[0]);
|
|
83
84
|
if (existing !== undefined) {
|
|
84
85
|
assigned.set(record.dedup_key, existing.offset);
|
|
@@ -89,11 +90,13 @@ const appendRecords = (sql, records, appendedAt) => {
|
|
|
89
90
|
if (distinct.length === 0) {
|
|
90
91
|
return records.map((record) => assigned.get(record.dedup_key));
|
|
91
92
|
}
|
|
93
|
+
// A channel's counter row is created by its first append.
|
|
92
94
|
const first = seqRowSchema.parse(sql
|
|
93
|
-
.exec(`
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
95
|
+
.exec(`INSERT INTO durability_log_seq (channel, next_offset, total_bytes)
|
|
96
|
+
VALUES (?, ?, 0)
|
|
97
|
+
ON CONFLICT(channel) DO UPDATE
|
|
98
|
+
SET next_offset = durability_log_seq.next_offset + excluded.next_offset
|
|
99
|
+
RETURNING next_offset - ? AS seq`, channel, distinct.length, distinct.length)
|
|
97
100
|
.toArray()[0]).seq;
|
|
98
101
|
let added = 0;
|
|
99
102
|
distinct.forEach(([dedupKey, row], index) => {
|
|
@@ -101,27 +104,31 @@ const appendRecords = (sql, records, appendedAt) => {
|
|
|
101
104
|
assigned.set(dedupKey, offset);
|
|
102
105
|
added += row.bytes;
|
|
103
106
|
sql.exec(`INSERT INTO durability_log_records
|
|
104
|
-
("offset", dedup_key, payload, bytes, appended_at)
|
|
105
|
-
VALUES (?, ?, ?, ?, ?)
|
|
106
|
-
ON CONFLICT(dedup_key) DO NOTHING`, offset, dedupKey, row.payload, row.bytes, appendedAt);
|
|
107
|
+
(channel, "offset", dedup_key, payload, bytes, appended_at)
|
|
108
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
109
|
+
ON CONFLICT(channel, dedup_key) DO NOTHING`, channel, offset, dedupKey, row.payload, row.bytes, appendedAt);
|
|
107
110
|
});
|
|
108
111
|
sql.exec(`UPDATE durability_log_seq SET total_bytes = total_bytes + ?
|
|
109
|
-
WHERE
|
|
112
|
+
WHERE channel = ?`, added, channel);
|
|
110
113
|
return records.map((record) => assigned.get(record.dedup_key));
|
|
111
114
|
};
|
|
112
|
-
const readRecords = (sql, from, limit) => logRecordRowSchema.array().parse(sql
|
|
115
|
+
const readRecords = (sql, channel, from, limit) => logRecordRowSchema.array().parse(sql
|
|
113
116
|
.exec(`SELECT * FROM durability_log_records
|
|
114
|
-
WHERE "offset" >= ?
|
|
117
|
+
WHERE channel = ? AND "offset" >= ?
|
|
115
118
|
ORDER BY "offset" ASC
|
|
116
|
-
LIMIT ?`, from, limit)
|
|
119
|
+
LIMIT ?`, channel, from, limit)
|
|
117
120
|
.toArray());
|
|
118
|
-
const readBounds = (sql) => {
|
|
121
|
+
const readBounds = (sql, channel) => {
|
|
119
122
|
const row = boundsRowSchema.parse(sql
|
|
120
123
|
.exec(`SELECT
|
|
121
|
-
(SELECT MIN("offset") FROM durability_log_records
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
124
|
+
(SELECT MIN("offset") FROM durability_log_records
|
|
125
|
+
WHERE channel = ?1) AS hot,
|
|
126
|
+
(SELECT MIN(first_offset) FROM durability_log_segments
|
|
127
|
+
WHERE channel = ?1) AS cold,
|
|
128
|
+
COALESCE(
|
|
129
|
+
(SELECT next_offset FROM durability_log_seq WHERE channel = ?1),
|
|
130
|
+
0
|
|
131
|
+
) AS next_offset`, channel)
|
|
125
132
|
.toArray()[0]);
|
|
126
133
|
const hotOffset = row.hot ?? row.next_offset;
|
|
127
134
|
return {
|
|
@@ -130,138 +137,148 @@ const readBounds = (sql) => {
|
|
|
130
137
|
nextOffset: row.next_offset,
|
|
131
138
|
};
|
|
132
139
|
};
|
|
133
|
-
const countRecords = (sql) => countRowSchema.parse(sql
|
|
134
|
-
.exec('SELECT COUNT(*) AS count FROM durability_log_records')
|
|
140
|
+
const countRecords = (sql, channel) => countRowSchema.parse(sql
|
|
141
|
+
.exec('SELECT COUNT(*) AS count FROM durability_log_records WHERE channel = ?', channel)
|
|
135
142
|
.toArray()[0]).count;
|
|
136
143
|
/** Bytes of record payloads held in the object, excluding flushed segments. */
|
|
137
|
-
const totalBytes = (sql) => totalBytesRowSchema.parse(sql
|
|
138
|
-
.exec(
|
|
144
|
+
const totalBytes = (sql, channel) => totalBytesRowSchema.parse(sql
|
|
145
|
+
.exec(`SELECT COALESCE(
|
|
146
|
+
(SELECT total_bytes FROM durability_log_seq WHERE channel = ?),
|
|
147
|
+
0
|
|
148
|
+
) AS total_bytes`, channel)
|
|
139
149
|
.toArray()[0]).total_bytes;
|
|
140
150
|
/** Oldest retained records in offset order, for flush-or-delete decisions. */
|
|
141
|
-
const listOldest = (sql, limit) => logRecordRowSchema.array().parse(sql
|
|
151
|
+
const listOldest = (sql, channel, limit) => logRecordRowSchema.array().parse(sql
|
|
142
152
|
.exec(`SELECT * FROM durability_log_records
|
|
153
|
+
WHERE channel = ?
|
|
143
154
|
ORDER BY "offset" ASC
|
|
144
|
-
LIMIT ?`, limit)
|
|
155
|
+
LIMIT ?`, channel, limit)
|
|
145
156
|
.toArray());
|
|
146
157
|
/**
|
|
147
158
|
* Deletes at most `limit` records below `through`, oldest first, and returns
|
|
148
159
|
* how many were removed. Runs inside the caller's transaction.
|
|
149
160
|
*/
|
|
150
|
-
const trimThrough = (sql, through, limit) => {
|
|
161
|
+
const trimThrough = (sql, channel, through, limit) => {
|
|
151
162
|
const removed = logRecordRowSchema.array().parse(sql
|
|
152
163
|
.exec(`DELETE FROM durability_log_records
|
|
153
|
-
WHERE "offset" IN (
|
|
164
|
+
WHERE channel = ?1 AND "offset" IN (
|
|
154
165
|
SELECT "offset" FROM durability_log_records
|
|
155
|
-
WHERE "offset" < ?
|
|
166
|
+
WHERE channel = ?1 AND "offset" < ?2
|
|
156
167
|
ORDER BY "offset" ASC
|
|
157
|
-
LIMIT ?
|
|
168
|
+
LIMIT ?3
|
|
158
169
|
)
|
|
159
|
-
RETURNING *`, through, limit)
|
|
170
|
+
RETURNING *`, channel, through, limit)
|
|
160
171
|
.toArray());
|
|
161
172
|
if (removed.length === 0) {
|
|
162
173
|
return 0;
|
|
163
174
|
}
|
|
164
175
|
sql.exec(`UPDATE durability_log_seq SET total_bytes = MAX(0, total_bytes - ?)
|
|
165
|
-
WHERE
|
|
176
|
+
WHERE channel = ?`, removed.reduce((total, row) => total + row.bytes, 0), channel);
|
|
166
177
|
return removed.length;
|
|
167
178
|
};
|
|
168
179
|
/** Records where a flushed run now lives, so a read can rehydrate it. */
|
|
169
|
-
const insertSegment = (sql, row) => {
|
|
180
|
+
const insertSegment = (sql, channel, row) => {
|
|
170
181
|
sql.exec(`INSERT INTO durability_log_segments
|
|
171
|
-
(first_offset, last_offset, locator, bytes, flushed_at)
|
|
172
|
-
VALUES (?, ?, ?, ?, ?)
|
|
173
|
-
ON CONFLICT(first_offset) DO UPDATE SET
|
|
182
|
+
(channel, first_offset, last_offset, locator, bytes, flushed_at)
|
|
183
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
184
|
+
ON CONFLICT(channel, first_offset) DO UPDATE SET
|
|
174
185
|
last_offset = excluded.last_offset,
|
|
175
186
|
locator = excluded.locator,
|
|
176
187
|
bytes = excluded.bytes,
|
|
177
|
-
flushed_at = excluded.flushed_at`, row.first_offset, row.last_offset, row.locator, row.bytes, row.flushed_at);
|
|
188
|
+
flushed_at = excluded.flushed_at`, channel, row.first_offset, row.last_offset, row.locator, row.bytes, row.flushed_at);
|
|
178
189
|
};
|
|
179
190
|
/** The segment holding an offset, if that offset was flushed. */
|
|
180
|
-
const findSegment = (sql, offset) => logSegmentRowSchema.optional().parse(sql
|
|
191
|
+
const findSegment = (sql, channel, offset) => logSegmentRowSchema.optional().parse(sql
|
|
181
192
|
.exec(`SELECT * FROM durability_log_segments
|
|
182
|
-
WHERE first_offset <= ? AND last_offset >= ?
|
|
183
|
-
LIMIT 1`,
|
|
193
|
+
WHERE channel = ?1 AND first_offset <= ?2 AND last_offset >= ?2
|
|
194
|
+
LIMIT 1`, channel, offset)
|
|
184
195
|
.toArray()[0]);
|
|
185
|
-
const readCursors = (sql) => logCursorRowSchema
|
|
186
|
-
.
|
|
187
|
-
|
|
188
|
-
|
|
196
|
+
const readCursors = (sql, channel) => logCursorRowSchema.array().parse(sql
|
|
197
|
+
.exec(`SELECT * FROM durability_log_cursors
|
|
198
|
+
WHERE channel = ?
|
|
199
|
+
ORDER BY consumer`, channel)
|
|
189
200
|
.toArray());
|
|
190
|
-
const readCursor = (sql, consumer) => offsetRowSchema.optional().parse(sql
|
|
201
|
+
const readCursor = (sql, channel, consumer) => offsetRowSchema.optional().parse(sql
|
|
191
202
|
.exec(`SELECT "offset" FROM durability_log_cursors
|
|
192
|
-
WHERE consumer = ? LIMIT 1`, consumer)
|
|
203
|
+
WHERE channel = ? AND consumer = ? LIMIT 1`, channel, consumer)
|
|
193
204
|
.toArray()[0])?.offset;
|
|
194
205
|
/** Advances a cursor, ignoring regressions so a late commit cannot rewind it. */
|
|
195
|
-
const commitCursor = (sql, consumer, offset, committedAt) => {
|
|
196
|
-
sql.exec(`INSERT INTO durability_log_cursors
|
|
197
|
-
|
|
198
|
-
|
|
206
|
+
const commitCursor = (sql, channel, consumer, offset, committedAt) => {
|
|
207
|
+
sql.exec(`INSERT INTO durability_log_cursors
|
|
208
|
+
(channel, consumer, "offset", committed_at)
|
|
209
|
+
VALUES (?, ?, ?, ?)
|
|
210
|
+
ON CONFLICT(channel, consumer) DO UPDATE
|
|
199
211
|
SET "offset" = excluded."offset", committed_at = excluded.committed_at
|
|
200
|
-
WHERE excluded."offset" > durability_log_cursors."offset"`, consumer, offset, committedAt);
|
|
212
|
+
WHERE excluded."offset" > durability_log_cursors."offset"`, channel, consumer, offset, committedAt);
|
|
201
213
|
};
|
|
202
214
|
/** Highest offset handed to a lease, which runs ahead of the commit. */
|
|
203
|
-
const readAllocation = (sql, consumer) => allocationRowSchema.optional().parse(sql
|
|
215
|
+
const readAllocation = (sql, channel, consumer) => allocationRowSchema.optional().parse(sql
|
|
204
216
|
.exec(`SELECT allocated_offset FROM durability_log_allocations
|
|
205
|
-
WHERE consumer = ? LIMIT 1`, consumer)
|
|
217
|
+
WHERE channel = ? AND consumer = ? LIMIT 1`, channel, consumer)
|
|
206
218
|
.toArray()[0])?.allocated_offset;
|
|
207
|
-
const setAllocation = (sql, consumer, offset) => {
|
|
208
|
-
sql.exec(`INSERT INTO durability_log_allocations
|
|
209
|
-
|
|
210
|
-
|
|
219
|
+
const setAllocation = (sql, channel, consumer, offset) => {
|
|
220
|
+
sql.exec(`INSERT INTO durability_log_allocations
|
|
221
|
+
(channel, consumer, allocated_offset)
|
|
222
|
+
VALUES (?, ?, ?)
|
|
223
|
+
ON CONFLICT(channel, consumer) DO UPDATE
|
|
224
|
+
SET allocated_offset = excluded.allocated_offset`, channel, consumer, offset);
|
|
211
225
|
};
|
|
212
|
-
const insertLease = (sql, row) => {
|
|
226
|
+
const insertLease = (sql, channel, row) => {
|
|
213
227
|
sql.exec(`INSERT INTO durability_log_leases
|
|
214
|
-
(batch_id, consumer, first_offset, last_offset, state, attempt,
|
|
228
|
+
(channel, batch_id, consumer, first_offset, last_offset, state, attempt,
|
|
215
229
|
expires_at, created_at)
|
|
216
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, row.batch_id, row.consumer, row.first_offset, row.last_offset, row.state, row.attempt, row.expires_at, row.created_at);
|
|
230
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, channel, row.batch_id, row.consumer, row.first_offset, row.last_offset, row.state, row.attempt, row.expires_at, row.created_at);
|
|
217
231
|
};
|
|
218
232
|
/**
|
|
219
233
|
* Oldest range owed redelivery: explicitly nacked, or held past its expiry.
|
|
220
234
|
* Claiming rewrites its `batch_id`, fencing the previous holder.
|
|
221
235
|
*/
|
|
222
|
-
const claimExpired = (sql, consumer, now, batchId, expiresAt) => logLeaseRowSchema.optional().parse(sql
|
|
236
|
+
const claimExpired = (sql, channel, consumer, now, batchId, expiresAt) => logLeaseRowSchema.optional().parse(sql
|
|
223
237
|
.exec(`UPDATE durability_log_leases
|
|
224
|
-
SET batch_id =
|
|
225
|
-
expires_at = ?
|
|
226
|
-
WHERE batch_id = (
|
|
238
|
+
SET batch_id = ?1, state = 'active', attempt = attempt + 1,
|
|
239
|
+
expires_at = ?2
|
|
240
|
+
WHERE channel = ?3 AND batch_id = (
|
|
227
241
|
SELECT batch_id FROM durability_log_leases
|
|
228
|
-
WHERE
|
|
242
|
+
WHERE channel = ?3
|
|
243
|
+
AND consumer = ?4
|
|
229
244
|
AND (state = 'pending'
|
|
230
|
-
OR (state = 'active' AND expires_at <= ?))
|
|
245
|
+
OR (state = 'active' AND expires_at <= ?5))
|
|
231
246
|
ORDER BY first_offset ASC
|
|
232
247
|
LIMIT 1
|
|
233
248
|
)
|
|
234
|
-
RETURNING *`, batchId, expiresAt, consumer, now)
|
|
249
|
+
RETURNING *`, batchId, expiresAt, channel, consumer, now)
|
|
235
250
|
.toArray()[0]);
|
|
236
251
|
/** Ranges still held, so parallelism can be bounded per consumer. */
|
|
237
|
-
const countHeld = (sql, consumer, now) => countRowSchema.parse(sql
|
|
252
|
+
const countHeld = (sql, channel, consumer, now) => countRowSchema.parse(sql
|
|
238
253
|
.exec(`SELECT COUNT(*) AS count FROM durability_log_leases
|
|
239
|
-
WHERE
|
|
240
|
-
AND
|
|
254
|
+
WHERE channel = ?
|
|
255
|
+
AND consumer = ?
|
|
256
|
+
AND (state = 'pending' OR (state = 'active' AND expires_at > ?))`, channel, consumer, now)
|
|
241
257
|
.toArray()[0]).count;
|
|
242
258
|
/** Settles one range, failing when the batch was fenced or already settled. */
|
|
243
|
-
const settleLease = (sql, batchId, state, expiresAt) => sql
|
|
259
|
+
const settleLease = (sql, channel, batchId, state, expiresAt) => sql
|
|
244
260
|
.exec(`UPDATE durability_log_leases
|
|
245
261
|
SET state = ?, expires_at = ?
|
|
246
|
-
WHERE batch_id = ? AND state = 'active'
|
|
247
|
-
RETURNING batch_id`, state, expiresAt, batchId)
|
|
262
|
+
WHERE channel = ? AND batch_id = ? AND state = 'active'
|
|
263
|
+
RETURNING batch_id`, state, expiresAt, channel, batchId)
|
|
248
264
|
.toArray().length > 0;
|
|
249
265
|
/**
|
|
250
266
|
* Ranges from `from` in offset order, so a commit can advance over the
|
|
251
267
|
* unbroken run of settled ones and no further.
|
|
252
268
|
*/
|
|
253
|
-
const leasesFrom = (sql, consumer, from, limit) => logLeaseRowSchema.array().parse(sql
|
|
269
|
+
const leasesFrom = (sql, channel, consumer, from, limit) => logLeaseRowSchema.array().parse(sql
|
|
254
270
|
.exec(`SELECT * FROM durability_log_leases
|
|
255
|
-
WHERE consumer = ? AND last_offset >= ?
|
|
271
|
+
WHERE channel = ? AND consumer = ? AND last_offset >= ?
|
|
256
272
|
ORDER BY first_offset ASC
|
|
257
|
-
LIMIT ?`, consumer, from, limit)
|
|
273
|
+
LIMIT ?`, channel, consumer, from, limit)
|
|
258
274
|
.toArray());
|
|
259
|
-
const deleteLeases = (sql, batchIds) => {
|
|
275
|
+
const deleteLeases = (sql, channel, batchIds) => {
|
|
260
276
|
if (batchIds.length === 0) {
|
|
261
277
|
return;
|
|
262
278
|
}
|
|
263
279
|
sql.exec(`DELETE FROM durability_log_leases
|
|
264
|
-
WHERE
|
|
280
|
+
WHERE channel = ?
|
|
281
|
+
AND batch_id IN (${batchIds.map(() => '?').join(', ')})`, channel, ...batchIds);
|
|
265
282
|
};
|
|
266
283
|
/**
|
|
267
284
|
* A single object holds a hot window, not an unbounded log, so the byte
|
|
@@ -282,6 +299,7 @@ export class DurabilityLog {
|
|
|
282
299
|
constructor(config) {
|
|
283
300
|
_DurabilityLog_instances.add(this);
|
|
284
301
|
_DurabilityLog_engine.set(this, void 0);
|
|
302
|
+
_DurabilityLog_channel.set(this, void 0);
|
|
285
303
|
_DurabilityLog_routing.set(this, void 0);
|
|
286
304
|
_DurabilityLog_retention.set(this, void 0);
|
|
287
305
|
_DurabilityLog_cold.set(this, void 0);
|
|
@@ -290,6 +308,7 @@ export class DurabilityLog {
|
|
|
290
308
|
_DurabilityLog_leaseMs.set(this, 30000);
|
|
291
309
|
_DurabilityLog_maxBatchSize.set(this, void 0);
|
|
292
310
|
_DurabilityLog_encoder.set(this, new TextEncoder());
|
|
311
|
+
__classPrivateFieldSet(this, _DurabilityLog_channel, assertChannel('DurabilityLog', config.channel ?? defaultChannel), "f");
|
|
293
312
|
__classPrivateFieldSet(this, _DurabilityLog_maxBatchSize, config.maxBatchSize ?? 100, "f");
|
|
294
313
|
assertPositiveInteger('maxBatchSize', __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
295
314
|
if (config.retention?.maxAgeMs !== undefined) {
|
|
@@ -304,7 +323,6 @@ export class DurabilityLog {
|
|
|
304
323
|
throw new RangeError(`retention.maxBytes must not exceed ${maxLogBytes} bytes`);
|
|
305
324
|
}
|
|
306
325
|
}
|
|
307
|
-
__classPrivateFieldSet(this, _DurabilityLog_engine, engineFor(config), "f");
|
|
308
326
|
__classPrivateFieldSet(this, _DurabilityLog_routing, config.routing ?? new RoutingLoad(), "f");
|
|
309
327
|
__classPrivateFieldSet(this, _DurabilityLog_retention, config.retention, "f");
|
|
310
328
|
__classPrivateFieldSet(this, _DurabilityLog_cold, config.cold, "f");
|
|
@@ -317,9 +335,13 @@ export class DurabilityLog {
|
|
|
317
335
|
if (__classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxAttempts !== undefined) {
|
|
318
336
|
assertPositiveInteger('leases.maxAttempts', __classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxAttempts);
|
|
319
337
|
}
|
|
320
|
-
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('logLeases', durabilityLogLeaseMigrations);
|
|
321
338
|
}
|
|
339
|
+
__classPrivateFieldSet(this, _DurabilityLog_engine, engineFor(config), "f");
|
|
322
340
|
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('log', durabilityLogMigrations);
|
|
341
|
+
if (__classPrivateFieldGet(this, _DurabilityLog_leases, "f")) {
|
|
342
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('logLeases', durabilityLogLeaseMigrations);
|
|
343
|
+
}
|
|
344
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").claimChannel('durability_log_records', __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), 'DurabilityLog');
|
|
323
345
|
}
|
|
324
346
|
static migrate(context, target) {
|
|
325
347
|
return migrateCapability(context.storage, 'log', durabilityLogMigrations, target);
|
|
@@ -364,7 +386,7 @@ export class DurabilityLog {
|
|
|
364
386
|
};
|
|
365
387
|
}
|
|
366
388
|
const appendedAt = Date.now();
|
|
367
|
-
const offsets = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => appendRecords(sql, rows, appendedAt));
|
|
389
|
+
const offsets = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => appendRecords(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), rows, appendedAt));
|
|
368
390
|
return { success: true, value: offsets, load: await this.load() };
|
|
369
391
|
}
|
|
370
392
|
finally {
|
|
@@ -382,14 +404,14 @@ export class DurabilityLog {
|
|
|
382
404
|
if (!Number.isInteger(options.from) || options.from < 0) {
|
|
383
405
|
throw new LogAppendError('Read offset must be a non-negative integer');
|
|
384
406
|
}
|
|
385
|
-
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
407
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
386
408
|
if (options.from < bounds.oldestOffset) {
|
|
387
409
|
throw new LogTruncatedError(options.from, bounds.oldestOffset);
|
|
388
410
|
}
|
|
389
411
|
const capped = Math.min(limit, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
390
412
|
const records = options.from < bounds.hotOffset
|
|
391
413
|
? await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, options.from, capped, bounds)
|
|
392
|
-
: readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, options.from, capped).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
414
|
+
: readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), options.from, capped).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
393
415
|
const nextOffset = records.length === 0
|
|
394
416
|
? Math.max(options.from, bounds.nextOffset)
|
|
395
417
|
: records[records.length - 1].offset + 1;
|
|
@@ -401,21 +423,24 @@ export class DurabilityLog {
|
|
|
401
423
|
}
|
|
402
424
|
/** The retained window. `oldestOffset` equals `nextOffset` when empty. */
|
|
403
425
|
async bounds() {
|
|
404
|
-
return readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
426
|
+
return readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
405
427
|
}
|
|
406
428
|
/** Undefined until the consumer commits, so callers choose their own start. */
|
|
407
429
|
async cursor(consumer) {
|
|
408
|
-
return readCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, consumer);
|
|
430
|
+
return readCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer);
|
|
409
431
|
}
|
|
410
432
|
/** Records progress. Regressions are ignored, so a late commit cannot rewind. */
|
|
411
433
|
async commit(consumer, offset) {
|
|
412
434
|
if (!Number.isInteger(offset) || offset < 0) {
|
|
413
435
|
throw new LogAppendError('Commit offset must be a non-negative integer');
|
|
414
436
|
}
|
|
415
|
-
commitCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, consumer, offset, Date.now());
|
|
437
|
+
commitCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, offset, Date.now());
|
|
416
438
|
}
|
|
417
439
|
async cursors() {
|
|
418
|
-
return Object.fromEntries(readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql).map((row) => [
|
|
440
|
+
return Object.fromEntries(readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f")).map((row) => [
|
|
441
|
+
row.consumer,
|
|
442
|
+
row.offset,
|
|
443
|
+
]));
|
|
419
444
|
}
|
|
420
445
|
/**
|
|
421
446
|
* Enforces retention on the oldest records: with cold storage they are
|
|
@@ -434,14 +459,16 @@ export class DurabilityLog {
|
|
|
434
459
|
const cutoff = maxAgeMs === undefined ? undefined : Date.now() - maxAgeMs;
|
|
435
460
|
let removed = 0;
|
|
436
461
|
for (;;) {
|
|
437
|
-
const oldest = listOldest(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
462
|
+
const oldest = listOldest(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
438
463
|
if (oldest.length === 0) {
|
|
439
464
|
return removed;
|
|
440
465
|
}
|
|
441
466
|
const excess = maxRecords === undefined
|
|
442
467
|
? 0
|
|
443
|
-
: Math.max(0, countRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql) - maxRecords);
|
|
444
|
-
const heldBytes = maxBytes === undefined
|
|
468
|
+
: Math.max(0, countRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f")) - maxRecords);
|
|
469
|
+
const heldBytes = maxBytes === undefined
|
|
470
|
+
? 0
|
|
471
|
+
: totalBytes(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
445
472
|
let overBytes = Math.max(0, heldBytes - (maxBytes ?? 0));
|
|
446
473
|
const expiring = takeWhile(oldest, (row, index) => {
|
|
447
474
|
const reclaiming = overBytes > 0;
|
|
@@ -459,6 +486,7 @@ export class DurabilityLog {
|
|
|
459
486
|
const records = expiring.map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
460
487
|
// eslint-disable-next-line no-await-in-loop
|
|
461
488
|
const locator = await __classPrivateFieldGet(this, _DurabilityLog_cold, "f").write({
|
|
489
|
+
channel: __classPrivateFieldGet(this, _DurabilityLog_channel, "f"),
|
|
462
490
|
firstOffset: first.offset,
|
|
463
491
|
lastOffset: last.offset,
|
|
464
492
|
records,
|
|
@@ -468,7 +496,7 @@ export class DurabilityLog {
|
|
|
468
496
|
}
|
|
469
497
|
// Indexed before deletion: a crash in between leaves the segment
|
|
470
498
|
// written and readable, never records that point nowhere.
|
|
471
|
-
insertSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, {
|
|
499
|
+
insertSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), {
|
|
472
500
|
first_offset: first.offset,
|
|
473
501
|
last_offset: last.offset,
|
|
474
502
|
locator,
|
|
@@ -476,7 +504,7 @@ export class DurabilityLog {
|
|
|
476
504
|
flushed_at: Date.now(),
|
|
477
505
|
});
|
|
478
506
|
}
|
|
479
|
-
const deleted = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => trimThrough(sql, last.offset + 1, expiring.length));
|
|
507
|
+
const deleted = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => trimThrough(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), last.offset + 1, expiring.length));
|
|
480
508
|
removed += deleted;
|
|
481
509
|
// Records are flushed before deletion, so stopping on partial progress
|
|
482
510
|
// keeps the next pass from writing the remainder a second time.
|
|
@@ -495,8 +523,8 @@ export class DurabilityLog {
|
|
|
495
523
|
}
|
|
496
524
|
return __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql
|
|
497
525
|
.exec(`DELETE FROM durability_log_segments
|
|
498
|
-
WHERE last_offset < ?
|
|
499
|
-
RETURNING first_offset`, offset)
|
|
526
|
+
WHERE channel = ? AND last_offset < ?
|
|
527
|
+
RETURNING first_offset`, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), offset)
|
|
500
528
|
.toArray().length;
|
|
501
529
|
}
|
|
502
530
|
/**
|
|
@@ -517,9 +545,9 @@ export class DurabilityLog {
|
|
|
517
545
|
assertConsumer(consumer);
|
|
518
546
|
const now = Date.now();
|
|
519
547
|
const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
|
|
520
|
-
const reclaimed = claimExpired(sql, consumer, now, crypto.randomUUID(), now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
|
|
548
|
+
const reclaimed = claimExpired(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, now, crypto.randomUUID(), now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
|
|
521
549
|
if (reclaimed) {
|
|
522
|
-
const { oldestOffset } = readBounds(sql);
|
|
550
|
+
const { oldestOffset } = readBounds(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
523
551
|
if (reclaimed.first_offset < oldestOffset) {
|
|
524
552
|
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_dropTrimmed).call(this, consumer, reclaimed, oldestOffset);
|
|
525
553
|
throw new LogTruncatedError(reclaimed.first_offset, oldestOffset);
|
|
@@ -550,16 +578,16 @@ export class DurabilityLog {
|
|
|
550
578
|
await leases.onPoison(lease);
|
|
551
579
|
}
|
|
552
580
|
catch (error) {
|
|
553
|
-
reportFailure('durability.log.poison_handler.failed', { consumer }, error);
|
|
581
|
+
reportFailure('durability.log.poison_handler.failed', { channel: __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer }, error);
|
|
554
582
|
}
|
|
555
583
|
return undefined;
|
|
556
584
|
}
|
|
557
|
-
if (countHeld(sql, consumer, now) >= __classPrivateFieldGet(this, _DurabilityLog_maxParallelism, "f")) {
|
|
585
|
+
if (countHeld(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, now) >= __classPrivateFieldGet(this, _DurabilityLog_maxParallelism, "f")) {
|
|
558
586
|
return undefined;
|
|
559
587
|
}
|
|
560
|
-
const bounds = readBounds(sql);
|
|
561
|
-
const committed = readCursor(sql, consumer);
|
|
562
|
-
const allocated = readAllocation(sql, consumer);
|
|
588
|
+
const bounds = readBounds(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
589
|
+
const committed = readCursor(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer);
|
|
590
|
+
const allocated = readAllocation(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer);
|
|
563
591
|
const from = Math.max(bounds.oldestOffset, committed ?? 0, allocated ?? 0);
|
|
564
592
|
const records = await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_readFrom).call(this, from, size, bounds);
|
|
565
593
|
if (records.length === 0) {
|
|
@@ -569,7 +597,7 @@ export class DurabilityLog {
|
|
|
569
597
|
const last = records[records.length - 1].offset;
|
|
570
598
|
const batchId = crypto.randomUUID();
|
|
571
599
|
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((transaction) => {
|
|
572
|
-
insertLease(transaction, {
|
|
600
|
+
insertLease(transaction, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), {
|
|
573
601
|
batch_id: batchId,
|
|
574
602
|
consumer,
|
|
575
603
|
first_offset: first,
|
|
@@ -579,7 +607,7 @@ export class DurabilityLog {
|
|
|
579
607
|
expires_at: now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"),
|
|
580
608
|
created_at: now,
|
|
581
609
|
});
|
|
582
|
-
setAllocation(transaction, consumer, last + 1);
|
|
610
|
+
setAllocation(transaction, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, last + 1);
|
|
583
611
|
});
|
|
584
612
|
return {
|
|
585
613
|
batchId,
|
|
@@ -613,8 +641,8 @@ export class DurabilityLog {
|
|
|
613
641
|
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
|
|
614
642
|
return logLeaseRowSchema.array().parse(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql
|
|
615
643
|
.exec(`SELECT * FROM durability_log_leases
|
|
616
|
-
WHERE consumer = ?
|
|
617
|
-
ORDER BY first_offset ASC`, consumer)
|
|
644
|
+
WHERE channel = ? AND consumer = ?
|
|
645
|
+
ORDER BY first_offset ASC`, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer)
|
|
618
646
|
.toArray());
|
|
619
647
|
}
|
|
620
648
|
/**
|
|
@@ -622,18 +650,18 @@ export class DurabilityLog {
|
|
|
622
650
|
* partitions by consumer lag. Without cursors there is no backlog to report.
|
|
623
651
|
*/
|
|
624
652
|
async load() {
|
|
625
|
-
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
626
|
-
const slowest = readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql).reduce((lowest, row) => lowest === undefined ? row.offset : Math.min(lowest, row.offset), undefined);
|
|
653
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
654
|
+
const slowest = readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f")).reduce((lowest, row) => lowest === undefined ? row.offset : Math.min(lowest, row.offset), undefined);
|
|
627
655
|
return __classPrivateFieldGet(this, _DurabilityLog_routing, "f").snapshot(slowest === undefined ? 0 : Math.max(0, bounds.nextOffset - slowest));
|
|
628
656
|
}
|
|
629
657
|
}
|
|
630
|
-
_DurabilityLog_engine = new WeakMap(), _DurabilityLog_routing = new WeakMap(), _DurabilityLog_retention = new WeakMap(), _DurabilityLog_cold = new WeakMap(), _DurabilityLog_leases = new WeakMap(), _DurabilityLog_maxParallelism = new WeakMap(), _DurabilityLog_leaseMs = new WeakMap(), _DurabilityLog_maxBatchSize = new WeakMap(), _DurabilityLog_encoder = new WeakMap(), _DurabilityLog_instances = new WeakSet(), _DurabilityLog_rehydrate =
|
|
658
|
+
_DurabilityLog_engine = new WeakMap(), _DurabilityLog_channel = new WeakMap(), _DurabilityLog_routing = new WeakMap(), _DurabilityLog_retention = new WeakMap(), _DurabilityLog_cold = new WeakMap(), _DurabilityLog_leases = new WeakMap(), _DurabilityLog_maxParallelism = new WeakMap(), _DurabilityLog_leaseMs = new WeakMap(), _DurabilityLog_maxBatchSize = new WeakMap(), _DurabilityLog_encoder = new WeakMap(), _DurabilityLog_instances = new WeakSet(), _DurabilityLog_rehydrate =
|
|
631
659
|
/**
|
|
632
660
|
* Cold reads cost a round trip to external storage and open input gates, so
|
|
633
661
|
* they serve only consumers that fell behind the hot window.
|
|
634
662
|
*/
|
|
635
663
|
async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
636
|
-
const segment = findSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, from);
|
|
664
|
+
const segment = findSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), from);
|
|
637
665
|
const records = segment
|
|
638
666
|
? await __classPrivateFieldGet(this, _DurabilityLog_cold, "f")?.read(segment.locator)
|
|
639
667
|
: undefined;
|
|
@@ -654,7 +682,7 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
654
682
|
};
|
|
655
683
|
}, _DurabilityLog_settle = function _DurabilityLog_settle(batchId, state) {
|
|
656
684
|
// Pending ranges are immediately claimable, so their expiry is in the past.
|
|
657
|
-
const settled = settleLease(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, batchId, state, state === 'pending' ? 0 : Date.now());
|
|
685
|
+
const settled = settleLease(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), batchId, state, state === 'pending' ? 0 : Date.now());
|
|
658
686
|
if (settled) {
|
|
659
687
|
return;
|
|
660
688
|
}
|
|
@@ -664,13 +692,13 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
664
692
|
}, _DurabilityLog_dropTrimmed = function _DurabilityLog_dropTrimmed(consumer, lease, oldestOffset) {
|
|
665
693
|
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => {
|
|
666
694
|
if (lease.last_offset < oldestOffset) {
|
|
667
|
-
deleteLeases(sql, [lease.batch_id]);
|
|
695
|
+
deleteLeases(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), [lease.batch_id]);
|
|
668
696
|
}
|
|
669
697
|
else {
|
|
670
698
|
sql.exec(`UPDATE durability_log_leases
|
|
671
699
|
SET first_offset = ?, state = 'pending', attempt = attempt - 1,
|
|
672
700
|
expires_at = 0
|
|
673
|
-
WHERE batch_id = ?`, oldestOffset, lease.batch_id);
|
|
701
|
+
WHERE channel = ? AND batch_id = ?`, oldestOffset, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), lease.batch_id);
|
|
674
702
|
}
|
|
675
703
|
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_advance).call(this, consumer);
|
|
676
704
|
});
|
|
@@ -678,13 +706,13 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
678
706
|
const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
|
|
679
707
|
// Offsets below the retained window are gone for everyone; a cursor
|
|
680
708
|
// stranded there could never meet the ranges leased after them.
|
|
681
|
-
const from = Math.max(readCursor(sql, consumer) ?? 0, readBounds(sql).oldestOffset);
|
|
709
|
+
const from = Math.max(readCursor(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer) ?? 0, readBounds(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f")).oldestOffset);
|
|
682
710
|
let cursor = from;
|
|
683
711
|
const settled = [];
|
|
684
712
|
// Ranges settled while an earlier one was outstanding wait behind it; when
|
|
685
713
|
// it settles the cursor must pass all of them, however many pages that is.
|
|
686
714
|
for (let page = true; page;) {
|
|
687
|
-
const candidates = leasesFrom(sql, consumer, cursor, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
715
|
+
const candidates = leasesFrom(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, cursor, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
688
716
|
page = candidates.length === __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f");
|
|
689
717
|
for (const row of candidates) {
|
|
690
718
|
const contiguous = row.first_offset <= cursor;
|
|
@@ -700,12 +728,12 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
700
728
|
if (settled.length === 0) {
|
|
701
729
|
return;
|
|
702
730
|
}
|
|
703
|
-
commitCursor(sql, consumer, cursor, Date.now());
|
|
704
|
-
deleteLeases(sql, settled);
|
|
731
|
+
commitCursor(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, cursor, Date.now());
|
|
732
|
+
deleteLeases(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), settled);
|
|
705
733
|
}, _DurabilityLog_between =
|
|
706
734
|
/** Reads a stored range, rehydrating it when retention already flushed it. */
|
|
707
735
|
async function _DurabilityLog_between(first, last) {
|
|
708
|
-
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
736
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
709
737
|
// One read stops at the end of a cold segment, and a range redelivered
|
|
710
738
|
// after a flush can span several segments and the hot window.
|
|
711
739
|
const records = [];
|
|
@@ -725,7 +753,7 @@ async function _DurabilityLog_between(first, last) {
|
|
|
725
753
|
if (from < bounds.hotOffset && from < bounds.nextOffset) {
|
|
726
754
|
return __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, from, limit, bounds);
|
|
727
755
|
}
|
|
728
|
-
return readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, from, limit).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
756
|
+
return readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), from, limit).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
729
757
|
}, _DurabilityLog_requireLeases = function _DurabilityLog_requireLeases() {
|
|
730
758
|
if (!__classPrivateFieldGet(this, _DurabilityLog_leases, "f")) {
|
|
731
759
|
throw new TypeError('Leases require the leases option');
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ordered SQLite migrations for the retained log and its consumer cursors.
|
|
3
|
+
* Applied automatically on construction.
|
|
4
|
+
*/
|
|
5
|
+
export declare const durabilityLogMigrations: {
|
|
6
|
+
name: string;
|
|
7
|
+
up: string;
|
|
8
|
+
down: string;
|
|
9
|
+
}[];
|
|
10
|
+
/**
|
|
11
|
+
* Leases let several workers consume one log in parallel without handing the
|
|
12
|
+
* same offsets to two of them. Applied automatically when leases are enabled.
|
|
13
|
+
*/
|
|
14
|
+
export declare const durabilityLogLeaseMigrations: {
|
|
15
|
+
name: string;
|
|
16
|
+
up: string;
|
|
17
|
+
down: string;
|
|
18
|
+
}[];
|