durability 5.1.0 → 5.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -21
- package/dist/{durability-alarms.d.ts → alarms/index.d.ts} +13 -5
- package/dist/{durability-alarms.js → alarms/index.js} +85 -74
- package/dist/alarms/migrations.d.ts +9 -0
- package/dist/alarms/migrations.js +133 -0
- package/dist/experimental.d.ts +10 -10
- package/dist/experimental.js +7 -7
- package/dist/{fanout.d.ts → fanout/index.d.ts} +13 -6
- package/dist/{fanout.js → fanout/index.js} +91 -83
- package/dist/fanout/migrations.d.ts +9 -0
- package/dist/fanout/migrations.js +181 -0
- package/dist/{stored-message.d.ts → fanout/stored-message.d.ts} +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.js +13 -8
- package/dist/{log.d.ts → log/index.d.ts} +27 -20
- package/dist/{log.js → log/index.js} +200 -166
- package/dist/log/migrations.d.ts +18 -0
- package/dist/log/migrations.js +297 -0
- package/dist/{memoize.d.ts → memoize/index.d.ts} +1 -1
- package/dist/{memoize.js → memoize/index.js} +23 -21
- package/dist/memoize/migrations.d.ts +10 -0
- package/dist/memoize/migrations.js +21 -0
- package/dist/migrations.d.ts +0 -111
- package/dist/migrations.js +12 -628
- package/dist/{durability.d.ts → operations/index.d.ts} +13 -5
- package/dist/{durability.js → operations/index.js} +76 -70
- package/dist/operations/migrations.d.ts +10 -0
- package/dist/operations/migrations.js +150 -0
- package/dist/{outbox.d.ts → outbox/index.d.ts} +4 -4
- package/dist/{outbox.js → outbox/index.js} +35 -77
- package/dist/outbox/migrations.d.ts +9 -0
- package/dist/outbox/migrations.js +43 -0
- package/dist/policy.d.ts +19 -19
- package/dist/policy.js +13 -12
- package/dist/{routing-client.js → routing/client.js} +1 -1
- package/dist/{routing.d.ts → routing/index.d.ts} +3 -3
- package/dist/{routing.js → routing/index.js} +3 -3
- package/dist/{load.d.ts → routing/load.d.ts} +1 -1
- package/dist/{schedule.d.ts → schedule/index.d.ts} +16 -8
- package/dist/{schedule.js → schedule/index.js} +97 -83
- package/dist/schedule/migrations.d.ts +9 -0
- package/dist/schedule/migrations.js +118 -0
- package/dist/{scheduler.d.ts → scheduler/index.d.ts} +75 -25
- package/dist/{scheduler.js → scheduler/index.js} +308 -70
- package/dist/{semaphore-client.d.ts → semaphore/client.d.ts} +1 -1
- package/dist/{semaphore-client.js → semaphore/client.js} +8 -7
- package/dist/{semaphore.d.ts → semaphore/index.d.ts} +10 -10
- package/dist/{semaphore.js → semaphore/index.js} +7 -7
- package/dist/{semaphore-leases.d.ts → semaphore/leases.d.ts} +17 -6
- package/dist/{semaphore-leases.js → semaphore/leases.js} +28 -12
- package/dist/semaphore/migrations.d.ts +18 -0
- package/dist/semaphore/migrations.js +444 -0
- package/dist/semaphore/protocol.d.ts +451 -0
- package/dist/semaphore/protocol.js +114 -0
- package/dist/semaphore/queue-store.d.ts +107 -0
- package/dist/semaphore/queue-store.js +258 -0
- package/dist/{semaphore-queue.d.ts → semaphore/queue.d.ts} +15 -5
- package/dist/{semaphore-queue.js → semaphore/queue.js} +105 -46
- package/dist/{semaphore-server.d.ts → semaphore/server.d.ts} +12 -4
- package/dist/{semaphore-server.js → semaphore/server.js} +56 -28
- package/dist/semaphore/store.d.ts +53 -0
- package/dist/semaphore/store.js +107 -0
- package/dist/serialization.d.ts +5 -0
- package/dist/serialization.js +41 -16
- package/dist/{client.d.ts → socket/client.d.ts} +3 -3
- package/dist/{client.js → socket/client.js} +4 -4
- package/dist/{socket.d.ts → socket/index.d.ts} +15 -4
- package/dist/{socket.js → socket/index.js} +66 -20
- package/dist/socket/migrations.d.ts +9 -0
- package/dist/socket/migrations.js +57 -0
- package/dist/{socket-protocol.d.ts → socket/protocol.d.ts} +1 -1
- package/dist/storage.d.ts +4 -0
- package/dist/storage.js +22 -15
- package/dist/{topology-client.d.ts → topology/client.d.ts} +2 -2
- package/dist/{topology-client.js → topology/client.js} +3 -3
- package/dist/{topology.d.ts → topology/index.d.ts} +17 -7
- package/dist/{topology.js → topology/index.js} +106 -72
- package/dist/topology/migrations.d.ts +10 -0
- package/dist/topology/migrations.js +138 -0
- package/dist/topology/snapshot.d.ts +50 -0
- package/dist/topology/snapshot.js +38 -0
- package/dist/{tracker.d.ts → tracker/index.d.ts} +3 -3
- package/dist/{tracker.js → tracker/index.js} +9 -9
- package/package.json +11 -11
- package/dist/semaphore-protocol.d.ts +0 -231
- package/dist/semaphore-protocol.js +0 -106
- package/dist/semaphore-queue-store.d.ts +0 -95
- package/dist/semaphore-queue-store.js +0 -252
- package/dist/semaphore-store.d.ts +0 -43
- package/dist/semaphore-store.js +0 -108
- package/dist/topology-snapshot.d.ts +0 -70
- package/dist/topology-snapshot.js +0 -48
- /package/dist/{stored-message.js → fanout/stored-message.js} +0 -0
- /package/dist/{routing-client.d.ts → routing/client.d.ts} +0 -0
- /package/dist/{load.js → routing/load.js} +0 -0
- /package/dist/{socket-protocol.js → socket/protocol.js} +0 -0
- /package/dist/{tracked-source.d.ts → tracker/source.d.ts} +0 -0
- /package/dist/{tracked-source.js → tracker/source.js} +0 -0
|
@@ -9,63 +9,70 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
9
9
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
10
10
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
11
|
};
|
|
12
|
-
var _DurabilityLog_instances, _DurabilityLog_engine, _DurabilityLog_routing, _DurabilityLog_retention, _DurabilityLog_cold, _DurabilityLog_leases, _DurabilityLog_maxParallelism, _DurabilityLog_leaseMs, _DurabilityLog_maxBatchSize, _DurabilityLog_encoder, _DurabilityLog_rehydrate, _DurabilityLog_record, _DurabilityLog_settle, _DurabilityLog_dropTrimmed, _DurabilityLog_advance, _DurabilityLog_between, _DurabilityLog_readFrom, _DurabilityLog_requireLeases;
|
|
13
|
-
import
|
|
14
|
-
import { traceMethods } from '
|
|
15
|
-
import { LogAppendError, LogLeaseError, LogTruncatedError, reportFailure, serializeError, } from '
|
|
16
|
-
import { RoutingLoad } from '
|
|
17
|
-
import { durabilityLogLeaseMigrations, durabilityLogMigrations,
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
12
|
+
var _DurabilityLog_instances, _DurabilityLog_engine, _DurabilityLog_channel, _DurabilityLog_routing, _DurabilityLog_retention, _DurabilityLog_cold, _DurabilityLog_leases, _DurabilityLog_maxParallelism, _DurabilityLog_leaseMs, _DurabilityLog_maxBatchSize, _DurabilityLog_encoder, _DurabilityLog_rehydrate, _DurabilityLog_record, _DurabilityLog_settle, _DurabilityLog_dropTrimmed, _DurabilityLog_advance, _DurabilityLog_between, _DurabilityLog_readFrom, _DurabilityLog_requireLeases;
|
|
13
|
+
import * as v from 'valibot';
|
|
14
|
+
import { traceMethods } from '../dev.js';
|
|
15
|
+
import { LogAppendError, LogLeaseError, LogTruncatedError, reportFailure, serializeError, } from '../errors.js';
|
|
16
|
+
import { RoutingLoad, } from '../routing/load.js';
|
|
17
|
+
import { durabilityLogLeaseMigrations, durabilityLogMigrations, } from './migrations.js';
|
|
18
|
+
import { migrateCapability } from '../migrations.js';
|
|
19
|
+
import { assertPositiveInteger } from '../policy.js';
|
|
20
|
+
import { assertChannel, defaultChannel, engineFor, } from '../scheduler/index.js';
|
|
21
|
+
import { deserialize, serialize } from '../serialization.js';
|
|
22
|
+
const logRecordRowSchema = v.object({
|
|
23
|
+
offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
24
|
+
dedup_key: v.string(),
|
|
25
|
+
payload: v.string(),
|
|
26
|
+
bytes: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
27
|
+
appended_at: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
27
28
|
});
|
|
28
|
-
const logCursorRowSchema =
|
|
29
|
-
consumer:
|
|
30
|
-
offset:
|
|
31
|
-
committed_at:
|
|
29
|
+
const logCursorRowSchema = v.object({
|
|
30
|
+
consumer: v.string(),
|
|
31
|
+
offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
32
|
+
committed_at: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
32
33
|
});
|
|
33
34
|
/** One contiguous run of records moved out of the object into cold storage. */
|
|
34
|
-
const logSegmentRowSchema =
|
|
35
|
-
first_offset:
|
|
36
|
-
last_offset:
|
|
37
|
-
locator:
|
|
38
|
-
bytes:
|
|
39
|
-
flushed_at:
|
|
35
|
+
const logSegmentRowSchema = v.object({
|
|
36
|
+
first_offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
37
|
+
last_offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
38
|
+
locator: v.string(),
|
|
39
|
+
bytes: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
40
|
+
flushed_at: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
40
41
|
});
|
|
41
42
|
/**
|
|
42
43
|
* One in-flight or settled range of offsets held by a consumer. A range is
|
|
43
44
|
* redelivered under a fresh `batch_id`, so an ack from a previous holder is
|
|
44
45
|
* rejected rather than advancing a range someone else now owns.
|
|
45
46
|
*/
|
|
46
|
-
const logLeaseRowSchema =
|
|
47
|
-
batch_id:
|
|
48
|
-
consumer:
|
|
49
|
-
first_offset:
|
|
50
|
-
last_offset:
|
|
51
|
-
state:
|
|
52
|
-
attempt:
|
|
53
|
-
expires_at:
|
|
54
|
-
created_at:
|
|
47
|
+
const logLeaseRowSchema = v.object({
|
|
48
|
+
batch_id: v.string(),
|
|
49
|
+
consumer: v.string(),
|
|
50
|
+
first_offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
51
|
+
last_offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
52
|
+
state: v.picklist(['active', 'pending', 'acked', 'skipped']),
|
|
53
|
+
attempt: v.pipe(v.number(), v.safeInteger(), v.gtValue(0)),
|
|
54
|
+
expires_at: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
55
|
+
created_at: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
55
56
|
});
|
|
56
|
-
const allocationRowSchema =
|
|
57
|
-
allocated_offset:
|
|
57
|
+
const allocationRowSchema = v.object({
|
|
58
|
+
allocated_offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
58
59
|
});
|
|
59
|
-
const offsetRowSchema =
|
|
60
|
-
|
|
61
|
-
const countRowSchema = z.object({ count: z.number().int().nonnegative() });
|
|
62
|
-
const totalBytesRowSchema = z.object({
|
|
63
|
-
total_bytes: z.number().int().nonnegative(),
|
|
60
|
+
const offsetRowSchema = v.object({
|
|
61
|
+
offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
64
62
|
});
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
63
|
+
const seqRowSchema = v.object({
|
|
64
|
+
seq: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
65
|
+
});
|
|
66
|
+
const countRowSchema = v.object({
|
|
67
|
+
count: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
68
|
+
});
|
|
69
|
+
const totalBytesRowSchema = v.object({
|
|
70
|
+
total_bytes: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
71
|
+
});
|
|
72
|
+
const boundsRowSchema = v.object({
|
|
73
|
+
hot: v.nullable(v.pipe(v.number(), v.safeInteger(), v.minValue(0))),
|
|
74
|
+
cold: v.nullable(v.pipe(v.number(), v.safeInteger(), v.minValue(0))),
|
|
75
|
+
next_offset: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
69
76
|
});
|
|
70
77
|
/**
|
|
71
78
|
* Inserts the batch and returns each record's offset in input order. A record
|
|
@@ -73,12 +80,12 @@ const boundsRowSchema = z.object({
|
|
|
73
80
|
* consumes no new one, so a repeated append is idempotent while that record is
|
|
74
81
|
* still held in the object. Runs inside the caller's transaction.
|
|
75
82
|
*/
|
|
76
|
-
const appendRecords = (sql, records, appendedAt) => {
|
|
83
|
+
const appendRecords = (sql, channel, records, appendedAt) => {
|
|
77
84
|
const assigned = new Map();
|
|
78
85
|
for (const record of records) {
|
|
79
|
-
const existing =
|
|
86
|
+
const existing = v.parse(v.optional(offsetRowSchema), sql
|
|
80
87
|
.exec(`SELECT "offset" FROM durability_log_records
|
|
81
|
-
WHERE dedup_key = ? LIMIT 1`, record.dedup_key)
|
|
88
|
+
WHERE channel = ? AND dedup_key = ? LIMIT 1`, channel, record.dedup_key)
|
|
82
89
|
.toArray()[0]);
|
|
83
90
|
if (existing !== undefined) {
|
|
84
91
|
assigned.set(record.dedup_key, existing.offset);
|
|
@@ -89,11 +96,13 @@ const appendRecords = (sql, records, appendedAt) => {
|
|
|
89
96
|
if (distinct.length === 0) {
|
|
90
97
|
return records.map((record) => assigned.get(record.dedup_key));
|
|
91
98
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
99
|
+
// A channel's counter row is created by its first append.
|
|
100
|
+
const first = v.parse(seqRowSchema, sql
|
|
101
|
+
.exec(`INSERT INTO durability_log_seq (channel, next_offset, total_bytes)
|
|
102
|
+
VALUES (?, ?, 0)
|
|
103
|
+
ON CONFLICT(channel) DO UPDATE
|
|
104
|
+
SET next_offset = durability_log_seq.next_offset + excluded.next_offset
|
|
105
|
+
RETURNING next_offset - ? AS seq`, channel, distinct.length, distinct.length)
|
|
97
106
|
.toArray()[0]).seq;
|
|
98
107
|
let added = 0;
|
|
99
108
|
distinct.forEach(([dedupKey, row], index) => {
|
|
@@ -101,27 +110,31 @@ const appendRecords = (sql, records, appendedAt) => {
|
|
|
101
110
|
assigned.set(dedupKey, offset);
|
|
102
111
|
added += row.bytes;
|
|
103
112
|
sql.exec(`INSERT INTO durability_log_records
|
|
104
|
-
("offset", dedup_key, payload, bytes, appended_at)
|
|
105
|
-
VALUES (?, ?, ?, ?, ?)
|
|
106
|
-
ON CONFLICT(dedup_key) DO NOTHING`, offset, dedupKey, row.payload, row.bytes, appendedAt);
|
|
113
|
+
(channel, "offset", dedup_key, payload, bytes, appended_at)
|
|
114
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
115
|
+
ON CONFLICT(channel, dedup_key) DO NOTHING`, channel, offset, dedupKey, row.payload, row.bytes, appendedAt);
|
|
107
116
|
});
|
|
108
117
|
sql.exec(`UPDATE durability_log_seq SET total_bytes = total_bytes + ?
|
|
109
|
-
WHERE
|
|
118
|
+
WHERE channel = ?`, added, channel);
|
|
110
119
|
return records.map((record) => assigned.get(record.dedup_key));
|
|
111
120
|
};
|
|
112
|
-
const readRecords = (sql, from, limit) =>
|
|
121
|
+
const readRecords = (sql, channel, from, limit) => v.parse(v.array(logRecordRowSchema), sql
|
|
113
122
|
.exec(`SELECT * FROM durability_log_records
|
|
114
|
-
WHERE "offset" >= ?
|
|
123
|
+
WHERE channel = ? AND "offset" >= ?
|
|
115
124
|
ORDER BY "offset" ASC
|
|
116
|
-
LIMIT ?`, from, limit)
|
|
125
|
+
LIMIT ?`, channel, from, limit)
|
|
117
126
|
.toArray());
|
|
118
|
-
const readBounds = (sql) => {
|
|
119
|
-
const row =
|
|
127
|
+
const readBounds = (sql, channel) => {
|
|
128
|
+
const row = v.parse(boundsRowSchema, sql
|
|
120
129
|
.exec(`SELECT
|
|
121
|
-
(SELECT MIN("offset") FROM durability_log_records
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
130
|
+
(SELECT MIN("offset") FROM durability_log_records
|
|
131
|
+
WHERE channel = ?1) AS hot,
|
|
132
|
+
(SELECT MIN(first_offset) FROM durability_log_segments
|
|
133
|
+
WHERE channel = ?1) AS cold,
|
|
134
|
+
COALESCE(
|
|
135
|
+
(SELECT next_offset FROM durability_log_seq WHERE channel = ?1),
|
|
136
|
+
0
|
|
137
|
+
) AS next_offset`, channel)
|
|
125
138
|
.toArray()[0]);
|
|
126
139
|
const hotOffset = row.hot ?? row.next_offset;
|
|
127
140
|
return {
|
|
@@ -130,138 +143,148 @@ const readBounds = (sql) => {
|
|
|
130
143
|
nextOffset: row.next_offset,
|
|
131
144
|
};
|
|
132
145
|
};
|
|
133
|
-
const countRecords = (sql) =>
|
|
134
|
-
.exec('SELECT COUNT(*) AS count FROM durability_log_records')
|
|
146
|
+
const countRecords = (sql, channel) => v.parse(countRowSchema, sql
|
|
147
|
+
.exec('SELECT COUNT(*) AS count FROM durability_log_records WHERE channel = ?', channel)
|
|
135
148
|
.toArray()[0]).count;
|
|
136
149
|
/** Bytes of record payloads held in the object, excluding flushed segments. */
|
|
137
|
-
const totalBytes = (sql) =>
|
|
138
|
-
.exec(
|
|
150
|
+
const totalBytes = (sql, channel) => v.parse(totalBytesRowSchema, sql
|
|
151
|
+
.exec(`SELECT COALESCE(
|
|
152
|
+
(SELECT total_bytes FROM durability_log_seq WHERE channel = ?),
|
|
153
|
+
0
|
|
154
|
+
) AS total_bytes`, channel)
|
|
139
155
|
.toArray()[0]).total_bytes;
|
|
140
156
|
/** Oldest retained records in offset order, for flush-or-delete decisions. */
|
|
141
|
-
const listOldest = (sql, limit) =>
|
|
157
|
+
const listOldest = (sql, channel, limit) => v.parse(v.array(logRecordRowSchema), sql
|
|
142
158
|
.exec(`SELECT * FROM durability_log_records
|
|
159
|
+
WHERE channel = ?
|
|
143
160
|
ORDER BY "offset" ASC
|
|
144
|
-
LIMIT ?`, limit)
|
|
161
|
+
LIMIT ?`, channel, limit)
|
|
145
162
|
.toArray());
|
|
146
163
|
/**
|
|
147
164
|
* Deletes at most `limit` records below `through`, oldest first, and returns
|
|
148
165
|
* how many were removed. Runs inside the caller's transaction.
|
|
149
166
|
*/
|
|
150
|
-
const trimThrough = (sql, through, limit) => {
|
|
151
|
-
const removed =
|
|
167
|
+
const trimThrough = (sql, channel, through, limit) => {
|
|
168
|
+
const removed = v.parse(v.array(logRecordRowSchema), sql
|
|
152
169
|
.exec(`DELETE FROM durability_log_records
|
|
153
|
-
WHERE "offset" IN (
|
|
170
|
+
WHERE channel = ?1 AND "offset" IN (
|
|
154
171
|
SELECT "offset" FROM durability_log_records
|
|
155
|
-
WHERE "offset" < ?
|
|
172
|
+
WHERE channel = ?1 AND "offset" < ?2
|
|
156
173
|
ORDER BY "offset" ASC
|
|
157
|
-
LIMIT ?
|
|
174
|
+
LIMIT ?3
|
|
158
175
|
)
|
|
159
|
-
RETURNING *`, through, limit)
|
|
176
|
+
RETURNING *`, channel, through, limit)
|
|
160
177
|
.toArray());
|
|
161
178
|
if (removed.length === 0) {
|
|
162
179
|
return 0;
|
|
163
180
|
}
|
|
164
181
|
sql.exec(`UPDATE durability_log_seq SET total_bytes = MAX(0, total_bytes - ?)
|
|
165
|
-
WHERE
|
|
182
|
+
WHERE channel = ?`, removed.reduce((total, row) => total + row.bytes, 0), channel);
|
|
166
183
|
return removed.length;
|
|
167
184
|
};
|
|
168
185
|
/** Records where a flushed run now lives, so a read can rehydrate it. */
|
|
169
|
-
const insertSegment = (sql, row) => {
|
|
186
|
+
const insertSegment = (sql, channel, row) => {
|
|
170
187
|
sql.exec(`INSERT INTO durability_log_segments
|
|
171
|
-
(first_offset, last_offset, locator, bytes, flushed_at)
|
|
172
|
-
VALUES (?, ?, ?, ?, ?)
|
|
173
|
-
ON CONFLICT(first_offset) DO UPDATE SET
|
|
188
|
+
(channel, first_offset, last_offset, locator, bytes, flushed_at)
|
|
189
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
190
|
+
ON CONFLICT(channel, first_offset) DO UPDATE SET
|
|
174
191
|
last_offset = excluded.last_offset,
|
|
175
192
|
locator = excluded.locator,
|
|
176
193
|
bytes = excluded.bytes,
|
|
177
|
-
flushed_at = excluded.flushed_at`, row.first_offset, row.last_offset, row.locator, row.bytes, row.flushed_at);
|
|
194
|
+
flushed_at = excluded.flushed_at`, channel, row.first_offset, row.last_offset, row.locator, row.bytes, row.flushed_at);
|
|
178
195
|
};
|
|
179
196
|
/** The segment holding an offset, if that offset was flushed. */
|
|
180
|
-
const findSegment = (sql, offset) =>
|
|
197
|
+
const findSegment = (sql, channel, offset) => v.parse(v.optional(logSegmentRowSchema), sql
|
|
181
198
|
.exec(`SELECT * FROM durability_log_segments
|
|
182
|
-
WHERE first_offset <= ? AND last_offset >= ?
|
|
183
|
-
LIMIT 1`,
|
|
199
|
+
WHERE channel = ?1 AND first_offset <= ?2 AND last_offset >= ?2
|
|
200
|
+
LIMIT 1`, channel, offset)
|
|
184
201
|
.toArray()[0]);
|
|
185
|
-
const readCursors = (sql) => logCursorRowSchema
|
|
186
|
-
.
|
|
187
|
-
|
|
188
|
-
|
|
202
|
+
const readCursors = (sql, channel) => v.parse(v.array(logCursorRowSchema), sql
|
|
203
|
+
.exec(`SELECT * FROM durability_log_cursors
|
|
204
|
+
WHERE channel = ?
|
|
205
|
+
ORDER BY consumer`, channel)
|
|
189
206
|
.toArray());
|
|
190
|
-
const readCursor = (sql, consumer) =>
|
|
207
|
+
const readCursor = (sql, channel, consumer) => v.parse(v.optional(offsetRowSchema), sql
|
|
191
208
|
.exec(`SELECT "offset" FROM durability_log_cursors
|
|
192
|
-
WHERE consumer = ? LIMIT 1`, consumer)
|
|
209
|
+
WHERE channel = ? AND consumer = ? LIMIT 1`, channel, consumer)
|
|
193
210
|
.toArray()[0])?.offset;
|
|
194
211
|
/** Advances a cursor, ignoring regressions so a late commit cannot rewind it. */
|
|
195
|
-
const commitCursor = (sql, consumer, offset, committedAt) => {
|
|
196
|
-
sql.exec(`INSERT INTO durability_log_cursors
|
|
197
|
-
|
|
198
|
-
|
|
212
|
+
const commitCursor = (sql, channel, consumer, offset, committedAt) => {
|
|
213
|
+
sql.exec(`INSERT INTO durability_log_cursors
|
|
214
|
+
(channel, consumer, "offset", committed_at)
|
|
215
|
+
VALUES (?, ?, ?, ?)
|
|
216
|
+
ON CONFLICT(channel, consumer) DO UPDATE
|
|
199
217
|
SET "offset" = excluded."offset", committed_at = excluded.committed_at
|
|
200
|
-
WHERE excluded."offset" > durability_log_cursors."offset"`, consumer, offset, committedAt);
|
|
218
|
+
WHERE excluded."offset" > durability_log_cursors."offset"`, channel, consumer, offset, committedAt);
|
|
201
219
|
};
|
|
202
220
|
/** Highest offset handed to a lease, which runs ahead of the commit. */
|
|
203
|
-
const readAllocation = (sql, consumer) =>
|
|
221
|
+
const readAllocation = (sql, channel, consumer) => v.parse(v.optional(allocationRowSchema), sql
|
|
204
222
|
.exec(`SELECT allocated_offset FROM durability_log_allocations
|
|
205
|
-
WHERE consumer = ? LIMIT 1`, consumer)
|
|
223
|
+
WHERE channel = ? AND consumer = ? LIMIT 1`, channel, consumer)
|
|
206
224
|
.toArray()[0])?.allocated_offset;
|
|
207
|
-
const setAllocation = (sql, consumer, offset) => {
|
|
208
|
-
sql.exec(`INSERT INTO durability_log_allocations
|
|
209
|
-
|
|
210
|
-
|
|
225
|
+
const setAllocation = (sql, channel, consumer, offset) => {
|
|
226
|
+
sql.exec(`INSERT INTO durability_log_allocations
|
|
227
|
+
(channel, consumer, allocated_offset)
|
|
228
|
+
VALUES (?, ?, ?)
|
|
229
|
+
ON CONFLICT(channel, consumer) DO UPDATE
|
|
230
|
+
SET allocated_offset = excluded.allocated_offset`, channel, consumer, offset);
|
|
211
231
|
};
|
|
212
|
-
const insertLease = (sql, row) => {
|
|
232
|
+
const insertLease = (sql, channel, row) => {
|
|
213
233
|
sql.exec(`INSERT INTO durability_log_leases
|
|
214
|
-
(batch_id, consumer, first_offset, last_offset, state, attempt,
|
|
234
|
+
(channel, batch_id, consumer, first_offset, last_offset, state, attempt,
|
|
215
235
|
expires_at, created_at)
|
|
216
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, row.batch_id, row.consumer, row.first_offset, row.last_offset, row.state, row.attempt, row.expires_at, row.created_at);
|
|
236
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, channel, row.batch_id, row.consumer, row.first_offset, row.last_offset, row.state, row.attempt, row.expires_at, row.created_at);
|
|
217
237
|
};
|
|
218
238
|
/**
|
|
219
239
|
* Oldest range owed redelivery: explicitly nacked, or held past its expiry.
|
|
220
240
|
* Claiming rewrites its `batch_id`, fencing the previous holder.
|
|
221
241
|
*/
|
|
222
|
-
const claimExpired = (sql, consumer, now, batchId, expiresAt) =>
|
|
242
|
+
const claimExpired = (sql, channel, consumer, now, batchId, expiresAt) => v.parse(v.optional(logLeaseRowSchema), sql
|
|
223
243
|
.exec(`UPDATE durability_log_leases
|
|
224
|
-
SET batch_id =
|
|
225
|
-
expires_at = ?
|
|
226
|
-
WHERE batch_id = (
|
|
244
|
+
SET batch_id = ?1, state = 'active', attempt = attempt + 1,
|
|
245
|
+
expires_at = ?2
|
|
246
|
+
WHERE channel = ?3 AND batch_id = (
|
|
227
247
|
SELECT batch_id FROM durability_log_leases
|
|
228
|
-
WHERE
|
|
248
|
+
WHERE channel = ?3
|
|
249
|
+
AND consumer = ?4
|
|
229
250
|
AND (state = 'pending'
|
|
230
|
-
OR (state = 'active' AND expires_at <= ?))
|
|
251
|
+
OR (state = 'active' AND expires_at <= ?5))
|
|
231
252
|
ORDER BY first_offset ASC
|
|
232
253
|
LIMIT 1
|
|
233
254
|
)
|
|
234
|
-
RETURNING *`, batchId, expiresAt, consumer, now)
|
|
255
|
+
RETURNING *`, batchId, expiresAt, channel, consumer, now)
|
|
235
256
|
.toArray()[0]);
|
|
236
257
|
/** Ranges still held, so parallelism can be bounded per consumer. */
|
|
237
|
-
const countHeld = (sql, consumer, now) =>
|
|
258
|
+
const countHeld = (sql, channel, consumer, now) => v.parse(countRowSchema, sql
|
|
238
259
|
.exec(`SELECT COUNT(*) AS count FROM durability_log_leases
|
|
239
|
-
WHERE
|
|
240
|
-
AND
|
|
260
|
+
WHERE channel = ?
|
|
261
|
+
AND consumer = ?
|
|
262
|
+
AND (state = 'pending' OR (state = 'active' AND expires_at > ?))`, channel, consumer, now)
|
|
241
263
|
.toArray()[0]).count;
|
|
242
264
|
/** Settles one range, failing when the batch was fenced or already settled. */
|
|
243
|
-
const settleLease = (sql, batchId, state, expiresAt) => sql
|
|
265
|
+
const settleLease = (sql, channel, batchId, state, expiresAt) => sql
|
|
244
266
|
.exec(`UPDATE durability_log_leases
|
|
245
267
|
SET state = ?, expires_at = ?
|
|
246
|
-
WHERE batch_id = ? AND state = 'active'
|
|
247
|
-
RETURNING batch_id`, state, expiresAt, batchId)
|
|
268
|
+
WHERE channel = ? AND batch_id = ? AND state = 'active'
|
|
269
|
+
RETURNING batch_id`, state, expiresAt, channel, batchId)
|
|
248
270
|
.toArray().length > 0;
|
|
249
271
|
/**
|
|
250
272
|
* Ranges from `from` in offset order, so a commit can advance over the
|
|
251
273
|
* unbroken run of settled ones and no further.
|
|
252
274
|
*/
|
|
253
|
-
const leasesFrom = (sql, consumer, from, limit) =>
|
|
275
|
+
const leasesFrom = (sql, channel, consumer, from, limit) => v.parse(v.array(logLeaseRowSchema), sql
|
|
254
276
|
.exec(`SELECT * FROM durability_log_leases
|
|
255
|
-
WHERE consumer = ? AND last_offset >= ?
|
|
277
|
+
WHERE channel = ? AND consumer = ? AND last_offset >= ?
|
|
256
278
|
ORDER BY first_offset ASC
|
|
257
|
-
LIMIT ?`, consumer, from, limit)
|
|
279
|
+
LIMIT ?`, channel, consumer, from, limit)
|
|
258
280
|
.toArray());
|
|
259
|
-
const deleteLeases = (sql, batchIds) => {
|
|
281
|
+
const deleteLeases = (sql, channel, batchIds) => {
|
|
260
282
|
if (batchIds.length === 0) {
|
|
261
283
|
return;
|
|
262
284
|
}
|
|
263
285
|
sql.exec(`DELETE FROM durability_log_leases
|
|
264
|
-
WHERE
|
|
286
|
+
WHERE channel = ?
|
|
287
|
+
AND batch_id IN (${batchIds.map(() => '?').join(', ')})`, channel, ...batchIds);
|
|
265
288
|
};
|
|
266
289
|
/**
|
|
267
290
|
* A single object holds a hot window, not an unbounded log, so the byte
|
|
@@ -282,6 +305,7 @@ export class DurabilityLog {
|
|
|
282
305
|
constructor(config) {
|
|
283
306
|
_DurabilityLog_instances.add(this);
|
|
284
307
|
_DurabilityLog_engine.set(this, void 0);
|
|
308
|
+
_DurabilityLog_channel.set(this, void 0);
|
|
285
309
|
_DurabilityLog_routing.set(this, void 0);
|
|
286
310
|
_DurabilityLog_retention.set(this, void 0);
|
|
287
311
|
_DurabilityLog_cold.set(this, void 0);
|
|
@@ -290,6 +314,7 @@ export class DurabilityLog {
|
|
|
290
314
|
_DurabilityLog_leaseMs.set(this, 30000);
|
|
291
315
|
_DurabilityLog_maxBatchSize.set(this, void 0);
|
|
292
316
|
_DurabilityLog_encoder.set(this, new TextEncoder());
|
|
317
|
+
__classPrivateFieldSet(this, _DurabilityLog_channel, assertChannel('DurabilityLog', config.channel ?? defaultChannel), "f");
|
|
293
318
|
__classPrivateFieldSet(this, _DurabilityLog_maxBatchSize, config.maxBatchSize ?? 100, "f");
|
|
294
319
|
assertPositiveInteger('maxBatchSize', __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
295
320
|
if (config.retention?.maxAgeMs !== undefined) {
|
|
@@ -304,7 +329,6 @@ export class DurabilityLog {
|
|
|
304
329
|
throw new RangeError(`retention.maxBytes must not exceed ${maxLogBytes} bytes`);
|
|
305
330
|
}
|
|
306
331
|
}
|
|
307
|
-
__classPrivateFieldSet(this, _DurabilityLog_engine, engineFor(config), "f");
|
|
308
332
|
__classPrivateFieldSet(this, _DurabilityLog_routing, config.routing ?? new RoutingLoad(), "f");
|
|
309
333
|
__classPrivateFieldSet(this, _DurabilityLog_retention, config.retention, "f");
|
|
310
334
|
__classPrivateFieldSet(this, _DurabilityLog_cold, config.cold, "f");
|
|
@@ -317,9 +341,13 @@ export class DurabilityLog {
|
|
|
317
341
|
if (__classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxAttempts !== undefined) {
|
|
318
342
|
assertPositiveInteger('leases.maxAttempts', __classPrivateFieldGet(this, _DurabilityLog_leases, "f").maxAttempts);
|
|
319
343
|
}
|
|
320
|
-
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('logLeases', durabilityLogLeaseMigrations);
|
|
321
344
|
}
|
|
345
|
+
__classPrivateFieldSet(this, _DurabilityLog_engine, engineFor(config), "f");
|
|
322
346
|
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('log', durabilityLogMigrations);
|
|
347
|
+
if (__classPrivateFieldGet(this, _DurabilityLog_leases, "f")) {
|
|
348
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('logLeases', durabilityLogLeaseMigrations);
|
|
349
|
+
}
|
|
350
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").claimChannel('durability_log_records', __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), 'DurabilityLog');
|
|
323
351
|
}
|
|
324
352
|
static migrate(context, target) {
|
|
325
353
|
return migrateCapability(context.storage, 'log', durabilityLogMigrations, target);
|
|
@@ -364,7 +392,7 @@ export class DurabilityLog {
|
|
|
364
392
|
};
|
|
365
393
|
}
|
|
366
394
|
const appendedAt = Date.now();
|
|
367
|
-
const offsets = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => appendRecords(sql, rows, appendedAt));
|
|
395
|
+
const offsets = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => appendRecords(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), rows, appendedAt));
|
|
368
396
|
return { success: true, value: offsets, load: await this.load() };
|
|
369
397
|
}
|
|
370
398
|
finally {
|
|
@@ -382,14 +410,14 @@ export class DurabilityLog {
|
|
|
382
410
|
if (!Number.isInteger(options.from) || options.from < 0) {
|
|
383
411
|
throw new LogAppendError('Read offset must be a non-negative integer');
|
|
384
412
|
}
|
|
385
|
-
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
413
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
386
414
|
if (options.from < bounds.oldestOffset) {
|
|
387
415
|
throw new LogTruncatedError(options.from, bounds.oldestOffset);
|
|
388
416
|
}
|
|
389
417
|
const capped = Math.min(limit, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
390
418
|
const records = options.from < bounds.hotOffset
|
|
391
419
|
? await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, options.from, capped, bounds)
|
|
392
|
-
: readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, options.from, capped).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
420
|
+
: readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), options.from, capped).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
393
421
|
const nextOffset = records.length === 0
|
|
394
422
|
? Math.max(options.from, bounds.nextOffset)
|
|
395
423
|
: records[records.length - 1].offset + 1;
|
|
@@ -401,21 +429,24 @@ export class DurabilityLog {
|
|
|
401
429
|
}
|
|
402
430
|
/** The retained window. `oldestOffset` equals `nextOffset` when empty. */
|
|
403
431
|
async bounds() {
|
|
404
|
-
return readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
432
|
+
return readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
405
433
|
}
|
|
406
434
|
/** Undefined until the consumer commits, so callers choose their own start. */
|
|
407
435
|
async cursor(consumer) {
|
|
408
|
-
return readCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, consumer);
|
|
436
|
+
return readCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer);
|
|
409
437
|
}
|
|
410
438
|
/** Records progress. Regressions are ignored, so a late commit cannot rewind. */
|
|
411
439
|
async commit(consumer, offset) {
|
|
412
440
|
if (!Number.isInteger(offset) || offset < 0) {
|
|
413
441
|
throw new LogAppendError('Commit offset must be a non-negative integer');
|
|
414
442
|
}
|
|
415
|
-
commitCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, consumer, offset, Date.now());
|
|
443
|
+
commitCursor(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, offset, Date.now());
|
|
416
444
|
}
|
|
417
445
|
async cursors() {
|
|
418
|
-
return Object.fromEntries(readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql).map((row) => [
|
|
446
|
+
return Object.fromEntries(readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f")).map((row) => [
|
|
447
|
+
row.consumer,
|
|
448
|
+
row.offset,
|
|
449
|
+
]));
|
|
419
450
|
}
|
|
420
451
|
/**
|
|
421
452
|
* Enforces retention on the oldest records: with cold storage they are
|
|
@@ -434,14 +465,16 @@ export class DurabilityLog {
|
|
|
434
465
|
const cutoff = maxAgeMs === undefined ? undefined : Date.now() - maxAgeMs;
|
|
435
466
|
let removed = 0;
|
|
436
467
|
for (;;) {
|
|
437
|
-
const oldest = listOldest(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
468
|
+
const oldest = listOldest(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
438
469
|
if (oldest.length === 0) {
|
|
439
470
|
return removed;
|
|
440
471
|
}
|
|
441
472
|
const excess = maxRecords === undefined
|
|
442
473
|
? 0
|
|
443
|
-
: Math.max(0, countRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql) - maxRecords);
|
|
444
|
-
const heldBytes = maxBytes === undefined
|
|
474
|
+
: Math.max(0, countRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f")) - maxRecords);
|
|
475
|
+
const heldBytes = maxBytes === undefined
|
|
476
|
+
? 0
|
|
477
|
+
: totalBytes(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
445
478
|
let overBytes = Math.max(0, heldBytes - (maxBytes ?? 0));
|
|
446
479
|
const expiring = takeWhile(oldest, (row, index) => {
|
|
447
480
|
const reclaiming = overBytes > 0;
|
|
@@ -459,6 +492,7 @@ export class DurabilityLog {
|
|
|
459
492
|
const records = expiring.map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
460
493
|
// eslint-disable-next-line no-await-in-loop
|
|
461
494
|
const locator = await __classPrivateFieldGet(this, _DurabilityLog_cold, "f").write({
|
|
495
|
+
channel: __classPrivateFieldGet(this, _DurabilityLog_channel, "f"),
|
|
462
496
|
firstOffset: first.offset,
|
|
463
497
|
lastOffset: last.offset,
|
|
464
498
|
records,
|
|
@@ -468,7 +502,7 @@ export class DurabilityLog {
|
|
|
468
502
|
}
|
|
469
503
|
// Indexed before deletion: a crash in between leaves the segment
|
|
470
504
|
// written and readable, never records that point nowhere.
|
|
471
|
-
insertSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, {
|
|
505
|
+
insertSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), {
|
|
472
506
|
first_offset: first.offset,
|
|
473
507
|
last_offset: last.offset,
|
|
474
508
|
locator,
|
|
@@ -476,7 +510,7 @@ export class DurabilityLog {
|
|
|
476
510
|
flushed_at: Date.now(),
|
|
477
511
|
});
|
|
478
512
|
}
|
|
479
|
-
const deleted = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => trimThrough(sql, last.offset + 1, expiring.length));
|
|
513
|
+
const deleted = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => trimThrough(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), last.offset + 1, expiring.length));
|
|
480
514
|
removed += deleted;
|
|
481
515
|
// Records are flushed before deletion, so stopping on partial progress
|
|
482
516
|
// keeps the next pass from writing the remainder a second time.
|
|
@@ -495,8 +529,8 @@ export class DurabilityLog {
|
|
|
495
529
|
}
|
|
496
530
|
return __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql
|
|
497
531
|
.exec(`DELETE FROM durability_log_segments
|
|
498
|
-
WHERE last_offset < ?
|
|
499
|
-
RETURNING first_offset`, offset)
|
|
532
|
+
WHERE channel = ? AND last_offset < ?
|
|
533
|
+
RETURNING first_offset`, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), offset)
|
|
500
534
|
.toArray().length;
|
|
501
535
|
}
|
|
502
536
|
/**
|
|
@@ -517,9 +551,9 @@ export class DurabilityLog {
|
|
|
517
551
|
assertConsumer(consumer);
|
|
518
552
|
const now = Date.now();
|
|
519
553
|
const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
|
|
520
|
-
const reclaimed = claimExpired(sql, consumer, now, crypto.randomUUID(), now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
|
|
554
|
+
const reclaimed = claimExpired(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, now, crypto.randomUUID(), now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
|
|
521
555
|
if (reclaimed) {
|
|
522
|
-
const { oldestOffset } = readBounds(sql);
|
|
556
|
+
const { oldestOffset } = readBounds(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
523
557
|
if (reclaimed.first_offset < oldestOffset) {
|
|
524
558
|
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_dropTrimmed).call(this, consumer, reclaimed, oldestOffset);
|
|
525
559
|
throw new LogTruncatedError(reclaimed.first_offset, oldestOffset);
|
|
@@ -550,16 +584,16 @@ export class DurabilityLog {
|
|
|
550
584
|
await leases.onPoison(lease);
|
|
551
585
|
}
|
|
552
586
|
catch (error) {
|
|
553
|
-
reportFailure('durability.log.poison_handler.failed', { consumer }, error);
|
|
587
|
+
reportFailure('durability.log.poison_handler.failed', { channel: __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer }, error);
|
|
554
588
|
}
|
|
555
589
|
return undefined;
|
|
556
590
|
}
|
|
557
|
-
if (countHeld(sql, consumer, now) >= __classPrivateFieldGet(this, _DurabilityLog_maxParallelism, "f")) {
|
|
591
|
+
if (countHeld(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, now) >= __classPrivateFieldGet(this, _DurabilityLog_maxParallelism, "f")) {
|
|
558
592
|
return undefined;
|
|
559
593
|
}
|
|
560
|
-
const bounds = readBounds(sql);
|
|
561
|
-
const committed = readCursor(sql, consumer);
|
|
562
|
-
const allocated = readAllocation(sql, consumer);
|
|
594
|
+
const bounds = readBounds(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
595
|
+
const committed = readCursor(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer);
|
|
596
|
+
const allocated = readAllocation(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer);
|
|
563
597
|
const from = Math.max(bounds.oldestOffset, committed ?? 0, allocated ?? 0);
|
|
564
598
|
const records = await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_readFrom).call(this, from, size, bounds);
|
|
565
599
|
if (records.length === 0) {
|
|
@@ -569,7 +603,7 @@ export class DurabilityLog {
|
|
|
569
603
|
const last = records[records.length - 1].offset;
|
|
570
604
|
const batchId = crypto.randomUUID();
|
|
571
605
|
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((transaction) => {
|
|
572
|
-
insertLease(transaction, {
|
|
606
|
+
insertLease(transaction, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), {
|
|
573
607
|
batch_id: batchId,
|
|
574
608
|
consumer,
|
|
575
609
|
first_offset: first,
|
|
@@ -579,7 +613,7 @@ export class DurabilityLog {
|
|
|
579
613
|
expires_at: now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"),
|
|
580
614
|
created_at: now,
|
|
581
615
|
});
|
|
582
|
-
setAllocation(transaction, consumer, last + 1);
|
|
616
|
+
setAllocation(transaction, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, last + 1);
|
|
583
617
|
});
|
|
584
618
|
return {
|
|
585
619
|
batchId,
|
|
@@ -611,10 +645,10 @@ export class DurabilityLog {
|
|
|
611
645
|
/** In-flight and settled-but-uncommitted ranges, for operators and tests. */
|
|
612
646
|
async leases(consumer) {
|
|
613
647
|
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_requireLeases).call(this);
|
|
614
|
-
return
|
|
648
|
+
return v.parse(v.array(logLeaseRowSchema), __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql
|
|
615
649
|
.exec(`SELECT * FROM durability_log_leases
|
|
616
|
-
WHERE consumer = ?
|
|
617
|
-
ORDER BY first_offset ASC`, consumer)
|
|
650
|
+
WHERE channel = ? AND consumer = ?
|
|
651
|
+
ORDER BY first_offset ASC`, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer)
|
|
618
652
|
.toArray());
|
|
619
653
|
}
|
|
620
654
|
/**
|
|
@@ -622,18 +656,18 @@ export class DurabilityLog {
|
|
|
622
656
|
* partitions by consumer lag. Without cursors there is no backlog to report.
|
|
623
657
|
*/
|
|
624
658
|
async load() {
|
|
625
|
-
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
626
|
-
const slowest = readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql).reduce((lowest, row) => lowest === undefined ? row.offset : Math.min(lowest, row.offset), undefined);
|
|
659
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
660
|
+
const slowest = readCursors(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f")).reduce((lowest, row) => lowest === undefined ? row.offset : Math.min(lowest, row.offset), undefined);
|
|
627
661
|
return __classPrivateFieldGet(this, _DurabilityLog_routing, "f").snapshot(slowest === undefined ? 0 : Math.max(0, bounds.nextOffset - slowest));
|
|
628
662
|
}
|
|
629
663
|
}
|
|
630
|
-
_DurabilityLog_engine = new WeakMap(), _DurabilityLog_routing = new WeakMap(), _DurabilityLog_retention = new WeakMap(), _DurabilityLog_cold = new WeakMap(), _DurabilityLog_leases = new WeakMap(), _DurabilityLog_maxParallelism = new WeakMap(), _DurabilityLog_leaseMs = new WeakMap(), _DurabilityLog_maxBatchSize = new WeakMap(), _DurabilityLog_encoder = new WeakMap(), _DurabilityLog_instances = new WeakSet(), _DurabilityLog_rehydrate =
|
|
664
|
+
_DurabilityLog_engine = new WeakMap(), _DurabilityLog_channel = new WeakMap(), _DurabilityLog_routing = new WeakMap(), _DurabilityLog_retention = new WeakMap(), _DurabilityLog_cold = new WeakMap(), _DurabilityLog_leases = new WeakMap(), _DurabilityLog_maxParallelism = new WeakMap(), _DurabilityLog_leaseMs = new WeakMap(), _DurabilityLog_maxBatchSize = new WeakMap(), _DurabilityLog_encoder = new WeakMap(), _DurabilityLog_instances = new WeakSet(), _DurabilityLog_rehydrate =
|
|
631
665
|
/**
|
|
632
666
|
* Cold reads cost a round trip to external storage and open input gates, so
|
|
633
667
|
* they serve only consumers that fell behind the hot window.
|
|
634
668
|
*/
|
|
635
669
|
async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
636
|
-
const segment = findSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, from);
|
|
670
|
+
const segment = findSegment(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), from);
|
|
637
671
|
const records = segment
|
|
638
672
|
? await __classPrivateFieldGet(this, _DurabilityLog_cold, "f")?.read(segment.locator)
|
|
639
673
|
: undefined;
|
|
@@ -654,7 +688,7 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
654
688
|
};
|
|
655
689
|
}, _DurabilityLog_settle = function _DurabilityLog_settle(batchId, state) {
|
|
656
690
|
// Pending ranges are immediately claimable, so their expiry is in the past.
|
|
657
|
-
const settled = settleLease(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, batchId, state, state === 'pending' ? 0 : Date.now());
|
|
691
|
+
const settled = settleLease(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), batchId, state, state === 'pending' ? 0 : Date.now());
|
|
658
692
|
if (settled) {
|
|
659
693
|
return;
|
|
660
694
|
}
|
|
@@ -664,13 +698,13 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
664
698
|
}, _DurabilityLog_dropTrimmed = function _DurabilityLog_dropTrimmed(consumer, lease, oldestOffset) {
|
|
665
699
|
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => {
|
|
666
700
|
if (lease.last_offset < oldestOffset) {
|
|
667
|
-
deleteLeases(sql, [lease.batch_id]);
|
|
701
|
+
deleteLeases(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), [lease.batch_id]);
|
|
668
702
|
}
|
|
669
703
|
else {
|
|
670
704
|
sql.exec(`UPDATE durability_log_leases
|
|
671
705
|
SET first_offset = ?, state = 'pending', attempt = attempt - 1,
|
|
672
706
|
expires_at = 0
|
|
673
|
-
WHERE batch_id = ?`, oldestOffset, lease.batch_id);
|
|
707
|
+
WHERE channel = ? AND batch_id = ?`, oldestOffset, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), lease.batch_id);
|
|
674
708
|
}
|
|
675
709
|
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_advance).call(this, consumer);
|
|
676
710
|
});
|
|
@@ -678,13 +712,13 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
678
712
|
const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
|
|
679
713
|
// Offsets below the retained window are gone for everyone; a cursor
|
|
680
714
|
// stranded there could never meet the ranges leased after them.
|
|
681
|
-
const from = Math.max(readCursor(sql, consumer) ?? 0, readBounds(sql).oldestOffset);
|
|
715
|
+
const from = Math.max(readCursor(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer) ?? 0, readBounds(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f")).oldestOffset);
|
|
682
716
|
let cursor = from;
|
|
683
717
|
const settled = [];
|
|
684
718
|
// Ranges settled while an earlier one was outstanding wait behind it; when
|
|
685
719
|
// it settles the cursor must pass all of them, however many pages that is.
|
|
686
720
|
for (let page = true; page;) {
|
|
687
|
-
const candidates = leasesFrom(sql, consumer, cursor, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
721
|
+
const candidates = leasesFrom(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, cursor, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
688
722
|
page = candidates.length === __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f");
|
|
689
723
|
for (const row of candidates) {
|
|
690
724
|
const contiguous = row.first_offset <= cursor;
|
|
@@ -700,12 +734,12 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
700
734
|
if (settled.length === 0) {
|
|
701
735
|
return;
|
|
702
736
|
}
|
|
703
|
-
commitCursor(sql, consumer, cursor, Date.now());
|
|
704
|
-
deleteLeases(sql, settled);
|
|
737
|
+
commitCursor(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), consumer, cursor, Date.now());
|
|
738
|
+
deleteLeases(sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), settled);
|
|
705
739
|
}, _DurabilityLog_between =
|
|
706
740
|
/** Reads a stored range, rehydrating it when retention already flushed it. */
|
|
707
741
|
async function _DurabilityLog_between(first, last) {
|
|
708
|
-
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
742
|
+
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"));
|
|
709
743
|
// One read stops at the end of a cold segment, and a range redelivered
|
|
710
744
|
// after a flush can span several segments and the hot window.
|
|
711
745
|
const records = [];
|
|
@@ -725,7 +759,7 @@ async function _DurabilityLog_between(first, last) {
|
|
|
725
759
|
if (from < bounds.hotOffset && from < bounds.nextOffset) {
|
|
726
760
|
return __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, from, limit, bounds);
|
|
727
761
|
}
|
|
728
|
-
return readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, from, limit).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
762
|
+
return readRecords(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql, __classPrivateFieldGet(this, _DurabilityLog_channel, "f"), from, limit).map((row) => __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_record).call(this, row));
|
|
729
763
|
}, _DurabilityLog_requireLeases = function _DurabilityLog_requireLeases() {
|
|
730
764
|
if (!__classPrivateFieldGet(this, _DurabilityLog_leases, "f")) {
|
|
731
765
|
throw new TypeError('Leases require the leases option');
|