durability 5.1.0 → 5.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -21
- package/dist/{durability-alarms.d.ts → alarms/index.d.ts} +13 -5
- package/dist/{durability-alarms.js → alarms/index.js} +85 -74
- package/dist/alarms/migrations.d.ts +9 -0
- package/dist/alarms/migrations.js +133 -0
- package/dist/experimental.d.ts +10 -10
- package/dist/experimental.js +7 -7
- package/dist/{fanout.d.ts → fanout/index.d.ts} +13 -6
- package/dist/{fanout.js → fanout/index.js} +91 -83
- package/dist/fanout/migrations.d.ts +9 -0
- package/dist/fanout/migrations.js +181 -0
- package/dist/{stored-message.d.ts → fanout/stored-message.d.ts} +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.js +13 -8
- package/dist/{log.d.ts → log/index.d.ts} +27 -20
- package/dist/{log.js → log/index.js} +200 -166
- package/dist/log/migrations.d.ts +18 -0
- package/dist/log/migrations.js +297 -0
- package/dist/{memoize.d.ts → memoize/index.d.ts} +1 -1
- package/dist/{memoize.js → memoize/index.js} +23 -21
- package/dist/memoize/migrations.d.ts +10 -0
- package/dist/memoize/migrations.js +21 -0
- package/dist/migrations.d.ts +0 -111
- package/dist/migrations.js +12 -628
- package/dist/{durability.d.ts → operations/index.d.ts} +13 -5
- package/dist/{durability.js → operations/index.js} +76 -70
- package/dist/operations/migrations.d.ts +10 -0
- package/dist/operations/migrations.js +150 -0
- package/dist/{outbox.d.ts → outbox/index.d.ts} +4 -4
- package/dist/{outbox.js → outbox/index.js} +35 -77
- package/dist/outbox/migrations.d.ts +9 -0
- package/dist/outbox/migrations.js +43 -0
- package/dist/policy.d.ts +19 -19
- package/dist/policy.js +13 -12
- package/dist/{routing-client.js → routing/client.js} +1 -1
- package/dist/{routing.d.ts → routing/index.d.ts} +3 -3
- package/dist/{routing.js → routing/index.js} +3 -3
- package/dist/{load.d.ts → routing/load.d.ts} +1 -1
- package/dist/{schedule.d.ts → schedule/index.d.ts} +16 -8
- package/dist/{schedule.js → schedule/index.js} +97 -83
- package/dist/schedule/migrations.d.ts +9 -0
- package/dist/schedule/migrations.js +118 -0
- package/dist/{scheduler.d.ts → scheduler/index.d.ts} +75 -25
- package/dist/{scheduler.js → scheduler/index.js} +308 -70
- package/dist/{semaphore-client.d.ts → semaphore/client.d.ts} +1 -1
- package/dist/{semaphore-client.js → semaphore/client.js} +8 -7
- package/dist/{semaphore.d.ts → semaphore/index.d.ts} +10 -10
- package/dist/{semaphore.js → semaphore/index.js} +7 -7
- package/dist/{semaphore-leases.d.ts → semaphore/leases.d.ts} +17 -6
- package/dist/{semaphore-leases.js → semaphore/leases.js} +28 -12
- package/dist/semaphore/migrations.d.ts +18 -0
- package/dist/semaphore/migrations.js +444 -0
- package/dist/semaphore/protocol.d.ts +451 -0
- package/dist/semaphore/protocol.js +114 -0
- package/dist/semaphore/queue-store.d.ts +107 -0
- package/dist/semaphore/queue-store.js +258 -0
- package/dist/{semaphore-queue.d.ts → semaphore/queue.d.ts} +15 -5
- package/dist/{semaphore-queue.js → semaphore/queue.js} +105 -46
- package/dist/{semaphore-server.d.ts → semaphore/server.d.ts} +12 -4
- package/dist/{semaphore-server.js → semaphore/server.js} +56 -28
- package/dist/semaphore/store.d.ts +53 -0
- package/dist/semaphore/store.js +107 -0
- package/dist/serialization.d.ts +5 -0
- package/dist/serialization.js +41 -16
- package/dist/{client.d.ts → socket/client.d.ts} +3 -3
- package/dist/{client.js → socket/client.js} +4 -4
- package/dist/{socket.d.ts → socket/index.d.ts} +15 -4
- package/dist/{socket.js → socket/index.js} +66 -20
- package/dist/socket/migrations.d.ts +9 -0
- package/dist/socket/migrations.js +57 -0
- package/dist/{socket-protocol.d.ts → socket/protocol.d.ts} +1 -1
- package/dist/storage.d.ts +4 -0
- package/dist/storage.js +22 -15
- package/dist/{topology-client.d.ts → topology/client.d.ts} +2 -2
- package/dist/{topology-client.js → topology/client.js} +3 -3
- package/dist/{topology.d.ts → topology/index.d.ts} +17 -7
- package/dist/{topology.js → topology/index.js} +106 -72
- package/dist/topology/migrations.d.ts +10 -0
- package/dist/topology/migrations.js +138 -0
- package/dist/topology/snapshot.d.ts +50 -0
- package/dist/topology/snapshot.js +38 -0
- package/dist/{tracker.d.ts → tracker/index.d.ts} +3 -3
- package/dist/{tracker.js → tracker/index.js} +9 -9
- package/package.json +11 -11
- package/dist/semaphore-protocol.d.ts +0 -231
- package/dist/semaphore-protocol.js +0 -106
- package/dist/semaphore-queue-store.d.ts +0 -95
- package/dist/semaphore-queue-store.js +0 -252
- package/dist/semaphore-store.d.ts +0 -43
- package/dist/semaphore-store.js +0 -108
- package/dist/topology-snapshot.d.ts +0 -70
- package/dist/topology-snapshot.js +0 -48
- /package/dist/{stored-message.js → fanout/stored-message.js} +0 -0
- /package/dist/{routing-client.d.ts → routing/client.d.ts} +0 -0
- /package/dist/{load.js → routing/load.js} +0 -0
- /package/dist/{socket-protocol.js → socket/protocol.js} +0 -0
- /package/dist/{tracked-source.d.ts → tracker/source.d.ts} +0 -0
- /package/dist/{tracked-source.js → tracker/source.js} +0 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ordered SQLite migrations for the retained log and its consumer cursors.
|
|
3
|
+
* Applied automatically on construction.
|
|
4
|
+
*/
|
|
5
|
+
export declare const durabilityLogMigrations: {
|
|
6
|
+
name: string;
|
|
7
|
+
up: string;
|
|
8
|
+
down: string;
|
|
9
|
+
}[];
|
|
10
|
+
/**
|
|
11
|
+
* Leases let several workers consume one log in parallel without handing the
|
|
12
|
+
* same offsets to two of them. Applied automatically when leases are enabled.
|
|
13
|
+
*/
|
|
14
|
+
export declare const durabilityLogLeaseMigrations: {
|
|
15
|
+
name: string;
|
|
16
|
+
up: string;
|
|
17
|
+
down: string;
|
|
18
|
+
}[];
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ordered SQLite migrations for the retained log and its consumer cursors.
|
|
3
|
+
* Applied automatically on construction.
|
|
4
|
+
*/
|
|
5
|
+
export const durabilityLogMigrations = [
|
|
6
|
+
{
|
|
7
|
+
name: 'durability_0007_create_log',
|
|
8
|
+
up: `
|
|
9
|
+
CREATE TABLE IF NOT EXISTS durability_log_records (
|
|
10
|
+
"offset" INTEGER PRIMARY KEY,
|
|
11
|
+
dedup_key TEXT NOT NULL UNIQUE,
|
|
12
|
+
payload TEXT NOT NULL,
|
|
13
|
+
bytes INTEGER NOT NULL,
|
|
14
|
+
appended_at INTEGER NOT NULL
|
|
15
|
+
);
|
|
16
|
+
CREATE INDEX IF NOT EXISTS durability_log_records_appended_idx
|
|
17
|
+
ON durability_log_records (appended_at, "offset");
|
|
18
|
+
CREATE TABLE IF NOT EXISTS durability_log_cursors (
|
|
19
|
+
consumer TEXT PRIMARY KEY,
|
|
20
|
+
"offset" INTEGER NOT NULL,
|
|
21
|
+
committed_at INTEGER NOT NULL
|
|
22
|
+
);
|
|
23
|
+
CREATE TABLE IF NOT EXISTS durability_log_segments (
|
|
24
|
+
first_offset INTEGER PRIMARY KEY,
|
|
25
|
+
last_offset INTEGER NOT NULL,
|
|
26
|
+
locator TEXT NOT NULL,
|
|
27
|
+
bytes INTEGER NOT NULL,
|
|
28
|
+
flushed_at INTEGER NOT NULL
|
|
29
|
+
);
|
|
30
|
+
CREATE TABLE IF NOT EXISTS durability_log_seq (
|
|
31
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
32
|
+
next_offset INTEGER NOT NULL,
|
|
33
|
+
total_bytes INTEGER NOT NULL
|
|
34
|
+
);
|
|
35
|
+
INSERT OR IGNORE INTO durability_log_seq (id, next_offset, total_bytes)
|
|
36
|
+
VALUES (1, 0, 0);
|
|
37
|
+
`,
|
|
38
|
+
down: `
|
|
39
|
+
DROP INDEX IF EXISTS durability_log_records_appended_idx;
|
|
40
|
+
DROP TABLE IF EXISTS durability_log_records;
|
|
41
|
+
DROP TABLE IF EXISTS durability_log_cursors;
|
|
42
|
+
DROP TABLE IF EXISTS durability_log_segments;
|
|
43
|
+
DROP TABLE IF EXISTS durability_log_seq;
|
|
44
|
+
`,
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
name: 'durability_0019_log_channels',
|
|
48
|
+
up: `
|
|
49
|
+
ALTER TABLE durability_log_records RENAME TO durability_log_records_before_channels;
|
|
50
|
+
CREATE TABLE durability_log_records (
|
|
51
|
+
channel TEXT NOT NULL,
|
|
52
|
+
"offset" INTEGER NOT NULL,
|
|
53
|
+
dedup_key TEXT NOT NULL,
|
|
54
|
+
payload TEXT NOT NULL,
|
|
55
|
+
bytes INTEGER NOT NULL,
|
|
56
|
+
appended_at INTEGER NOT NULL,
|
|
57
|
+
PRIMARY KEY (channel, "offset"),
|
|
58
|
+
UNIQUE (channel, dedup_key)
|
|
59
|
+
);
|
|
60
|
+
INSERT INTO durability_log_records
|
|
61
|
+
(channel, "offset", dedup_key, payload, bytes, appended_at)
|
|
62
|
+
SELECT 'default', "offset", dedup_key, payload, bytes, appended_at
|
|
63
|
+
FROM durability_log_records_before_channels;
|
|
64
|
+
DROP TABLE durability_log_records_before_channels;
|
|
65
|
+
CREATE INDEX durability_log_records_appended_idx
|
|
66
|
+
ON durability_log_records (channel, appended_at, "offset");
|
|
67
|
+
|
|
68
|
+
ALTER TABLE durability_log_cursors RENAME TO durability_log_cursors_before_channels;
|
|
69
|
+
CREATE TABLE durability_log_cursors (
|
|
70
|
+
channel TEXT NOT NULL,
|
|
71
|
+
consumer TEXT NOT NULL,
|
|
72
|
+
"offset" INTEGER NOT NULL,
|
|
73
|
+
committed_at INTEGER NOT NULL,
|
|
74
|
+
PRIMARY KEY (channel, consumer)
|
|
75
|
+
);
|
|
76
|
+
INSERT INTO durability_log_cursors (channel, consumer, "offset", committed_at)
|
|
77
|
+
SELECT 'default', consumer, "offset", committed_at
|
|
78
|
+
FROM durability_log_cursors_before_channels;
|
|
79
|
+
DROP TABLE durability_log_cursors_before_channels;
|
|
80
|
+
|
|
81
|
+
ALTER TABLE durability_log_segments RENAME TO durability_log_segments_before_channels;
|
|
82
|
+
CREATE TABLE durability_log_segments (
|
|
83
|
+
channel TEXT NOT NULL,
|
|
84
|
+
first_offset INTEGER NOT NULL,
|
|
85
|
+
last_offset INTEGER NOT NULL,
|
|
86
|
+
locator TEXT NOT NULL,
|
|
87
|
+
bytes INTEGER NOT NULL,
|
|
88
|
+
flushed_at INTEGER NOT NULL,
|
|
89
|
+
PRIMARY KEY (channel, first_offset)
|
|
90
|
+
);
|
|
91
|
+
INSERT INTO durability_log_segments
|
|
92
|
+
(channel, first_offset, last_offset, locator, bytes, flushed_at)
|
|
93
|
+
SELECT 'default', first_offset, last_offset, locator, bytes, flushed_at
|
|
94
|
+
FROM durability_log_segments_before_channels;
|
|
95
|
+
DROP TABLE durability_log_segments_before_channels;
|
|
96
|
+
|
|
97
|
+
ALTER TABLE durability_log_seq RENAME TO durability_log_seq_before_channels;
|
|
98
|
+
CREATE TABLE durability_log_seq (
|
|
99
|
+
channel TEXT PRIMARY KEY,
|
|
100
|
+
next_offset INTEGER NOT NULL,
|
|
101
|
+
total_bytes INTEGER NOT NULL
|
|
102
|
+
);
|
|
103
|
+
INSERT INTO durability_log_seq (channel, next_offset, total_bytes)
|
|
104
|
+
SELECT 'default', next_offset, total_bytes
|
|
105
|
+
FROM durability_log_seq_before_channels WHERE id = 1;
|
|
106
|
+
DROP TABLE durability_log_seq_before_channels;
|
|
107
|
+
`,
|
|
108
|
+
// A channel's offset counter is kept even once its records are gone, so
|
|
109
|
+
// it also blocks the downgrade: dropping it would let offsets repeat.
|
|
110
|
+
down: `
|
|
111
|
+
CREATE TABLE durability_log_channel_guard (
|
|
112
|
+
channel TEXT CONSTRAINT cannot_downgrade_nondefault_channels CHECK (channel = 'default')
|
|
113
|
+
);
|
|
114
|
+
INSERT INTO durability_log_channel_guard
|
|
115
|
+
SELECT channel FROM durability_log_records
|
|
116
|
+
UNION SELECT channel FROM durability_log_cursors
|
|
117
|
+
UNION SELECT channel FROM durability_log_segments
|
|
118
|
+
UNION SELECT channel FROM durability_log_seq;
|
|
119
|
+
DROP TABLE durability_log_channel_guard;
|
|
120
|
+
|
|
121
|
+
ALTER TABLE durability_log_records RENAME TO durability_log_records_with_channels;
|
|
122
|
+
CREATE TABLE durability_log_records (
|
|
123
|
+
"offset" INTEGER PRIMARY KEY,
|
|
124
|
+
dedup_key TEXT NOT NULL UNIQUE,
|
|
125
|
+
payload TEXT NOT NULL,
|
|
126
|
+
bytes INTEGER NOT NULL,
|
|
127
|
+
appended_at INTEGER NOT NULL
|
|
128
|
+
);
|
|
129
|
+
INSERT INTO durability_log_records
|
|
130
|
+
("offset", dedup_key, payload, bytes, appended_at)
|
|
131
|
+
SELECT "offset", dedup_key, payload, bytes, appended_at
|
|
132
|
+
FROM durability_log_records_with_channels;
|
|
133
|
+
DROP TABLE durability_log_records_with_channels;
|
|
134
|
+
CREATE INDEX durability_log_records_appended_idx
|
|
135
|
+
ON durability_log_records (appended_at, "offset");
|
|
136
|
+
|
|
137
|
+
ALTER TABLE durability_log_cursors RENAME TO durability_log_cursors_with_channels;
|
|
138
|
+
CREATE TABLE durability_log_cursors (
|
|
139
|
+
consumer TEXT PRIMARY KEY,
|
|
140
|
+
"offset" INTEGER NOT NULL,
|
|
141
|
+
committed_at INTEGER NOT NULL
|
|
142
|
+
);
|
|
143
|
+
INSERT INTO durability_log_cursors (consumer, "offset", committed_at)
|
|
144
|
+
SELECT consumer, "offset", committed_at
|
|
145
|
+
FROM durability_log_cursors_with_channels;
|
|
146
|
+
DROP TABLE durability_log_cursors_with_channels;
|
|
147
|
+
|
|
148
|
+
ALTER TABLE durability_log_segments RENAME TO durability_log_segments_with_channels;
|
|
149
|
+
CREATE TABLE durability_log_segments (
|
|
150
|
+
first_offset INTEGER PRIMARY KEY,
|
|
151
|
+
last_offset INTEGER NOT NULL,
|
|
152
|
+
locator TEXT NOT NULL,
|
|
153
|
+
bytes INTEGER NOT NULL,
|
|
154
|
+
flushed_at INTEGER NOT NULL
|
|
155
|
+
);
|
|
156
|
+
INSERT INTO durability_log_segments
|
|
157
|
+
(first_offset, last_offset, locator, bytes, flushed_at)
|
|
158
|
+
SELECT first_offset, last_offset, locator, bytes, flushed_at
|
|
159
|
+
FROM durability_log_segments_with_channels;
|
|
160
|
+
DROP TABLE durability_log_segments_with_channels;
|
|
161
|
+
|
|
162
|
+
ALTER TABLE durability_log_seq RENAME TO durability_log_seq_with_channels;
|
|
163
|
+
CREATE TABLE durability_log_seq (
|
|
164
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
165
|
+
next_offset INTEGER NOT NULL,
|
|
166
|
+
total_bytes INTEGER NOT NULL
|
|
167
|
+
);
|
|
168
|
+
INSERT INTO durability_log_seq (id, next_offset, total_bytes)
|
|
169
|
+
SELECT 1, next_offset, total_bytes
|
|
170
|
+
FROM durability_log_seq_with_channels WHERE channel = 'default';
|
|
171
|
+
INSERT OR IGNORE INTO durability_log_seq (id, next_offset, total_bytes)
|
|
172
|
+
VALUES (1, 0, 0);
|
|
173
|
+
DROP TABLE durability_log_seq_with_channels;
|
|
174
|
+
`,
|
|
175
|
+
},
|
|
176
|
+
];
|
|
177
|
+
/**
|
|
178
|
+
* Leases let several workers consume one log in parallel without handing the
|
|
179
|
+
* same offsets to two of them. Applied automatically when leases are enabled.
|
|
180
|
+
*/
|
|
181
|
+
export const durabilityLogLeaseMigrations = [
|
|
182
|
+
{
|
|
183
|
+
name: 'durability_0008_create_log_leases',
|
|
184
|
+
up: `
|
|
185
|
+
CREATE TABLE IF NOT EXISTS durability_log_leases (
|
|
186
|
+
batch_id TEXT PRIMARY KEY,
|
|
187
|
+
consumer TEXT NOT NULL,
|
|
188
|
+
first_offset INTEGER NOT NULL,
|
|
189
|
+
last_offset INTEGER NOT NULL,
|
|
190
|
+
state TEXT NOT NULL
|
|
191
|
+
CHECK (state IN ('active', 'pending', 'acked', 'skipped')),
|
|
192
|
+
attempt INTEGER NOT NULL,
|
|
193
|
+
expires_at INTEGER NOT NULL,
|
|
194
|
+
created_at INTEGER NOT NULL
|
|
195
|
+
);
|
|
196
|
+
CREATE INDEX IF NOT EXISTS durability_log_leases_claim_idx
|
|
197
|
+
ON durability_log_leases (consumer, state, expires_at, first_offset);
|
|
198
|
+
CREATE INDEX IF NOT EXISTS durability_log_leases_order_idx
|
|
199
|
+
ON durability_log_leases (consumer, first_offset);
|
|
200
|
+
CREATE TABLE IF NOT EXISTS durability_log_allocations (
|
|
201
|
+
consumer TEXT PRIMARY KEY,
|
|
202
|
+
allocated_offset INTEGER NOT NULL
|
|
203
|
+
);
|
|
204
|
+
`,
|
|
205
|
+
down: `
|
|
206
|
+
DROP INDEX IF EXISTS durability_log_leases_claim_idx;
|
|
207
|
+
DROP INDEX IF EXISTS durability_log_leases_order_idx;
|
|
208
|
+
DROP TABLE IF EXISTS durability_log_leases;
|
|
209
|
+
DROP TABLE IF EXISTS durability_log_allocations;
|
|
210
|
+
`,
|
|
211
|
+
},
|
|
212
|
+
{
|
|
213
|
+
name: 'durability_0024_log_lease_channels',
|
|
214
|
+
up: `
|
|
215
|
+
ALTER TABLE durability_log_leases RENAME TO durability_log_leases_before_channels;
|
|
216
|
+
CREATE TABLE durability_log_leases (
|
|
217
|
+
channel TEXT NOT NULL,
|
|
218
|
+
batch_id TEXT NOT NULL,
|
|
219
|
+
consumer TEXT NOT NULL,
|
|
220
|
+
first_offset INTEGER NOT NULL,
|
|
221
|
+
last_offset INTEGER NOT NULL,
|
|
222
|
+
state TEXT NOT NULL
|
|
223
|
+
CHECK (state IN ('active', 'pending', 'acked', 'skipped')),
|
|
224
|
+
attempt INTEGER NOT NULL,
|
|
225
|
+
expires_at INTEGER NOT NULL,
|
|
226
|
+
created_at INTEGER NOT NULL,
|
|
227
|
+
PRIMARY KEY (channel, batch_id)
|
|
228
|
+
);
|
|
229
|
+
INSERT INTO durability_log_leases
|
|
230
|
+
(channel, batch_id, consumer, first_offset, last_offset, state,
|
|
231
|
+
attempt, expires_at, created_at)
|
|
232
|
+
SELECT 'default', batch_id, consumer, first_offset, last_offset, state,
|
|
233
|
+
attempt, expires_at, created_at
|
|
234
|
+
FROM durability_log_leases_before_channels;
|
|
235
|
+
DROP TABLE durability_log_leases_before_channels;
|
|
236
|
+
CREATE INDEX durability_log_leases_claim_idx
|
|
237
|
+
ON durability_log_leases (channel, consumer, state, expires_at, first_offset);
|
|
238
|
+
CREATE INDEX durability_log_leases_order_idx
|
|
239
|
+
ON durability_log_leases (channel, consumer, first_offset);
|
|
240
|
+
|
|
241
|
+
ALTER TABLE durability_log_allocations RENAME TO durability_log_allocations_before_channels;
|
|
242
|
+
CREATE TABLE durability_log_allocations (
|
|
243
|
+
channel TEXT NOT NULL,
|
|
244
|
+
consumer TEXT NOT NULL,
|
|
245
|
+
allocated_offset INTEGER NOT NULL,
|
|
246
|
+
PRIMARY KEY (channel, consumer)
|
|
247
|
+
);
|
|
248
|
+
INSERT INTO durability_log_allocations (channel, consumer, allocated_offset)
|
|
249
|
+
SELECT 'default', consumer, allocated_offset
|
|
250
|
+
FROM durability_log_allocations_before_channels;
|
|
251
|
+
DROP TABLE durability_log_allocations_before_channels;
|
|
252
|
+
`,
|
|
253
|
+
down: `
|
|
254
|
+
CREATE TABLE durability_log_lease_channel_guard (
|
|
255
|
+
channel TEXT CONSTRAINT cannot_downgrade_nondefault_channels CHECK (channel = 'default')
|
|
256
|
+
);
|
|
257
|
+
INSERT INTO durability_log_lease_channel_guard
|
|
258
|
+
SELECT channel FROM durability_log_leases
|
|
259
|
+
UNION SELECT channel FROM durability_log_allocations;
|
|
260
|
+
DROP TABLE durability_log_lease_channel_guard;
|
|
261
|
+
|
|
262
|
+
ALTER TABLE durability_log_leases RENAME TO durability_log_leases_with_channels;
|
|
263
|
+
CREATE TABLE durability_log_leases (
|
|
264
|
+
batch_id TEXT PRIMARY KEY,
|
|
265
|
+
consumer TEXT NOT NULL,
|
|
266
|
+
first_offset INTEGER NOT NULL,
|
|
267
|
+
last_offset INTEGER NOT NULL,
|
|
268
|
+
state TEXT NOT NULL
|
|
269
|
+
CHECK (state IN ('active', 'pending', 'acked', 'skipped')),
|
|
270
|
+
attempt INTEGER NOT NULL,
|
|
271
|
+
expires_at INTEGER NOT NULL,
|
|
272
|
+
created_at INTEGER NOT NULL
|
|
273
|
+
);
|
|
274
|
+
INSERT INTO durability_log_leases
|
|
275
|
+
(batch_id, consumer, first_offset, last_offset, state, attempt,
|
|
276
|
+
expires_at, created_at)
|
|
277
|
+
SELECT batch_id, consumer, first_offset, last_offset, state, attempt,
|
|
278
|
+
expires_at, created_at
|
|
279
|
+
FROM durability_log_leases_with_channels;
|
|
280
|
+
DROP TABLE durability_log_leases_with_channels;
|
|
281
|
+
CREATE INDEX durability_log_leases_claim_idx
|
|
282
|
+
ON durability_log_leases (consumer, state, expires_at, first_offset);
|
|
283
|
+
CREATE INDEX durability_log_leases_order_idx
|
|
284
|
+
ON durability_log_leases (consumer, first_offset);
|
|
285
|
+
|
|
286
|
+
ALTER TABLE durability_log_allocations RENAME TO durability_log_allocations_with_channels;
|
|
287
|
+
CREATE TABLE durability_log_allocations (
|
|
288
|
+
consumer TEXT PRIMARY KEY,
|
|
289
|
+
allocated_offset INTEGER NOT NULL
|
|
290
|
+
);
|
|
291
|
+
INSERT INTO durability_log_allocations (consumer, allocated_offset)
|
|
292
|
+
SELECT consumer, allocated_offset
|
|
293
|
+
FROM durability_log_allocations_with_channels;
|
|
294
|
+
DROP TABLE durability_log_allocations_with_channels;
|
|
295
|
+
`,
|
|
296
|
+
},
|
|
297
|
+
];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AttemptRef, type DurableTable } from '
|
|
1
|
+
import { type AttemptRef, type DurableTable } from '../storage.js';
|
|
2
2
|
/**
|
|
3
3
|
* Runs `callback` once per key for a durable operation or named alarm
|
|
4
4
|
* occurrence and persists its result, so a retried attempt returns the result
|
|
@@ -1,21 +1,22 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { traceFunction } from '
|
|
3
|
-
import { DurableMemoLimitError, DurableMemoSerializationError, } from '
|
|
4
|
-
import { deserialize, serialize } from '
|
|
5
|
-
import { pendingGeneration, } from '
|
|
1
|
+
import * as v from 'valibot';
|
|
2
|
+
import { traceFunction } from '../dev.js';
|
|
3
|
+
import { DurableMemoLimitError, DurableMemoSerializationError, } from '../errors.js';
|
|
4
|
+
import { deserialize, serialize } from '../serialization.js';
|
|
5
|
+
import { channelBindings, pendingGeneration, tableOwner, } from '../storage.js';
|
|
6
6
|
/** The most keys one operation, alarm occurrence, or fanout delivery may memoize. */
|
|
7
7
|
export const maxMemosPerRecord = 1000;
|
|
8
8
|
/** The largest serialized memoized result, in UTF-8 bytes. */
|
|
9
9
|
export const maxMemoBytes = 1048576;
|
|
10
|
-
const memoLookupSchema =
|
|
11
|
-
result:
|
|
12
|
-
owned:
|
|
13
|
-
saved:
|
|
10
|
+
const memoLookupSchema = v.object({
|
|
11
|
+
result: v.nullable(v.string()),
|
|
12
|
+
owned: v.pipe(v.number(), v.safeInteger()),
|
|
13
|
+
saved: v.pipe(v.number(), v.safeInteger()),
|
|
14
14
|
});
|
|
15
15
|
const supersededError = (key) => new Error(`Memoized key ${JSON.stringify(key)} belongs to a superseded attempt`);
|
|
16
16
|
/** Memoization for one attempt; only that attempt may persist new results. */
|
|
17
17
|
export const createMemoize = (sql, table, ref) => {
|
|
18
18
|
const running = new Set();
|
|
19
|
+
const owner = tableOwner(table);
|
|
19
20
|
/** Serializes, checks, and writes a callback's result, then returns its persisted form. */
|
|
20
21
|
const save = (key, value) => {
|
|
21
22
|
let serialized;
|
|
@@ -38,7 +39,7 @@ export const createMemoize = (sql, table, ref) => {
|
|
|
38
39
|
SELECT 1 FROM ${table.name}
|
|
39
40
|
WHERE ${pendingGeneration(table)} AND attempt = ?
|
|
40
41
|
)
|
|
41
|
-
RETURNING memo_key`,
|
|
42
|
+
RETURNING memo_key`, owner, ref.key, ref.generation, key, serialized, ref.key, ref.generation, ...channelBindings(table), ref.attempt)
|
|
42
43
|
.toArray().length > 0;
|
|
43
44
|
if (!written) {
|
|
44
45
|
throw supersededError(key);
|
|
@@ -52,7 +53,7 @@ export const createMemoize = (sql, table, ref) => {
|
|
|
52
53
|
if (typeof key !== 'string' || key === '') {
|
|
53
54
|
throw new TypeError('Memoize keys must be non-empty strings');
|
|
54
55
|
}
|
|
55
|
-
const lookup =
|
|
56
|
+
const lookup = v.parse(memoLookupSchema, sql
|
|
56
57
|
.exec(`SELECT
|
|
57
58
|
(SELECT result FROM durability_memos
|
|
58
59
|
WHERE owner = ? AND key = ? AND generation_id = ? AND memo_key = ?)
|
|
@@ -62,7 +63,7 @@ export const createMemoize = (sql, table, ref) => {
|
|
|
62
63
|
WHERE ${pendingGeneration(table)} AND attempt = ?
|
|
63
64
|
) AS owned,
|
|
64
65
|
(SELECT COUNT(*) FROM durability_memos
|
|
65
|
-
WHERE owner = ? AND key = ? AND generation_id = ?) AS saved`,
|
|
66
|
+
WHERE owner = ? AND key = ? AND generation_id = ?) AS saved`, owner, ref.key, ref.generation, key, ref.key, ref.generation, ...channelBindings(table), ref.attempt, owner, ref.key, ref.generation)
|
|
66
67
|
.toArray()[0]);
|
|
67
68
|
if (lookup.result !== null) {
|
|
68
69
|
return deserialize(lookup.result);
|
|
@@ -90,25 +91,24 @@ export const createMemoize = (sql, table, ref) => {
|
|
|
90
91
|
}
|
|
91
92
|
};
|
|
92
93
|
return (key, callback) => traceFunction('memoize', () => run(key, callback), {
|
|
93
|
-
owner
|
|
94
|
+
owner,
|
|
94
95
|
key: ref.key,
|
|
95
96
|
attempt: ref.attempt,
|
|
96
97
|
memoKey: key,
|
|
97
98
|
})();
|
|
98
99
|
};
|
|
99
|
-
const memoKeySchema =
|
|
100
|
+
const memoKeySchema = v.object({ memo_key: v.string() });
|
|
100
101
|
/** Keys a record's current generation has results for, in the order they were saved. */
|
|
101
|
-
export const memoizedKeys = (sql, table, key, generation) =>
|
|
102
|
-
.array()
|
|
103
|
-
.parse(sql
|
|
102
|
+
export const memoizedKeys = (sql, table, key, generation) => v
|
|
103
|
+
.parse(v.array(memoKeySchema), sql
|
|
104
104
|
.exec(`SELECT memo_key FROM durability_memos
|
|
105
105
|
WHERE owner = ? AND key = ? AND generation_id = ?
|
|
106
|
-
ORDER BY rowid`, table
|
|
106
|
+
ORDER BY rowid`, tableOwner(table), key, generation)
|
|
107
107
|
.toArray())
|
|
108
108
|
.map((row) => row.memo_key);
|
|
109
109
|
/** Deletes every memoized result of a record, whichever generation wrote it. */
|
|
110
110
|
export const deleteMemos = (sql, table, key) => {
|
|
111
|
-
sql.exec('DELETE FROM durability_memos WHERE owner = ? AND key = ?', table
|
|
111
|
+
sql.exec('DELETE FROM durability_memos WHERE owner = ? AND key = ?', tableOwner(table), key);
|
|
112
112
|
};
|
|
113
113
|
/** Deletes memoized results whose record generation is no longer pending. */
|
|
114
114
|
export const deleteStaleMemos = (sql, table) => {
|
|
@@ -118,7 +118,8 @@ export const deleteStaleMemos = (sql, table) => {
|
|
|
118
118
|
WHERE record.${table.key} = durability_memos.key
|
|
119
119
|
AND record.generation_id = durability_memos.generation_id
|
|
120
120
|
AND record.status = 'pending'
|
|
121
|
-
|
|
121
|
+
${table.channel === undefined ? '' : 'AND record.channel = ?'}
|
|
122
|
+
)`, tableOwner(table), ...channelBindings(table));
|
|
122
123
|
};
|
|
123
124
|
const tableExists = (sql, name) => sql
|
|
124
125
|
.exec(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`, name)
|
|
@@ -132,5 +133,6 @@ export const deleteMemosOfDroppedTable = (sql, table) => {
|
|
|
132
133
|
if (!tableExists(sql, 'durability_memos') || tableExists(sql, table.name)) {
|
|
133
134
|
return;
|
|
134
135
|
}
|
|
135
|
-
sql.exec(
|
|
136
|
+
sql.exec(`DELETE FROM durability_memos
|
|
137
|
+
WHERE owner = ? OR (json_valid(owner) AND json_extract(owner, '$[0]') = ?)`, table.name, table.name);
|
|
136
138
|
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ordered SQLite migrations for `durability_memos`, the memoized results that
|
|
3
|
+
* {@link Durability} and {@link DurabilityAlarms} share. Applied by whichever
|
|
4
|
+
* helper is constructed first.
|
|
5
|
+
*/
|
|
6
|
+
export declare const durabilityMemoMigrations: {
|
|
7
|
+
name: string;
|
|
8
|
+
up: string;
|
|
9
|
+
down: string;
|
|
10
|
+
}[];
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ordered SQLite migrations for `durability_memos`, the memoized results that
|
|
3
|
+
* {@link Durability} and {@link DurabilityAlarms} share. Applied by whichever
|
|
4
|
+
* helper is constructed first.
|
|
5
|
+
*/
|
|
6
|
+
export const durabilityMemoMigrations = [
|
|
7
|
+
{
|
|
8
|
+
name: 'durability_0012_create_memos',
|
|
9
|
+
up: `
|
|
10
|
+
CREATE TABLE IF NOT EXISTS durability_memos (
|
|
11
|
+
owner TEXT NOT NULL,
|
|
12
|
+
key TEXT NOT NULL,
|
|
13
|
+
generation_id TEXT NOT NULL,
|
|
14
|
+
memo_key TEXT NOT NULL,
|
|
15
|
+
result TEXT NOT NULL,
|
|
16
|
+
PRIMARY KEY (owner, key, generation_id, memo_key)
|
|
17
|
+
);
|
|
18
|
+
`,
|
|
19
|
+
down: 'DROP TABLE IF EXISTS durability_memos;',
|
|
20
|
+
},
|
|
21
|
+
];
|
package/dist/migrations.d.ts
CHANGED
|
@@ -1,116 +1,5 @@
|
|
|
1
1
|
import type { DurableMigrations } from '@durability/storage';
|
|
2
2
|
export declare const migrationTableName = "durability_migrations";
|
|
3
|
-
/**
|
|
4
|
-
* Ordered SQLite migrations for the `durability_calls` table owned by
|
|
5
|
-
* {@link Durability}. Applied automatically on construction and tracked in the
|
|
6
|
-
* namespaced `durability_migrations` table alongside application migrations.
|
|
7
|
-
*/
|
|
8
|
-
export declare const durabilityOperationMigrations: {
|
|
9
|
-
name: string;
|
|
10
|
-
up: string;
|
|
11
|
-
down: string;
|
|
12
|
-
}[];
|
|
13
|
-
/**
|
|
14
|
-
* Ordered SQLite migrations for the `durability_alarms` table owned by
|
|
15
|
-
* {@link DurabilityAlarms}. Applied automatically on construction.
|
|
16
|
-
*/
|
|
17
|
-
export declare const durabilityNamedAlarmMigrations: {
|
|
18
|
-
name: string;
|
|
19
|
-
up: string;
|
|
20
|
-
down: string;
|
|
21
|
-
}[];
|
|
22
|
-
/**
|
|
23
|
-
* Ordered SQLite migrations for `durability_memos`, the memoized results that
|
|
24
|
-
* {@link Durability} and {@link DurabilityAlarms} share. Applied by whichever
|
|
25
|
-
* helper is constructed first.
|
|
26
|
-
*/
|
|
27
|
-
export declare const durabilityMemoMigrations: {
|
|
28
|
-
name: string;
|
|
29
|
-
up: string;
|
|
30
|
-
down: string;
|
|
31
|
-
}[];
|
|
32
|
-
/**
|
|
33
|
-
* Ordered SQLite migrations for fanout manifests and deliveries. Applied
|
|
34
|
-
* automatically on construction.
|
|
35
|
-
*/
|
|
36
|
-
export declare const durabilityFanoutMigrations: {
|
|
37
|
-
name: string;
|
|
38
|
-
up: string;
|
|
39
|
-
down: string;
|
|
40
|
-
}[];
|
|
41
|
-
/**
|
|
42
|
-
* Ordered SQLite migrations for the outbound event log owned by
|
|
43
|
-
* {@link DurabilitySocket}. Applied automatically on construction.
|
|
44
|
-
*/
|
|
45
|
-
export declare const durabilitySocketMigrations: {
|
|
46
|
-
name: string;
|
|
47
|
-
up: string;
|
|
48
|
-
down: string;
|
|
49
|
-
}[];
|
|
50
|
-
/**
|
|
51
|
-
* Ordered SQLite migrations for the retained log and its consumer cursors.
|
|
52
|
-
* Applied automatically on construction.
|
|
53
|
-
*/
|
|
54
|
-
export declare const durabilityLogMigrations: {
|
|
55
|
-
name: string;
|
|
56
|
-
up: string;
|
|
57
|
-
down: string;
|
|
58
|
-
}[];
|
|
59
|
-
/**
|
|
60
|
-
* Leases let several workers consume one log in parallel without handing the
|
|
61
|
-
* same offsets to two of them. Applied automatically when leases are enabled.
|
|
62
|
-
*/
|
|
63
|
-
export declare const durabilityLogLeaseMigrations: {
|
|
64
|
-
name: string;
|
|
65
|
-
up: string;
|
|
66
|
-
down: string;
|
|
67
|
-
}[];
|
|
68
|
-
/**
|
|
69
|
-
* Ordered SQLite migrations for outbox channels. Applied automatically on
|
|
70
|
-
* construction.
|
|
71
|
-
*/
|
|
72
|
-
export declare const durabilityOutboxMigrations: {
|
|
73
|
-
name: string;
|
|
74
|
-
up: string;
|
|
75
|
-
down: string;
|
|
76
|
-
}[];
|
|
77
|
-
/**
|
|
78
|
-
* Ordered SQLite migrations for semaphore grants. Applied automatically on
|
|
79
|
-
* construction.
|
|
80
|
-
*/
|
|
81
|
-
export declare const durabilitySemaphoreMigrations: {
|
|
82
|
-
name: string;
|
|
83
|
-
up: string;
|
|
84
|
-
down: string;
|
|
85
|
-
}[];
|
|
86
|
-
/**
|
|
87
|
-
* Ordered SQLite migrations for a semaphore client's queue. Applied
|
|
88
|
-
* automatically on construction.
|
|
89
|
-
*/
|
|
90
|
-
export declare const durabilitySemaphoreQueueMigrations: {
|
|
91
|
-
name: string;
|
|
92
|
-
up: string;
|
|
93
|
-
down: string;
|
|
94
|
-
}[];
|
|
95
|
-
/**
|
|
96
|
-
* Ordered SQLite migrations for the experimental topology coordinator.
|
|
97
|
-
* Publication and sync ride on outbox channels. Applied automatically on
|
|
98
|
-
* construction.
|
|
99
|
-
*/
|
|
100
|
-
export declare const durabilityTopologyMigrations: {
|
|
101
|
-
name: string;
|
|
102
|
-
up: string;
|
|
103
|
-
down: string;
|
|
104
|
-
}[];
|
|
105
|
-
/**
|
|
106
|
-
* Ordered SQLite migrations for the experimental schedule. Applied
|
|
107
|
-
* automatically on construction.
|
|
108
|
-
*/
|
|
109
|
-
export declare const durabilityScheduleMigrations: {
|
|
110
|
-
name: string;
|
|
111
|
-
up: string;
|
|
112
|
-
down: string;
|
|
113
|
-
}[];
|
|
114
3
|
/** Migration names changed by one migrate call. */
|
|
115
4
|
export type DurabilityMigrationResult = {
|
|
116
5
|
/** Migrations applied in ascending order. */
|