@effect-agent/storage-sqlite 0.1.0-beta.42 → 0.1.0-beta.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.mjs.map +1 -1
- package/dist/testing.mjs.map +1 -1
- package/package.json +1 -49
- package/src/errors.ts +1 -0
- package/src/sqlite-activity-store.ts +53 -0
- package/src/sqlite-journal.ts +52 -0
- package/src/sqlite-ledger.ts +307 -0
- package/src/sqlite-schedule-store.ts +39 -0
- package/src/sqlite-storage-failpoint-testing.ts +1 -0
- package/src/sqlite-subscription-store.ts +129 -0
- package/src/sqlite-thread-store.ts +74 -0
|
@@ -33,12 +33,15 @@ import type { SqliteStorageInitializationError } from "./sqlite-thread-store.ts"
|
|
|
33
33
|
const StoredJson = Schema.String.check(Schema.isMaxLength(16 * 1024 * 1024));
|
|
34
34
|
const CountRow = Schema.Struct({ count: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) });
|
|
35
35
|
const SequenceRow = Schema.Struct({ sequence: Schema.Natural });
|
|
36
|
+
|
|
36
37
|
const ScanRow = Schema.Struct({
|
|
37
38
|
event_scan_cursor: Schema.String,
|
|
38
39
|
delivery_scan_cursor: Schema.String,
|
|
39
40
|
recovery_scan_cursor: Schema.Natural,
|
|
40
41
|
});
|
|
42
|
+
|
|
41
43
|
const JsonRow = Schema.Struct({ record_json: StoredJson });
|
|
44
|
+
|
|
42
45
|
const RegistrationRow = Schema.Struct({
|
|
43
46
|
owner_id: Schema.String,
|
|
44
47
|
subscription_id: Schema.String,
|
|
@@ -51,6 +54,7 @@ const RegistrationRow = Schema.Struct({
|
|
|
51
54
|
recovery_at_millis: Schema.NullOr(Schema.Number),
|
|
52
55
|
record_json: StoredJson,
|
|
53
56
|
});
|
|
57
|
+
|
|
54
58
|
const EventRow = Schema.Struct({
|
|
55
59
|
event_id: Schema.String,
|
|
56
60
|
source_name: Schema.String,
|
|
@@ -63,6 +67,7 @@ const EventRow = Schema.Struct({
|
|
|
63
67
|
next_attempt_at_millis: Schema.Number,
|
|
64
68
|
record_json: StoredJson,
|
|
65
69
|
});
|
|
70
|
+
|
|
66
71
|
const DeliveryRow = Schema.Struct({
|
|
67
72
|
owner_id: Schema.String,
|
|
68
73
|
subscription_id: Schema.String,
|
|
@@ -75,6 +80,7 @@ const DeliveryRow = Schema.Struct({
|
|
|
75
80
|
|
|
76
81
|
const error = (reason: SubscriptionError["reason"], code: string) =>
|
|
77
82
|
SubscriptionError.make({ reason, code });
|
|
83
|
+
|
|
78
84
|
const unavailable = (operation: string) => error("storage", operation);
|
|
79
85
|
const corrupt = (operation: string) => error("corrupt", operation);
|
|
80
86
|
const bytes = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)).byteLength;
|
|
@@ -111,6 +117,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
111
117
|
const partition = yield* validate(SourcePartition, owned, "partition");
|
|
112
118
|
const sql = yield* SqlClientService.SqlClient;
|
|
113
119
|
const failpoint = yield* SubscriptionFailpoint;
|
|
120
|
+
|
|
114
121
|
yield* initializeSqliteJournal();
|
|
115
122
|
yield* sql`
|
|
116
123
|
INSERT INTO effect_agent_subscription_sequences (
|
|
@@ -122,22 +129,28 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
122
129
|
sql
|
|
123
130
|
.withTransaction(effect)
|
|
124
131
|
.pipe(Effect.catchTag("SqlError", () => Effect.fail(unavailable("transaction"))));
|
|
132
|
+
|
|
125
133
|
const query = <A extends object>(
|
|
126
134
|
effect: Effect.Effect<ReadonlyArray<A>, SqlError>,
|
|
127
135
|
code: string,
|
|
128
136
|
) => effect.pipe(Effect.mapError(() => unavailable(code)));
|
|
137
|
+
|
|
129
138
|
const requirePartition = (candidate: SourcePartition, code: string) =>
|
|
130
139
|
sameSourcePartition(candidate, partition)
|
|
131
140
|
? Effect.void
|
|
132
141
|
: Effect.fail(error("validation", code));
|
|
142
|
+
|
|
133
143
|
const requireKey = Effect.fn("SqliteSubscriptionStore.requireKey")(function* (
|
|
134
144
|
input: SubscriptionKey,
|
|
135
145
|
code: string,
|
|
136
146
|
) {
|
|
137
147
|
const key = yield* validate(SubscriptionKey, input, code);
|
|
148
|
+
|
|
138
149
|
yield* requirePartition(key.partition, code);
|
|
150
|
+
|
|
139
151
|
return key;
|
|
140
152
|
});
|
|
153
|
+
|
|
141
154
|
const readRegistration = Effect.fn("SqliteSubscriptionStore.readRegistration")(function* (
|
|
142
155
|
key: SubscriptionKey,
|
|
143
156
|
code: string,
|
|
@@ -151,11 +164,15 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
151
164
|
`,
|
|
152
165
|
code,
|
|
153
166
|
);
|
|
167
|
+
|
|
154
168
|
const decodedRows = yield* decodeRows(RegistrationRow, rows, code);
|
|
169
|
+
|
|
155
170
|
if (decodedRows.length > 1) return yield* corrupt(code);
|
|
156
171
|
const row = decodedRows[0];
|
|
172
|
+
|
|
157
173
|
if (row === undefined) return null;
|
|
158
174
|
const record = yield* decode(SubscriptionRecord, row.record_json, code);
|
|
175
|
+
|
|
159
176
|
if (
|
|
160
177
|
!sameSourcePartition(record.key.partition, partition) ||
|
|
161
178
|
record.key.ownerId !== row.owner_id ||
|
|
@@ -169,8 +186,10 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
169
186
|
(record.recovery?.nextAttemptAtMillis ?? null) !== row.recovery_at_millis
|
|
170
187
|
)
|
|
171
188
|
return yield* corrupt(`${code}-projection`);
|
|
189
|
+
|
|
172
190
|
return record;
|
|
173
191
|
});
|
|
192
|
+
|
|
174
193
|
const readEvent = Effect.fn("SqliteSubscriptionStore.readEvent")(function* (
|
|
175
194
|
eventId: string,
|
|
176
195
|
code: string,
|
|
@@ -183,11 +202,15 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
183
202
|
`,
|
|
184
203
|
code,
|
|
185
204
|
);
|
|
205
|
+
|
|
186
206
|
const decodedRows = yield* decodeRows(EventRow, rows, code);
|
|
207
|
+
|
|
187
208
|
if (decodedRows.length > 1) return yield* corrupt(code);
|
|
188
209
|
const row = decodedRows[0];
|
|
210
|
+
|
|
189
211
|
if (row === undefined) return null;
|
|
190
212
|
const event = yield* decode(AcceptedEvent, row.record_json, code);
|
|
213
|
+
|
|
191
214
|
if (
|
|
192
215
|
!sameSourcePartition(event.partition, partition) ||
|
|
193
216
|
event.eventId !== row.event_id ||
|
|
@@ -201,8 +224,10 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
201
224
|
event.nextAttemptAtMillis !== row.next_attempt_at_millis
|
|
202
225
|
)
|
|
203
226
|
return yield* corrupt(`${code}-projection`);
|
|
227
|
+
|
|
204
228
|
return event;
|
|
205
229
|
});
|
|
230
|
+
|
|
206
231
|
const readDelivery = Effect.fn("SqliteSubscriptionStore.readDelivery")(function* (
|
|
207
232
|
key: SubscriptionDeliveryKey,
|
|
208
233
|
code: string,
|
|
@@ -217,11 +242,15 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
217
242
|
`,
|
|
218
243
|
code,
|
|
219
244
|
);
|
|
245
|
+
|
|
220
246
|
const decodedRows = yield* decodeRows(DeliveryRow, rows, code);
|
|
247
|
+
|
|
221
248
|
if (decodedRows.length > 1) return yield* corrupt(code);
|
|
222
249
|
const row = decodedRows[0];
|
|
250
|
+
|
|
223
251
|
if (row === undefined) return null;
|
|
224
252
|
const delivery = yield* decode(SubscriptionDelivery, row.record_json, code);
|
|
253
|
+
|
|
225
254
|
if (
|
|
226
255
|
!sameSourcePartition(delivery.key.subscription.partition, partition) ||
|
|
227
256
|
delivery.key.subscription.ownerId !== row.owner_id ||
|
|
@@ -232,17 +261,22 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
232
261
|
delivery.retry.nextAttemptAtMillis !== row.next_attempt_at_millis
|
|
233
262
|
)
|
|
234
263
|
return yield* corrupt(`${code}-projection`);
|
|
264
|
+
|
|
235
265
|
return delivery;
|
|
236
266
|
});
|
|
267
|
+
|
|
237
268
|
const count = Effect.fn("SqliteSubscriptionStore.count")(function* (
|
|
238
269
|
statement: Effect.Effect<ReadonlyArray<Record<string, unknown>>, SqlError>,
|
|
239
270
|
code: string,
|
|
240
271
|
) {
|
|
241
272
|
const rows = yield* query(statement, code);
|
|
242
273
|
const decoded = yield* decodeRows(CountRow, rows, code);
|
|
274
|
+
|
|
243
275
|
if (decoded.length !== 1) return yield* corrupt(code);
|
|
276
|
+
|
|
244
277
|
return decoded[0].count;
|
|
245
278
|
});
|
|
279
|
+
|
|
246
280
|
const nextSequence = Effect.fn("SqliteSubscriptionStore.nextSequence")(function* () {
|
|
247
281
|
const rows = yield* query(
|
|
248
282
|
sql<Record<string, unknown>>`
|
|
@@ -252,14 +286,19 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
252
286
|
`,
|
|
253
287
|
"advance subscription sequence",
|
|
254
288
|
);
|
|
289
|
+
|
|
255
290
|
const decoded = yield* decodeRows(SequenceRow, rows, "advance subscription sequence");
|
|
291
|
+
|
|
256
292
|
if (decoded.length !== 1) return yield* corrupt("subscription sequence");
|
|
293
|
+
|
|
257
294
|
return decoded[0].sequence;
|
|
258
295
|
});
|
|
296
|
+
|
|
259
297
|
const writeRegistration = Effect.fn("SqliteSubscriptionStore.writeRegistration")(function* (
|
|
260
298
|
record: SubscriptionRecord,
|
|
261
299
|
) {
|
|
262
300
|
const json = yield* encode(SubscriptionRecord, record, "encode subscription");
|
|
301
|
+
|
|
263
302
|
yield* query(
|
|
264
303
|
sql<Record<string, unknown>>`
|
|
265
304
|
UPDATE effect_agent_subscriptions SET state=${record.state}, expires_at_millis=${record.configuration.expiresAtMillis},
|
|
@@ -270,10 +309,12 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
270
309
|
"write subscription",
|
|
271
310
|
);
|
|
272
311
|
});
|
|
312
|
+
|
|
273
313
|
const writeEvent = Effect.fn("SqliteSubscriptionStore.writeEvent")(function* (
|
|
274
314
|
event: AcceptedEvent,
|
|
275
315
|
) {
|
|
276
316
|
const json = yield* encode(AcceptedEvent, event, "encode event");
|
|
317
|
+
|
|
277
318
|
yield* query(
|
|
278
319
|
sql<Record<string, unknown>>`
|
|
279
320
|
UPDATE effect_agent_subscription_events SET cursor=${event.cursor}, routing_complete=${event.routingComplete ? 1 : 0},
|
|
@@ -283,10 +324,12 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
283
324
|
"write event",
|
|
284
325
|
);
|
|
285
326
|
});
|
|
327
|
+
|
|
286
328
|
const writeDelivery = Effect.fn("SqliteSubscriptionStore.writeDelivery")(function* (
|
|
287
329
|
delivery: SubscriptionDelivery,
|
|
288
330
|
) {
|
|
289
331
|
const json = yield* encode(SubscriptionDelivery, delivery, "encode delivery");
|
|
332
|
+
|
|
290
333
|
yield* query(
|
|
291
334
|
sql<Record<string, unknown>>`
|
|
292
335
|
UPDATE effect_agent_subscription_deliveries SET state=${delivery.state},
|
|
@@ -304,13 +347,17 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
304
347
|
)(function* (input, inputLimits) {
|
|
305
348
|
const record = yield* validate(SubscriptionRecord, input, "register-record");
|
|
306
349
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "register-limits");
|
|
350
|
+
|
|
307
351
|
yield* requirePartition(record.key.partition, "register-partition");
|
|
352
|
+
|
|
308
353
|
const result = yield* transact(
|
|
309
354
|
Effect.gen(function* () {
|
|
310
355
|
const existing = yield* readRegistration(record.key, "register-existing");
|
|
356
|
+
|
|
311
357
|
if (existing !== null) {
|
|
312
358
|
if (existing.creationFingerprint !== record.creationFingerprint)
|
|
313
359
|
return yield* error("conflict", "registration-identity");
|
|
360
|
+
|
|
314
361
|
return { value: existing, changed: false } as const;
|
|
315
362
|
}
|
|
316
363
|
if (bytes(record.configuration.context) > limits.maxContextBytes)
|
|
@@ -342,6 +389,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
342
389
|
return yield* error("capacity", "owner-registrations");
|
|
343
390
|
const assigned = { ...record, ordinal: yield* nextSequence() };
|
|
344
391
|
const json = yield* encode(SubscriptionRecord, assigned, "encode registration");
|
|
392
|
+
|
|
345
393
|
yield* failpoint.hit("subscription:register:before");
|
|
346
394
|
yield* query(
|
|
347
395
|
sql<Record<string, unknown>>`
|
|
@@ -353,10 +401,13 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
353
401
|
`,
|
|
354
402
|
"insert registration",
|
|
355
403
|
);
|
|
404
|
+
|
|
356
405
|
return { value: assigned, changed: true } as const;
|
|
357
406
|
}),
|
|
358
407
|
);
|
|
408
|
+
|
|
359
409
|
if (result.changed) yield* failpoint.hit("subscription:register:after");
|
|
410
|
+
|
|
360
411
|
return result.value;
|
|
361
412
|
});
|
|
362
413
|
|
|
@@ -375,6 +426,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
375
426
|
`,
|
|
376
427
|
"list subscriptions",
|
|
377
428
|
);
|
|
429
|
+
|
|
378
430
|
const decoded = yield* decodeRows(
|
|
379
431
|
Schema.Struct({
|
|
380
432
|
owner_id: Schema.String,
|
|
@@ -384,6 +436,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
384
436
|
rows,
|
|
385
437
|
"list subscriptions",
|
|
386
438
|
);
|
|
439
|
+
|
|
387
440
|
return yield* Effect.forEach(
|
|
388
441
|
decoded,
|
|
389
442
|
Effect.fn("SqliteSubscriptionStore.listRecord")(function* (row) {
|
|
@@ -391,8 +444,10 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
391
444
|
{ partition, ownerId: row.owner_id, subscriptionId: row.subscription_id },
|
|
392
445
|
"list subscription",
|
|
393
446
|
);
|
|
447
|
+
|
|
394
448
|
if (record === null || record.ordinal !== row.ordinal)
|
|
395
449
|
return yield* corrupt("list subscription projection");
|
|
450
|
+
|
|
396
451
|
return record;
|
|
397
452
|
}),
|
|
398
453
|
);
|
|
@@ -403,18 +458,24 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
403
458
|
"SqliteSubscriptionStore.cancel",
|
|
404
459
|
)(function* (input) {
|
|
405
460
|
const key = yield* requireKey(input, "cancel-key");
|
|
461
|
+
|
|
406
462
|
const result = yield* transact(
|
|
407
463
|
Effect.gen(function* () {
|
|
408
464
|
const current = yield* readRegistration(key, "cancel subscription");
|
|
465
|
+
|
|
409
466
|
if (current === null) return yield* error("not-found", "subscription");
|
|
410
467
|
if (current.state === "cancelled") return { value: current, changed: false } as const;
|
|
411
468
|
const updated = { ...current, state: "cancelled" as const, recovery: null };
|
|
469
|
+
|
|
412
470
|
yield* failpoint.hit("subscription:cancel:before");
|
|
413
471
|
yield* writeRegistration(updated);
|
|
472
|
+
|
|
414
473
|
return { value: updated, changed: true } as const;
|
|
415
474
|
}),
|
|
416
475
|
);
|
|
476
|
+
|
|
417
477
|
if (result.changed) yield* failpoint.hit("subscription:cancel:after");
|
|
478
|
+
|
|
418
479
|
return result.value;
|
|
419
480
|
});
|
|
420
481
|
|
|
@@ -423,13 +484,17 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
423
484
|
)(function* (input, inputLimits) {
|
|
424
485
|
const event = yield* validate(AcceptedEvent, input, "accept-event");
|
|
425
486
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "accept-limits");
|
|
487
|
+
|
|
426
488
|
yield* requirePartition(event.partition, "accept-partition");
|
|
489
|
+
|
|
427
490
|
const result = yield* transact(
|
|
428
491
|
Effect.gen(function* () {
|
|
429
492
|
const existing = yield* readEvent(event.eventId, "accept event");
|
|
493
|
+
|
|
430
494
|
if (existing !== null) {
|
|
431
495
|
if (!sameAcceptedEventIdentity(existing, event))
|
|
432
496
|
return yield* error("conflict", "event-identity");
|
|
497
|
+
|
|
433
498
|
return { value: existing, changed: false } as const;
|
|
434
499
|
}
|
|
435
500
|
if (bytes(event.payload) > limits.maxPayloadBytes)
|
|
@@ -443,6 +508,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
443
508
|
)) >= limits.maxEvents
|
|
444
509
|
)
|
|
445
510
|
return yield* error("capacity", "events");
|
|
511
|
+
|
|
446
512
|
const accepted: AcceptedEvent = {
|
|
447
513
|
...event,
|
|
448
514
|
cutoff: yield* nextSequence(),
|
|
@@ -450,7 +516,9 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
450
516
|
routingComplete: false,
|
|
451
517
|
routingFailure: null,
|
|
452
518
|
};
|
|
519
|
+
|
|
453
520
|
const json = yield* encode(AcceptedEvent, accepted, "encode accepted event");
|
|
521
|
+
|
|
454
522
|
yield* failpoint.hit("subscription:accept:before");
|
|
455
523
|
yield* query(
|
|
456
524
|
sql<Record<string, unknown>>`
|
|
@@ -461,10 +529,13 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
461
529
|
`,
|
|
462
530
|
"insert event",
|
|
463
531
|
);
|
|
532
|
+
|
|
464
533
|
return { value: accepted, changed: true } as const;
|
|
465
534
|
}),
|
|
466
535
|
);
|
|
536
|
+
|
|
467
537
|
if (result.changed) yield* failpoint.hit("subscription:accept:after");
|
|
538
|
+
|
|
468
539
|
return result.value;
|
|
469
540
|
});
|
|
470
541
|
|
|
@@ -482,6 +553,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
482
553
|
`,
|
|
483
554
|
"pending events",
|
|
484
555
|
);
|
|
556
|
+
|
|
485
557
|
return yield* decodeRows(
|
|
486
558
|
Schema.Struct({ event_id: Schema.String }),
|
|
487
559
|
rows,
|
|
@@ -493,10 +565,13 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
493
565
|
"SqliteSubscriptionStore.candidates",
|
|
494
566
|
)(function* (input, limit) {
|
|
495
567
|
const supplied = yield* validate(AcceptedEvent, input, "candidates-event");
|
|
568
|
+
|
|
496
569
|
yield* requirePartition(supplied.partition, "candidates-partition");
|
|
497
570
|
const stored = yield* readEvent(supplied.eventId, "candidates event");
|
|
571
|
+
|
|
498
572
|
if (stored === null) return yield* error("not-found", "event");
|
|
499
573
|
if (!sameAcceptedEventIdentity(stored, supplied)) return yield* error("conflict", "event");
|
|
574
|
+
|
|
500
575
|
const rows = yield* query(
|
|
501
576
|
sql<Record<string, unknown>>`
|
|
502
577
|
SELECT owner_id, subscription_id, ordinal FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
@@ -505,6 +580,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
505
580
|
`,
|
|
506
581
|
"subscription candidates",
|
|
507
582
|
);
|
|
583
|
+
|
|
508
584
|
const decoded = yield* decodeRows(
|
|
509
585
|
Schema.Struct({
|
|
510
586
|
owner_id: Schema.String,
|
|
@@ -514,6 +590,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
514
590
|
rows,
|
|
515
591
|
"subscription candidates",
|
|
516
592
|
);
|
|
593
|
+
|
|
517
594
|
return yield* Effect.forEach(
|
|
518
595
|
decoded,
|
|
519
596
|
Effect.fn("SqliteSubscriptionStore.candidateRecord")(function* (row) {
|
|
@@ -521,8 +598,10 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
521
598
|
{ partition, ownerId: row.owner_id, subscriptionId: row.subscription_id },
|
|
522
599
|
"subscription candidate",
|
|
523
600
|
);
|
|
601
|
+
|
|
524
602
|
if (record === null || record.ordinal !== row.ordinal)
|
|
525
603
|
return yield* corrupt("subscription candidate projection");
|
|
604
|
+
|
|
526
605
|
return record;
|
|
527
606
|
}),
|
|
528
607
|
);
|
|
@@ -532,6 +611,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
532
611
|
delivery: SubscriptionDelivery,
|
|
533
612
|
) {
|
|
534
613
|
const json = yield* encode(SubscriptionDelivery, delivery, "encode selected delivery");
|
|
614
|
+
|
|
535
615
|
yield* query(
|
|
536
616
|
sql<Record<string, unknown>>`
|
|
537
617
|
INSERT INTO effect_agent_subscription_deliveries (tenant_id, source_address, owner_id, subscription_id, event_id,
|
|
@@ -547,18 +627,23 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
547
627
|
"SqliteSubscriptionStore.select",
|
|
548
628
|
)(function* (inputEvent, inputDeliveries, cursor, complete, nowMillis, inputLimits) {
|
|
549
629
|
const supplied = yield* validate(AcceptedEvent, inputEvent, "select-event");
|
|
630
|
+
|
|
550
631
|
const deliveries = yield* validate(
|
|
551
632
|
Schema.Array(SubscriptionDelivery),
|
|
552
633
|
inputDeliveries,
|
|
553
634
|
"select-deliveries",
|
|
554
635
|
);
|
|
636
|
+
|
|
555
637
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "select-limits");
|
|
638
|
+
|
|
556
639
|
yield* requirePartition(supplied.partition, "select-partition");
|
|
557
640
|
for (const candidate of deliveries)
|
|
558
641
|
yield* requirePartition(candidate.key.subscription.partition, "select-delivery-partition");
|
|
642
|
+
|
|
559
643
|
const changed = yield* transact(
|
|
560
644
|
Effect.gen(function* () {
|
|
561
645
|
const accepted = yield* readEvent(supplied.eventId, "select event");
|
|
646
|
+
|
|
562
647
|
if (accepted === null) return yield* error("not-found", "event");
|
|
563
648
|
if (!sameAcceptedEventIdentity(accepted, supplied) || accepted.cursor !== supplied.cursor)
|
|
564
649
|
return yield* error("conflict", "event-cursor");
|
|
@@ -568,8 +653,10 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
568
653
|
yield* failpoint.hit("subscription:select:before");
|
|
569
654
|
const effectiveNowMillis = Math.max(nowMillis, yield* Clock.currentTimeMillis);
|
|
570
655
|
const additions: Array<{ delivery: SubscriptionDelivery; record: SubscriptionRecord }> = [];
|
|
656
|
+
|
|
571
657
|
for (const delivery of deliveries) {
|
|
572
658
|
const record = yield* readRegistration(delivery.key.subscription, "select registration");
|
|
659
|
+
|
|
573
660
|
if (record === null) return yield* error("not-found", "subscription");
|
|
574
661
|
if (
|
|
575
662
|
!subscriptionDeliveryCanSelect(delivery, record, accepted) ||
|
|
@@ -581,6 +668,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
581
668
|
)
|
|
582
669
|
return yield* error("conflict", "selection");
|
|
583
670
|
const existing = yield* readDelivery(delivery.key, "select existing delivery");
|
|
671
|
+
|
|
584
672
|
if (existing !== null) {
|
|
585
673
|
if (!sameDeliveryIdentity(existing, delivery))
|
|
586
674
|
return yield* error("conflict", "delivery-identity");
|
|
@@ -589,12 +677,14 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
589
677
|
if (!subscriptionCanSelect(record, accepted, effectiveNowMillis, false)) continue;
|
|
590
678
|
additions.push({ delivery, record });
|
|
591
679
|
}
|
|
680
|
+
|
|
592
681
|
const total = yield* count(
|
|
593
682
|
sql<
|
|
594
683
|
Record<string, unknown>
|
|
595
684
|
>`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`,
|
|
596
685
|
"count deliveries",
|
|
597
686
|
);
|
|
687
|
+
|
|
598
688
|
if (total + additions.length > limits.maxDeliveries)
|
|
599
689
|
return yield* error("capacity", "deliveries");
|
|
600
690
|
for (const ownerId of new Set(additions.map(({ record }) => record.key.ownerId))) {
|
|
@@ -604,6 +694,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
604
694
|
>`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND owner_id=${ownerId}`,
|
|
605
695
|
"count owner deliveries",
|
|
606
696
|
);
|
|
697
|
+
|
|
607
698
|
if (
|
|
608
699
|
existing + additions.filter(({ record }) => record.key.ownerId === ownerId).length >
|
|
609
700
|
limits.maxDeliveriesPerOwner
|
|
@@ -616,9 +707,11 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
616
707
|
yield* writeRegistration({ ...addition.record, state: "consumed", recovery: null });
|
|
617
708
|
}
|
|
618
709
|
yield* writeEvent({ ...accepted, cursor, routingComplete: complete, routingFailure: null });
|
|
710
|
+
|
|
619
711
|
return true;
|
|
620
712
|
}),
|
|
621
713
|
);
|
|
714
|
+
|
|
622
715
|
if (changed) yield* failpoint.hit("subscription:select:after");
|
|
623
716
|
});
|
|
624
717
|
|
|
@@ -628,12 +721,15 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
628
721
|
const supplied = yield* validate(AcceptedEvent, inputEvent, "catch-up-event");
|
|
629
722
|
const delivery = yield* validate(SubscriptionDelivery, inputDelivery, "catch-up-delivery");
|
|
630
723
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "catch-up-limits");
|
|
724
|
+
|
|
631
725
|
yield* requirePartition(supplied.partition, "catch-up-partition");
|
|
632
726
|
yield* requirePartition(delivery.key.subscription.partition, "catch-up-delivery-partition");
|
|
727
|
+
|
|
633
728
|
const changed = yield* transact(
|
|
634
729
|
Effect.gen(function* () {
|
|
635
730
|
const accepted = yield* readEvent(supplied.eventId, "catch-up event");
|
|
636
731
|
const record = yield* readRegistration(delivery.key.subscription, "catch-up subscription");
|
|
732
|
+
|
|
637
733
|
if (accepted === null || record === null)
|
|
638
734
|
return yield* error("not-found", accepted === null ? "event" : "subscription");
|
|
639
735
|
if (
|
|
@@ -646,13 +742,16 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
646
742
|
)
|
|
647
743
|
return yield* error("conflict", "catch-up-identity");
|
|
648
744
|
const existing = yield* readDelivery(delivery.key, "catch-up existing delivery");
|
|
745
|
+
|
|
649
746
|
if (existing !== null) {
|
|
650
747
|
if (!sameDeliveryIdentity(existing, delivery))
|
|
651
748
|
return yield* error("conflict", "delivery-identity");
|
|
749
|
+
|
|
652
750
|
return false;
|
|
653
751
|
}
|
|
654
752
|
yield* failpoint.hit("subscription:catch-up:before");
|
|
655
753
|
const effectiveNowMillis = Math.max(nowMillis, yield* Clock.currentTimeMillis);
|
|
754
|
+
|
|
656
755
|
if (!subscriptionCanSelect(record, accepted, effectiveNowMillis, true))
|
|
657
756
|
return yield* error("conflict", "catch-up-eligibility");
|
|
658
757
|
if (
|
|
@@ -675,9 +774,11 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
675
774
|
return yield* error("capacity", "owner-deliveries");
|
|
676
775
|
yield* insertDelivery(delivery);
|
|
677
776
|
yield* writeRegistration({ ...record, state: "consumed", recovery: null });
|
|
777
|
+
|
|
678
778
|
return true;
|
|
679
779
|
}),
|
|
680
780
|
);
|
|
781
|
+
|
|
681
782
|
if (changed) yield* failpoint.hit("subscription:catch-up:after");
|
|
682
783
|
});
|
|
683
784
|
|
|
@@ -688,9 +789,11 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
688
789
|
code === undefined
|
|
689
790
|
? "routing-failed"
|
|
690
791
|
: yield* validate(SubscriptionName, code, "routing-failure");
|
|
792
|
+
|
|
691
793
|
yield* transact(
|
|
692
794
|
Effect.gen(function* () {
|
|
693
795
|
const accepted = yield* readEvent(eventId, "defer event");
|
|
796
|
+
|
|
694
797
|
if (accepted === null) return yield* error("not-found", "event");
|
|
695
798
|
yield* failpoint.hit("subscription:defer-event:before");
|
|
696
799
|
yield* writeEvent({ ...accepted, nextAttemptAtMillis, routingFailure });
|
|
@@ -703,7 +806,9 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
703
806
|
"SqliteSubscriptionStore.delivery",
|
|
704
807
|
)(function* (input) {
|
|
705
808
|
const key = yield* validate(SubscriptionDeliveryKey, input, "delivery-key");
|
|
809
|
+
|
|
706
810
|
yield* requirePartition(key.subscription.partition, "delivery-partition");
|
|
811
|
+
|
|
707
812
|
return yield* readDelivery(key, "get delivery");
|
|
708
813
|
});
|
|
709
814
|
|
|
@@ -718,11 +823,13 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
718
823
|
`,
|
|
719
824
|
"pending deliveries",
|
|
720
825
|
);
|
|
826
|
+
|
|
721
827
|
const rowSchema = Schema.Struct({
|
|
722
828
|
owner_id: Schema.String,
|
|
723
829
|
subscription_id: Schema.String,
|
|
724
830
|
event_id: Schema.String,
|
|
725
831
|
});
|
|
832
|
+
|
|
726
833
|
return yield* decodeRows(rowSchema, rows, "pending delivery keys").pipe(
|
|
727
834
|
Effect.map((items) =>
|
|
728
835
|
items.map((item) => ({
|
|
@@ -737,6 +844,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
737
844
|
"SqliteSubscriptionStore.listDeliveries",
|
|
738
845
|
)(function* (input, after, limit) {
|
|
739
846
|
const key = yield* requireKey(input, "list-deliveries-key");
|
|
847
|
+
|
|
740
848
|
const rows = yield* query(
|
|
741
849
|
sql<Record<string, unknown>>`
|
|
742
850
|
SELECT record_json FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
@@ -744,7 +852,9 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
744
852
|
`,
|
|
745
853
|
"list deliveries",
|
|
746
854
|
);
|
|
855
|
+
|
|
747
856
|
const decoded = yield* decodeRows(JsonRow, rows, "list deliveries");
|
|
857
|
+
|
|
748
858
|
return yield* Effect.forEach(decoded, (row) =>
|
|
749
859
|
decode(SubscriptionDelivery, row.record_json, "list delivery"),
|
|
750
860
|
);
|
|
@@ -756,32 +866,41 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
756
866
|
const key = yield* validate(SubscriptionDeliveryKey, inputKey, "change-delivery-key");
|
|
757
867
|
const deliveryId = yield* validate(Digest, inputDeliveryId, "change-delivery-id");
|
|
758
868
|
const change = yield* validate(DeliveryChange, inputChange, "change-delivery-change");
|
|
869
|
+
|
|
759
870
|
yield* requirePartition(key.subscription.partition, "change-delivery-partition");
|
|
871
|
+
|
|
760
872
|
const result = yield* transact(
|
|
761
873
|
Effect.gen(function* () {
|
|
762
874
|
const existing = yield* readDelivery(key, "change delivery");
|
|
763
875
|
const record = yield* readRegistration(key.subscription, "change delivery subscription");
|
|
876
|
+
|
|
764
877
|
if (existing === null || record === null)
|
|
765
878
|
return yield* error("not-found", existing === null ? "delivery" : "subscription");
|
|
766
879
|
yield* failpoint.hit(`subscription:delivery-${change._tag.toLowerCase()}:before`);
|
|
880
|
+
|
|
767
881
|
const effectiveChange =
|
|
768
882
|
change._tag === "Prepare"
|
|
769
883
|
? { ...change, nowMillis: Math.max(change.nowMillis, yield* Clock.currentTimeMillis) }
|
|
770
884
|
: change;
|
|
885
|
+
|
|
771
886
|
const transition = applySubscriptionDeliveryChange(
|
|
772
887
|
existing,
|
|
773
888
|
record,
|
|
774
889
|
deliveryId,
|
|
775
890
|
effectiveChange,
|
|
776
891
|
);
|
|
892
|
+
|
|
777
893
|
if (Result.isFailure(transition)) return yield* transition.failure;
|
|
778
894
|
if (transition.success === existing) return { value: existing, changed: false } as const;
|
|
779
895
|
yield* writeDelivery(transition.success);
|
|
896
|
+
|
|
780
897
|
return { value: transition.success, changed: true } as const;
|
|
781
898
|
}),
|
|
782
899
|
);
|
|
900
|
+
|
|
783
901
|
if (result.changed)
|
|
784
902
|
yield* failpoint.hit(`subscription:delivery-${change._tag.toLowerCase()}:after`);
|
|
903
|
+
|
|
785
904
|
return result.value;
|
|
786
905
|
});
|
|
787
906
|
|
|
@@ -797,11 +916,13 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
797
916
|
`,
|
|
798
917
|
"recovering subscriptions",
|
|
799
918
|
);
|
|
919
|
+
|
|
800
920
|
const rowSchema = Schema.Struct({
|
|
801
921
|
owner_id: Schema.String,
|
|
802
922
|
subscription_id: Schema.String,
|
|
803
923
|
ordinal: Schema.Natural,
|
|
804
924
|
});
|
|
925
|
+
|
|
805
926
|
return yield* decodeRows(rowSchema, rows, "recovering subscription keys").pipe(
|
|
806
927
|
Effect.map((items) =>
|
|
807
928
|
items.map((item) => ({
|
|
@@ -816,9 +937,11 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
816
937
|
"SqliteSubscriptionStore.deferRecovery",
|
|
817
938
|
)(function* (input, recovery) {
|
|
818
939
|
const key = yield* requireKey(input, "defer-recovery-key");
|
|
940
|
+
|
|
819
941
|
yield* transact(
|
|
820
942
|
Effect.gen(function* () {
|
|
821
943
|
const record = yield* readRegistration(key, "defer recovery");
|
|
944
|
+
|
|
822
945
|
if (record === null) return yield* error("not-found", "subscription");
|
|
823
946
|
yield* failpoint.hit("subscription:defer-recovery:before");
|
|
824
947
|
yield* writeRegistration({
|
|
@@ -839,8 +962,11 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
839
962
|
`,
|
|
840
963
|
"read subscription scan cursors",
|
|
841
964
|
);
|
|
965
|
+
|
|
842
966
|
const decoded = yield* decodeRows(ScanRow, rows, "read subscription scan cursors");
|
|
967
|
+
|
|
843
968
|
if (decoded.length !== 1) return yield* corrupt("subscription scan cursors");
|
|
969
|
+
|
|
844
970
|
return {
|
|
845
971
|
events: decoded[0].event_scan_cursor,
|
|
846
972
|
deliveries: decoded[0].delivery_scan_cursor,
|
|
@@ -852,6 +978,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
852
978
|
"SqliteSubscriptionStore.advanceScanCursors",
|
|
853
979
|
)(function* (input) {
|
|
854
980
|
const cursors = yield* validate(SubscriptionScanCursors, input, "scan-cursors");
|
|
981
|
+
|
|
855
982
|
yield* transact(
|
|
856
983
|
Effect.gen(function* () {
|
|
857
984
|
yield* failpoint.hit("subscription:advance-scan-cursors:before");
|
|
@@ -897,7 +1024,9 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
|
|
|
897
1024
|
|
|
898
1025
|
const nextDeadline = Effect.gen(function* () {
|
|
899
1026
|
const cursors = yield* readScanCursors;
|
|
1027
|
+
|
|
900
1028
|
if (cursors.events !== "" || cursors.deliveries !== "" || cursors.recovery !== 0) return 0;
|
|
1029
|
+
|
|
901
1030
|
return yield* indexedDeadline;
|
|
902
1031
|
});
|
|
903
1032
|
|