@effect-agent/storage-cloudflare 0.1.0-beta.49 → 0.1.0-beta.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,58 +1,20 @@
1
1
  import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
2
- import { Clock, Context, Effect, Layer, Result, Schema } from "effect";
2
+ import { Context, Effect, Layer, Schema } from "effect";
3
3
  import * as SqlClientService from "effect/unstable/sql/SqlClient";
4
- import { Digest } from "@effect-agent/thread/Records";
5
- import { AcceptedEvent, DeliveryChange, SourcePartition, SubscriptionDelivery, SubscriptionDeliveryKey, SubscriptionError, SubscriptionFailpoint, SubscriptionKey, SubscriptionLimits, SubscriptionName, SubscriptionRecord, SubscriptionScanCursors, SubscriptionStore, subscriptionDeliveryKeyString } from "@effect-agent/thread/Subscription";
6
- import { applySubscriptionDeliveryChange, sameAcceptedEventIdentity, sameSourcePartition, subscriptionCanSelect, subscriptionDeliveryCanSelect } from "@effect-agent/thread/SubscriptionTransition";
4
+ import { SqlSubscriptionTransaction, makeSqlSubscriptionStore } from "@effect-agent/thread/SqlSubscriptionStore";
5
+ import { SourcePartition, SubscriptionError, SubscriptionFailpoint, SubscriptionStore } from "@effect-agent/thread/Subscription";
7
6
  //#region src/DoSubscriptionStore.ts
8
7
  var DoSubscriptionStore_exports = /* @__PURE__ */ __exportAll({
9
8
  DoSubscriptionAlarmControl: () => DoSubscriptionAlarmControl,
10
9
  DoSubscriptionTransaction: () => DoSubscriptionTransaction,
11
10
  doSubscriptionStoreLayer: () => doSubscriptionStoreLayer
12
11
  });
13
- const CURRENT_SUBSCRIPTION_STORE_VERSION = 2;
14
- const StoredJson = Schema.String.check(Schema.isMaxLength(19e5));
15
- const CountRow = Schema.Struct({ count: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) });
16
- const SequenceRow = Schema.Struct({ sequence: Schema.Natural });
12
+ const CURRENT_SUBSCRIPTION_STORE_VERSION = 3;
17
13
  const ScanRow = Schema.Struct({
18
14
  event_scan_cursor: Schema.String,
19
15
  delivery_scan_cursor: Schema.String,
20
16
  recovery_scan_cursor: Schema.Natural
21
17
  });
22
- const JsonRow = Schema.Struct({ record_json: StoredJson });
23
- const RegistrationRow = Schema.Struct({
24
- owner_id: Schema.String,
25
- subscription_id: Schema.String,
26
- ordinal: Schema.Natural,
27
- source_name: Schema.String,
28
- source_version: Schema.String,
29
- matching_key: Schema.String,
30
- state: SubscriptionRecord.fields.state,
31
- expires_at_millis: Schema.Number,
32
- recovery_at_millis: Schema.NullOr(Schema.Number),
33
- record_json: StoredJson
34
- });
35
- const EventRow = Schema.Struct({
36
- event_id: Schema.String,
37
- source_name: Schema.String,
38
- source_version: Schema.String,
39
- matching_key: Schema.String,
40
- payload_digest: Digest,
41
- cutoff: Schema.Natural,
42
- cursor: Schema.Natural,
43
- routing_complete: Schema.Number,
44
- next_attempt_at_millis: Schema.Number,
45
- record_json: StoredJson
46
- });
47
- const DeliveryRow = Schema.Struct({
48
- owner_id: Schema.String,
49
- subscription_id: Schema.String,
50
- event_id: Schema.String,
51
- delivery_key: Schema.String,
52
- state: SubscriptionDelivery.fields.state,
53
- next_attempt_at_millis: Schema.Number,
54
- record_json: StoredJson
55
- });
56
18
  const StoreStateRow = Schema.Struct({
57
19
  storage_version: Schema.Int,
58
20
  alarm_generation: Schema.Natural
@@ -66,12 +28,8 @@ const error = (reason, code) => SubscriptionError.make({
66
28
  });
67
29
  const unavailable = (operation) => error("storage", operation);
68
30
  const corrupt = (operation) => error("corrupt", operation);
69
- const bytes = (value) => new TextEncoder().encode(JSON.stringify(value)).byteLength;
70
31
  const validate = (schema, value, code) => Schema.decodeUnknownEffect(schema)(value).pipe(Effect.mapError(() => error("validation", code)));
71
- const encode = (schema, value, code) => Schema.encodeEffect(Schema.fromJsonString(schema))(value).pipe(Effect.mapError(() => corrupt(code)));
72
- const decode = (schema, value, code) => Schema.decodeEffect(Schema.fromJsonString(schema))(value).pipe(Effect.mapError(() => corrupt(code)));
73
32
  const decodeRows = (schema, rows, code) => Schema.decodeUnknownEffect(Schema.Array(schema))(rows).pipe(Effect.mapError(() => corrupt(code)));
74
- const sameDeliveryIdentity = (left, right) => subscriptionDeliveryKeyString(left.key) === subscriptionDeliveryKeyString(right.key) && left.deliveryId === right.deliveryId && left.source.name === right.source.name && left.source.version === right.source.version && left.threadId === right.threadId && left.admissionKey === right.admissionKey && left.subscriptionFingerprint === right.subscriptionFingerprint && left.eventDigest === right.eventDigest;
75
33
  const initializeDoSubscriptionStore = Effect.fn("DoSubscriptionStore.initialize")(function* () {
76
34
  const sql = yield* SqlClientService.SqlClient;
77
35
  const names = yield* sql`
@@ -100,7 +58,7 @@ const initializeDoSubscriptionStore = Effect.fn("DoSubscriptionStore.initialize"
100
58
  yield* sql`CREATE TABLE effect_agent_subscriptions (
101
59
  tenant_id TEXT NOT NULL, source_address TEXT NOT NULL, owner_id TEXT NOT NULL, subscription_id TEXT NOT NULL,
102
60
  ordinal INTEGER NOT NULL, source_name TEXT NOT NULL, source_version TEXT NOT NULL, matching_key TEXT NOT NULL,
103
- state TEXT NOT NULL, expires_at_millis INTEGER NOT NULL, recovery_at_millis INTEGER, record_json TEXT NOT NULL,
61
+ state TEXT NOT NULL, expires_at_millis INTEGER, recovery_at_millis INTEGER, recovery_present INTEGER NOT NULL DEFAULT 0, record_json TEXT NOT NULL,
104
62
  PRIMARY KEY (tenant_id, source_address, owner_id, subscription_id), UNIQUE (tenant_id, source_address, ordinal)
105
63
  )`.withoutTransform;
106
64
  yield* sql`CREATE INDEX effect_agent_subscriptions_owner ON effect_agent_subscriptions (tenant_id, source_address, owner_id, ordinal)`.withoutTransform;
@@ -109,7 +67,7 @@ const initializeDoSubscriptionStore = Effect.fn("DoSubscriptionStore.initialize"
109
67
  yield* sql`CREATE TABLE effect_agent_subscription_events (
110
68
  tenant_id TEXT NOT NULL, source_address TEXT NOT NULL, event_id TEXT NOT NULL, source_name TEXT NOT NULL,
111
69
  source_version TEXT NOT NULL, matching_key TEXT NOT NULL, payload_digest TEXT NOT NULL, cutoff INTEGER NOT NULL,
112
- cursor INTEGER NOT NULL, routing_complete INTEGER NOT NULL, next_attempt_at_millis INTEGER NOT NULL, record_json TEXT NOT NULL,
70
+ cursor INTEGER NOT NULL, routing_complete INTEGER NOT NULL, tombstone INTEGER NOT NULL DEFAULT 0, next_attempt_at_millis INTEGER NOT NULL, record_json TEXT NOT NULL,
113
71
  PRIMARY KEY (tenant_id, source_address, event_id)
114
72
  )`.withoutTransform;
115
73
  yield* sql`CREATE INDEX effect_agent_subscription_events_pending ON effect_agent_subscription_events (tenant_id, source_address, routing_complete, next_attempt_at_millis, event_id)`.withoutTransform;
@@ -131,17 +89,12 @@ const initializeDoSubscriptionStore = Effect.fn("DoSubscriptionStore.initialize"
131
89
  const state = yield* decodeRows(StoreStateRow, rows, "read subscription storage version");
132
90
  if (state.length !== 1 || state[0].storage_version !== CURRENT_SUBSCRIPTION_STORE_VERSION) return yield* corrupt(state.length === 1 ? `incompatible subscription storage version ${state[0].storage_version}; expected ${CURRENT_SUBSCRIPTION_STORE_VERSION}` : "invalid subscription storage version row");
133
91
  });
134
- const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function* (owned) {
92
+ const makeSubscriptionStore = Effect.fn("DoSubscriptionStore.make")(function* (owned) {
135
93
  const partition = yield* validate(SourcePartition, owned, "partition");
136
94
  const sql = yield* SqlClientService.SqlClient;
137
95
  const failpoint = yield* SubscriptionFailpoint;
138
96
  const transactions = yield* DoSubscriptionTransaction;
139
97
  yield* initializeDoSubscriptionStore();
140
- yield* sql`
141
- INSERT INTO effect_agent_subscription_sequences (
142
- tenant_id, source_address, sequence, event_scan_cursor, delivery_scan_cursor, recovery_scan_cursor
143
- ) VALUES (${partition.tenantId}, ${partition.address}, 0, '', '', 0) ON CONFLICT DO NOTHING
144
- `.pipe(Effect.mapError(() => unavailable("initialize subscription partition")));
145
98
  const query = (effect, code) => effect.pipe(Effect.mapError(() => unavailable(code)));
146
99
  const readIndexedDeadline = Effect.fn("DoSubscriptionStore.readIndexedDeadline")(function* () {
147
100
  const scanRows = yield* query(sql`
@@ -156,7 +109,11 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
156
109
  SELECT next_attempt_at_millis AS deadline FROM effect_agent_subscription_events
157
110
  WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND routing_complete=0
158
111
  UNION ALL SELECT next_attempt_at_millis FROM effect_agent_subscription_deliveries
159
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state NOT IN ('delivered','refused')
112
+ WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND CASE WHEN json_valid(record_json) THEN
113
+ ((state NOT IN ('delivered','refused') AND COALESCE(json_extract(record_json, '$.retry.parked'), 0)=0) OR (state='delivered' AND json_extract(record_json, '$.observeSettlement')=1))
114
+ ELSE state<>'refused' END
115
+ UNION ALL SELECT next_maintenance_at_millis FROM effect_agent_event_retention
116
+ WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
160
117
  UNION ALL SELECT recovery_at_millis FROM effect_agent_subscriptions
161
118
  WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state='active' AND recovery_at_millis IS NOT NULL
162
119
  )
@@ -184,498 +141,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
184
141
  yield* replaceAlarm(replace, yield* readIndexedDeadline());
185
142
  return value;
186
143
  }));
187
- const requirePartition = (candidate, code) => sameSourcePartition(candidate, partition) ? Effect.void : Effect.fail(error("validation", code));
188
- const requireKey = Effect.fn("SqliteSubscriptionStore.requireKey")(function* (input, code) {
189
- const key = yield* validate(SubscriptionKey, input, code);
190
- yield* requirePartition(key.partition, code);
191
- return key;
192
- });
193
- const readRegistration = Effect.fn("SqliteSubscriptionStore.readRegistration")(function* (key, code) {
194
- const rows = yield* query(sql`
195
- SELECT owner_id, subscription_id, ordinal, source_name, source_version, matching_key, state,
196
- expires_at_millis, recovery_at_millis, record_json FROM effect_agent_subscriptions
197
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
198
- AND owner_id=${key.ownerId} AND subscription_id=${key.subscriptionId}
199
- `, code);
200
- const decodedRows = yield* decodeRows(RegistrationRow, rows, code);
201
- if (decodedRows.length > 1) return yield* corrupt(code);
202
- const row = decodedRows[0];
203
- if (row === void 0) return null;
204
- const record = yield* decode(SubscriptionRecord, row.record_json, code);
205
- if (!sameSourcePartition(record.key.partition, partition) || record.key.ownerId !== row.owner_id || record.key.subscriptionId !== row.subscription_id || record.ordinal !== row.ordinal || record.configuration.source.name !== row.source_name || record.configuration.source.version !== row.source_version || record.configuration.matchingKey !== row.matching_key || record.state !== row.state || record.configuration.expiresAtMillis !== row.expires_at_millis || (record.recovery?.nextAttemptAtMillis ?? null) !== row.recovery_at_millis) return yield* corrupt(`${code}-projection`);
206
- return record;
207
- });
208
- const readEvent = Effect.fn("SqliteSubscriptionStore.readEvent")(function* (eventId, code) {
209
- const rows = yield* query(sql`
210
- SELECT event_id, source_name, source_version, matching_key, payload_digest, cutoff, cursor,
211
- routing_complete, next_attempt_at_millis, record_json FROM effect_agent_subscription_events
212
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND event_id=${eventId}
213
- `, code);
214
- const decodedRows = yield* decodeRows(EventRow, rows, code);
215
- if (decodedRows.length > 1) return yield* corrupt(code);
216
- const row = decodedRows[0];
217
- if (row === void 0) return null;
218
- const event = yield* decode(AcceptedEvent, row.record_json, code);
219
- if (!sameSourcePartition(event.partition, partition) || event.eventId !== row.event_id || event.source.name !== row.source_name || event.source.version !== row.source_version || event.matchingKey !== row.matching_key || event.payloadDigest !== row.payload_digest || event.cutoff !== row.cutoff || event.cursor !== row.cursor || (event.routingComplete ? 1 : 0) !== row.routing_complete || event.nextAttemptAtMillis !== row.next_attempt_at_millis) return yield* corrupt(`${code}-projection`);
220
- return event;
221
- });
222
- const readDelivery = Effect.fn("SqliteSubscriptionStore.readDelivery")(function* (key, code) {
223
- const rows = yield* query(sql`
224
- SELECT owner_id, subscription_id, event_id, delivery_key, state, next_attempt_at_millis, record_json
225
- FROM effect_agent_subscription_deliveries
226
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
227
- AND owner_id=${key.subscription.ownerId} AND subscription_id=${key.subscription.subscriptionId}
228
- AND event_id=${key.eventId}
229
- `, code);
230
- const decodedRows = yield* decodeRows(DeliveryRow, rows, code);
231
- if (decodedRows.length > 1) return yield* corrupt(code);
232
- const row = decodedRows[0];
233
- if (row === void 0) return null;
234
- const delivery = yield* decode(SubscriptionDelivery, row.record_json, code);
235
- if (!sameSourcePartition(delivery.key.subscription.partition, partition) || delivery.key.subscription.ownerId !== row.owner_id || delivery.key.subscription.subscriptionId !== row.subscription_id || delivery.key.eventId !== row.event_id || subscriptionDeliveryKeyString(delivery.key) !== row.delivery_key || delivery.state !== row.state || delivery.retry.nextAttemptAtMillis !== row.next_attempt_at_millis) return yield* corrupt(`${code}-projection`);
236
- return delivery;
237
- });
238
- const count = Effect.fn("SqliteSubscriptionStore.count")(function* (statement, code) {
239
- const rows = yield* query(statement, code);
240
- const decoded = yield* decodeRows(CountRow, rows, code);
241
- if (decoded.length !== 1) return yield* corrupt(code);
242
- return decoded[0].count;
243
- });
244
- const nextSequence = Effect.fn("SqliteSubscriptionStore.nextSequence")(function* () {
245
- const rows = yield* query(sql`
246
- UPDATE effect_agent_subscription_sequences SET sequence=sequence+1
247
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
248
- RETURNING sequence
249
- `, "advance subscription sequence");
250
- const decoded = yield* decodeRows(SequenceRow, rows, "advance subscription sequence");
251
- if (decoded.length !== 1) return yield* corrupt("subscription sequence");
252
- return decoded[0].sequence;
253
- });
254
- const writeRegistration = Effect.fn("SqliteSubscriptionStore.writeRegistration")(function* (record) {
255
- const json = yield* encode(SubscriptionRecord, record, "encode subscription");
256
- yield* query(sql`
257
- UPDATE effect_agent_subscriptions SET state=${record.state}, expires_at_millis=${record.configuration.expiresAtMillis},
258
- recovery_at_millis=${record.recovery?.nextAttemptAtMillis ?? null}, record_json=${json}
259
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
260
- AND owner_id=${record.key.ownerId} AND subscription_id=${record.key.subscriptionId}
261
- `, "write subscription");
262
- });
263
- const writeEvent = Effect.fn("SqliteSubscriptionStore.writeEvent")(function* (event) {
264
- const json = yield* encode(AcceptedEvent, event, "encode event");
265
- yield* query(sql`
266
- UPDATE effect_agent_subscription_events SET cursor=${event.cursor}, routing_complete=${event.routingComplete ? 1 : 0},
267
- next_attempt_at_millis=${event.nextAttemptAtMillis}, record_json=${json}
268
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND event_id=${event.eventId}
269
- `, "write event");
270
- });
271
- const writeDelivery = Effect.fn("SqliteSubscriptionStore.writeDelivery")(function* (delivery) {
272
- const json = yield* encode(SubscriptionDelivery, delivery, "encode delivery");
273
- yield* query(sql`
274
- UPDATE effect_agent_subscription_deliveries SET state=${delivery.state},
275
- next_attempt_at_millis=${delivery.retry.nextAttemptAtMillis}, record_json=${json}
276
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
277
- AND owner_id=${delivery.key.subscription.ownerId} AND subscription_id=${delivery.key.subscription.subscriptionId}
278
- AND event_id=${delivery.key.eventId}
279
- `, "write delivery");
280
- });
281
- const register = Effect.fn("SqliteSubscriptionStore.register")(function* (input, inputLimits) {
282
- const record = yield* validate(SubscriptionRecord, input, "register-record");
283
- const limits = yield* validate(SubscriptionLimits, inputLimits, "register-limits");
284
- yield* requirePartition(record.key.partition, "register-partition");
285
- const result = yield* transact(Effect.gen(function* () {
286
- const existing = yield* readRegistration(record.key, "register-existing");
287
- if (existing !== null) {
288
- if (existing.creationFingerprint !== record.creationFingerprint) return yield* error("conflict", "registration-identity");
289
- return {
290
- value: existing,
291
- changed: false
292
- };
293
- }
294
- if (bytes(record.configuration.context) > limits.maxContextBytes) return yield* error("capacity", "context-bytes");
295
- if (bytes(record.configuration.parameters) > limits.maxPayloadBytes) return yield* error("capacity", "parameters-bytes");
296
- if (record.configuration.expiresAtMillis - record.createdAtMillis > limits.maxLifetimeMillis) return yield* error("capacity", "lifetime");
297
- if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`, "count registrations")) >= limits.maxRegistrations) return yield* error("capacity", "registrations");
298
- if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND owner_id=${record.key.ownerId}`, "count owner registrations")) >= limits.maxRegistrationsPerOwner) return yield* error("capacity", "owner-registrations");
299
- const assigned = {
300
- ...record,
301
- ordinal: yield* nextSequence()
302
- };
303
- const json = yield* encode(SubscriptionRecord, assigned, "encode registration");
304
- yield* failpoint.hit("subscription:register:before");
305
- yield* query(sql`
306
- INSERT INTO effect_agent_subscriptions (tenant_id, source_address, owner_id, subscription_id, ordinal,
307
- source_name, source_version, matching_key, state, expires_at_millis, recovery_at_millis, record_json)
308
- VALUES (${partition.tenantId}, ${partition.address}, ${assigned.key.ownerId}, ${assigned.key.subscriptionId}, ${assigned.ordinal},
309
- ${assigned.configuration.source.name}, ${assigned.configuration.source.version}, ${assigned.configuration.matchingKey}, ${assigned.state},
310
- ${assigned.configuration.expiresAtMillis}, ${assigned.recovery?.nextAttemptAtMillis ?? null}, ${json})
311
- `, "insert registration");
312
- return {
313
- value: assigned,
314
- changed: true
315
- };
316
- }));
317
- if (result.changed) yield* failpoint.hit("subscription:register:after");
318
- return result.value;
319
- });
320
- const get = Effect.fn("SqliteSubscriptionStore.get")(function* (input) {
321
- return yield* readRegistration(yield* requireKey(input, "get-key"), "get subscription");
322
- });
323
- const list = Effect.fn("SqliteSubscriptionStore.list")(function* (ownerId, after, limit) {
324
- const rows = yield* query(sql`
325
- SELECT owner_id, subscription_id, ordinal FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
326
- AND owner_id=${ownerId} AND ordinal>${after} ORDER BY ordinal LIMIT ${limit}
327
- `, "list subscriptions");
328
- const decoded = yield* decodeRows(Schema.Struct({
329
- owner_id: Schema.String,
330
- subscription_id: Schema.String,
331
- ordinal: Schema.Natural
332
- }), rows, "list subscriptions");
333
- return yield* Effect.forEach(decoded, Effect.fn("SqliteSubscriptionStore.listRecord")(function* (row) {
334
- const record = yield* readRegistration({
335
- partition,
336
- ownerId: row.owner_id,
337
- subscriptionId: row.subscription_id
338
- }, "list subscription");
339
- if (record === null || record.ordinal !== row.ordinal) return yield* corrupt("list subscription projection");
340
- return record;
341
- }));
342
- });
343
- const cancel = Effect.fn("SqliteSubscriptionStore.cancel")(function* (input) {
344
- const key = yield* requireKey(input, "cancel-key");
345
- const result = yield* transact(Effect.gen(function* () {
346
- const current = yield* readRegistration(key, "cancel subscription");
347
- if (current === null) return yield* error("not-found", "subscription");
348
- if (current.state === "cancelled") return {
349
- value: current,
350
- changed: false
351
- };
352
- const updated = {
353
- ...current,
354
- state: "cancelled",
355
- recovery: null
356
- };
357
- yield* failpoint.hit("subscription:cancel:before");
358
- yield* writeRegistration(updated);
359
- return {
360
- value: updated,
361
- changed: true
362
- };
363
- }));
364
- if (result.changed) yield* failpoint.hit("subscription:cancel:after");
365
- return result.value;
366
- });
367
- const accept = Effect.fn("SqliteSubscriptionStore.accept")(function* (input, inputLimits) {
368
- const event = yield* validate(AcceptedEvent, input, "accept-event");
369
- const limits = yield* validate(SubscriptionLimits, inputLimits, "accept-limits");
370
- yield* requirePartition(event.partition, "accept-partition");
371
- const result = yield* transact(Effect.gen(function* () {
372
- const existing = yield* readEvent(event.eventId, "accept event");
373
- if (existing !== null) {
374
- if (!sameAcceptedEventIdentity(existing, event)) return yield* error("conflict", "event-identity");
375
- return {
376
- value: existing,
377
- changed: false
378
- };
379
- }
380
- if (bytes(event.payload) > limits.maxPayloadBytes) return yield* error("capacity", "payload-bytes");
381
- if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_events WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`, "count events")) >= limits.maxEvents) return yield* error("capacity", "events");
382
- const accepted = {
383
- ...event,
384
- cutoff: yield* nextSequence(),
385
- cursor: 0,
386
- routingComplete: false,
387
- routingFailure: null
388
- };
389
- const json = yield* encode(AcceptedEvent, accepted, "encode accepted event");
390
- yield* failpoint.hit("subscription:accept:before");
391
- yield* query(sql`
392
- INSERT INTO effect_agent_subscription_events (tenant_id, source_address, event_id, source_name, source_version,
393
- matching_key, payload_digest, cutoff, cursor, routing_complete, next_attempt_at_millis, record_json)
394
- VALUES (${partition.tenantId}, ${partition.address}, ${accepted.eventId}, ${accepted.source.name}, ${accepted.source.version},
395
- ${accepted.matchingKey}, ${accepted.payloadDigest}, ${accepted.cutoff}, ${accepted.cursor}, 0, ${accepted.nextAttemptAtMillis}, ${json})
396
- `, "insert event");
397
- return {
398
- value: accepted,
399
- changed: true
400
- };
401
- }));
402
- if (result.changed) yield* failpoint.hit("subscription:accept:after");
403
- return result.value;
404
- });
405
- const event = Effect.fn("SqliteSubscriptionStore.event")((eventId) => readEvent(eventId, "get event"));
406
- const pendingEvents = Effect.fn("SqliteSubscriptionStore.pendingEvents")(function* (nowMillis, after, limit) {
407
- const rows = yield* query(sql`
408
- SELECT event_id FROM effect_agent_subscription_events WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
409
- AND routing_complete=0 AND next_attempt_at_millis<=${nowMillis} AND event_id>${after} ORDER BY event_id LIMIT ${limit}
410
- `, "pending events");
411
- return yield* decodeRows(Schema.Struct({ event_id: Schema.String }), rows, "pending event keys").pipe(Effect.map((items) => items.map((item) => item.event_id)));
412
- });
413
- const candidates = Effect.fn("SqliteSubscriptionStore.candidates")(function* (input, limit) {
414
- const supplied = yield* validate(AcceptedEvent, input, "candidates-event");
415
- yield* requirePartition(supplied.partition, "candidates-partition");
416
- const stored = yield* readEvent(supplied.eventId, "candidates event");
417
- if (stored === null) return yield* error("not-found", "event");
418
- if (!sameAcceptedEventIdentity(stored, supplied)) return yield* error("conflict", "event");
419
- const rows = yield* query(sql`
420
- SELECT owner_id, subscription_id, ordinal FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
421
- AND source_name=${stored.source.name} AND source_version=${stored.source.version} AND matching_key=${stored.matchingKey}
422
- AND ordinal>${stored.cursor} AND ordinal<=${stored.cutoff} ORDER BY ordinal LIMIT ${limit}
423
- `, "subscription candidates");
424
- const decoded = yield* decodeRows(Schema.Struct({
425
- owner_id: Schema.String,
426
- subscription_id: Schema.String,
427
- ordinal: Schema.Natural
428
- }), rows, "subscription candidates");
429
- return yield* Effect.forEach(decoded, Effect.fn("SqliteSubscriptionStore.candidateRecord")(function* (row) {
430
- const record = yield* readRegistration({
431
- partition,
432
- ownerId: row.owner_id,
433
- subscriptionId: row.subscription_id
434
- }, "subscription candidate");
435
- if (record === null || record.ordinal !== row.ordinal) return yield* corrupt("subscription candidate projection");
436
- return record;
437
- }));
438
- });
439
- const insertDelivery = Effect.fn("SqliteSubscriptionStore.insertDelivery")(function* (delivery) {
440
- const json = yield* encode(SubscriptionDelivery, delivery, "encode selected delivery");
441
- yield* query(sql`
442
- INSERT INTO effect_agent_subscription_deliveries (tenant_id, source_address, owner_id, subscription_id, event_id,
443
- delivery_key, state, next_attempt_at_millis, record_json)
444
- VALUES (${partition.tenantId}, ${partition.address}, ${delivery.key.subscription.ownerId}, ${delivery.key.subscription.subscriptionId},
445
- ${delivery.key.eventId}, ${subscriptionDeliveryKeyString(delivery.key)}, ${delivery.state}, ${delivery.retry.nextAttemptAtMillis}, ${json})
446
- `, "insert delivery");
447
- });
448
- const select = Effect.fn("SqliteSubscriptionStore.select")(function* (inputEvent, inputDeliveries, cursor, complete, nowMillis, inputLimits) {
449
- const supplied = yield* validate(AcceptedEvent, inputEvent, "select-event");
450
- const deliveries = yield* validate(Schema.Array(SubscriptionDelivery), inputDeliveries, "select-deliveries");
451
- const limits = yield* validate(SubscriptionLimits, inputLimits, "select-limits");
452
- yield* requirePartition(supplied.partition, "select-partition");
453
- for (const candidate of deliveries) yield* requirePartition(candidate.key.subscription.partition, "select-delivery-partition");
454
- if (yield* transact(Effect.gen(function* () {
455
- const accepted = yield* readEvent(supplied.eventId, "select event");
456
- if (accepted === null) return yield* error("not-found", "event");
457
- if (!sameAcceptedEventIdentity(accepted, supplied) || accepted.cursor !== supplied.cursor) return yield* error("conflict", "event-cursor");
458
- if (accepted.routingComplete) return false;
459
- if (!Number.isSafeInteger(cursor) || cursor < accepted.cursor || cursor > accepted.cutoff) return yield* error("validation", "cursor");
460
- yield* failpoint.hit("subscription:select:before");
461
- const effectiveNowMillis = Math.max(nowMillis, yield* Clock.currentTimeMillis);
462
- const additions = [];
463
- for (const delivery of deliveries) {
464
- const record = yield* readRegistration(delivery.key.subscription, "select registration");
465
- if (record === null) return yield* error("not-found", "subscription");
466
- if (!subscriptionDeliveryCanSelect(delivery, record, accepted) || delivery.key.eventId !== accepted.eventId || delivery.source.name !== accepted.source.name || delivery.source.version !== accepted.source.version || record.ordinal <= accepted.cursor || record.ordinal > cursor) return yield* error("conflict", "selection");
467
- const existing = yield* readDelivery(delivery.key, "select existing delivery");
468
- if (existing !== null) {
469
- if (!sameDeliveryIdentity(existing, delivery)) return yield* error("conflict", "delivery-identity");
470
- continue;
471
- }
472
- if (!subscriptionCanSelect(record, accepted, effectiveNowMillis, false)) continue;
473
- additions.push({
474
- delivery,
475
- record
476
- });
477
- }
478
- if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`, "count deliveries")) + additions.length > limits.maxDeliveries) return yield* error("capacity", "deliveries");
479
- for (const ownerId of new Set(additions.map(({ record }) => record.key.ownerId))) if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND owner_id=${ownerId}`, "count owner deliveries")) + additions.filter(({ record }) => record.key.ownerId === ownerId).length > limits.maxDeliveriesPerOwner) return yield* error("capacity", "owner-deliveries");
480
- for (const addition of additions) {
481
- yield* insertDelivery(addition.delivery);
482
- if (addition.record.configuration.mode === "once") yield* writeRegistration({
483
- ...addition.record,
484
- state: "consumed",
485
- recovery: null
486
- });
487
- }
488
- yield* writeEvent({
489
- ...accepted,
490
- cursor,
491
- routingComplete: complete,
492
- routingFailure: null
493
- });
494
- return true;
495
- }))) yield* failpoint.hit("subscription:select:after");
496
- });
497
- const catchUp = Effect.fn("SqliteSubscriptionStore.catchUp")(function* (inputEvent, inputDelivery, nowMillis, inputLimits) {
498
- const supplied = yield* validate(AcceptedEvent, inputEvent, "catch-up-event");
499
- const delivery = yield* validate(SubscriptionDelivery, inputDelivery, "catch-up-delivery");
500
- const limits = yield* validate(SubscriptionLimits, inputLimits, "catch-up-limits");
501
- yield* requirePartition(supplied.partition, "catch-up-partition");
502
- yield* requirePartition(delivery.key.subscription.partition, "catch-up-delivery-partition");
503
- if (yield* transact(Effect.gen(function* () {
504
- const accepted = yield* readEvent(supplied.eventId, "catch-up event");
505
- const record = yield* readRegistration(delivery.key.subscription, "catch-up subscription");
506
- if (accepted === null || record === null) return yield* error("not-found", accepted === null ? "event" : "subscription");
507
- if (!sameAcceptedEventIdentity(accepted, supplied) || !subscriptionDeliveryCanSelect(delivery, record, accepted) || delivery.key.eventId !== accepted.eventId || delivery.source.name !== accepted.source.name || delivery.source.version !== accepted.source.version || record.configuration.mode !== "once") return yield* error("conflict", "catch-up-identity");
508
- const existing = yield* readDelivery(delivery.key, "catch-up existing delivery");
509
- if (existing !== null) {
510
- if (!sameDeliveryIdentity(existing, delivery)) return yield* error("conflict", "delivery-identity");
511
- return false;
512
- }
513
- yield* failpoint.hit("subscription:catch-up:before");
514
- const effectiveNowMillis = Math.max(nowMillis, yield* Clock.currentTimeMillis);
515
- if (!subscriptionCanSelect(record, accepted, effectiveNowMillis, true)) return yield* error("conflict", "catch-up-eligibility");
516
- if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`, "count deliveries")) >= limits.maxDeliveries) return yield* error("capacity", "deliveries");
517
- if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND owner_id=${record.key.ownerId}`, "count owner deliveries")) >= limits.maxDeliveriesPerOwner) return yield* error("capacity", "owner-deliveries");
518
- yield* insertDelivery(delivery);
519
- yield* writeRegistration({
520
- ...record,
521
- state: "consumed",
522
- recovery: null
523
- });
524
- return true;
525
- }))) yield* failpoint.hit("subscription:catch-up:after");
526
- });
527
- const deferEvent = Effect.fn("SqliteSubscriptionStore.deferEvent")(function* (eventId, nextAttemptAtMillis, code) {
528
- const routingFailure = code === void 0 ? "routing-failed" : yield* validate(SubscriptionName, code, "routing-failure");
529
- yield* transact(Effect.gen(function* () {
530
- const accepted = yield* readEvent(eventId, "defer event");
531
- if (accepted === null) return yield* error("not-found", "event");
532
- yield* failpoint.hit("subscription:defer-event:before");
533
- yield* writeEvent({
534
- ...accepted,
535
- nextAttemptAtMillis,
536
- routingFailure
537
- });
538
- }));
539
- yield* failpoint.hit("subscription:defer-event:after");
540
- });
541
- const delivery = Effect.fn("SqliteSubscriptionStore.delivery")(function* (input) {
542
- const key = yield* validate(SubscriptionDeliveryKey, input, "delivery-key");
543
- yield* requirePartition(key.subscription.partition, "delivery-partition");
544
- return yield* readDelivery(key, "get delivery");
545
- });
546
- const pendingDeliveries = Effect.fn("SqliteSubscriptionStore.pendingDeliveries")(function* (nowMillis, after, limit) {
547
- const rows = yield* query(sql`
548
- SELECT owner_id, subscription_id, event_id FROM effect_agent_subscription_deliveries
549
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state NOT IN ('delivered','refused')
550
- AND next_attempt_at_millis<=${nowMillis} AND delivery_key>${after} ORDER BY delivery_key LIMIT ${limit}
551
- `, "pending deliveries");
552
- const rowSchema = Schema.Struct({
553
- owner_id: Schema.String,
554
- subscription_id: Schema.String,
555
- event_id: Schema.String
556
- });
557
- return yield* decodeRows(rowSchema, rows, "pending delivery keys").pipe(Effect.map((items) => items.map((item) => ({
558
- subscription: {
559
- partition,
560
- ownerId: item.owner_id,
561
- subscriptionId: item.subscription_id
562
- },
563
- eventId: item.event_id
564
- }))));
565
- });
566
- const listDeliveries = Effect.fn("SqliteSubscriptionStore.listDeliveries")(function* (input, after, limit) {
567
- const key = yield* requireKey(input, "list-deliveries-key");
568
- const rows = yield* query(sql`
569
- SELECT record_json FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
570
- AND owner_id=${key.ownerId} AND subscription_id=${key.subscriptionId} AND delivery_key>${after} ORDER BY delivery_key LIMIT ${limit}
571
- `, "list deliveries");
572
- const decoded = yield* decodeRows(JsonRow, rows, "list deliveries");
573
- return yield* Effect.forEach(decoded, (row) => decode(SubscriptionDelivery, row.record_json, "list delivery"));
574
- });
575
- const changeDelivery = Effect.fn("SqliteSubscriptionStore.changeDelivery")(function* (inputKey, inputDeliveryId, inputChange) {
576
- const key = yield* validate(SubscriptionDeliveryKey, inputKey, "change-delivery-key");
577
- const deliveryId = yield* validate(Digest, inputDeliveryId, "change-delivery-id");
578
- const change = yield* validate(DeliveryChange, inputChange, "change-delivery-change");
579
- yield* requirePartition(key.subscription.partition, "change-delivery-partition");
580
- const result = yield* transact(Effect.gen(function* () {
581
- const existing = yield* readDelivery(key, "change delivery");
582
- const record = yield* readRegistration(key.subscription, "change delivery subscription");
583
- if (existing === null || record === null) return yield* error("not-found", existing === null ? "delivery" : "subscription");
584
- yield* failpoint.hit(`subscription:delivery-${change._tag.toLowerCase()}:before`);
585
- const effectiveChange = change._tag === "Prepare" ? {
586
- ...change,
587
- nowMillis: Math.max(change.nowMillis, yield* Clock.currentTimeMillis)
588
- } : change;
589
- const transition = applySubscriptionDeliveryChange(existing, record, deliveryId, effectiveChange);
590
- if (Result.isFailure(transition)) return yield* transition.failure;
591
- if (transition.success === existing) return {
592
- value: existing,
593
- changed: false
594
- };
595
- yield* writeDelivery(transition.success);
596
- return {
597
- value: transition.success,
598
- changed: true
599
- };
600
- }));
601
- if (result.changed) yield* failpoint.hit(`subscription:delivery-${change._tag.toLowerCase()}:after`);
602
- return result.value;
603
- });
604
- const recovering = Effect.fn("SqliteSubscriptionStore.recovering")(function* (nowMillis, after, limit) {
605
- const rows = yield* query(sql`
606
- SELECT owner_id, subscription_id, ordinal FROM effect_agent_subscriptions
607
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state='active'
608
- AND recovery_at_millis IS NOT NULL AND recovery_at_millis<=${nowMillis} AND ordinal>${after}
609
- ORDER BY ordinal LIMIT ${limit}
610
- `, "recovering subscriptions");
611
- const rowSchema = Schema.Struct({
612
- owner_id: Schema.String,
613
- subscription_id: Schema.String,
614
- ordinal: Schema.Natural
615
- });
616
- return yield* decodeRows(rowSchema, rows, "recovering subscription keys").pipe(Effect.map((items) => items.map((item) => ({
617
- key: {
618
- partition,
619
- ownerId: item.owner_id,
620
- subscriptionId: item.subscription_id
621
- },
622
- ordinal: item.ordinal
623
- }))));
624
- });
625
- const deferRecovery = Effect.fn("SqliteSubscriptionStore.deferRecovery")(function* (input, recovery) {
626
- const key = yield* requireKey(input, "defer-recovery-key");
627
- yield* transact(Effect.gen(function* () {
628
- const record = yield* readRegistration(key, "defer recovery");
629
- if (record === null) return yield* error("not-found", "subscription");
630
- yield* failpoint.hit("subscription:defer-recovery:before");
631
- yield* writeRegistration({
632
- ...record,
633
- recovery: record.state === "active" ? recovery : null
634
- });
635
- }));
636
- yield* failpoint.hit("subscription:defer-recovery:after");
637
- });
638
- const readScanCursors = Effect.gen(function* () {
639
- const rows = yield* query(sql`
640
- SELECT event_scan_cursor, delivery_scan_cursor, recovery_scan_cursor
641
- FROM effect_agent_subscription_sequences
642
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
643
- `, "read subscription scan cursors");
644
- const decoded = yield* decodeRows(ScanRow, rows, "read subscription scan cursors");
645
- if (decoded.length !== 1) return yield* corrupt("subscription scan cursors");
646
- return {
647
- events: decoded[0].event_scan_cursor,
648
- deliveries: decoded[0].delivery_scan_cursor,
649
- recovery: decoded[0].recovery_scan_cursor
650
- };
651
- });
652
- const advanceScanCursors = Effect.fn("SqliteSubscriptionStore.advanceScanCursors")(function* (input) {
653
- const cursors = yield* validate(SubscriptionScanCursors, input, "scan-cursors");
654
- yield* transact(Effect.gen(function* () {
655
- yield* failpoint.hit("subscription:advance-scan-cursors:before");
656
- yield* query(sql`
657
- UPDATE effect_agent_subscription_sequences
658
- SET event_scan_cursor=${cursors.events}, delivery_scan_cursor=${cursors.deliveries}, recovery_scan_cursor=${cursors.recovery}
659
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
660
- `, "advance subscription scan cursors");
661
- }));
662
- yield* failpoint.hit("subscription:advance-scan-cursors:after");
663
- });
664
- const indexedDeadline = query(sql`
665
- SELECT MIN(deadline) AS deadline FROM (
666
- SELECT next_attempt_at_millis AS deadline FROM effect_agent_subscription_events
667
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND routing_complete=0
668
- UNION ALL SELECT next_attempt_at_millis FROM effect_agent_subscription_deliveries
669
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state NOT IN ('delivered','refused')
670
- UNION ALL SELECT recovery_at_millis FROM effect_agent_subscriptions
671
- WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state='active' AND recovery_at_millis IS NOT NULL
672
- )
673
- `, "next subscription deadline").pipe(Effect.flatMap((rows) => decodeRows(Schema.Struct({ deadline: Schema.NullOr(Schema.Number) }), rows, "next subscription deadline")), Effect.flatMap((rows) => rows.length === 1 ? Effect.succeed(rows[0].deadline) : Effect.fail(corrupt("next subscription deadline"))));
674
- const nextDeadline = Effect.gen(function* () {
675
- const cursors = yield* readScanCursors;
676
- if (cursors.events !== "" || cursors.deliveries !== "" || cursors.recovery !== 0) return 0;
677
- return yield* indexedDeadline;
678
- });
144
+ const store = yield* makeSqlSubscriptionStore(partition, { maxStoredJsonLength: 19e5 }).pipe(Effect.provide(Layer.succeed(SqlSubscriptionTransaction)({ run: transact })));
679
145
  const prearm = Effect.fn("DoSubscriptionStore.prearm")(function* (deadlineAtMillis) {
680
146
  yield* transactions.run((replace) => replaceAlarm(replace, deadlineAtMillis));
681
147
  yield* failpoint.hit("subscription:prearm:after");
@@ -686,29 +152,7 @@ const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function
686
152
  }));
687
153
  yield* failpoint.hit("subscription:reconcile:after");
688
154
  });
689
- return Context.make(SubscriptionStore, {
690
- partition,
691
- register,
692
- get,
693
- list,
694
- cancel,
695
- accept,
696
- event,
697
- pendingEvents,
698
- candidates,
699
- select,
700
- catchUp,
701
- deferEvent,
702
- delivery,
703
- pendingDeliveries,
704
- listDeliveries,
705
- changeDelivery,
706
- recovering,
707
- deferRecovery,
708
- readScanCursors,
709
- advanceScanCursors,
710
- nextDeadline
711
- }).pipe(Context.add(DoSubscriptionAlarmControl, {
155
+ return Context.make(SubscriptionStore, store).pipe(Context.add(DoSubscriptionAlarmControl, {
712
156
  prearm,
713
157
  reconcile
714
158
  }));