@effect-agent/storage-memory 0.1.0-beta.41 → 0.1.0-beta.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -48
- package/src/memory-ledger.ts +192 -0
- package/src/memory-schedule-store.ts +42 -0
- package/src/memory-storage.ts +48 -0
- package/src/memory-subscription-store.ts +149 -0
- package/src/semantic-memory-index.ts +37 -0
- package/src/testing.ts +1 -0
|
@@ -106,6 +106,7 @@ const candidateIndexKey = (record: SubscriptionRecord): string =>
|
|
|
106
106
|
record.configuration.source.version,
|
|
107
107
|
record.configuration.matchingKey,
|
|
108
108
|
]);
|
|
109
|
+
|
|
109
110
|
const eventCandidateIndexKey = (event: AcceptedEvent): string =>
|
|
110
111
|
JSON.stringify([event.source.name, event.source.version, event.matchingKey]);
|
|
111
112
|
|
|
@@ -129,6 +130,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
129
130
|
ownedPartition: SourcePartition,
|
|
130
131
|
) {
|
|
131
132
|
const partition = yield* validate(SourcePartition, ownedPartition, "partition");
|
|
133
|
+
|
|
132
134
|
const state = yield* Ref.make<MemorySubscriptionState>({
|
|
133
135
|
sequence: 0,
|
|
134
136
|
registrations: new Map(),
|
|
@@ -142,6 +144,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
142
144
|
ownerDeliveryCounts: new Map(),
|
|
143
145
|
scanCursors: { events: "", deliveries: "", recovery: 0 },
|
|
144
146
|
});
|
|
147
|
+
|
|
145
148
|
const failpoint = yield* SubscriptionFailpoint;
|
|
146
149
|
|
|
147
150
|
const requirePartition = <A extends { readonly partition: SourcePartition }>(
|
|
@@ -162,8 +165,10 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
162
165
|
)(function* (input, inputLimits) {
|
|
163
166
|
const record = yield* validate(SubscriptionRecord, input, "register-record");
|
|
164
167
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "register-limits");
|
|
168
|
+
|
|
165
169
|
yield* requirePartition(record.key, "register-partition");
|
|
166
170
|
yield* failpoint.hit("subscription:register:before");
|
|
171
|
+
|
|
167
172
|
const result = yield* Effect.uninterruptible(
|
|
168
173
|
Ref.modify(
|
|
169
174
|
state,
|
|
@@ -175,9 +180,12 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
175
180
|
] => {
|
|
176
181
|
const key = subscriptionKeyString(record.key);
|
|
177
182
|
const existingText = current.registrations.get(key);
|
|
183
|
+
|
|
178
184
|
if (existingText !== undefined) {
|
|
179
185
|
const existing = decode(SubscriptionRecord, existingText, "register-existing");
|
|
186
|
+
|
|
180
187
|
if (Result.isFailure(existing)) return [Result.fail(existing.failure), current];
|
|
188
|
+
|
|
181
189
|
return existing.success.creationFingerprint === record.creationFingerprint
|
|
182
190
|
? [Result.succeed(existing.success), current]
|
|
183
191
|
: [Result.fail(error("conflict", "registration-identity")), current];
|
|
@@ -194,14 +202,18 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
194
202
|
if (current.registrations.size >= limits.maxRegistrations)
|
|
195
203
|
return [Result.fail(error("capacity", "registrations")), current];
|
|
196
204
|
const ownerCount = current.ownerRegistrationCounts.get(record.key.ownerId) ?? 0;
|
|
205
|
+
|
|
197
206
|
if (ownerCount >= limits.maxRegistrationsPerOwner)
|
|
198
207
|
return [Result.fail(error("capacity", "owner-registrations")), current];
|
|
199
208
|
const assigned = { ...record, ordinal: current.sequence + 1 };
|
|
200
209
|
const encoded = encode(SubscriptionRecord, assigned, "register-encode");
|
|
210
|
+
|
|
201
211
|
if (Result.isFailure(encoded)) return [Result.fail(encoded.failure), current];
|
|
202
212
|
const registrations = new Map(current.registrations);
|
|
213
|
+
|
|
203
214
|
registrations.set(key, encoded.success);
|
|
204
215
|
const registrationIndex = new Map(current.registrationIndex);
|
|
216
|
+
|
|
205
217
|
registrationIndex.set(key, {
|
|
206
218
|
key: assigned.key,
|
|
207
219
|
ordinal: assigned.ordinal,
|
|
@@ -210,9 +222,12 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
210
222
|
});
|
|
211
223
|
const candidateIndex = new Map(current.candidateIndex);
|
|
212
224
|
const candidateKey = candidateIndexKey(assigned);
|
|
225
|
+
|
|
213
226
|
candidateIndex.set(candidateKey, [...(candidateIndex.get(candidateKey) ?? []), key]);
|
|
214
227
|
const ownerRegistrationCounts = new Map(current.ownerRegistrationCounts);
|
|
228
|
+
|
|
215
229
|
ownerRegistrationCounts.set(assigned.key.ownerId, ownerCount + 1);
|
|
230
|
+
|
|
216
231
|
return [
|
|
217
232
|
Result.succeed(assigned),
|
|
218
233
|
{
|
|
@@ -227,7 +242,9 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
227
242
|
},
|
|
228
243
|
),
|
|
229
244
|
).pipe(Effect.flatMap(Effect.fromResult));
|
|
245
|
+
|
|
230
246
|
yield* failpoint.hit("subscription:register:after");
|
|
247
|
+
|
|
231
248
|
return result;
|
|
232
249
|
});
|
|
233
250
|
|
|
@@ -235,6 +252,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
235
252
|
function* (input) {
|
|
236
253
|
const key = yield* requireKey(input, "get-key");
|
|
237
254
|
const text = (yield* Ref.get(state)).registrations.get(subscriptionKeyString(key));
|
|
255
|
+
|
|
238
256
|
return text === undefined
|
|
239
257
|
? null
|
|
240
258
|
: yield* decodeEffect(SubscriptionRecord, text, "get-record");
|
|
@@ -247,13 +265,16 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
247
265
|
return yield* error("validation", "list-page");
|
|
248
266
|
const current = yield* Ref.get(state);
|
|
249
267
|
const records: Array<SubscriptionRecord> = [];
|
|
268
|
+
|
|
250
269
|
for (const [storageKey, indexed] of current.registrationIndex) {
|
|
251
270
|
if (indexed.key.ownerId !== ownerId || indexed.ordinal <= after) continue;
|
|
252
271
|
const text = current.registrations.get(storageKey);
|
|
272
|
+
|
|
253
273
|
if (text === undefined) return yield* error("corrupt", "list-index");
|
|
254
274
|
records.push(yield* decodeEffect(SubscriptionRecord, text, "list-record"));
|
|
255
275
|
}
|
|
256
276
|
records.sort((a, b) => a.ordinal - b.ordinal);
|
|
277
|
+
|
|
257
278
|
return records.slice(0, limit);
|
|
258
279
|
},
|
|
259
280
|
);
|
|
@@ -262,7 +283,9 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
262
283
|
"MemorySubscriptionStore.cancel",
|
|
263
284
|
)(function* (input) {
|
|
264
285
|
const key = yield* requireKey(input, "cancel-key");
|
|
286
|
+
|
|
265
287
|
yield* failpoint.hit("subscription:cancel:before");
|
|
288
|
+
|
|
266
289
|
const result = yield* Effect.uninterruptible(
|
|
267
290
|
Ref.modify(
|
|
268
291
|
state,
|
|
@@ -274,26 +297,34 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
274
297
|
] => {
|
|
275
298
|
const storageKey = subscriptionKeyString(key);
|
|
276
299
|
const text = current.registrations.get(storageKey);
|
|
300
|
+
|
|
277
301
|
if (text === undefined) return [Result.fail(error("not-found", "subscription")), current];
|
|
278
302
|
const decoded = decode(SubscriptionRecord, text, "cancel-record");
|
|
303
|
+
|
|
279
304
|
if (Result.isFailure(decoded)) return [decoded, current];
|
|
280
305
|
if (decoded.success.state === "cancelled")
|
|
281
306
|
return [Result.succeed(decoded.success), current];
|
|
282
307
|
const cancelled = { ...decoded.success, state: "cancelled" as const, recovery: null };
|
|
283
308
|
const encoded = encode(SubscriptionRecord, cancelled, "cancel-encode");
|
|
309
|
+
|
|
284
310
|
if (Result.isFailure(encoded)) return [Result.fail(encoded.failure), current];
|
|
285
311
|
const registrations = new Map(current.registrations);
|
|
312
|
+
|
|
286
313
|
registrations.set(storageKey, encoded.success);
|
|
287
314
|
const registrationIndex = new Map(current.registrationIndex);
|
|
288
315
|
const indexed = registrationIndex.get(storageKey);
|
|
316
|
+
|
|
289
317
|
if (indexed === undefined)
|
|
290
318
|
return [Result.fail(error("corrupt", "cancel-index")), current];
|
|
291
319
|
registrationIndex.set(storageKey, { ...indexed, state: "cancelled", recoveryAt: null });
|
|
320
|
+
|
|
292
321
|
return [Result.succeed(cancelled), { ...current, registrations, registrationIndex }];
|
|
293
322
|
},
|
|
294
323
|
),
|
|
295
324
|
).pipe(Effect.flatMap(Effect.fromResult));
|
|
325
|
+
|
|
296
326
|
yield* failpoint.hit("subscription:cancel:after");
|
|
327
|
+
|
|
297
328
|
return result;
|
|
298
329
|
});
|
|
299
330
|
|
|
@@ -302,8 +333,10 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
302
333
|
)(function* (input, inputLimits) {
|
|
303
334
|
const event = yield* validate(AcceptedEvent, input, "accept-event");
|
|
304
335
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "accept-limits");
|
|
336
|
+
|
|
305
337
|
yield* requirePartition(event, "accept-partition");
|
|
306
338
|
yield* failpoint.hit("subscription:accept:before");
|
|
339
|
+
|
|
307
340
|
const result = yield* Effect.uninterruptible(
|
|
308
341
|
Ref.modify(
|
|
309
342
|
state,
|
|
@@ -311,9 +344,12 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
311
344
|
current,
|
|
312
345
|
): readonly [Result.Result<AcceptedEvent, SubscriptionError>, MemorySubscriptionState] => {
|
|
313
346
|
const existingText = current.events.get(event.eventId);
|
|
347
|
+
|
|
314
348
|
if (existingText !== undefined) {
|
|
315
349
|
const existing = decode(AcceptedEvent, existingText, "accept-existing");
|
|
350
|
+
|
|
316
351
|
if (Result.isFailure(existing)) return [Result.fail(existing.failure), current];
|
|
352
|
+
|
|
317
353
|
return sameEventIdentity(existing.success, event)
|
|
318
354
|
? [Result.succeed(existing.success), current]
|
|
319
355
|
: [Result.fail(error("conflict", "event-identity")), current];
|
|
@@ -322,6 +358,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
322
358
|
return [Result.fail(error("capacity", "payload-bytes")), current];
|
|
323
359
|
if (current.events.size >= limits.maxEvents)
|
|
324
360
|
return [Result.fail(error("capacity", "events")), current];
|
|
361
|
+
|
|
325
362
|
const accepted: AcceptedEvent = {
|
|
326
363
|
...event,
|
|
327
364
|
cutoff: current.sequence + 1,
|
|
@@ -329,15 +366,20 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
329
366
|
routingComplete: false,
|
|
330
367
|
routingFailure: null,
|
|
331
368
|
};
|
|
369
|
+
|
|
332
370
|
const encoded = encode(AcceptedEvent, accepted, "accept-encode");
|
|
371
|
+
|
|
333
372
|
if (Result.isFailure(encoded)) return [Result.fail(encoded.failure), current];
|
|
334
373
|
const events = new Map(current.events);
|
|
374
|
+
|
|
335
375
|
events.set(accepted.eventId, encoded.success);
|
|
336
376
|
const eventIndex = new Map(current.eventIndex);
|
|
377
|
+
|
|
337
378
|
eventIndex.set(accepted.eventId, {
|
|
338
379
|
routingComplete: false,
|
|
339
380
|
nextAttemptAtMillis: accepted.nextAttemptAtMillis,
|
|
340
381
|
});
|
|
382
|
+
|
|
341
383
|
return [
|
|
342
384
|
Result.succeed(accepted),
|
|
343
385
|
{ ...current, sequence: accepted.cutoff, events, eventIndex },
|
|
@@ -345,13 +387,16 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
345
387
|
},
|
|
346
388
|
),
|
|
347
389
|
).pipe(Effect.flatMap(Effect.fromResult));
|
|
390
|
+
|
|
348
391
|
yield* failpoint.hit("subscription:accept:after");
|
|
392
|
+
|
|
349
393
|
return result;
|
|
350
394
|
});
|
|
351
395
|
|
|
352
396
|
const event: SubscriptionStore["Service"]["event"] = Effect.fn("MemorySubscriptionStore.event")(
|
|
353
397
|
function* (eventId) {
|
|
354
398
|
const text = (yield* Ref.get(state)).events.get(eventId);
|
|
399
|
+
|
|
355
400
|
return text === undefined ? null : yield* decodeEffect(AcceptedEvent, text, "event-record");
|
|
356
401
|
},
|
|
357
402
|
);
|
|
@@ -360,6 +405,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
360
405
|
"MemorySubscriptionStore.pendingEvents",
|
|
361
406
|
)(function* (nowMillis, after, limit) {
|
|
362
407
|
const events: Array<string> = [];
|
|
408
|
+
|
|
363
409
|
for (const [eventId, indexed] of (yield* Ref.get(state)).eventIndex) {
|
|
364
410
|
if (
|
|
365
411
|
!indexed.routingComplete &&
|
|
@@ -369,6 +415,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
369
415
|
events.push(eventId);
|
|
370
416
|
}
|
|
371
417
|
events.sort(compareScheduleNames);
|
|
418
|
+
|
|
372
419
|
return events.slice(0, limit);
|
|
373
420
|
});
|
|
374
421
|
|
|
@@ -376,21 +423,27 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
376
423
|
"MemorySubscriptionStore.candidates",
|
|
377
424
|
)(function* (input, limit) {
|
|
378
425
|
const accepted = yield* validate(AcceptedEvent, input, "candidates-event");
|
|
426
|
+
|
|
379
427
|
yield* requirePartition(accepted, "candidates-partition");
|
|
380
428
|
const stored = yield* event(accepted.eventId);
|
|
429
|
+
|
|
381
430
|
if (stored === null || !sameEventIdentity(stored, accepted))
|
|
382
431
|
return yield* error(stored === null ? "not-found" : "conflict", "event");
|
|
383
432
|
const current = yield* Ref.get(state);
|
|
384
433
|
const records: Array<SubscriptionRecord> = [];
|
|
434
|
+
|
|
385
435
|
for (const storageKey of current.candidateIndex.get(eventCandidateIndexKey(stored)) ?? []) {
|
|
386
436
|
const indexed = current.registrationIndex.get(storageKey);
|
|
437
|
+
|
|
387
438
|
if (indexed === undefined) return yield* error("corrupt", "candidate-index");
|
|
388
439
|
if (indexed.ordinal <= stored.cursor || indexed.ordinal > stored.cutoff) continue;
|
|
389
440
|
const text = current.registrations.get(storageKey);
|
|
441
|
+
|
|
390
442
|
if (text === undefined) return yield* error("corrupt", "candidate-record");
|
|
391
443
|
records.push(yield* decodeEffect(SubscriptionRecord, text, "candidate-record"));
|
|
392
444
|
}
|
|
393
445
|
records.sort((a, b) => a.ordinal - b.ordinal);
|
|
446
|
+
|
|
394
447
|
return records.slice(0, limit);
|
|
395
448
|
});
|
|
396
449
|
|
|
@@ -398,24 +451,31 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
398
451
|
"MemorySubscriptionStore.select",
|
|
399
452
|
)(function* (inputEvent, inputDeliveries, cursor, complete, nowMillis, inputLimits) {
|
|
400
453
|
const suppliedEvent = yield* validate(AcceptedEvent, inputEvent, "select-event");
|
|
454
|
+
|
|
401
455
|
const deliveries = yield* validate(
|
|
402
456
|
Schema.Array(SubscriptionDelivery),
|
|
403
457
|
inputDeliveries,
|
|
404
458
|
"select-deliveries",
|
|
405
459
|
);
|
|
460
|
+
|
|
406
461
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "select-limits");
|
|
462
|
+
|
|
407
463
|
yield* requirePartition(suppliedEvent, "select-partition");
|
|
408
464
|
yield* failpoint.hit("subscription:select:before");
|
|
409
465
|
const effectiveNowMillis = Math.max(nowMillis, yield* Clock.currentTimeMillis);
|
|
466
|
+
|
|
410
467
|
yield* Effect.uninterruptible(
|
|
411
468
|
Ref.modify(
|
|
412
469
|
state,
|
|
413
470
|
(current): readonly [Result.Result<void, SubscriptionError>, MemorySubscriptionState] => {
|
|
414
471
|
const eventText = current.events.get(suppliedEvent.eventId);
|
|
472
|
+
|
|
415
473
|
if (eventText === undefined) return [Result.fail(error("not-found", "event")), current];
|
|
416
474
|
const decodedEvent = decode(AcceptedEvent, eventText, "select-event-record");
|
|
475
|
+
|
|
417
476
|
if (Result.isFailure(decodedEvent)) return [Result.fail(decodedEvent.failure), current];
|
|
418
477
|
const accepted = decodedEvent.success;
|
|
478
|
+
|
|
419
479
|
if (
|
|
420
480
|
!sameEventIdentity(accepted, suppliedEvent) ||
|
|
421
481
|
suppliedEvent.cursor !== accepted.cursor
|
|
@@ -436,13 +496,16 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
436
496
|
const additionIndex: Array<readonly [string, DeliveryIndex, string]> = [];
|
|
437
497
|
const registrationUpdates: Array<readonly [string, RegistrationIndex]> = [];
|
|
438
498
|
const owners = new Map(current.ownerDeliveryCounts);
|
|
499
|
+
|
|
439
500
|
for (const delivery of deliveries) {
|
|
440
501
|
const recordText = current.registrations.get(
|
|
441
502
|
subscriptionKeyString(delivery.key.subscription),
|
|
442
503
|
);
|
|
504
|
+
|
|
443
505
|
if (recordText === undefined)
|
|
444
506
|
return [Result.fail(error("not-found", "subscription")), current];
|
|
445
507
|
const record = decode(SubscriptionRecord, recordText, "select-registration");
|
|
508
|
+
|
|
446
509
|
if (Result.isFailure(record)) return [Result.fail(record.failure), current];
|
|
447
510
|
if (
|
|
448
511
|
!deliveryBelongsTo(delivery, record.success, accepted) ||
|
|
@@ -455,18 +518,22 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
455
518
|
continue;
|
|
456
519
|
const deliveryKey = subscriptionDeliveryKeyString(delivery.key);
|
|
457
520
|
const existingText = current.deliveries.get(deliveryKey);
|
|
521
|
+
|
|
458
522
|
if (existingText !== undefined) {
|
|
459
523
|
const existing = decode(SubscriptionDelivery, existingText, "select-existing");
|
|
524
|
+
|
|
460
525
|
if (Result.isFailure(existing)) return [Result.fail(existing.failure), current];
|
|
461
526
|
if (!sameDeliveryIdentity(existing.success, delivery))
|
|
462
527
|
return [Result.fail(error("conflict", "delivery-identity")), current];
|
|
463
528
|
continue;
|
|
464
529
|
}
|
|
530
|
+
|
|
465
531
|
const encodedDelivery = encode(
|
|
466
532
|
SubscriptionDelivery,
|
|
467
533
|
delivery,
|
|
468
534
|
"select-delivery-encode",
|
|
469
535
|
);
|
|
536
|
+
|
|
470
537
|
if (Result.isFailure(encodedDelivery))
|
|
471
538
|
return [Result.fail(encodedDelivery.failure), current];
|
|
472
539
|
additions.push([deliveryKey, encodedDelivery.success]);
|
|
@@ -480,21 +547,26 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
480
547
|
record.success.key.ownerId,
|
|
481
548
|
]);
|
|
482
549
|
const ownerId = record.success.key.ownerId;
|
|
550
|
+
|
|
483
551
|
owners.set(ownerId, (owners.get(ownerId) ?? 0) + 1);
|
|
484
552
|
if ((owners.get(ownerId) ?? 0) > limits.maxDeliveriesPerOwner)
|
|
485
553
|
return [Result.fail(error("capacity", "owner-deliveries")), current];
|
|
486
554
|
if (record.success.configuration.mode === "once") {
|
|
487
555
|
const consumed = { ...record.success, state: "consumed" as const, recovery: null };
|
|
556
|
+
|
|
488
557
|
const encodedRecord = encode(
|
|
489
558
|
SubscriptionRecord,
|
|
490
559
|
consumed,
|
|
491
560
|
"select-registration-encode",
|
|
492
561
|
);
|
|
562
|
+
|
|
493
563
|
if (Result.isFailure(encodedRecord))
|
|
494
564
|
return [Result.fail(encodedRecord.failure), current];
|
|
495
565
|
const consumedKey = subscriptionKeyString(consumed.key);
|
|
566
|
+
|
|
496
567
|
updates.push([consumedKey, encodedRecord.success]);
|
|
497
568
|
const indexed = current.registrationIndex.get(consumedKey);
|
|
569
|
+
|
|
498
570
|
if (indexed === undefined)
|
|
499
571
|
return [Result.fail(error("corrupt", "selection-index")), current];
|
|
500
572
|
registrationUpdates.push([
|
|
@@ -506,28 +578,38 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
506
578
|
if (current.deliveries.size + additions.length > limits.maxDeliveries)
|
|
507
579
|
return [Result.fail(error("capacity", "deliveries")), current];
|
|
508
580
|
const registrations = new Map(current.registrations);
|
|
581
|
+
|
|
509
582
|
for (const [key, value] of updates) registrations.set(key, value);
|
|
510
583
|
const nextDeliveries = new Map(current.deliveries);
|
|
584
|
+
|
|
511
585
|
for (const [key, value] of additions) nextDeliveries.set(key, value);
|
|
512
586
|
const deliveryIndex = new Map(current.deliveryIndex);
|
|
587
|
+
|
|
513
588
|
for (const [key, value] of additionIndex) deliveryIndex.set(key, value);
|
|
514
589
|
const registrationIndex = new Map(current.registrationIndex);
|
|
590
|
+
|
|
515
591
|
for (const [key, value] of registrationUpdates) registrationIndex.set(key, value);
|
|
592
|
+
|
|
516
593
|
const nextEvent: AcceptedEvent = {
|
|
517
594
|
...accepted,
|
|
518
595
|
cursor,
|
|
519
596
|
routingComplete: complete,
|
|
520
597
|
routingFailure: null,
|
|
521
598
|
};
|
|
599
|
+
|
|
522
600
|
const encodedEvent = encode(AcceptedEvent, nextEvent, "select-event-encode");
|
|
601
|
+
|
|
523
602
|
if (Result.isFailure(encodedEvent)) return [Result.fail(encodedEvent.failure), current];
|
|
524
603
|
const events = new Map(current.events);
|
|
604
|
+
|
|
525
605
|
events.set(nextEvent.eventId, encodedEvent.success);
|
|
526
606
|
const eventIndex = new Map(current.eventIndex);
|
|
607
|
+
|
|
527
608
|
eventIndex.set(nextEvent.eventId, {
|
|
528
609
|
routingComplete: nextEvent.routingComplete,
|
|
529
610
|
nextAttemptAtMillis: nextEvent.nextAttemptAtMillis,
|
|
530
611
|
});
|
|
612
|
+
|
|
531
613
|
return [
|
|
532
614
|
Result.void,
|
|
533
615
|
{
|
|
@@ -553,25 +635,31 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
553
635
|
const suppliedEvent = yield* validate(AcceptedEvent, inputEvent, "catch-up-event");
|
|
554
636
|
const delivery = yield* validate(SubscriptionDelivery, inputDelivery, "catch-up-delivery");
|
|
555
637
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "catch-up-limits");
|
|
638
|
+
|
|
556
639
|
yield* requirePartition(suppliedEvent, "catch-up-partition");
|
|
557
640
|
yield* failpoint.hit("subscription:catch-up:before");
|
|
558
641
|
const effectiveNowMillis = Math.max(nowMillis, yield* Clock.currentTimeMillis);
|
|
642
|
+
|
|
559
643
|
yield* Effect.uninterruptible(
|
|
560
644
|
Ref.modify(
|
|
561
645
|
state,
|
|
562
646
|
(current): readonly [Result.Result<void, SubscriptionError>, MemorySubscriptionState] => {
|
|
563
647
|
const eventText = current.events.get(suppliedEvent.eventId);
|
|
648
|
+
|
|
564
649
|
const recordText = current.registrations.get(
|
|
565
650
|
subscriptionKeyString(delivery.key.subscription),
|
|
566
651
|
);
|
|
652
|
+
|
|
567
653
|
if (eventText === undefined || recordText === undefined)
|
|
568
654
|
return [
|
|
569
655
|
Result.fail(error("not-found", eventText === undefined ? "event" : "subscription")),
|
|
570
656
|
current,
|
|
571
657
|
];
|
|
572
658
|
const accepted = decode(AcceptedEvent, eventText, "catch-up-event-record");
|
|
659
|
+
|
|
573
660
|
if (Result.isFailure(accepted)) return [Result.fail(accepted.failure), current];
|
|
574
661
|
const record = decode(SubscriptionRecord, recordText, "catch-up-registration");
|
|
662
|
+
|
|
575
663
|
if (Result.isFailure(record)) return [Result.fail(record.failure), current];
|
|
576
664
|
if (
|
|
577
665
|
!sameEventIdentity(accepted.success, suppliedEvent) ||
|
|
@@ -582,9 +670,12 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
582
670
|
return [Result.fail(error("conflict", "catch-up-identity")), current];
|
|
583
671
|
const key = subscriptionDeliveryKeyString(delivery.key);
|
|
584
672
|
const existingText = current.deliveries.get(key);
|
|
673
|
+
|
|
585
674
|
if (existingText !== undefined) {
|
|
586
675
|
const existing = decode(SubscriptionDelivery, existingText, "catch-up-existing");
|
|
676
|
+
|
|
587
677
|
if (Result.isFailure(existing)) return [Result.fail(existing.failure), current];
|
|
678
|
+
|
|
588
679
|
return sameDeliveryIdentity(existing.success, delivery)
|
|
589
680
|
? [Result.void, current]
|
|
590
681
|
: [Result.fail(error("conflict", "delivery-identity")), current];
|
|
@@ -594,40 +685,51 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
594
685
|
if (current.deliveries.size >= limits.maxDeliveries)
|
|
595
686
|
return [Result.fail(error("capacity", "deliveries")), current];
|
|
596
687
|
const ownerCount = current.ownerDeliveryCounts.get(record.success.key.ownerId) ?? 0;
|
|
688
|
+
|
|
597
689
|
if (ownerCount >= limits.maxDeliveriesPerOwner)
|
|
598
690
|
return [Result.fail(error("capacity", "owner-deliveries")), current];
|
|
691
|
+
|
|
599
692
|
const encodedDelivery = encode(
|
|
600
693
|
SubscriptionDelivery,
|
|
601
694
|
delivery,
|
|
602
695
|
"catch-up-delivery-encode",
|
|
603
696
|
);
|
|
697
|
+
|
|
604
698
|
if (Result.isFailure(encodedDelivery))
|
|
605
699
|
return [Result.fail(encodedDelivery.failure), current];
|
|
606
700
|
const consumed = { ...record.success, state: "consumed" as const, recovery: null };
|
|
701
|
+
|
|
607
702
|
const encodedRecord = encode(
|
|
608
703
|
SubscriptionRecord,
|
|
609
704
|
consumed,
|
|
610
705
|
"catch-up-registration-encode",
|
|
611
706
|
);
|
|
707
|
+
|
|
612
708
|
if (Result.isFailure(encodedRecord)) return [Result.fail(encodedRecord.failure), current];
|
|
613
709
|
const deliveries = new Map(current.deliveries);
|
|
710
|
+
|
|
614
711
|
deliveries.set(key, encodedDelivery.success);
|
|
615
712
|
const registrations = new Map(current.registrations);
|
|
616
713
|
const consumedKey = subscriptionKeyString(consumed.key);
|
|
714
|
+
|
|
617
715
|
registrations.set(consumedKey, encodedRecord.success);
|
|
618
716
|
const registrationIndex = new Map(current.registrationIndex);
|
|
619
717
|
const indexed = registrationIndex.get(consumedKey);
|
|
718
|
+
|
|
620
719
|
if (indexed === undefined)
|
|
621
720
|
return [Result.fail(error("corrupt", "catch-up-index")), current];
|
|
622
721
|
registrationIndex.set(consumedKey, { ...indexed, state: "consumed", recoveryAt: null });
|
|
623
722
|
const deliveryIndex = new Map(current.deliveryIndex);
|
|
723
|
+
|
|
624
724
|
deliveryIndex.set(key, {
|
|
625
725
|
key: delivery.key,
|
|
626
726
|
state: delivery.state,
|
|
627
727
|
nextAttemptAtMillis: delivery.retry.nextAttemptAtMillis,
|
|
628
728
|
});
|
|
629
729
|
const ownerDeliveryCounts = new Map(current.ownerDeliveryCounts);
|
|
730
|
+
|
|
630
731
|
ownerDeliveryCounts.set(consumed.key.ownerId, ownerCount + 1);
|
|
732
|
+
|
|
631
733
|
return [
|
|
632
734
|
Result.void,
|
|
633
735
|
{
|
|
@@ -652,25 +754,32 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
652
754
|
code === undefined
|
|
653
755
|
? "routing-failed"
|
|
654
756
|
: yield* validate(SubscriptionName, code, "routing-failure");
|
|
757
|
+
|
|
655
758
|
yield* failpoint.hit("subscription:defer-event:before");
|
|
656
759
|
yield* Effect.uninterruptible(
|
|
657
760
|
Ref.modify(
|
|
658
761
|
state,
|
|
659
762
|
(current): readonly [Result.Result<void, SubscriptionError>, MemorySubscriptionState] => {
|
|
660
763
|
const text = current.events.get(eventId);
|
|
764
|
+
|
|
661
765
|
if (text === undefined) return [Result.fail(error("not-found", "event")), current];
|
|
662
766
|
const accepted = decode(AcceptedEvent, text, "defer-event-record");
|
|
767
|
+
|
|
663
768
|
if (Result.isFailure(accepted)) return [Result.fail(accepted.failure), current];
|
|
664
769
|
const updated = { ...accepted.success, nextAttemptAtMillis, routingFailure };
|
|
665
770
|
const encoded = encode(AcceptedEvent, updated, "defer-event-encode");
|
|
771
|
+
|
|
666
772
|
if (Result.isFailure(encoded)) return [Result.fail(encoded.failure), current];
|
|
667
773
|
const events = new Map(current.events);
|
|
774
|
+
|
|
668
775
|
events.set(eventId, encoded.success);
|
|
669
776
|
const eventIndex = new Map(current.eventIndex);
|
|
670
777
|
const indexed = eventIndex.get(eventId);
|
|
778
|
+
|
|
671
779
|
if (indexed === undefined)
|
|
672
780
|
return [Result.fail(error("corrupt", "defer-event-index")), current];
|
|
673
781
|
eventIndex.set(eventId, { ...indexed, nextAttemptAtMillis });
|
|
782
|
+
|
|
674
783
|
return [Result.void, { ...current, events, eventIndex }];
|
|
675
784
|
},
|
|
676
785
|
),
|
|
@@ -682,8 +791,10 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
682
791
|
"MemorySubscriptionStore.delivery",
|
|
683
792
|
)(function* (input) {
|
|
684
793
|
const key = yield* validate(SubscriptionDeliveryKey, input, "delivery-key");
|
|
794
|
+
|
|
685
795
|
yield* requirePartition(key.subscription, "delivery-partition");
|
|
686
796
|
const text = (yield* Ref.get(state)).deliveries.get(subscriptionDeliveryKeyString(key));
|
|
797
|
+
|
|
687
798
|
return text === undefined
|
|
688
799
|
? null
|
|
689
800
|
: yield* decodeEffect(SubscriptionDelivery, text, "delivery-record");
|
|
@@ -693,6 +804,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
693
804
|
"MemorySubscriptionStore.pendingDeliveries",
|
|
694
805
|
)(function* (nowMillis, after, limit) {
|
|
695
806
|
const items: Array<SubscriptionDeliveryKey> = [];
|
|
807
|
+
|
|
696
808
|
for (const [storageKey, item] of (yield* Ref.get(state)).deliveryIndex) {
|
|
697
809
|
if (
|
|
698
810
|
item.state !== "delivered" &&
|
|
@@ -705,6 +817,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
705
817
|
items.sort((a, b) =>
|
|
706
818
|
compareScheduleNames(subscriptionDeliveryKeyString(a), subscriptionDeliveryKeyString(b)),
|
|
707
819
|
);
|
|
820
|
+
|
|
708
821
|
return items.slice(0, limit);
|
|
709
822
|
});
|
|
710
823
|
|
|
@@ -713,9 +826,11 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
713
826
|
)(function* (input, after, limit) {
|
|
714
827
|
const key = yield* requireKey(input, "list-deliveries-key");
|
|
715
828
|
const items: Array<SubscriptionDelivery> = [];
|
|
829
|
+
|
|
716
830
|
for (const text of (yield* Ref.get(state)).deliveries.values()) {
|
|
717
831
|
const item = yield* decodeEffect(SubscriptionDelivery, text, "list-delivery");
|
|
718
832
|
const itemKey = subscriptionDeliveryKeyString(item.key);
|
|
833
|
+
|
|
719
834
|
if (
|
|
720
835
|
subscriptionKeyString(item.key.subscription) === subscriptionKeyString(key) &&
|
|
721
836
|
compareScheduleNames(itemKey, after) > 0
|
|
@@ -728,6 +843,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
728
843
|
subscriptionDeliveryKeyString(b.key),
|
|
729
844
|
),
|
|
730
845
|
);
|
|
846
|
+
|
|
731
847
|
return items.slice(0, limit);
|
|
732
848
|
});
|
|
733
849
|
|
|
@@ -737,12 +853,15 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
737
853
|
const key = yield* validate(SubscriptionDeliveryKey, inputKey, "change-delivery-key");
|
|
738
854
|
const deliveryId = yield* validate(Digest, inputDeliveryId, "change-delivery-id");
|
|
739
855
|
const change = yield* validate(DeliveryChange, inputChange, "change-delivery-change");
|
|
856
|
+
|
|
740
857
|
yield* requirePartition(key.subscription, "change-delivery-partition");
|
|
741
858
|
yield* failpoint.hit(`subscription:delivery-${change._tag.toLowerCase()}:before`);
|
|
859
|
+
|
|
742
860
|
const effectiveChange =
|
|
743
861
|
change._tag === "Prepare"
|
|
744
862
|
? { ...change, nowMillis: Math.max(change.nowMillis, yield* Clock.currentTimeMillis) }
|
|
745
863
|
: change;
|
|
864
|
+
|
|
746
865
|
const result = yield* Effect.uninterruptible(
|
|
747
866
|
Ref.modify(
|
|
748
867
|
state,
|
|
@@ -754,36 +873,47 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
754
873
|
] => {
|
|
755
874
|
const storageKey = subscriptionDeliveryKeyString(key);
|
|
756
875
|
const text = current.deliveries.get(storageKey);
|
|
876
|
+
|
|
757
877
|
if (text === undefined) return [Result.fail(error("not-found", "delivery")), current];
|
|
758
878
|
const decoded = decode(SubscriptionDelivery, text, "change-delivery-record");
|
|
879
|
+
|
|
759
880
|
if (Result.isFailure(decoded)) return [decoded, current];
|
|
881
|
+
|
|
760
882
|
const registrationText = current.registrations.get(
|
|
761
883
|
subscriptionKeyString(key.subscription),
|
|
762
884
|
);
|
|
885
|
+
|
|
763
886
|
if (registrationText === undefined)
|
|
764
887
|
return [Result.fail(error("corrupt", "delivery-registration")), current];
|
|
888
|
+
|
|
765
889
|
const registration = decode(
|
|
766
890
|
SubscriptionRecord,
|
|
767
891
|
registrationText,
|
|
768
892
|
"delivery-registration",
|
|
769
893
|
);
|
|
894
|
+
|
|
770
895
|
if (Result.isFailure(registration)) return [Result.fail(registration.failure), current];
|
|
896
|
+
|
|
771
897
|
const transition = applySubscriptionDeliveryChange(
|
|
772
898
|
decoded.success,
|
|
773
899
|
registration.success,
|
|
774
900
|
deliveryId,
|
|
775
901
|
effectiveChange,
|
|
776
902
|
);
|
|
903
|
+
|
|
777
904
|
if (Result.isFailure(transition)) return [Result.fail(transition.failure), current];
|
|
778
905
|
if (transition.success === decoded.success)
|
|
779
906
|
return [Result.succeed(decoded.success), current];
|
|
780
907
|
const updated = transition.success;
|
|
781
908
|
const encoded = encode(SubscriptionDelivery, updated, "change-delivery-encode");
|
|
909
|
+
|
|
782
910
|
if (Result.isFailure(encoded)) return [Result.fail(encoded.failure), current];
|
|
783
911
|
const deliveries = new Map(current.deliveries);
|
|
912
|
+
|
|
784
913
|
deliveries.set(storageKey, encoded.success);
|
|
785
914
|
const deliveryIndex = new Map(current.deliveryIndex);
|
|
786
915
|
const indexed = deliveryIndex.get(storageKey);
|
|
916
|
+
|
|
787
917
|
if (indexed === undefined)
|
|
788
918
|
return [Result.fail(error("corrupt", "delivery-index")), current];
|
|
789
919
|
deliveryIndex.set(storageKey, {
|
|
@@ -791,11 +921,14 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
791
921
|
state: updated.state,
|
|
792
922
|
nextAttemptAtMillis: updated.retry.nextAttemptAtMillis,
|
|
793
923
|
});
|
|
924
|
+
|
|
794
925
|
return [Result.succeed(updated), { ...current, deliveries, deliveryIndex }];
|
|
795
926
|
},
|
|
796
927
|
),
|
|
797
928
|
).pipe(Effect.flatMap(Effect.fromResult));
|
|
929
|
+
|
|
798
930
|
yield* failpoint.hit(`subscription:delivery-${change._tag.toLowerCase()}:after`);
|
|
931
|
+
|
|
799
932
|
return result;
|
|
800
933
|
});
|
|
801
934
|
|
|
@@ -803,6 +936,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
803
936
|
"MemorySubscriptionStore.recovering",
|
|
804
937
|
)(function* (nowMillis, after, limit) {
|
|
805
938
|
const records: Array<{ readonly key: SubscriptionKey; readonly ordinal: number }> = [];
|
|
939
|
+
|
|
806
940
|
for (const item of (yield* Ref.get(state)).registrationIndex.values()) {
|
|
807
941
|
if (
|
|
808
942
|
item.ordinal > after &&
|
|
@@ -813,6 +947,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
813
947
|
records.push({ key: item.key, ordinal: item.ordinal });
|
|
814
948
|
}
|
|
815
949
|
records.sort((a, b) => a.ordinal - b.ordinal);
|
|
950
|
+
|
|
816
951
|
return records.slice(0, limit);
|
|
817
952
|
});
|
|
818
953
|
|
|
@@ -820,6 +955,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
820
955
|
"MemorySubscriptionStore.deferRecovery",
|
|
821
956
|
)(function* (input, recovery) {
|
|
822
957
|
const key = yield* requireKey(input, "defer-recovery-key");
|
|
958
|
+
|
|
823
959
|
yield* failpoint.hit("subscription:defer-recovery:before");
|
|
824
960
|
yield* Effect.uninterruptible(
|
|
825
961
|
Ref.modify(
|
|
@@ -827,25 +963,33 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
827
963
|
(current): readonly [Result.Result<void, SubscriptionError>, MemorySubscriptionState] => {
|
|
828
964
|
const storageKey = subscriptionKeyString(key);
|
|
829
965
|
const text = current.registrations.get(storageKey);
|
|
966
|
+
|
|
830
967
|
if (text === undefined) return [Result.fail(error("not-found", "subscription")), current];
|
|
831
968
|
const record = decode(SubscriptionRecord, text, "defer-recovery-record");
|
|
969
|
+
|
|
832
970
|
if (Result.isFailure(record)) return [Result.fail(record.failure), current];
|
|
971
|
+
|
|
833
972
|
const updated = {
|
|
834
973
|
...record.success,
|
|
835
974
|
recovery: record.success.state === "active" ? recovery : null,
|
|
836
975
|
};
|
|
976
|
+
|
|
837
977
|
const encoded = encode(SubscriptionRecord, updated, "defer-recovery-encode");
|
|
978
|
+
|
|
838
979
|
if (Result.isFailure(encoded)) return [Result.fail(encoded.failure), current];
|
|
839
980
|
const registrations = new Map(current.registrations);
|
|
981
|
+
|
|
840
982
|
registrations.set(storageKey, encoded.success);
|
|
841
983
|
const registrationIndex = new Map(current.registrationIndex);
|
|
842
984
|
const indexed = registrationIndex.get(storageKey);
|
|
985
|
+
|
|
843
986
|
if (indexed === undefined)
|
|
844
987
|
return [Result.fail(error("corrupt", "recovery-index")), current];
|
|
845
988
|
registrationIndex.set(storageKey, {
|
|
846
989
|
...indexed,
|
|
847
990
|
recoveryAt: updated.recovery?.nextAttemptAtMillis ?? null,
|
|
848
991
|
});
|
|
992
|
+
|
|
849
993
|
return [Result.void, { ...current, registrations, registrationIndex }];
|
|
850
994
|
},
|
|
851
995
|
),
|
|
@@ -861,6 +1005,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
861
1005
|
"MemorySubscriptionStore.advanceScanCursors",
|
|
862
1006
|
)(function* (input) {
|
|
863
1007
|
const cursors = yield* validate(SubscriptionScanCursors, input, "scan-cursors");
|
|
1008
|
+
|
|
864
1009
|
yield* failpoint.hit("subscription:advance-scan-cursors:before");
|
|
865
1010
|
yield* Effect.uninterruptible(
|
|
866
1011
|
Ref.update(state, (current) => ({ ...current, scanCursors: cursors })),
|
|
@@ -871,15 +1016,18 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
871
1016
|
const nextDeadline = Effect.gen(function* () {
|
|
872
1017
|
let deadline: number | null = null;
|
|
873
1018
|
const current = yield* Ref.get(state);
|
|
1019
|
+
|
|
874
1020
|
if (
|
|
875
1021
|
current.scanCursors.events !== "" ||
|
|
876
1022
|
current.scanCursors.deliveries !== "" ||
|
|
877
1023
|
current.scanCursors.recovery !== 0
|
|
878
1024
|
)
|
|
879
1025
|
return 0;
|
|
1026
|
+
|
|
880
1027
|
const consider = (value: number) => {
|
|
881
1028
|
if (deadline === null || value < deadline) deadline = value;
|
|
882
1029
|
};
|
|
1030
|
+
|
|
883
1031
|
for (const accepted of current.eventIndex.values())
|
|
884
1032
|
if (!accepted.routingComplete) consider(accepted.nextAttemptAtMillis);
|
|
885
1033
|
for (const item of current.deliveryIndex.values())
|
|
@@ -887,6 +1035,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
887
1035
|
consider(item.nextAttemptAtMillis);
|
|
888
1036
|
for (const record of current.registrationIndex.values())
|
|
889
1037
|
if (record.state === "active" && record.recoveryAt !== null) consider(record.recoveryAt);
|
|
1038
|
+
|
|
890
1039
|
return deadline;
|
|
891
1040
|
}).pipe(Effect.withSpan("MemorySubscriptionStore.nextDeadline"));
|
|
892
1041
|
|