@effect-agent/storage-memory 0.1.0-beta.50 → 0.1.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/MemorySubmissionLedger.mjs +30 -5
- package/dist/MemorySubmissionLedger.mjs.map +1 -1
- package/dist/MemorySubscriptionStore.mjs +266 -17
- package/dist/MemorySubscriptionStore.mjs.map +1 -1
- package/package.json +1 -1
- package/src/MemorySubmissionLedger.ts +76 -3
- package/src/MemorySubscriptionStore.ts +499 -17
|
@@ -2,8 +2,8 @@ import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
|
2
2
|
import { compareScheduleNames } from "@effect-agent/thread/ScheduleTransition";
|
|
3
3
|
import { Clock, Effect, Layer, Ref, Result, Schema } from "effect";
|
|
4
4
|
import { Digest } from "@effect-agent/thread/Records";
|
|
5
|
-
import { AcceptedEvent, DeliveryChange, SourcePartition, SubscriptionDelivery, SubscriptionDeliveryKey, SubscriptionError, SubscriptionFailpoint, SubscriptionKey, SubscriptionLimits, SubscriptionName, SubscriptionRecord, SubscriptionScanCursors, SubscriptionStore, subscriptionDeliveryKeyString, subscriptionKeyString } from "@effect-agent/thread/Subscription";
|
|
6
|
-
import { applySubscriptionDeliveryChange, subscriptionCanSelect, subscriptionDeliveryCanSelect } from "@effect-agent/thread/SubscriptionTransition";
|
|
5
|
+
import { AcceptedEvent, DeliveryChange, SourcePartition, SubscriptionChange, SubscriptionDelivery, SubscriptionDeliveryKey, SubscriptionError, SubscriptionFailpoint, SubscriptionKey, SubscriptionLimits, SubscriptionName, SubscriptionRecord, SubscriptionRetentionPolicy, SubscriptionScanCursors, SubscriptionStore, subscriptionDeliveryKeyString, subscriptionKeyString } from "@effect-agent/thread/Subscription";
|
|
6
|
+
import { applySubscriptionChange, applySubscriptionDeliveryChange, subscriptionCanSelect, subscriptionDeliveryCanSelect, validateEventRetention } from "@effect-agent/thread/SubscriptionTransition";
|
|
7
7
|
//#region src/MemorySubscriptionStore.ts
|
|
8
8
|
var MemorySubscriptionStore_exports = /* @__PURE__ */ __exportAll({ memorySubscriptionStoreLayer: () => memorySubscriptionStoreLayer });
|
|
9
9
|
const error = (reason, code) => SubscriptionError.make({
|
|
@@ -34,12 +34,61 @@ const eventCandidateIndexKey = (event) => JSON.stringify([
|
|
|
34
34
|
event.source.version,
|
|
35
35
|
event.matchingKey
|
|
36
36
|
]);
|
|
37
|
-
const sameEventIdentity = (left, right) => samePartition(left.partition, right.partition) && left.eventId === right.eventId && sameSource(left.source, right.source) && left.matchingKey === right.matchingKey && left.payloadDigest === right.payloadDigest;
|
|
37
|
+
const sameEventIdentity = (left, right) => samePartition(left.partition, right.partition) && left.eventId === right.eventId && sameSource(left.source, right.source) && left.matchingKey === right.matchingKey && left.payloadDigest === right.payloadDigest && left.occurredAtMillis === right.occurredAtMillis;
|
|
38
38
|
const deliveryBelongsTo = (delivery, record, event) => subscriptionKeyString(delivery.key.subscription) === subscriptionKeyString(record.key) && delivery.key.eventId === event.eventId && sameSource(delivery.source, event.source);
|
|
39
|
+
const removeKeys = (keys, removed) => {
|
|
40
|
+
if (removed.size === 0) return keys;
|
|
41
|
+
const result = [...keys];
|
|
42
|
+
for (const key of removed) {
|
|
43
|
+
const index = upperBound(result, key) - 1;
|
|
44
|
+
if (result[index] === key) result.splice(index, 1);
|
|
45
|
+
}
|
|
46
|
+
return result;
|
|
47
|
+
};
|
|
48
|
+
const upperBound = (keys, after) => {
|
|
49
|
+
let low = 0;
|
|
50
|
+
let high = keys.length;
|
|
51
|
+
while (low < high) {
|
|
52
|
+
const middle = Math.floor((low + high) / 2);
|
|
53
|
+
const key = keys[middle];
|
|
54
|
+
if (key !== void 0 && compareScheduleNames(key, after) <= 0) low = middle + 1;
|
|
55
|
+
else high = middle;
|
|
56
|
+
}
|
|
57
|
+
return low;
|
|
58
|
+
};
|
|
59
|
+
const insertKey = (keys, key) => {
|
|
60
|
+
const at = upperBound(keys, key);
|
|
61
|
+
if (at > 0 && keys[at - 1] === key) return keys;
|
|
62
|
+
return [
|
|
63
|
+
...keys.slice(0, at),
|
|
64
|
+
key,
|
|
65
|
+
...keys.slice(at)
|
|
66
|
+
];
|
|
67
|
+
};
|
|
68
|
+
const recoveryCountsAfter = (current, next, keys) => {
|
|
69
|
+
const counts = new Map(current.recoveryCounts);
|
|
70
|
+
for (const key of keys) {
|
|
71
|
+
const before = current.registrationIndex.get(key)?.recoveryKey;
|
|
72
|
+
const after = next.get(key)?.recoveryKey;
|
|
73
|
+
if (before === after) continue;
|
|
74
|
+
if (before !== null && before !== void 0) counts.set(before, (counts.get(before) ?? 0) - 1);
|
|
75
|
+
if (after !== null && after !== void 0) counts.set(after, (counts.get(after) ?? 0) + 1);
|
|
76
|
+
}
|
|
77
|
+
return counts;
|
|
78
|
+
};
|
|
39
79
|
const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(function* (ownedPartition) {
|
|
40
80
|
const partition = yield* validate(SourcePartition, ownedPartition, "partition");
|
|
41
81
|
const state = yield* Ref.make({
|
|
42
82
|
sequence: 0,
|
|
83
|
+
retentionDeadline: null,
|
|
84
|
+
tombstoneCount: 0,
|
|
85
|
+
eventKeys: [],
|
|
86
|
+
deliveryKeys: [],
|
|
87
|
+
maintenanceEvents: "",
|
|
88
|
+
maintenanceDeliveries: "",
|
|
89
|
+
recoveryCounts: /* @__PURE__ */ new Map(),
|
|
90
|
+
eventDeliveryCounts: /* @__PURE__ */ new Map(),
|
|
91
|
+
retentionHorizon: null,
|
|
43
92
|
registrations: /* @__PURE__ */ new Map(),
|
|
44
93
|
events: /* @__PURE__ */ new Map(),
|
|
45
94
|
deliveries: /* @__PURE__ */ new Map(),
|
|
@@ -73,7 +122,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
73
122
|
}
|
|
74
123
|
if (jsonBytes(record.configuration.context) > limits.maxContextBytes) return [Result.fail(error("capacity", "context-bytes")), current];
|
|
75
124
|
if (jsonBytes(record.configuration.parameters) > limits.maxPayloadBytes) return [Result.fail(error("capacity", "parameters-bytes")), current];
|
|
76
|
-
if (record.configuration.expiresAtMillis - record.createdAtMillis > limits.maxLifetimeMillis) return [Result.fail(error("capacity", "lifetime")), current];
|
|
125
|
+
if (record.configuration.expiresAtMillis !== null && record.configuration.expiresAtMillis - record.createdAtMillis > limits.maxLifetimeMillis) return [Result.fail(error("capacity", "lifetime")), current];
|
|
77
126
|
if (current.registrations.size >= limits.maxRegistrations) return [Result.fail(error("capacity", "registrations")), current];
|
|
78
127
|
const ownerCount = current.ownerRegistrationCounts.get(record.key.ownerId) ?? 0;
|
|
79
128
|
if (ownerCount >= limits.maxRegistrationsPerOwner) return [Result.fail(error("capacity", "owner-registrations")), current];
|
|
@@ -90,6 +139,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
90
139
|
key: assigned.key,
|
|
91
140
|
ordinal: assigned.ordinal,
|
|
92
141
|
state: assigned.state,
|
|
142
|
+
recoveryKey: assigned.recovery === null ? null : candidateIndexKey(assigned),
|
|
93
143
|
recoveryAt: assigned.recovery?.nextAttemptAtMillis ?? null
|
|
94
144
|
});
|
|
95
145
|
const candidateIndex = new Map(current.candidateIndex);
|
|
@@ -102,6 +152,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
102
152
|
sequence: assigned.ordinal,
|
|
103
153
|
registrations,
|
|
104
154
|
registrationIndex,
|
|
155
|
+
recoveryCounts: recoveryCountsAfter(current, registrationIndex, [key]),
|
|
105
156
|
candidateIndex,
|
|
106
157
|
ownerRegistrationCounts
|
|
107
158
|
}];
|
|
@@ -127,7 +178,53 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
127
178
|
records.sort((a, b) => a.ordinal - b.ordinal);
|
|
128
179
|
return records.slice(0, limit);
|
|
129
180
|
});
|
|
130
|
-
const
|
|
181
|
+
const change = Effect.fn("MemorySubscriptionStore.change")(function* (input, expectedRevision, inputChange) {
|
|
182
|
+
const key = yield* requireKey(input, "change-key");
|
|
183
|
+
const change = yield* validate(SubscriptionChange, inputChange, "change");
|
|
184
|
+
yield* validate(Schema.Int.check(Schema.isGreaterThan(0)), expectedRevision, "revision");
|
|
185
|
+
yield* failpoint.hit("subscription:change:before");
|
|
186
|
+
const updated = yield* Effect.uninterruptible(Ref.modify(state, (current) => {
|
|
187
|
+
const storageKey = subscriptionKeyString(key);
|
|
188
|
+
const text = current.registrations.get(storageKey);
|
|
189
|
+
if (text === void 0) return [Result.fail(error("not-found", "subscription")), current];
|
|
190
|
+
const existing = decode(SubscriptionRecord, text, "change-record");
|
|
191
|
+
if (Result.isFailure(existing)) return [existing, current];
|
|
192
|
+
const updated = applySubscriptionChange(existing.success, expectedRevision, change);
|
|
193
|
+
if (Result.isFailure(updated)) return [updated, current];
|
|
194
|
+
const revised = {
|
|
195
|
+
...updated.success,
|
|
196
|
+
ordinal: current.sequence + 1
|
|
197
|
+
};
|
|
198
|
+
const encoded = encode(SubscriptionRecord, revised, "change-encode");
|
|
199
|
+
if (Result.isFailure(encoded)) return [Result.fail(encoded.failure), current];
|
|
200
|
+
const registrations = new Map(current.registrations);
|
|
201
|
+
registrations.set(storageKey, encoded.success);
|
|
202
|
+
const registrationIndex = new Map(current.registrationIndex);
|
|
203
|
+
registrationIndex.set(storageKey, {
|
|
204
|
+
key,
|
|
205
|
+
ordinal: revised.ordinal,
|
|
206
|
+
state: revised.state,
|
|
207
|
+
recoveryKey: revised.recovery === null ? null : candidateIndexKey(revised),
|
|
208
|
+
recoveryAt: revised.recovery?.nextAttemptAtMillis ?? null
|
|
209
|
+
});
|
|
210
|
+
const candidateIndex = new Map(current.candidateIndex);
|
|
211
|
+
const oldKey = candidateIndexKey(existing.success);
|
|
212
|
+
const nextKey = candidateIndexKey(revised);
|
|
213
|
+
candidateIndex.set(oldKey, (candidateIndex.get(oldKey) ?? []).filter((key) => key !== storageKey));
|
|
214
|
+
candidateIndex.set(nextKey, [...candidateIndex.get(nextKey) ?? [], storageKey].sort((a, b) => (registrationIndex.get(a)?.ordinal ?? 0) - (registrationIndex.get(b)?.ordinal ?? 0)));
|
|
215
|
+
return [Result.succeed(revised), {
|
|
216
|
+
...current,
|
|
217
|
+
sequence: revised.ordinal,
|
|
218
|
+
registrations,
|
|
219
|
+
registrationIndex,
|
|
220
|
+
candidateIndex,
|
|
221
|
+
recoveryCounts: recoveryCountsAfter(current, registrationIndex, [storageKey])
|
|
222
|
+
}];
|
|
223
|
+
})).pipe(Effect.flatMap(Effect.fromResult));
|
|
224
|
+
yield* failpoint.hit("subscription:change:after");
|
|
225
|
+
return updated;
|
|
226
|
+
});
|
|
227
|
+
const cancel = Effect.fn("MemorySubscriptionStore.cancel")(function* (input, expectedRevision) {
|
|
131
228
|
const key = yield* requireKey(input, "cancel-key");
|
|
132
229
|
yield* failpoint.hit("subscription:cancel:before");
|
|
133
230
|
const result = yield* Effect.uninterruptible(Ref.modify(state, (current) => {
|
|
@@ -136,9 +233,16 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
136
233
|
if (text === void 0) return [Result.fail(error("not-found", "subscription")), current];
|
|
137
234
|
const decoded = decode(SubscriptionRecord, text, "cancel-record");
|
|
138
235
|
if (Result.isFailure(decoded)) return [decoded, current];
|
|
236
|
+
if (expectedRevision !== void 0 && expectedRevision !== decoded.success.configurationRevision && !(decoded.success.state === "cancelled" && expectedRevision + 1 === decoded.success.configurationRevision)) return [Result.fail(SubscriptionError.make({
|
|
237
|
+
reason: "conflict",
|
|
238
|
+
code: "configuration-revision",
|
|
239
|
+
currentRevision: decoded.success.configurationRevision,
|
|
240
|
+
currentState: decoded.success.state
|
|
241
|
+
})), current];
|
|
139
242
|
if (decoded.success.state === "cancelled") return [Result.succeed(decoded.success), current];
|
|
140
243
|
const cancelled = {
|
|
141
244
|
...decoded.success,
|
|
245
|
+
configurationRevision: decoded.success.configurationRevision + 1,
|
|
142
246
|
state: "cancelled",
|
|
143
247
|
recovery: null
|
|
144
248
|
};
|
|
@@ -152,12 +256,14 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
152
256
|
registrationIndex.set(storageKey, {
|
|
153
257
|
...indexed,
|
|
154
258
|
state: "cancelled",
|
|
155
|
-
recoveryAt: null
|
|
259
|
+
recoveryAt: null,
|
|
260
|
+
recoveryKey: null
|
|
156
261
|
});
|
|
157
262
|
return [Result.succeed(cancelled), {
|
|
158
263
|
...current,
|
|
159
264
|
registrations,
|
|
160
|
-
registrationIndex
|
|
265
|
+
registrationIndex,
|
|
266
|
+
recoveryCounts: recoveryCountsAfter(current, registrationIndex, [storageKey])
|
|
161
267
|
}];
|
|
162
268
|
})).pipe(Effect.flatMap(Effect.fromResult));
|
|
163
269
|
yield* failpoint.hit("subscription:cancel:after");
|
|
@@ -165,6 +271,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
165
271
|
});
|
|
166
272
|
const accept = Effect.fn("MemorySubscriptionStore.accept")(function* (input, inputLimits) {
|
|
167
273
|
const event = yield* validate(AcceptedEvent, input, "accept-event");
|
|
274
|
+
const currentTimeMillis = yield* Clock.currentTimeMillis;
|
|
168
275
|
const limits = yield* validate(SubscriptionLimits, inputLimits, "accept-limits");
|
|
169
276
|
yield* requirePartition(event, "accept-partition");
|
|
170
277
|
yield* failpoint.hit("subscription:accept:before");
|
|
@@ -175,8 +282,11 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
175
282
|
if (Result.isFailure(existing)) return [Result.fail(existing.failure), current];
|
|
176
283
|
return sameEventIdentity(existing.success, event) ? [Result.succeed(existing.success), current] : [Result.fail(error("conflict", "event-identity")), current];
|
|
177
284
|
}
|
|
285
|
+
if (current.retentionHorizon !== null && current.retentionHorizon !== limits.retention?.replayHorizonMillis) return [Result.fail(error("conflict", "retention-horizon")), current];
|
|
286
|
+
const horizon = validateEventRetention(event, limits, currentTimeMillis);
|
|
287
|
+
if (Result.isFailure(horizon)) return [Result.fail(horizon.failure), current];
|
|
178
288
|
if (jsonBytes(event.payload) > limits.maxPayloadBytes) return [Result.fail(error("capacity", "payload-bytes")), current];
|
|
179
|
-
if (current.events.size >= limits.maxEvents) return [Result.fail(error("capacity", "events")), current];
|
|
289
|
+
if (current.events.size - current.tombstoneCount >= limits.maxEvents) return [Result.fail(error("capacity", "events")), current];
|
|
180
290
|
const accepted = {
|
|
181
291
|
...event,
|
|
182
292
|
cutoff: current.sequence + 1,
|
|
@@ -197,7 +307,10 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
197
307
|
...current,
|
|
198
308
|
sequence: accepted.cutoff,
|
|
199
309
|
events,
|
|
200
|
-
eventIndex
|
|
310
|
+
eventIndex,
|
|
311
|
+
eventKeys: insertKey(current.eventKeys, accepted.eventId),
|
|
312
|
+
retentionHorizon: limits.retention?.replayHorizonMillis ?? current.retentionHorizon,
|
|
313
|
+
retentionDeadline: limits.retention === void 0 ? current.retentionDeadline : accepted.acceptedAtMillis
|
|
201
314
|
}];
|
|
202
315
|
})).pipe(Effect.flatMap(Effect.fromResult));
|
|
203
316
|
yield* failpoint.hit("subscription:accept:after");
|
|
@@ -297,7 +410,8 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
297
410
|
registrationUpdates.push([consumedKey, {
|
|
298
411
|
...indexed,
|
|
299
412
|
state: "consumed",
|
|
300
|
-
recoveryAt: null
|
|
413
|
+
recoveryAt: null,
|
|
414
|
+
recoveryKey: null
|
|
301
415
|
}]);
|
|
302
416
|
}
|
|
303
417
|
}
|
|
@@ -310,6 +424,8 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
310
424
|
for (const [key, value] of additionIndex) deliveryIndex.set(key, value);
|
|
311
425
|
const registrationIndex = new Map(current.registrationIndex);
|
|
312
426
|
for (const [key, value] of registrationUpdates) registrationIndex.set(key, value);
|
|
427
|
+
const eventDeliveryCounts = new Map(current.eventDeliveryCounts);
|
|
428
|
+
eventDeliveryCounts.set(accepted.eventId, (eventDeliveryCounts.get(accepted.eventId) ?? 0) + additions.length);
|
|
313
429
|
const nextEvent = {
|
|
314
430
|
...accepted,
|
|
315
431
|
cursor,
|
|
@@ -329,9 +445,12 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
329
445
|
...current,
|
|
330
446
|
registrations,
|
|
331
447
|
registrationIndex,
|
|
448
|
+
recoveryCounts: recoveryCountsAfter(current, registrationIndex, registrationUpdates.map(([key]) => key)),
|
|
449
|
+
eventDeliveryCounts,
|
|
332
450
|
events,
|
|
333
451
|
eventIndex,
|
|
334
452
|
deliveries: nextDeliveries,
|
|
453
|
+
deliveryKeys: additions.reduce((keys, [key]) => insertKey(keys, key), current.deliveryKeys),
|
|
335
454
|
deliveryIndex,
|
|
336
455
|
ownerDeliveryCounts: owners
|
|
337
456
|
}];
|
|
@@ -385,7 +504,8 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
385
504
|
registrationIndex.set(consumedKey, {
|
|
386
505
|
...indexed,
|
|
387
506
|
state: "consumed",
|
|
388
|
-
recoveryAt: null
|
|
507
|
+
recoveryAt: null,
|
|
508
|
+
recoveryKey: null
|
|
389
509
|
});
|
|
390
510
|
const deliveryIndex = new Map(current.deliveryIndex);
|
|
391
511
|
deliveryIndex.set(key, {
|
|
@@ -394,14 +514,19 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
394
514
|
nextAttemptAtMillis: delivery.retry.nextAttemptAtMillis
|
|
395
515
|
});
|
|
396
516
|
const ownerDeliveryCounts = new Map(current.ownerDeliveryCounts);
|
|
517
|
+
const eventDeliveryCounts = new Map(current.eventDeliveryCounts);
|
|
518
|
+
eventDeliveryCounts.set(accepted.success.eventId, (eventDeliveryCounts.get(accepted.success.eventId) ?? 0) + 1);
|
|
397
519
|
ownerDeliveryCounts.set(consumed.key.ownerId, ownerCount + 1);
|
|
398
520
|
return [Result.void, {
|
|
399
521
|
...current,
|
|
400
522
|
deliveries,
|
|
401
523
|
deliveryIndex,
|
|
524
|
+
deliveryKeys: insertKey(current.deliveryKeys, key),
|
|
402
525
|
ownerDeliveryCounts,
|
|
403
526
|
registrations,
|
|
404
|
-
registrationIndex
|
|
527
|
+
registrationIndex,
|
|
528
|
+
recoveryCounts: recoveryCountsAfter(current, registrationIndex, [consumedKey]),
|
|
529
|
+
eventDeliveryCounts
|
|
405
530
|
}];
|
|
406
531
|
})).pipe(Effect.flatMap(Effect.fromResult));
|
|
407
532
|
yield* failpoint.hit("subscription:catch-up:after");
|
|
@@ -446,7 +571,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
446
571
|
});
|
|
447
572
|
const pendingDeliveries = Effect.fn("MemorySubscriptionStore.pendingDeliveries")(function* (nowMillis, after, limit) {
|
|
448
573
|
const items = [];
|
|
449
|
-
for (const [storageKey, item] of (yield* Ref.get(state)).deliveryIndex) if (item.state !== "delivered" && item.state !== "refused" && item.nextAttemptAtMillis <= nowMillis && compareScheduleNames(storageKey, after) > 0) items.push(item.key);
|
|
574
|
+
for (const [storageKey, item] of (yield* Ref.get(state)).deliveryIndex) if ((item.state !== "delivered" && item.state !== "refused" && item.parked !== true || item.state === "delivered" && item.observeSettlement === true) && item.nextAttemptAtMillis <= nowMillis && compareScheduleNames(storageKey, after) > 0) items.push(item.key);
|
|
450
575
|
items.sort((a, b) => compareScheduleNames(subscriptionDeliveryKeyString(a), subscriptionDeliveryKeyString(b)));
|
|
451
576
|
return items.slice(0, limit);
|
|
452
577
|
});
|
|
@@ -495,6 +620,8 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
495
620
|
deliveryIndex.set(storageKey, {
|
|
496
621
|
...indexed,
|
|
497
622
|
state: updated.state,
|
|
623
|
+
parked: updated.retry.parked ?? false,
|
|
624
|
+
observeSettlement: updated.observeSettlement ?? false,
|
|
498
625
|
nextAttemptAtMillis: updated.retry.nextAttemptAtMillis
|
|
499
626
|
});
|
|
500
627
|
return [Result.succeed(updated), {
|
|
@@ -515,7 +642,7 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
515
642
|
records.sort((a, b) => a.ordinal - b.ordinal);
|
|
516
643
|
return records.slice(0, limit);
|
|
517
644
|
});
|
|
518
|
-
const deferRecovery = Effect.fn("MemorySubscriptionStore.deferRecovery")(function* (input, recovery) {
|
|
645
|
+
const deferRecovery = Effect.fn("MemorySubscriptionStore.deferRecovery")(function* (input, expectedRevision, recovery) {
|
|
519
646
|
const key = yield* requireKey(input, "defer-recovery-key");
|
|
520
647
|
yield* failpoint.hit("subscription:defer-recovery:before");
|
|
521
648
|
yield* Effect.uninterruptible(Ref.modify(state, (current) => {
|
|
@@ -524,9 +651,10 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
524
651
|
if (text === void 0) return [Result.fail(error("not-found", "subscription")), current];
|
|
525
652
|
const record = decode(SubscriptionRecord, text, "defer-recovery-record");
|
|
526
653
|
if (Result.isFailure(record)) return [Result.fail(record.failure), current];
|
|
654
|
+
if (record.success.configurationRevision !== expectedRevision) return [Result.void, current];
|
|
527
655
|
const updated = {
|
|
528
656
|
...record.success,
|
|
529
|
-
recovery: record.success.state === "active" ? recovery : null
|
|
657
|
+
recovery: record.success.state === "active" || record.success.state === "paused" ? recovery : null
|
|
530
658
|
};
|
|
531
659
|
const encoded = encode(SubscriptionRecord, updated, "defer-recovery-encode");
|
|
532
660
|
if (Result.isFailure(encoded)) return [Result.fail(encoded.failure), current];
|
|
@@ -537,12 +665,14 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
537
665
|
if (indexed === void 0) return [Result.fail(error("corrupt", "recovery-index")), current];
|
|
538
666
|
registrationIndex.set(storageKey, {
|
|
539
667
|
...indexed,
|
|
668
|
+
recoveryKey: updated.recovery === null ? null : candidateIndexKey(updated),
|
|
540
669
|
recoveryAt: updated.recovery?.nextAttemptAtMillis ?? null
|
|
541
670
|
});
|
|
542
671
|
return [Result.void, {
|
|
543
672
|
...current,
|
|
544
673
|
registrations,
|
|
545
|
-
registrationIndex
|
|
674
|
+
registrationIndex,
|
|
675
|
+
recoveryCounts: recoveryCountsAfter(current, registrationIndex, [storageKey])
|
|
546
676
|
}];
|
|
547
677
|
})).pipe(Effect.flatMap(Effect.fromResult));
|
|
548
678
|
yield* failpoint.hit("subscription:defer-recovery:after");
|
|
@@ -564,17 +694,136 @@ const makeMemorySubscriptionStore = Effect.fn("makeMemorySubscriptionStore")(fun
|
|
|
564
694
|
const consider = (value) => {
|
|
565
695
|
if (deadline === null || value < deadline) deadline = value;
|
|
566
696
|
};
|
|
697
|
+
if (current.retentionDeadline !== null) consider(current.retentionDeadline);
|
|
567
698
|
for (const accepted of current.eventIndex.values()) if (!accepted.routingComplete) consider(accepted.nextAttemptAtMillis);
|
|
568
|
-
for (const item of current.deliveryIndex.values()) if (item.state !== "delivered" && item.state !== "refused") consider(item.nextAttemptAtMillis);
|
|
699
|
+
for (const item of current.deliveryIndex.values()) if (item.state !== "delivered" && item.state !== "refused" && item.parked !== true || item.state === "delivered" && item.observeSettlement === true) consider(item.nextAttemptAtMillis);
|
|
569
700
|
for (const record of current.registrationIndex.values()) if (record.state === "active" && record.recoveryAt !== null) consider(record.recoveryAt);
|
|
570
701
|
return deadline;
|
|
571
702
|
}).pipe(Effect.withSpan("MemorySubscriptionStore.nextDeadline"));
|
|
703
|
+
const compact = Effect.fn("MemorySubscriptionStore.compact")(function* (nowMillis, inputPolicy, requestedLimit) {
|
|
704
|
+
nowMillis = Math.min(nowMillis, yield* Clock.currentTimeMillis);
|
|
705
|
+
const policy = yield* validate(SubscriptionRetentionPolicy, inputPolicy, "retention-policy");
|
|
706
|
+
const limit = yield* validate(Schema.Int.check(Schema.isBetween({
|
|
707
|
+
minimum: 1,
|
|
708
|
+
maximum: 100
|
|
709
|
+
})), requestedLimit, "maintenance-limit");
|
|
710
|
+
yield* failpoint.hit("subscription:compact:before");
|
|
711
|
+
let corruptCandidates = 0;
|
|
712
|
+
const result = yield* Effect.uninterruptible(Ref.modify(state, (current) => {
|
|
713
|
+
if (current.retentionHorizon !== null && current.retentionHorizon !== policy.replayHorizonMillis) return [Result.fail(error("conflict", "retention-horizon")), current];
|
|
714
|
+
const events = new Map(current.events);
|
|
715
|
+
const eventIndex = new Map(current.eventIndex);
|
|
716
|
+
const deliveries = new Map(current.deliveries);
|
|
717
|
+
const deliveryIndex = new Map(current.deliveryIndex);
|
|
718
|
+
const eventDeliveryCounts = new Map(current.eventDeliveryCounts);
|
|
719
|
+
const ownerDeliveryCounts = new Map(current.ownerDeliveryCounts);
|
|
720
|
+
const deliveryPage = current.deliveryKeys.slice(upperBound(current.deliveryKeys, current.maintenanceDeliveries), upperBound(current.deliveryKeys, current.maintenanceDeliveries) + limit);
|
|
721
|
+
const eventPage = current.eventKeys.slice(upperBound(current.eventKeys, current.maintenanceEvents), upperBound(current.eventKeys, current.maintenanceEvents) + limit);
|
|
722
|
+
const cutoff = nowMillis - policy.completedRetentionMillis;
|
|
723
|
+
let removed = 0;
|
|
724
|
+
const removedEvents = /* @__PURE__ */ new Set();
|
|
725
|
+
const removedDeliveries = /* @__PURE__ */ new Set();
|
|
726
|
+
for (const key of deliveryPage) {
|
|
727
|
+
const text = deliveries.get(key);
|
|
728
|
+
if (text === void 0) continue;
|
|
729
|
+
const decoded = decode(SubscriptionDelivery, text, "compact-delivery");
|
|
730
|
+
if (Result.isFailure(decoded)) {
|
|
731
|
+
corruptCandidates++;
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
const delivery = decoded.success;
|
|
735
|
+
if (subscriptionDeliveryKeyString(delivery.key) !== key || !samePartition(delivery.key.subscription.partition, partition)) {
|
|
736
|
+
corruptCandidates++;
|
|
737
|
+
continue;
|
|
738
|
+
}
|
|
739
|
+
if (delivery.state !== "refused" && (delivery.state !== "delivered" || delivery.settledAtMillis === void 0)) continue;
|
|
740
|
+
if ((delivery.settledAtMillis ?? delivery.completedAtMillis ?? delivery.selectedAtMillis) > cutoff) continue;
|
|
741
|
+
const eventText = events.get(delivery.key.eventId);
|
|
742
|
+
if (eventText === void 0) continue;
|
|
743
|
+
const accepted = decode(AcceptedEvent, eventText, "compact-delivery-event");
|
|
744
|
+
if (Result.isFailure(accepted)) {
|
|
745
|
+
corruptCandidates++;
|
|
746
|
+
continue;
|
|
747
|
+
}
|
|
748
|
+
const event = accepted.success;
|
|
749
|
+
if (event.eventId !== delivery.key.eventId || !samePartition(event.partition, partition)) {
|
|
750
|
+
corruptCandidates++;
|
|
751
|
+
continue;
|
|
752
|
+
}
|
|
753
|
+
if (!event.routingComplete || event.occurredAtMillis === void 0 || event.acceptedAtMillis > cutoff || (current.recoveryCounts.get(eventCandidateIndexKey(event)) ?? 0) > 0) continue;
|
|
754
|
+
deliveries.delete(key);
|
|
755
|
+
removedDeliveries.add(key);
|
|
756
|
+
deliveryIndex.delete(key);
|
|
757
|
+
eventDeliveryCounts.set(event.eventId, (eventDeliveryCounts.get(event.eventId) ?? 1) - 1);
|
|
758
|
+
const owner = delivery.key.subscription.ownerId;
|
|
759
|
+
ownerDeliveryCounts.set(owner, (ownerDeliveryCounts.get(owner) ?? 1) - 1);
|
|
760
|
+
}
|
|
761
|
+
let tombstones = current.tombstoneCount;
|
|
762
|
+
for (const key of eventPage) {
|
|
763
|
+
const text = events.get(key);
|
|
764
|
+
if (text === void 0) continue;
|
|
765
|
+
const decoded = decode(AcceptedEvent, text, "compact-event");
|
|
766
|
+
if (Result.isFailure(decoded)) {
|
|
767
|
+
corruptCandidates++;
|
|
768
|
+
continue;
|
|
769
|
+
}
|
|
770
|
+
const event = decoded.success;
|
|
771
|
+
if (event.eventId !== key || !samePartition(event.partition, partition)) {
|
|
772
|
+
corruptCandidates++;
|
|
773
|
+
continue;
|
|
774
|
+
}
|
|
775
|
+
if (!event.routingComplete || event.occurredAtMillis === void 0) continue;
|
|
776
|
+
const expired = event.occurredAtMillis <= nowMillis - policy.replayHorizonMillis;
|
|
777
|
+
if (event.tombstone === true && !expired) continue;
|
|
778
|
+
if (event.acceptedAtMillis > cutoff || (eventDeliveryCounts.get(key) ?? 0) > 0 || (current.recoveryCounts.get(eventCandidateIndexKey(event)) ?? 0) > 0) continue;
|
|
779
|
+
if (expired) {
|
|
780
|
+
events.delete(key);
|
|
781
|
+
removedEvents.add(key);
|
|
782
|
+
eventIndex.delete(key);
|
|
783
|
+
eventDeliveryCounts.delete(key);
|
|
784
|
+
if (event.tombstone === true) tombstones--;
|
|
785
|
+
} else {
|
|
786
|
+
if (tombstones >= policy.maxTombstones) continue;
|
|
787
|
+
const encoded = encode(AcceptedEvent, {
|
|
788
|
+
...event,
|
|
789
|
+
payload: null,
|
|
790
|
+
tombstone: true
|
|
791
|
+
}, "compact-tombstone");
|
|
792
|
+
if (Result.isFailure(encoded)) continue;
|
|
793
|
+
events.set(key, encoded.success);
|
|
794
|
+
tombstones++;
|
|
795
|
+
}
|
|
796
|
+
removed++;
|
|
797
|
+
}
|
|
798
|
+
return [Result.succeed(removed), {
|
|
799
|
+
...current,
|
|
800
|
+
events,
|
|
801
|
+
eventIndex,
|
|
802
|
+
deliveries,
|
|
803
|
+
deliveryIndex,
|
|
804
|
+
eventDeliveryCounts,
|
|
805
|
+
ownerDeliveryCounts,
|
|
806
|
+
eventKeys: removeKeys(current.eventKeys, removedEvents),
|
|
807
|
+
deliveryKeys: removeKeys(current.deliveryKeys, removedDeliveries),
|
|
808
|
+
tombstoneCount: tombstones,
|
|
809
|
+
maintenanceEvents: eventPage.length < limit ? "" : eventPage.at(-1) ?? "",
|
|
810
|
+
maintenanceDeliveries: deliveryPage.length < limit ? "" : deliveryPage.at(-1) ?? "",
|
|
811
|
+
retentionHorizon: policy.replayHorizonMillis,
|
|
812
|
+
retentionDeadline: events.size === 0 ? null : nowMillis + 6e4
|
|
813
|
+
}];
|
|
814
|
+
})).pipe(Effect.flatMap(Effect.fromResult));
|
|
815
|
+
if (corruptCandidates > 0) yield* Effect.logWarning("Subscription retention preserved corrupt candidates", { count: corruptCandidates });
|
|
816
|
+
yield* failpoint.hit("subscription:compact:after");
|
|
817
|
+
return result;
|
|
818
|
+
});
|
|
572
819
|
return SubscriptionStore.of({
|
|
573
820
|
partition,
|
|
821
|
+
compact,
|
|
574
822
|
register,
|
|
575
823
|
get,
|
|
576
824
|
list,
|
|
577
825
|
cancel,
|
|
826
|
+
change,
|
|
578
827
|
accept,
|
|
579
828
|
event,
|
|
580
829
|
pendingEvents,
|