@effect-agent/storage-memory 0.1.0-beta.37 → 0.1.0-beta.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +9 -6
- package/dist/index.mjs +752 -160
- package/dist/index.mjs.map +1 -1
- package/dist/testing.d.mts +2 -2
- package/dist/testing.mjs +2 -2
- package/package.json +8 -5
- package/src/index.ts +1 -0
- package/src/memory-ledger.ts +28 -35
- package/src/memory-schedule-store.ts +1 -1
- package/src/memory-storage.ts +336 -366
- package/src/memory-subscription-store.ts +921 -0
- package/src/testing.ts +7 -6
package/src/memory-storage.ts
CHANGED
|
@@ -1,45 +1,46 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ThreadId } from "@effect-agent/core";
|
|
2
2
|
import {
|
|
3
|
+
type ThreadCheckpoint,
|
|
4
|
+
type ProducerEpoch,
|
|
5
|
+
type RecordId,
|
|
3
6
|
AppendConflict,
|
|
4
7
|
AppendResult,
|
|
5
8
|
CanonicalRecordEnvelope,
|
|
6
9
|
CanonicalSequence,
|
|
7
10
|
CheckpointRejected,
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
11
|
+
ThreadExportRequest,
|
|
12
|
+
ThreadExport,
|
|
13
|
+
ThreadMaterialization,
|
|
14
|
+
ThreadNotMaterialized,
|
|
15
|
+
ThreadObservation,
|
|
16
|
+
ThreadRead,
|
|
17
|
+
ThreadStore,
|
|
18
|
+
type ThreadCheckpoints,
|
|
19
|
+
ThreadStoreError,
|
|
20
|
+
ThreadTail,
|
|
21
|
+
ThreadTailRequest,
|
|
19
22
|
digestCanonicalBatch,
|
|
20
23
|
EMPTY_TAIL_DIGEST,
|
|
21
24
|
FenceRejected,
|
|
22
25
|
FencedAppendRequest,
|
|
23
26
|
LoadCheckpointRequest,
|
|
24
27
|
ObservationOffset,
|
|
25
|
-
ProducerEpoch,
|
|
26
|
-
RecordId,
|
|
27
28
|
SaveCheckpointRequest,
|
|
28
29
|
type BatchId,
|
|
29
30
|
type Digest,
|
|
30
|
-
} from "@effect-agent/
|
|
31
|
+
} from "@effect-agent/thread";
|
|
31
32
|
import { Crypto, Effect, Encoding, Layer, Option, PubSub, Ref, Schema, Stream } from "effect";
|
|
32
33
|
|
|
33
|
-
const
|
|
34
|
-
const
|
|
35
|
-
const
|
|
34
|
+
const MAX_THREADS = 256;
|
|
35
|
+
const MAX_RECORDS_PER_THREAD = 65_536;
|
|
36
|
+
const MAX_CHECKPOINTS_PER_THREAD = 1_024;
|
|
36
37
|
|
|
37
38
|
interface StoredBatch {
|
|
38
39
|
readonly digest: Digest;
|
|
39
40
|
readonly result: AppendResult;
|
|
40
41
|
}
|
|
41
42
|
|
|
42
|
-
interface
|
|
43
|
+
interface StoredThread {
|
|
43
44
|
readonly producerEpoch: ProducerEpoch;
|
|
44
45
|
readonly tailSequence: CanonicalSequence;
|
|
45
46
|
readonly tailDigest: Digest;
|
|
@@ -47,21 +48,17 @@ interface StoredConversation {
|
|
|
47
48
|
readonly recordIds: ReadonlySet<RecordId>;
|
|
48
49
|
readonly batches: ReadonlyMap<BatchId, StoredBatch>;
|
|
49
50
|
readonly tailDigests: ReadonlyMap<CanonicalSequence, Digest>;
|
|
50
|
-
readonly checkpoints: ReadonlyMap<CanonicalSequence,
|
|
51
|
+
readonly checkpoints: ReadonlyMap<CanonicalSequence, ThreadCheckpoint>;
|
|
51
52
|
}
|
|
52
53
|
|
|
53
54
|
interface MemoryState {
|
|
54
|
-
readonly
|
|
55
|
+
readonly threads: ReadonlyMap<ThreadId, StoredThread>;
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
type AppendDecision =
|
|
58
59
|
| {
|
|
59
60
|
readonly _tag: "failure";
|
|
60
|
-
readonly error:
|
|
61
|
-
| ConversationStoreError
|
|
62
|
-
| ConversationNotMaterialized
|
|
63
|
-
| AppendConflict
|
|
64
|
-
| FenceRejected;
|
|
61
|
+
readonly error: ThreadStoreError | ThreadNotMaterialized | AppendConflict | FenceRejected;
|
|
65
62
|
}
|
|
66
63
|
| {
|
|
67
64
|
readonly _tag: "success";
|
|
@@ -70,27 +67,27 @@ type AppendDecision =
|
|
|
70
67
|
};
|
|
71
68
|
|
|
72
69
|
type MaterializeDecision =
|
|
73
|
-
| { readonly _tag: "failure"; readonly error:
|
|
70
|
+
| { readonly _tag: "failure"; readonly error: ThreadStoreError | FenceRejected }
|
|
74
71
|
| { readonly _tag: "success" };
|
|
75
72
|
|
|
76
73
|
type CheckpointDecision =
|
|
77
74
|
| {
|
|
78
75
|
readonly _tag: "failure";
|
|
79
|
-
readonly error:
|
|
76
|
+
readonly error: ThreadNotMaterialized | ThreadStoreError | CheckpointRejected;
|
|
80
77
|
}
|
|
81
78
|
| { readonly _tag: "success" };
|
|
82
79
|
|
|
83
|
-
const storeError = (operation: string, message: string, cause?: unknown):
|
|
80
|
+
const storeError = (operation: string, message: string, cause?: unknown): ThreadStoreError =>
|
|
84
81
|
cause === undefined
|
|
85
|
-
?
|
|
86
|
-
:
|
|
82
|
+
? ThreadStoreError.make({ operation, message })
|
|
83
|
+
: ThreadStoreError.make({ operation, message, cause });
|
|
87
84
|
|
|
88
|
-
const validate = Effect.fn("
|
|
85
|
+
const validate = Effect.fn("MemoryThreadStore.validate")(
|
|
89
86
|
<A, I>(
|
|
90
87
|
schema: Schema.Codec<A, I>,
|
|
91
88
|
operation: string,
|
|
92
89
|
value: unknown,
|
|
93
|
-
): Effect.Effect<A,
|
|
90
|
+
): Effect.Effect<A, ThreadStoreError> =>
|
|
94
91
|
Schema.encodeUnknownEffect(schema)(value).pipe(
|
|
95
92
|
Effect.flatMap(Schema.decodeUnknownEffect(schema)),
|
|
96
93
|
Effect.mapError((error) => storeError(operation, `Invalid ${operation} request`, error)),
|
|
@@ -100,12 +97,12 @@ const validate = Effect.fn("MemoryConversationStore.validate")(
|
|
|
100
97
|
const decodeCanonicalSequence = Schema.decodeSync(CanonicalSequence);
|
|
101
98
|
const ZERO_CANONICAL_SEQUENCE = decodeCanonicalSequence(0);
|
|
102
99
|
|
|
103
|
-
const offsetSequence = Effect.fn("
|
|
104
|
-
|
|
100
|
+
const offsetSequence = Effect.fn("MemoryThreadStore.offsetSequence")((
|
|
101
|
+
threadId: ThreadId,
|
|
105
102
|
offset: ObservationOffset | undefined,
|
|
106
|
-
): Effect.Effect<CanonicalSequence,
|
|
103
|
+
): Effect.Effect<CanonicalSequence, ThreadStoreError> => {
|
|
107
104
|
if (offset === undefined) return Effect.succeed(ZERO_CANONICAL_SEQUENCE);
|
|
108
|
-
const prefix = `memory:v1:${Encoding.encodeBase64(
|
|
105
|
+
const prefix = `memory:v1:${Encoding.encodeBase64(threadId)}:`;
|
|
109
106
|
const encodedSequence = offset.startsWith(prefix) ? offset.slice(prefix.length) : "";
|
|
110
107
|
if (!/^\d+$/.test(encodedSequence)) {
|
|
111
108
|
return Effect.fail(storeError("observe", "Malformed observation offset"));
|
|
@@ -118,67 +115,62 @@ const offsetSequence = Effect.fn("MemoryConversationStore.offsetSequence")((
|
|
|
118
115
|
: Effect.fail(storeError("observe", "Malformed observation offset"));
|
|
119
116
|
});
|
|
120
117
|
|
|
121
|
-
const observationOffset = (
|
|
122
|
-
|
|
123
|
-
sequence: CanonicalSequence,
|
|
124
|
-
): ObservationOffset =>
|
|
125
|
-
Schema.decodeSync(ObservationOffset)(
|
|
126
|
-
`memory:v1:${Encoding.encodeBase64(conversationId)}:${sequence}`,
|
|
127
|
-
);
|
|
118
|
+
const observationOffset = (threadId: ThreadId, sequence: CanonicalSequence): ObservationOffset =>
|
|
119
|
+
Schema.decodeSync(ObservationOffset)(`memory:v1:${Encoding.encodeBase64(threadId)}:${sequence}`);
|
|
128
120
|
|
|
129
|
-
const
|
|
121
|
+
const findThread = Effect.fn("MemoryThreadStore.findThread")((
|
|
130
122
|
state: MemoryState,
|
|
131
|
-
|
|
132
|
-
): Effect.Effect<
|
|
133
|
-
const
|
|
134
|
-
return
|
|
135
|
-
? Effect.fail(
|
|
136
|
-
: Effect.succeed(
|
|
123
|
+
threadId: ThreadId,
|
|
124
|
+
): Effect.Effect<StoredThread, ThreadNotMaterialized> => {
|
|
125
|
+
const thread = state.threads.get(threadId);
|
|
126
|
+
return thread === undefined
|
|
127
|
+
? Effect.fail(ThreadNotMaterialized.make({ threadId }))
|
|
128
|
+
: Effect.succeed(thread);
|
|
137
129
|
});
|
|
138
130
|
|
|
139
131
|
const CheckpointVersionEnvelope = Schema.Struct({
|
|
140
132
|
checkpoint: Schema.Struct({
|
|
141
|
-
|
|
133
|
+
threadId: ThreadId,
|
|
142
134
|
schemaVersion: Schema.Natural,
|
|
143
135
|
}),
|
|
144
136
|
});
|
|
145
137
|
|
|
146
|
-
const validateCheckpointVersion = Effect.fn("
|
|
147
|
-
function* (value: unknown): Effect.fn.Return<void,
|
|
138
|
+
const validateCheckpointVersion = Effect.fn("MemoryThreadStore.validateCheckpointVersion")(
|
|
139
|
+
function* (value: unknown): Effect.fn.Return<void, ThreadStoreError | CheckpointRejected> {
|
|
148
140
|
const envelope = yield* Schema.decodeUnknownEffect(CheckpointVersionEnvelope)(value).pipe(
|
|
149
141
|
Effect.mapError(() => storeError("saveCheckpoint", "Invalid saveCheckpoint request")),
|
|
150
142
|
);
|
|
151
143
|
if (envelope.checkpoint.schemaVersion !== 1) {
|
|
152
144
|
return yield* CheckpointRejected.make({
|
|
153
|
-
|
|
145
|
+
threadId: envelope.checkpoint.threadId,
|
|
154
146
|
reason: "unsupported-version",
|
|
155
147
|
});
|
|
156
148
|
}
|
|
157
149
|
},
|
|
158
150
|
);
|
|
159
151
|
|
|
160
|
-
const
|
|
152
|
+
const makeThreadStore = Effect.gen(function* () {
|
|
161
153
|
const crypto = yield* Crypto.Crypto;
|
|
162
|
-
const state = yield* Ref.make<MemoryState>({
|
|
154
|
+
const state = yield* Ref.make<MemoryState>({ threads: new Map() });
|
|
163
155
|
const updates = yield* PubSub.sliding<void>(1);
|
|
164
156
|
yield* Effect.addFinalizer(() => PubSub.shutdown(updates));
|
|
165
157
|
|
|
166
|
-
const materialize:
|
|
167
|
-
"
|
|
158
|
+
const materialize: ThreadStore["Service"]["materialize"] = Effect.fn(
|
|
159
|
+
"MemoryThreadStore.materialize",
|
|
168
160
|
)((unvalidated) =>
|
|
169
161
|
Effect.gen(function* () {
|
|
170
|
-
const request = yield* validate(
|
|
162
|
+
const request = yield* validate(ThreadMaterialization, "materialize", unvalidated);
|
|
171
163
|
const decision = yield* Ref.modify(
|
|
172
164
|
state,
|
|
173
165
|
(current): readonly [MaterializeDecision, MemoryState] => {
|
|
174
|
-
const existing = current.
|
|
166
|
+
const existing = current.threads.get(request.threadId);
|
|
175
167
|
if (existing !== undefined) {
|
|
176
168
|
if (request.producerEpoch < existing.producerEpoch) {
|
|
177
169
|
return [
|
|
178
170
|
{
|
|
179
171
|
_tag: "failure",
|
|
180
172
|
error: FenceRejected.make({
|
|
181
|
-
|
|
173
|
+
threadId: request.threadId,
|
|
182
174
|
actualEpoch: existing.producerEpoch,
|
|
183
175
|
attemptedEpoch: request.producerEpoch,
|
|
184
176
|
}),
|
|
@@ -189,27 +181,24 @@ const makeConversationStore = Effect.gen(function* () {
|
|
|
189
181
|
if (request.producerEpoch === existing.producerEpoch) {
|
|
190
182
|
return [{ _tag: "success" }, current];
|
|
191
183
|
}
|
|
192
|
-
const
|
|
193
|
-
|
|
184
|
+
const threads = new Map(current.threads);
|
|
185
|
+
threads.set(request.threadId, {
|
|
194
186
|
...existing,
|
|
195
187
|
producerEpoch: request.producerEpoch,
|
|
196
188
|
});
|
|
197
|
-
return [{ _tag: "success" }, {
|
|
189
|
+
return [{ _tag: "success" }, { threads }];
|
|
198
190
|
}
|
|
199
|
-
if (current.
|
|
191
|
+
if (current.threads.size >= MAX_THREADS) {
|
|
200
192
|
return [
|
|
201
193
|
{
|
|
202
194
|
_tag: "failure",
|
|
203
|
-
error: storeError(
|
|
204
|
-
"materialize",
|
|
205
|
-
`In-memory conversation limit ${MAX_CONVERSATIONS} exceeded`,
|
|
206
|
-
),
|
|
195
|
+
error: storeError("materialize", `In-memory thread limit ${MAX_THREADS} exceeded`),
|
|
207
196
|
},
|
|
208
197
|
current,
|
|
209
198
|
];
|
|
210
199
|
}
|
|
211
|
-
const
|
|
212
|
-
|
|
200
|
+
const threads = new Map(current.threads);
|
|
201
|
+
threads.set(request.threadId, {
|
|
213
202
|
producerEpoch: request.producerEpoch,
|
|
214
203
|
tailSequence: ZERO_CANONICAL_SEQUENCE,
|
|
215
204
|
tailDigest: EMPTY_TAIL_DIGEST,
|
|
@@ -219,218 +208,207 @@ const makeConversationStore = Effect.gen(function* () {
|
|
|
219
208
|
tailDigests: new Map([[ZERO_CANONICAL_SEQUENCE, EMPTY_TAIL_DIGEST]]),
|
|
220
209
|
checkpoints: new Map(),
|
|
221
210
|
});
|
|
222
|
-
return [{ _tag: "success" }, {
|
|
211
|
+
return [{ _tag: "success" }, { threads }];
|
|
223
212
|
},
|
|
224
213
|
);
|
|
225
214
|
if (decision._tag === "failure") return yield* decision.error;
|
|
226
215
|
}),
|
|
227
216
|
);
|
|
228
217
|
|
|
229
|
-
const append:
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
);
|
|
218
|
+
const append: ThreadStore["Service"]["append"] = Effect.fn("MemoryThreadStore.append")(
|
|
219
|
+
(unvalidated) =>
|
|
220
|
+
Effect.gen(function* () {
|
|
221
|
+
const request = yield* validate(FencedAppendRequest, "append", unvalidated);
|
|
222
|
+
const digest = yield* digestCanonicalBatch(request.expectedTailDigest, request.batch).pipe(
|
|
223
|
+
Effect.provideService(Crypto.Crypto, crypto),
|
|
224
|
+
Effect.mapError((error) => storeError("append", error.message, error)),
|
|
225
|
+
);
|
|
238
226
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
return [
|
|
244
|
-
{
|
|
245
|
-
_tag: "failure",
|
|
246
|
-
error: ConversationNotMaterialized.make({
|
|
247
|
-
conversationId: request.conversationId,
|
|
248
|
-
}),
|
|
249
|
-
},
|
|
250
|
-
current,
|
|
251
|
-
];
|
|
252
|
-
}
|
|
253
|
-
if (request.producerEpoch !== conversation.producerEpoch) {
|
|
254
|
-
return [
|
|
255
|
-
{
|
|
256
|
-
_tag: "failure",
|
|
257
|
-
error: FenceRejected.make({
|
|
258
|
-
conversationId: request.conversationId,
|
|
259
|
-
actualEpoch: conversation.producerEpoch,
|
|
260
|
-
attemptedEpoch: request.producerEpoch,
|
|
261
|
-
}),
|
|
262
|
-
},
|
|
263
|
-
current,
|
|
264
|
-
];
|
|
265
|
-
}
|
|
266
|
-
const previous = conversation.batches.get(request.batch.batchId);
|
|
267
|
-
if (previous !== undefined) {
|
|
268
|
-
if (previous.digest !== digest) {
|
|
227
|
+
const decision = yield* Effect.uninterruptible(
|
|
228
|
+
Ref.modify(state, (current): readonly [AppendDecision, MemoryState] => {
|
|
229
|
+
const thread = current.threads.get(request.threadId);
|
|
230
|
+
if (thread === undefined) {
|
|
269
231
|
return [
|
|
270
232
|
{
|
|
271
233
|
_tag: "failure",
|
|
272
|
-
error:
|
|
273
|
-
|
|
274
|
-
batchId: request.batch.batchId,
|
|
275
|
-
reason: "batch-digest",
|
|
234
|
+
error: ThreadNotMaterialized.make({
|
|
235
|
+
threadId: request.threadId,
|
|
276
236
|
}),
|
|
277
237
|
},
|
|
278
238
|
current,
|
|
279
239
|
];
|
|
280
240
|
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
),
|
|
324
|
-
},
|
|
325
|
-
current,
|
|
326
|
-
];
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
const batchRecordIds = new Set<RecordId>();
|
|
330
|
-
for (const record of request.batch.records) {
|
|
241
|
+
if (request.producerEpoch !== thread.producerEpoch) {
|
|
242
|
+
return [
|
|
243
|
+
{
|
|
244
|
+
_tag: "failure",
|
|
245
|
+
error: FenceRejected.make({
|
|
246
|
+
threadId: request.threadId,
|
|
247
|
+
actualEpoch: thread.producerEpoch,
|
|
248
|
+
attemptedEpoch: request.producerEpoch,
|
|
249
|
+
}),
|
|
250
|
+
},
|
|
251
|
+
current,
|
|
252
|
+
];
|
|
253
|
+
}
|
|
254
|
+
const previous = thread.batches.get(request.batch.batchId);
|
|
255
|
+
if (previous !== undefined) {
|
|
256
|
+
if (previous.digest !== digest) {
|
|
257
|
+
return [
|
|
258
|
+
{
|
|
259
|
+
_tag: "failure",
|
|
260
|
+
error: AppendConflict.make({
|
|
261
|
+
threadId: request.threadId,
|
|
262
|
+
batchId: request.batch.batchId,
|
|
263
|
+
reason: "batch-digest",
|
|
264
|
+
}),
|
|
265
|
+
},
|
|
266
|
+
current,
|
|
267
|
+
];
|
|
268
|
+
}
|
|
269
|
+
return [
|
|
270
|
+
{
|
|
271
|
+
_tag: "success",
|
|
272
|
+
result: AppendResult.make({
|
|
273
|
+
firstSequence: previous.result.firstSequence,
|
|
274
|
+
lastSequence: previous.result.lastSequence,
|
|
275
|
+
tailDigest: previous.result.tailDigest,
|
|
276
|
+
replayed: true,
|
|
277
|
+
}),
|
|
278
|
+
records: [],
|
|
279
|
+
},
|
|
280
|
+
current,
|
|
281
|
+
];
|
|
282
|
+
}
|
|
331
283
|
if (
|
|
332
|
-
|
|
333
|
-
|
|
284
|
+
request.expectedTailSequence !== thread.tailSequence ||
|
|
285
|
+
request.expectedTailDigest !== thread.tailDigest
|
|
334
286
|
) {
|
|
335
287
|
return [
|
|
336
288
|
{
|
|
337
289
|
_tag: "failure",
|
|
338
290
|
error: AppendConflict.make({
|
|
339
|
-
|
|
291
|
+
threadId: request.threadId,
|
|
340
292
|
batchId: request.batch.batchId,
|
|
341
|
-
reason: "
|
|
293
|
+
reason: "tail",
|
|
294
|
+
actualTailSequence: thread.tailSequence,
|
|
295
|
+
actualTailDigest: thread.tailDigest,
|
|
342
296
|
}),
|
|
343
297
|
},
|
|
344
298
|
current,
|
|
345
299
|
];
|
|
346
300
|
}
|
|
347
|
-
|
|
348
|
-
|
|
301
|
+
if (thread.records.length + request.batch.records.length > MAX_RECORDS_PER_THREAD) {
|
|
302
|
+
return [
|
|
303
|
+
{
|
|
304
|
+
_tag: "failure",
|
|
305
|
+
error: storeError(
|
|
306
|
+
"append",
|
|
307
|
+
`In-memory record limit ${MAX_RECORDS_PER_THREAD} exceeded`,
|
|
308
|
+
),
|
|
309
|
+
},
|
|
310
|
+
current,
|
|
311
|
+
];
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const batchRecordIds = new Set<RecordId>();
|
|
315
|
+
for (const record of request.batch.records) {
|
|
316
|
+
if (thread.recordIds.has(record.recordId) || batchRecordIds.has(record.recordId)) {
|
|
317
|
+
return [
|
|
318
|
+
{
|
|
319
|
+
_tag: "failure",
|
|
320
|
+
error: AppendConflict.make({
|
|
321
|
+
threadId: request.threadId,
|
|
322
|
+
batchId: request.batch.batchId,
|
|
323
|
+
reason: "record-identity",
|
|
324
|
+
}),
|
|
325
|
+
},
|
|
326
|
+
current,
|
|
327
|
+
];
|
|
328
|
+
}
|
|
329
|
+
batchRecordIds.add(record.recordId);
|
|
330
|
+
}
|
|
349
331
|
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
332
|
+
const records = request.batch.records.map((record, index) => {
|
|
333
|
+
const sequence = decodeCanonicalSequence(thread.tailSequence + index + 1);
|
|
334
|
+
return CanonicalRecordEnvelope.make({
|
|
335
|
+
threadId: request.threadId,
|
|
336
|
+
batchId: request.batch.batchId,
|
|
337
|
+
sequence,
|
|
338
|
+
offset: observationOffset(request.threadId, sequence),
|
|
339
|
+
record,
|
|
340
|
+
});
|
|
358
341
|
});
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
342
|
+
const lastSequence = decodeCanonicalSequence(thread.tailSequence + records.length);
|
|
343
|
+
const result = AppendResult.make({
|
|
344
|
+
firstSequence: decodeCanonicalSequence(thread.tailSequence + 1),
|
|
345
|
+
lastSequence,
|
|
346
|
+
tailDigest: digest,
|
|
347
|
+
replayed: false,
|
|
348
|
+
});
|
|
349
|
+
const batches = new Map(thread.batches);
|
|
350
|
+
batches.set(request.batch.batchId, { digest, result });
|
|
351
|
+
const recordIds = new Set(thread.recordIds);
|
|
352
|
+
for (const recordId of batchRecordIds) recordIds.add(recordId);
|
|
353
|
+
const tailDigests = new Map(thread.tailDigests);
|
|
354
|
+
tailDigests.set(lastSequence, digest);
|
|
355
|
+
const threads = new Map(current.threads);
|
|
356
|
+
threads.set(request.threadId, {
|
|
357
|
+
...thread,
|
|
358
|
+
tailSequence: lastSequence,
|
|
359
|
+
tailDigest: digest,
|
|
360
|
+
records: [...thread.records, ...records],
|
|
361
|
+
recordIds,
|
|
362
|
+
batches,
|
|
363
|
+
tailDigests,
|
|
364
|
+
});
|
|
365
|
+
return [{ _tag: "success", result, records }, { threads }];
|
|
366
|
+
}).pipe(
|
|
367
|
+
Effect.tap((decision) =>
|
|
368
|
+
decision._tag === "success" && decision.records.length > 0
|
|
369
|
+
? PubSub.publish(updates, undefined)
|
|
370
|
+
: Effect.void,
|
|
371
|
+
),
|
|
389
372
|
),
|
|
390
|
-
)
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
}),
|
|
373
|
+
);
|
|
374
|
+
if (decision._tag === "failure") return yield* decision.error;
|
|
375
|
+
return decision.result;
|
|
376
|
+
}),
|
|
395
377
|
);
|
|
396
378
|
|
|
397
|
-
const readSnapshot = Effect.fn("
|
|
398
|
-
(
|
|
379
|
+
const readSnapshot = Effect.fn("MemoryThreadStore.readSnapshot")(
|
|
380
|
+
(threadId: ThreadId, afterSequence: CanonicalSequence | undefined, limit: number) =>
|
|
399
381
|
Ref.get(state).pipe(
|
|
400
|
-
Effect.flatMap((current) =>
|
|
401
|
-
Effect.map((
|
|
402
|
-
|
|
382
|
+
Effect.flatMap((current) => findThread(current, threadId)),
|
|
383
|
+
Effect.map((thread) =>
|
|
384
|
+
thread.records
|
|
403
385
|
.filter((record) => record.sequence > (afterSequence ?? ZERO_CANONICAL_SEQUENCE))
|
|
404
386
|
.slice(0, limit),
|
|
405
387
|
),
|
|
406
388
|
),
|
|
407
389
|
);
|
|
408
390
|
|
|
409
|
-
const read:
|
|
391
|
+
const read: ThreadStore["Service"]["read"] = (unvalidated) =>
|
|
410
392
|
Stream.unwrap(
|
|
411
393
|
Effect.gen(function* () {
|
|
412
|
-
const request = yield* validate(
|
|
413
|
-
const records = yield* readSnapshot(
|
|
414
|
-
request.conversationId,
|
|
415
|
-
request.afterSequence,
|
|
416
|
-
request.limit,
|
|
417
|
-
);
|
|
394
|
+
const request = yield* validate(ThreadRead, "read", unvalidated);
|
|
395
|
+
const records = yield* readSnapshot(request.threadId, request.afterSequence, request.limit);
|
|
418
396
|
return Stream.fromIterable(records);
|
|
419
397
|
}),
|
|
420
398
|
);
|
|
421
399
|
|
|
422
|
-
const observe:
|
|
400
|
+
const observe: ThreadStore["Service"]["observe"] = (unvalidated) =>
|
|
423
401
|
Stream.unwrap(
|
|
424
402
|
Effect.gen(function* () {
|
|
425
|
-
const request = yield* validate(
|
|
426
|
-
const afterSequence = yield* offsetSequence(request.
|
|
403
|
+
const request = yield* validate(ThreadObservation, "observe", unvalidated);
|
|
404
|
+
const afterSequence = yield* offsetSequence(request.threadId, request.afterOffset);
|
|
427
405
|
return Stream.unwrap(
|
|
428
406
|
Effect.gen(function* () {
|
|
429
407
|
const subscription = yield* PubSub.subscribe(updates);
|
|
430
408
|
const initial = yield* readSnapshot(
|
|
431
|
-
request.
|
|
409
|
+
request.threadId,
|
|
432
410
|
afterSequence,
|
|
433
|
-
|
|
411
|
+
MAX_RECORDS_PER_THREAD,
|
|
434
412
|
);
|
|
435
413
|
const highWater =
|
|
436
414
|
initial.length === 0 ? afterSequence : (initial.at(-1)?.sequence ?? afterSequence);
|
|
@@ -438,11 +416,7 @@ const makeConversationStore = Effect.gen(function* () {
|
|
|
438
416
|
Stream.mapAccumEffect(
|
|
439
417
|
() => highWater,
|
|
440
418
|
(lastSequence) =>
|
|
441
|
-
readSnapshot(
|
|
442
|
-
request.conversationId,
|
|
443
|
-
lastSequence,
|
|
444
|
-
MAX_RECORDS_PER_CONVERSATION,
|
|
445
|
-
).pipe(
|
|
419
|
+
readSnapshot(request.threadId, lastSequence, MAX_RECORDS_PER_THREAD).pipe(
|
|
446
420
|
Effect.map(
|
|
447
421
|
(records) => [records.at(-1)?.sequence ?? lastSequence, records] as const,
|
|
448
422
|
),
|
|
@@ -455,160 +429,156 @@ const makeConversationStore = Effect.gen(function* () {
|
|
|
455
429
|
}),
|
|
456
430
|
);
|
|
457
431
|
|
|
458
|
-
const
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
})
|
|
473
|
-
}),
|
|
432
|
+
const exportThread: ThreadStore["Service"]["export"] = Effect.fn("MemoryThreadStore.export")(
|
|
433
|
+
(unvalidated) =>
|
|
434
|
+
Effect.gen(function* () {
|
|
435
|
+
const request = yield* validate(ThreadExportRequest, "export", unvalidated);
|
|
436
|
+
const thread = yield* Ref.get(state).pipe(
|
|
437
|
+
Effect.flatMap((current) => findThread(current, request.threadId)),
|
|
438
|
+
);
|
|
439
|
+
return ThreadExport.make({
|
|
440
|
+
format: "effect-agent/thread@1",
|
|
441
|
+
threadId: request.threadId,
|
|
442
|
+
tailSequence: thread.tailSequence,
|
|
443
|
+
tailDigest: thread.tailDigest,
|
|
444
|
+
records: thread.records,
|
|
445
|
+
});
|
|
446
|
+
}),
|
|
474
447
|
);
|
|
475
448
|
|
|
476
|
-
const inspectTail:
|
|
477
|
-
"
|
|
449
|
+
const inspectTail: ThreadStore["Service"]["inspectTail"] = Effect.fn(
|
|
450
|
+
"MemoryThreadStore.inspectTail",
|
|
478
451
|
)((unvalidated) =>
|
|
479
452
|
Effect.gen(function* () {
|
|
480
|
-
const request = yield* validate(
|
|
481
|
-
const
|
|
482
|
-
Effect.flatMap((current) =>
|
|
453
|
+
const request = yield* validate(ThreadTailRequest, "inspectTail", unvalidated);
|
|
454
|
+
const thread = yield* Ref.get(state).pipe(
|
|
455
|
+
Effect.flatMap((current) => findThread(current, request.threadId)),
|
|
483
456
|
);
|
|
484
|
-
return
|
|
485
|
-
|
|
486
|
-
tailSequence:
|
|
487
|
-
tailDigest:
|
|
488
|
-
producerEpoch:
|
|
457
|
+
return ThreadTail.make({
|
|
458
|
+
threadId: request.threadId,
|
|
459
|
+
tailSequence: thread.tailSequence,
|
|
460
|
+
tailDigest: thread.tailDigest,
|
|
461
|
+
producerEpoch: thread.producerEpoch,
|
|
489
462
|
});
|
|
490
463
|
}),
|
|
491
464
|
);
|
|
492
465
|
|
|
493
|
-
const saveCheckpoint:
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
466
|
+
const saveCheckpoint: ThreadCheckpoints["save"] = Effect.fn("MemoryThreadStore.saveCheckpoint")(
|
|
467
|
+
(unvalidated) =>
|
|
468
|
+
Effect.gen(function* () {
|
|
469
|
+
yield* validateCheckpointVersion(unvalidated);
|
|
470
|
+
const request = yield* validate(SaveCheckpointRequest, "saveCheckpoint", unvalidated);
|
|
471
|
+
const decision = yield* Ref.modify(
|
|
472
|
+
state,
|
|
473
|
+
(current): readonly [CheckpointDecision, MemoryState] => {
|
|
474
|
+
const checkpoint = request.checkpoint;
|
|
475
|
+
const thread = current.threads.get(checkpoint.threadId);
|
|
476
|
+
if (thread === undefined) {
|
|
477
|
+
return [
|
|
478
|
+
{
|
|
479
|
+
_tag: "failure",
|
|
480
|
+
error: ThreadNotMaterialized.make({
|
|
481
|
+
threadId: checkpoint.threadId,
|
|
482
|
+
}),
|
|
483
|
+
},
|
|
484
|
+
current,
|
|
485
|
+
];
|
|
486
|
+
}
|
|
487
|
+
if (checkpoint.throughSequence > thread.tailSequence) {
|
|
488
|
+
return [
|
|
489
|
+
{
|
|
490
|
+
_tag: "failure",
|
|
491
|
+
error: CheckpointRejected.make({
|
|
492
|
+
threadId: checkpoint.threadId,
|
|
493
|
+
reason: "ahead-of-tail",
|
|
494
|
+
}),
|
|
495
|
+
},
|
|
496
|
+
current,
|
|
497
|
+
];
|
|
498
|
+
}
|
|
499
|
+
if (thread.tailDigests.get(checkpoint.throughSequence) !== checkpoint.tailDigest) {
|
|
500
|
+
return [
|
|
501
|
+
{
|
|
502
|
+
_tag: "failure",
|
|
503
|
+
error: CheckpointRejected.make({
|
|
504
|
+
threadId: checkpoint.threadId,
|
|
505
|
+
reason: "digest-mismatch",
|
|
506
|
+
}),
|
|
507
|
+
},
|
|
508
|
+
current,
|
|
509
|
+
];
|
|
510
|
+
}
|
|
511
|
+
if (
|
|
512
|
+
!thread.checkpoints.has(checkpoint.throughSequence) &&
|
|
513
|
+
thread.checkpoints.size >= MAX_CHECKPOINTS_PER_THREAD
|
|
514
|
+
) {
|
|
515
|
+
return [
|
|
516
|
+
{
|
|
517
|
+
_tag: "failure",
|
|
518
|
+
error: storeError(
|
|
519
|
+
"saveCheckpoint",
|
|
520
|
+
`In-memory checkpoint limit ${MAX_CHECKPOINTS_PER_THREAD} exceeded`,
|
|
521
|
+
),
|
|
522
|
+
},
|
|
523
|
+
current,
|
|
524
|
+
];
|
|
525
|
+
}
|
|
526
|
+
const checkpoints = new Map(thread.checkpoints);
|
|
527
|
+
checkpoints.set(checkpoint.throughSequence, checkpoint);
|
|
528
|
+
const threads = new Map(current.threads);
|
|
529
|
+
threads.set(checkpoint.threadId, { ...thread, checkpoints });
|
|
530
|
+
return [{ _tag: "success" }, { threads }];
|
|
531
|
+
},
|
|
532
|
+
);
|
|
533
|
+
if (decision._tag === "failure") return yield* decision.error;
|
|
534
|
+
}),
|
|
535
|
+
);
|
|
536
|
+
|
|
537
|
+
const loadCheckpoint: ThreadCheckpoints["load"] = Effect.fn("MemoryThreadStore.loadCheckpoint")(
|
|
538
|
+
(unvalidated) =>
|
|
539
|
+
Effect.gen(function* () {
|
|
540
|
+
const request = yield* validate(LoadCheckpointRequest, "loadCheckpoint", unvalidated);
|
|
541
|
+
const thread = yield* Ref.get(state).pipe(
|
|
542
|
+
Effect.flatMap((current) => findThread(current, request.threadId)),
|
|
543
|
+
);
|
|
544
|
+
const maximum = request.atOrBeforeSequence ?? thread.tailSequence;
|
|
545
|
+
let selected: ThreadCheckpoint | undefined;
|
|
546
|
+
for (const [sequence, checkpoint] of thread.checkpoints) {
|
|
539
547
|
if (
|
|
540
|
-
|
|
541
|
-
|
|
548
|
+
sequence <= maximum &&
|
|
549
|
+
(selected === undefined || sequence > selected.throughSequence)
|
|
542
550
|
) {
|
|
543
|
-
|
|
544
|
-
{
|
|
545
|
-
_tag: "failure",
|
|
546
|
-
error: storeError(
|
|
547
|
-
"saveCheckpoint",
|
|
548
|
-
`In-memory checkpoint limit ${MAX_CHECKPOINTS_PER_CONVERSATION} exceeded`,
|
|
549
|
-
),
|
|
550
|
-
},
|
|
551
|
-
current,
|
|
552
|
-
];
|
|
551
|
+
selected = checkpoint;
|
|
553
552
|
}
|
|
554
|
-
|
|
555
|
-
checkpoints.set(checkpoint.throughSequence, checkpoint);
|
|
556
|
-
const conversations = new Map(current.conversations);
|
|
557
|
-
conversations.set(checkpoint.conversationId, { ...conversation, checkpoints });
|
|
558
|
-
return [{ _tag: "success" }, { conversations }];
|
|
559
|
-
},
|
|
560
|
-
);
|
|
561
|
-
if (decision._tag === "failure") return yield* decision.error;
|
|
562
|
-
}),
|
|
563
|
-
);
|
|
564
|
-
|
|
565
|
-
const loadCheckpoint: ConversationStore["Service"]["loadCheckpoint"] = Effect.fn(
|
|
566
|
-
"MemoryConversationStore.loadCheckpoint",
|
|
567
|
-
)((unvalidated) =>
|
|
568
|
-
Effect.gen(function* () {
|
|
569
|
-
const request = yield* validate(LoadCheckpointRequest, "loadCheckpoint", unvalidated);
|
|
570
|
-
const conversation = yield* Ref.get(state).pipe(
|
|
571
|
-
Effect.flatMap((current) => findConversation(current, request.conversationId)),
|
|
572
|
-
);
|
|
573
|
-
const maximum = request.atOrBeforeSequence ?? conversation.tailSequence;
|
|
574
|
-
let selected: ConversationCheckpoint | undefined;
|
|
575
|
-
for (const [sequence, checkpoint] of conversation.checkpoints) {
|
|
553
|
+
}
|
|
576
554
|
if (
|
|
577
|
-
|
|
578
|
-
(selected
|
|
555
|
+
selected !== undefined &&
|
|
556
|
+
thread.tailDigests.get(selected.throughSequence) !== selected.tailDigest
|
|
579
557
|
) {
|
|
580
|
-
|
|
558
|
+
return yield* CheckpointRejected.make({
|
|
559
|
+
threadId: request.threadId,
|
|
560
|
+
reason: "digest-mismatch",
|
|
561
|
+
});
|
|
581
562
|
}
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
selected !== undefined &&
|
|
585
|
-
conversation.tailDigests.get(selected.throughSequence) !== selected.tailDigest
|
|
586
|
-
) {
|
|
587
|
-
return yield* CheckpointRejected.make({
|
|
588
|
-
conversationId: request.conversationId,
|
|
589
|
-
reason: "digest-mismatch",
|
|
590
|
-
});
|
|
591
|
-
}
|
|
592
|
-
return Option.fromNullishOr(selected);
|
|
593
|
-
}),
|
|
563
|
+
return Option.fromNullishOr(selected);
|
|
564
|
+
}),
|
|
594
565
|
);
|
|
595
566
|
|
|
596
|
-
return
|
|
567
|
+
return ThreadStore.of({
|
|
597
568
|
materialize,
|
|
598
569
|
append,
|
|
599
570
|
read,
|
|
600
571
|
observe,
|
|
601
|
-
export:
|
|
572
|
+
export: exportThread,
|
|
602
573
|
inspectTail,
|
|
603
|
-
saveCheckpoint,
|
|
604
|
-
loadCheckpoint,
|
|
574
|
+
checkpoints: { save: saveCheckpoint, load: loadCheckpoint },
|
|
605
575
|
});
|
|
606
576
|
});
|
|
607
577
|
|
|
608
|
-
export const
|
|
578
|
+
export const MemoryThreadStoreLive = Layer.effect(ThreadStore, makeThreadStore);
|
|
609
579
|
|
|
610
580
|
/**
|
|
611
|
-
* In-memory canonical
|
|
612
|
-
* SubmissionLedger port; this Layer deliberately provides only the
|
|
581
|
+
* In-memory canonical Thread persistence. Durable accepted work is served by the separate
|
|
582
|
+
* SubmissionLedger port; this Layer deliberately provides only the ThreadStore.
|
|
613
583
|
*/
|
|
614
|
-
export const MemoryStorageLive =
|
|
584
|
+
export const MemoryStorageLive = MemoryThreadStoreLive;
|