@effect-agent/storage-sqlite 0.1.0-beta.6 → 0.1.0-beta.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/SqliteActivityStore.d.mts +14 -0
- package/dist/SqliteActivityStore.mjs +310 -0
- package/dist/SqliteActivityStore.mjs.map +1 -0
- package/dist/SqliteMessageDeliveryStore.d.mts +14 -0
- package/dist/SqliteMessageDeliveryStore.mjs +24 -0
- package/dist/SqliteMessageDeliveryStore.mjs.map +1 -0
- package/dist/SqliteScheduleStore.d.mts +14 -0
- package/dist/SqliteScheduleStore.mjs +255 -0
- package/dist/SqliteScheduleStore.mjs.map +1 -0
- package/dist/SqliteStorageConfig.d.mts +34 -0
- package/dist/SqliteStorageConfig.mjs +39 -0
- package/dist/SqliteStorageConfig.mjs.map +1 -0
- package/dist/SqliteStorageError-CzKmwCLl.d.mts +86 -0
- package/dist/SqliteStorageError.d.mts +2 -0
- package/dist/SqliteStorageError.mjs +148 -0
- package/dist/SqliteStorageError.mjs.map +1 -0
- package/dist/SqliteStorageFailpoint.d.mts +17 -0
- package/dist/SqliteStorageFailpoint.mjs +14 -0
- package/dist/SqliteStorageFailpoint.mjs.map +1 -0
- package/dist/SqliteStorageFailpointTesting.d.mts +15 -0
- package/dist/SqliteStorageFailpointTesting.mjs +19 -0
- package/dist/SqliteStorageFailpointTesting.mjs.map +1 -0
- package/dist/SqliteStorageVersion-gA7_96xM.d.mts +9 -0
- package/dist/SqliteStorageVersion.d.mts +2 -0
- package/dist/SqliteStorageVersion.mjs +8 -0
- package/dist/SqliteStorageVersion.mjs.map +1 -0
- package/dist/SqliteSubmissionLedger.d.mts +23 -0
- package/dist/SqliteSubmissionLedger.mjs +1765 -0
- package/dist/SqliteSubmissionLedger.mjs.map +1 -0
- package/dist/SqliteSubscriptionStore.d.mts +13 -0
- package/dist/SqliteSubscriptionStore.mjs +28 -0
- package/dist/SqliteSubscriptionStore.mjs.map +1 -0
- package/dist/SqliteThreadStore.d.mts +49 -0
- package/dist/SqliteThreadStore.mjs +427 -0
- package/dist/SqliteThreadStore.mjs.map +1 -0
- package/dist/index.d.mts +11 -193
- package/dist/index.mjs +11 -3082
- package/dist/migrations-dnTR0RKq.mjs +326 -0
- package/dist/migrations-dnTR0RKq.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/dist/sqlite-journal-lDduEakl.mjs +887 -0
- package/dist/sqlite-journal-lDduEakl.mjs.map +1 -0
- package/package.json +1 -41
- package/src/SqliteActivityStore.ts +556 -0
- package/src/SqliteMessageDeliveryStore.ts +42 -0
- package/src/SqliteScheduleStore.ts +404 -0
- package/src/{sqlite-storage-config.ts → SqliteStorageConfig.ts} +1 -1
- package/src/{errors.ts → SqliteStorageError.ts} +8 -3
- package/src/SqliteStorageFailpoint.ts +23 -0
- package/src/{sqlite-storage-failpoint.ts → SqliteStorageFailpointTesting.ts} +7 -18
- package/src/SqliteStorageVersion.ts +2 -0
- package/src/{sqlite-ledger.ts → SqliteSubmissionLedger.ts} +753 -118
- package/src/SqliteSubscriptionStore.ts +59 -0
- package/src/{sqlite-conversation-store.ts → SqliteThreadStore.ts} +372 -302
- package/src/index.ts +10 -6
- package/src/internal/message-delivery-schema.ts +24 -0
- package/src/{migrations.ts → internal/migrations.ts} +145 -79
- package/src/{sqlite-journal.ts → internal/sqlite-journal.ts} +534 -222
- package/dist/index.mjs.map +0 -1
|
@@ -1,32 +1,34 @@
|
|
|
1
|
+
import { digestCanonicalBatch, EMPTY_TAIL_DIGEST } from "@effect-agent/thread/Digest";
|
|
1
2
|
import {
|
|
2
|
-
AppendConflict,
|
|
3
|
-
AppendResult,
|
|
4
3
|
CanonicalBatch,
|
|
5
4
|
CanonicalRecord,
|
|
6
5
|
CanonicalRecordEnvelope,
|
|
7
6
|
CanonicalSequence,
|
|
8
|
-
CheckpointRejected,
|
|
9
|
-
ConversationCheckpoint,
|
|
10
|
-
ConversationExport,
|
|
11
|
-
ConversationExportRequest,
|
|
12
|
-
ConversationMaterialization,
|
|
13
|
-
ConversationNotMaterialized,
|
|
14
|
-
ConversationObservation,
|
|
15
|
-
ConversationRead,
|
|
16
|
-
ConversationStore,
|
|
17
|
-
ConversationStoreError,
|
|
18
|
-
ConversationTail,
|
|
19
|
-
ConversationTailRequest,
|
|
20
|
-
DEFAULT_OWNERSHIP_LEASE_DURATION,
|
|
21
|
-
digestCanonicalBatch,
|
|
22
7
|
Digest,
|
|
23
|
-
|
|
8
|
+
ObservationOffset,
|
|
9
|
+
} from "@effect-agent/thread/Records";
|
|
10
|
+
import { DEFAULT_OWNERSHIP_LEASE_DURATION } from "@effect-agent/thread/SubmissionLedger";
|
|
11
|
+
import {
|
|
12
|
+
AppendConflict,
|
|
13
|
+
AppendResult,
|
|
14
|
+
CheckpointRejected,
|
|
15
|
+
ThreadCheckpoint,
|
|
16
|
+
ThreadExport,
|
|
17
|
+
ThreadExportRequest,
|
|
18
|
+
ThreadMaterialization,
|
|
19
|
+
ThreadNotMaterialized,
|
|
20
|
+
ThreadObservation,
|
|
21
|
+
ThreadRead,
|
|
22
|
+
ThreadStore,
|
|
23
|
+
type ThreadCheckpoints,
|
|
24
|
+
ThreadStoreError,
|
|
25
|
+
ThreadTail,
|
|
26
|
+
ThreadTailRequest,
|
|
24
27
|
FenceRejected,
|
|
25
28
|
FencedAppendRequest,
|
|
26
29
|
LoadCheckpointRequest,
|
|
27
|
-
ObservationOffset,
|
|
28
30
|
SaveCheckpointRequest,
|
|
29
|
-
} from "@effect-agent/
|
|
31
|
+
} from "@effect-agent/thread/ThreadStore";
|
|
30
32
|
import { NodeCrypto } from "@effect/platform-node";
|
|
31
33
|
import { SqliteClient } from "@effect/sql-sqlite-node";
|
|
32
34
|
import {
|
|
@@ -41,29 +43,29 @@ import {
|
|
|
41
43
|
Schema,
|
|
42
44
|
Stream,
|
|
43
45
|
} from "effect";
|
|
44
|
-
import * as SqlClientService from "effect/unstable/sql/SqlClient";
|
|
46
|
+
import type * as SqlClientService from "effect/unstable/sql/SqlClient";
|
|
45
47
|
|
|
46
48
|
import {
|
|
49
|
+
initializeSqliteJournal,
|
|
50
|
+
RawAppendRequest,
|
|
51
|
+
RawCheckpoint,
|
|
52
|
+
RawReadRequest,
|
|
53
|
+
type SqliteJournal,
|
|
54
|
+
} from "./internal/sqlite-journal.ts";
|
|
55
|
+
import { SqliteStorageConfig, SqliteStorageConfigValue } from "./SqliteStorageConfig.ts";
|
|
56
|
+
import {
|
|
57
|
+
type SqliteStorageCompatibilityError,
|
|
47
58
|
SqliteAppendConflict,
|
|
48
59
|
SqliteCheckpointConflict,
|
|
49
60
|
SqliteFenceRejected,
|
|
50
61
|
type SqliteStorageFailpointLocation,
|
|
51
|
-
SqliteStorageCompatibilityError,
|
|
52
62
|
SqliteStorageCorruptionError,
|
|
53
63
|
SqliteStorageError,
|
|
54
|
-
} from "./
|
|
55
|
-
import {
|
|
56
|
-
initializeSqliteJournal,
|
|
57
|
-
RawAppendRequest,
|
|
58
|
-
RawCheckpoint,
|
|
59
|
-
RawReadRequest,
|
|
60
|
-
type SqliteJournal,
|
|
61
|
-
} from "./sqlite-journal.ts";
|
|
62
|
-
import { SqliteStorageConfig, SqliteStorageConfigValue } from "./sqlite-storage-config.ts";
|
|
64
|
+
} from "./SqliteStorageError.ts";
|
|
63
65
|
import {
|
|
64
66
|
SqliteStorageFailpoint,
|
|
65
67
|
type SqliteStorageFailpointHandler,
|
|
66
|
-
} from "./
|
|
68
|
+
} from "./SqliteStorageFailpoint.ts";
|
|
67
69
|
|
|
68
70
|
export interface SqliteStorageOptions {
|
|
69
71
|
readonly filename: string;
|
|
@@ -72,7 +74,7 @@ export interface SqliteStorageOptions {
|
|
|
72
74
|
readonly busyTimeout?: number | undefined;
|
|
73
75
|
/**
|
|
74
76
|
* Submission ownership lease duration in milliseconds (D5). Defaults to
|
|
75
|
-
* `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@effect-agent/
|
|
77
|
+
* `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@effect-agent/thread`.
|
|
76
78
|
*/
|
|
77
79
|
readonly ownershipLeaseDuration?: number | undefined;
|
|
78
80
|
/**
|
|
@@ -93,100 +95,104 @@ const OffsetText = Schema.String.check(Schema.isMaxLength(4 * 1024));
|
|
|
93
95
|
const SQLITE_OFFSET_PREFIX = "effect-agent-sqlite@1:";
|
|
94
96
|
const ZERO_CANONICAL_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
95
97
|
const isDigest = Schema.is(Digest);
|
|
98
|
+
const isSqliteFenceRejected = Schema.is(SqliteFenceRejected);
|
|
99
|
+
const isSqliteAppendConflict = Schema.is(SqliteAppendConflict);
|
|
100
|
+
const isSqliteCheckpointConflict = Schema.is(SqliteCheckpointConflict);
|
|
96
101
|
|
|
97
102
|
const storeError = (operation: string, error: { readonly message: string }) =>
|
|
98
|
-
|
|
103
|
+
ThreadStoreError.make({
|
|
99
104
|
cause: error,
|
|
100
105
|
operation,
|
|
101
106
|
message: error.message,
|
|
102
107
|
});
|
|
103
108
|
|
|
104
109
|
const schemaStoreError = (operation: string, error: { readonly message: string }) =>
|
|
105
|
-
|
|
110
|
+
ThreadStoreError.make({
|
|
106
111
|
cause: error,
|
|
107
112
|
operation,
|
|
108
113
|
message: error.message,
|
|
109
114
|
});
|
|
110
115
|
|
|
111
|
-
const makeOffset = Effect.fn("
|
|
112
|
-
|
|
116
|
+
const makeOffset = Effect.fn("SqliteThreadStore.makeOffset")(function* (
|
|
117
|
+
threadId: ThreadMaterialization["threadId"],
|
|
113
118
|
sequence: number,
|
|
114
|
-
): Effect.fn.Return<ObservationOffset,
|
|
119
|
+
): Effect.fn.Return<ObservationOffset, ThreadStoreError> {
|
|
115
120
|
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(sequence).pipe(
|
|
116
121
|
Effect.flatMap((validatedSequence) =>
|
|
117
122
|
Schema.decodeUnknownEffect(ObservationOffset)(
|
|
118
|
-
`${SQLITE_OFFSET_PREFIX}${encodeURIComponent(
|
|
123
|
+
`${SQLITE_OFFSET_PREFIX}${encodeURIComponent(threadId)}:${validatedSequence}`,
|
|
119
124
|
),
|
|
120
125
|
),
|
|
121
126
|
Effect.mapError((error) => schemaStoreError("encode observation offset", error)),
|
|
122
127
|
);
|
|
123
128
|
});
|
|
124
129
|
|
|
125
|
-
const parseOffset = Effect.fn("
|
|
126
|
-
|
|
130
|
+
const parseOffset = Effect.fn("SqliteThreadStore.parseOffset")(function* (
|
|
131
|
+
threadId: ThreadMaterialization["threadId"],
|
|
127
132
|
offset: ObservationOffset | undefined,
|
|
128
|
-
): Effect.fn.Return<CanonicalSequence,
|
|
133
|
+
): Effect.fn.Return<CanonicalSequence, ThreadStoreError> {
|
|
129
134
|
if (offset === undefined) return ZERO_CANONICAL_SEQUENCE;
|
|
135
|
+
|
|
130
136
|
const text = yield* Schema.decodeUnknownEffect(OffsetText)(offset).pipe(
|
|
131
137
|
Effect.mapError((error) => schemaStoreError("decode observation offset", error)),
|
|
132
138
|
);
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
139
|
+
|
|
140
|
+
const threadPrefix = `${SQLITE_OFFSET_PREFIX}${encodeURIComponent(threadId)}:`;
|
|
141
|
+
|
|
142
|
+
if (!text.startsWith(threadPrefix)) {
|
|
143
|
+
return yield* ThreadStoreError.make({
|
|
136
144
|
operation: "decode observation offset",
|
|
137
|
-
message:
|
|
138
|
-
"The observation offset belongs to a different adapter, storage version, or Conversation.",
|
|
145
|
+
message: "The observation offset belongs to a different adapter, storage version, or Thread.",
|
|
139
146
|
});
|
|
140
147
|
}
|
|
141
|
-
const sequenceText = text.slice(
|
|
148
|
+
const sequenceText = text.slice(threadPrefix.length);
|
|
149
|
+
|
|
142
150
|
if (!/^(0|[1-9][0-9]*)$/.test(sequenceText)) {
|
|
143
|
-
return yield*
|
|
151
|
+
return yield* ThreadStoreError.make({
|
|
144
152
|
operation: "decode observation offset",
|
|
145
153
|
message: "The observation offset is malformed.",
|
|
146
154
|
});
|
|
147
155
|
}
|
|
156
|
+
|
|
148
157
|
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(Number(sequenceText)).pipe(
|
|
149
158
|
Effect.mapError((error) => schemaStoreError("decode observation offset", error)),
|
|
150
159
|
);
|
|
151
160
|
});
|
|
152
161
|
|
|
153
|
-
const mapFence = (
|
|
154
|
-
conversationId: ConversationMaterialization["conversationId"],
|
|
155
|
-
error: SqliteFenceRejected,
|
|
156
|
-
) =>
|
|
162
|
+
const mapFence = (threadId: ThreadMaterialization["threadId"], error: SqliteFenceRejected) =>
|
|
157
163
|
FenceRejected.make({
|
|
158
|
-
|
|
164
|
+
threadId,
|
|
159
165
|
actualEpoch: error.actualEpoch,
|
|
160
166
|
attemptedEpoch: error.producerEpoch,
|
|
161
167
|
});
|
|
162
168
|
|
|
163
|
-
const encodeCanonicalRecord = Effect.fn("
|
|
169
|
+
const encodeCanonicalRecord = Effect.fn("SqliteThreadStore.encodeCanonicalRecord")(function* (
|
|
164
170
|
record: CanonicalRecord,
|
|
165
|
-
): Effect.fn.Return<string,
|
|
171
|
+
): Effect.fn.Return<string, ThreadStoreError> {
|
|
166
172
|
return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(record).pipe(
|
|
167
173
|
Effect.mapError((error) => schemaStoreError("encode canonical record", error)),
|
|
168
174
|
);
|
|
169
175
|
});
|
|
170
176
|
|
|
171
|
-
const encodeCanonicalBatch = Effect.fn("
|
|
177
|
+
const encodeCanonicalBatch = Effect.fn("SqliteThreadStore.encodeCanonicalBatch")(function* (
|
|
172
178
|
batch: CanonicalBatch,
|
|
173
|
-
): Effect.fn.Return<string,
|
|
179
|
+
): Effect.fn.Return<string, ThreadStoreError> {
|
|
174
180
|
return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalBatch))(batch).pipe(
|
|
175
181
|
Effect.mapError((error) => schemaStoreError("encode canonical batch", error)),
|
|
176
182
|
);
|
|
177
183
|
});
|
|
178
184
|
|
|
179
|
-
const encodeCheckpoint = Effect.fn("
|
|
180
|
-
checkpoint:
|
|
181
|
-
): Effect.fn.Return<string,
|
|
182
|
-
return yield* Schema.encodeEffect(Schema.fromJsonString(
|
|
185
|
+
const encodeCheckpoint = Effect.fn("SqliteThreadStore.encodeCheckpoint")(function* (
|
|
186
|
+
checkpoint: ThreadCheckpoint,
|
|
187
|
+
): Effect.fn.Return<string, ThreadStoreError> {
|
|
188
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint).pipe(
|
|
183
189
|
Effect.mapError((error) => schemaStoreError("encode checkpoint", error)),
|
|
184
190
|
);
|
|
185
191
|
});
|
|
186
192
|
|
|
187
|
-
const decodeEnvelope = Effect.fn("
|
|
193
|
+
const decodeEnvelope = Effect.fn("SqliteThreadStore.decodeEnvelope")(function* (row: {
|
|
188
194
|
readonly batch_id: string;
|
|
189
|
-
readonly
|
|
195
|
+
readonly thread_id: string;
|
|
190
196
|
readonly record_json: string;
|
|
191
197
|
readonly sequence: CanonicalSequence;
|
|
192
198
|
}) {
|
|
@@ -194,23 +200,25 @@ const decodeEnvelope = Effect.fn("SqliteConversationStore.decodeEnvelope")(funct
|
|
|
194
200
|
row.record_json,
|
|
195
201
|
).pipe(
|
|
196
202
|
Effect.mapError((error) =>
|
|
197
|
-
|
|
203
|
+
ThreadStoreError.make({
|
|
198
204
|
operation: "decode canonical record",
|
|
199
205
|
message: error.message,
|
|
200
206
|
}),
|
|
201
207
|
),
|
|
202
208
|
);
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
const offset = yield* makeOffset(
|
|
209
|
+
|
|
210
|
+
const threadId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.threadId)(
|
|
211
|
+
row.thread_id,
|
|
212
|
+
).pipe(Effect.mapError((error) => schemaStoreError("decode thread identity", error)));
|
|
213
|
+
|
|
214
|
+
const offset = yield* makeOffset(threadId, row.sequence);
|
|
215
|
+
|
|
209
216
|
const batchId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.batchId)(
|
|
210
217
|
row.batch_id,
|
|
211
218
|
).pipe(Effect.mapError((error) => schemaStoreError("decode batch identity", error)));
|
|
219
|
+
|
|
212
220
|
return CanonicalRecordEnvelope.make({
|
|
213
|
-
|
|
221
|
+
threadId,
|
|
214
222
|
batchId,
|
|
215
223
|
sequence: row.sequence,
|
|
216
224
|
offset,
|
|
@@ -218,42 +226,47 @@ const decodeEnvelope = Effect.fn("SqliteConversationStore.decodeEnvelope")(funct
|
|
|
218
226
|
});
|
|
219
227
|
});
|
|
220
228
|
|
|
221
|
-
const decodeCheckpoint = Effect.fn("
|
|
229
|
+
const decodeCheckpoint = Effect.fn("SqliteThreadStore.decodeCheckpoint")(function* (
|
|
222
230
|
checkpointJson: string,
|
|
223
|
-
): Effect.fn.Return<
|
|
224
|
-
return yield* Schema.decodeEffect(Schema.fromJsonString(
|
|
225
|
-
|
|
226
|
-
)
|
|
231
|
+
): Effect.fn.Return<ThreadCheckpoint, ThreadStoreError> {
|
|
232
|
+
return yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpointJson).pipe(
|
|
233
|
+
Effect.mapError((error) => schemaStoreError("decode checkpoint", error)),
|
|
234
|
+
);
|
|
227
235
|
});
|
|
228
236
|
|
|
229
|
-
const
|
|
237
|
+
const requireThread = Effect.fn("SqliteThreadStore.requireThread")(function* (
|
|
230
238
|
journal: SqliteJournal,
|
|
231
|
-
|
|
239
|
+
threadId: ThreadMaterialization["threadId"],
|
|
232
240
|
) {
|
|
233
241
|
const rows = yield* journal
|
|
234
|
-
.
|
|
235
|
-
.pipe(Effect.mapError((error) => storeError("read
|
|
242
|
+
.getThread(threadId)
|
|
243
|
+
.pipe(Effect.mapError((error) => storeError("read thread", error)));
|
|
244
|
+
|
|
236
245
|
if (rows.length === 0) {
|
|
237
|
-
return yield*
|
|
246
|
+
return yield* ThreadNotMaterialized.make({ threadId });
|
|
238
247
|
}
|
|
248
|
+
|
|
239
249
|
return rows[0];
|
|
240
250
|
});
|
|
241
251
|
|
|
242
|
-
const tailDigestAt = Effect.fn("
|
|
252
|
+
const tailDigestAt = Effect.fn("SqliteThreadStore.tailDigestAt")(function* (
|
|
243
253
|
journal: SqliteJournal,
|
|
244
|
-
|
|
254
|
+
threadId: ThreadMaterialization["threadId"],
|
|
245
255
|
sequence: CanonicalSequence,
|
|
246
256
|
) {
|
|
247
257
|
if (sequence === 0) return EMPTY_TAIL_DIGEST;
|
|
258
|
+
|
|
248
259
|
const digests = yield* journal
|
|
249
|
-
.getTailDigestAt(
|
|
260
|
+
.getTailDigestAt(threadId, sequence)
|
|
250
261
|
.pipe(Effect.mapError((error) => storeError("read checkpoint digest", error)));
|
|
262
|
+
|
|
251
263
|
if (digests.length !== 1) {
|
|
252
264
|
return yield* CheckpointRejected.make({
|
|
253
|
-
|
|
265
|
+
threadId,
|
|
254
266
|
reason: "digest-mismatch",
|
|
255
267
|
});
|
|
256
268
|
}
|
|
269
|
+
|
|
257
270
|
return yield* Schema.decodeUnknownEffect(Digest)(digests[0]).pipe(
|
|
258
271
|
Effect.mapError((error) => schemaStoreError("decode checkpoint digest", error)),
|
|
259
272
|
);
|
|
@@ -264,14 +277,17 @@ const groupByKey = <A>(
|
|
|
264
277
|
key: (row: A) => string,
|
|
265
278
|
): ReadonlyMap<string, ReadonlyArray<A>> => {
|
|
266
279
|
const grouped = new Map<string, Array<A>>();
|
|
280
|
+
|
|
267
281
|
for (const row of rows) {
|
|
268
282
|
const existing = grouped.get(key(row));
|
|
283
|
+
|
|
269
284
|
if (existing === undefined) {
|
|
270
285
|
grouped.set(key(row), [row]);
|
|
271
286
|
} else {
|
|
272
287
|
existing.push(row);
|
|
273
288
|
}
|
|
274
289
|
}
|
|
290
|
+
|
|
275
291
|
return grouped;
|
|
276
292
|
};
|
|
277
293
|
|
|
@@ -280,69 +296,68 @@ const groupByKey = <A>(
|
|
|
280
296
|
* and re-digested against the canonical chain. Routine opens skip this scan: per-operation
|
|
281
297
|
* Schema decoding plus the digest chain already fail clearly on corrupt rows.
|
|
282
298
|
*/
|
|
283
|
-
const decodeStartupPayloads = Effect.fn("
|
|
299
|
+
const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads")(function* (
|
|
284
300
|
journal: SqliteJournal,
|
|
285
301
|
crypto: Crypto.Crypto,
|
|
286
302
|
) {
|
|
287
303
|
const stored = yield* journal.scanStoredPayloads();
|
|
304
|
+
|
|
288
305
|
const batches = yield* Effect.forEach(stored.batches, (batch) =>
|
|
289
306
|
Schema.decodeEffect(Schema.fromJsonString(CanonicalBatch))(batch.batch_json).pipe(
|
|
290
307
|
Effect.map((decoded) => ({ decoded, row: batch })),
|
|
291
308
|
Effect.mapError((error) =>
|
|
292
309
|
SqliteStorageCorruptionError.make({
|
|
293
310
|
table: "effect_agent_canonical_batches",
|
|
294
|
-
rowKey: `${batch.
|
|
311
|
+
rowKey: `${batch.thread_id}/${batch.batch_id}`,
|
|
295
312
|
message: error.message,
|
|
296
313
|
}),
|
|
297
314
|
),
|
|
298
315
|
),
|
|
299
316
|
);
|
|
317
|
+
|
|
300
318
|
const records = yield* Effect.forEach(stored.records, (record) =>
|
|
301
319
|
Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(record.record_json).pipe(
|
|
302
320
|
Effect.map((decoded) => ({ decoded, row: record })),
|
|
303
321
|
Effect.mapError((error) =>
|
|
304
322
|
SqliteStorageCorruptionError.make({
|
|
305
323
|
table: "effect_agent_canonical_records",
|
|
306
|
-
rowKey: `${record.
|
|
324
|
+
rowKey: `${record.thread_id}/${record.sequence}`,
|
|
307
325
|
message: error.message,
|
|
308
326
|
}),
|
|
309
327
|
),
|
|
310
328
|
),
|
|
311
329
|
);
|
|
330
|
+
|
|
312
331
|
const checkpoints = yield* Effect.forEach(stored.checkpoints, (checkpoint) =>
|
|
313
|
-
Schema.decodeEffect(Schema.fromJsonString(
|
|
314
|
-
checkpoint.checkpoint_json,
|
|
315
|
-
).pipe(
|
|
332
|
+
Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint.checkpoint_json).pipe(
|
|
316
333
|
Effect.map((decoded) => ({ decoded, row: checkpoint })),
|
|
317
334
|
Effect.mapError((error) =>
|
|
318
335
|
SqliteStorageCorruptionError.make({
|
|
319
336
|
table: "effect_agent_checkpoints",
|
|
320
|
-
rowKey: `${checkpoint.
|
|
337
|
+
rowKey: `${checkpoint.thread_id}/${checkpoint.through_sequence}`,
|
|
321
338
|
message: error.message,
|
|
322
339
|
}),
|
|
323
340
|
),
|
|
324
341
|
),
|
|
325
342
|
);
|
|
326
343
|
|
|
327
|
-
const
|
|
328
|
-
const
|
|
329
|
-
const
|
|
330
|
-
const materializedIds = new Set(
|
|
331
|
-
stored.conversations.map((conversation) => conversation.conversation_id),
|
|
332
|
-
);
|
|
344
|
+
const batchesByThread = groupByKey(batches, ({ row }) => row.thread_id);
|
|
345
|
+
const recordsByThread = groupByKey(records, ({ row }) => row.thread_id);
|
|
346
|
+
const checkpointsByThread = groupByKey(checkpoints, ({ row }) => row.thread_id);
|
|
347
|
+
const materializedIds = new Set(stored.threads.map((thread) => thread.thread_id));
|
|
333
348
|
|
|
334
|
-
for (const
|
|
335
|
-
const
|
|
336
|
-
const
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
const recordsByBatch = groupByKey(conversationRecords, ({ row }) => row.batch_id);
|
|
349
|
+
for (const thread of stored.threads) {
|
|
350
|
+
const threadBatches = batchesByThread.get(thread.thread_id) ?? [];
|
|
351
|
+
const threadRecords = recordsByThread.get(thread.thread_id) ?? [];
|
|
352
|
+
const threadCheckpoints = checkpointsByThread.get(thread.thread_id) ?? [];
|
|
353
|
+
const recordsByBatch = groupByKey(threadRecords, ({ row }) => row.batch_id);
|
|
340
354
|
let previousDigest = EMPTY_TAIL_DIGEST;
|
|
341
355
|
let expectedSequence = 1;
|
|
342
356
|
const tailDigests = new Map<number, string>([[0, EMPTY_TAIL_DIGEST]]);
|
|
343
357
|
|
|
344
|
-
for (const { decoded: canonicalBatch, row: batchRow } of
|
|
345
|
-
const key = `${batchRow.
|
|
358
|
+
for (const { decoded: canonicalBatch, row: batchRow } of threadBatches) {
|
|
359
|
+
const key = `${batchRow.thread_id}/${batchRow.batch_id}`;
|
|
360
|
+
|
|
346
361
|
if (
|
|
347
362
|
canonicalBatch.batchId !== batchRow.batch_id ||
|
|
348
363
|
batchRow.first_sequence !== expectedSequence ||
|
|
@@ -365,6 +380,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
365
380
|
}),
|
|
366
381
|
),
|
|
367
382
|
);
|
|
383
|
+
|
|
368
384
|
if (batchRow.batch_digest !== digest || batchRow.tail_digest !== digest) {
|
|
369
385
|
return yield* SqliteStorageCorruptionError.make({
|
|
370
386
|
table: "effect_agent_canonical_batches",
|
|
@@ -374,6 +390,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
374
390
|
}
|
|
375
391
|
|
|
376
392
|
const batchRecords = recordsByBatch.get(batchRow.batch_id) ?? [];
|
|
393
|
+
|
|
377
394
|
if (batchRecords.length !== canonicalBatch.records.length) {
|
|
378
395
|
return yield* SqliteStorageCorruptionError.make({
|
|
379
396
|
table: "effect_agent_canonical_records",
|
|
@@ -384,6 +401,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
384
401
|
for (let index = 0; index < canonicalBatch.records.length; index++) {
|
|
385
402
|
const expectedRecord = canonicalBatch.records[index];
|
|
386
403
|
const storedRecord = batchRecords[index];
|
|
404
|
+
|
|
387
405
|
const expectedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(
|
|
388
406
|
expectedRecord,
|
|
389
407
|
).pipe(
|
|
@@ -395,6 +413,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
395
413
|
}),
|
|
396
414
|
),
|
|
397
415
|
);
|
|
416
|
+
|
|
398
417
|
const storedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(
|
|
399
418
|
storedRecord.decoded,
|
|
400
419
|
).pipe(
|
|
@@ -406,6 +425,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
406
425
|
}),
|
|
407
426
|
),
|
|
408
427
|
);
|
|
428
|
+
|
|
409
429
|
if (
|
|
410
430
|
storedRecord.row.sequence !== batchRow.first_sequence + index ||
|
|
411
431
|
storedRecord.row.record_id !== expectedRecord.recordId ||
|
|
@@ -425,27 +445,27 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
425
445
|
}
|
|
426
446
|
|
|
427
447
|
if (
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
448
|
+
threadRecords.length !== thread.tail_sequence ||
|
|
449
|
+
thread.tail_sequence !== expectedSequence - 1 ||
|
|
450
|
+
thread.tail_digest !== previousDigest
|
|
431
451
|
) {
|
|
432
452
|
return yield* SqliteStorageCorruptionError.make({
|
|
433
|
-
table: "
|
|
434
|
-
rowKey:
|
|
435
|
-
message: "
|
|
453
|
+
table: "effect_agent_threads",
|
|
454
|
+
rowKey: thread.thread_id,
|
|
455
|
+
message: "Thread tail does not match its canonical batch chain.",
|
|
436
456
|
});
|
|
437
457
|
}
|
|
438
458
|
|
|
439
|
-
for (const checkpoint of
|
|
459
|
+
for (const checkpoint of threadCheckpoints) {
|
|
440
460
|
if (
|
|
441
|
-
checkpoint.decoded.
|
|
461
|
+
checkpoint.decoded.threadId !== thread.thread_id ||
|
|
442
462
|
checkpoint.decoded.throughSequence !== checkpoint.row.through_sequence ||
|
|
443
463
|
checkpoint.decoded.tailDigest !== checkpoint.row.tail_digest ||
|
|
444
464
|
tailDigests.get(checkpoint.row.through_sequence) !== checkpoint.row.tail_digest
|
|
445
465
|
) {
|
|
446
466
|
return yield* SqliteStorageCorruptionError.make({
|
|
447
467
|
table: "effect_agent_checkpoints",
|
|
448
|
-
rowKey: `${
|
|
468
|
+
rowKey: `${thread.thread_id}/${checkpoint.row.through_sequence}`,
|
|
449
469
|
message: "Checkpoint identity or digest is not bound to a canonical batch tail.",
|
|
450
470
|
});
|
|
451
471
|
}
|
|
@@ -453,48 +473,51 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
453
473
|
}
|
|
454
474
|
|
|
455
475
|
if (
|
|
456
|
-
batches.some(({ row }) => !materializedIds.has(row.
|
|
457
|
-
records.some(({ row }) => !materializedIds.has(row.
|
|
458
|
-
checkpoints.some(({ row }) => !materializedIds.has(row.
|
|
476
|
+
batches.some(({ row }) => !materializedIds.has(row.thread_id)) ||
|
|
477
|
+
records.some(({ row }) => !materializedIds.has(row.thread_id)) ||
|
|
478
|
+
checkpoints.some(({ row }) => !materializedIds.has(row.thread_id))
|
|
459
479
|
) {
|
|
460
480
|
return yield* SqliteStorageCorruptionError.make({
|
|
461
|
-
table: "
|
|
481
|
+
table: "effect_agent_threads",
|
|
462
482
|
rowKey: "startup_scan",
|
|
463
|
-
message: "Canonical rows exist without a materialized
|
|
483
|
+
message: "Canonical rows exist without a materialized Thread.",
|
|
464
484
|
});
|
|
465
485
|
}
|
|
466
486
|
});
|
|
467
487
|
|
|
468
|
-
const makeServices = Effect.fn("
|
|
488
|
+
const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
|
|
469
489
|
const config = yield* SqliteStorageConfig;
|
|
470
490
|
const failpoint = yield* SqliteStorageFailpoint;
|
|
471
|
-
const sql = yield* SqlClientService.SqlClient;
|
|
472
491
|
const crypto = yield* Crypto.Crypto;
|
|
473
|
-
const journal = yield* initializeSqliteJournal(
|
|
492
|
+
const journal = yield* initializeSqliteJournal();
|
|
493
|
+
|
|
474
494
|
if (config.verifyOnOpen) {
|
|
475
495
|
yield* decodeStartupPayloads(journal, crypto);
|
|
476
496
|
}
|
|
477
497
|
|
|
478
498
|
const provideCrypto = <A, E>(effect: Effect.Effect<A, E, Crypto.Crypto>) =>
|
|
479
499
|
Effect.provideService(effect, Crypto.Crypto, crypto);
|
|
480
|
-
|
|
481
|
-
|
|
500
|
+
|
|
501
|
+
const hitFailpoint = Effect.fn("SqliteThreadStore.hitFailpoint")(
|
|
502
|
+
(location: SqliteStorageFailpointLocation): Effect.Effect<void, ThreadStoreError> =>
|
|
482
503
|
failpoint
|
|
483
504
|
.hit(location)
|
|
484
505
|
.pipe(Effect.mapError((error) => storeError(`storage failpoint ${location}`, error))),
|
|
485
506
|
);
|
|
486
507
|
|
|
487
|
-
const materialize:
|
|
488
|
-
"
|
|
489
|
-
)(function* (request:
|
|
490
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
508
|
+
const materialize: ThreadStore["Service"]["materialize"] = Effect.fn(
|
|
509
|
+
"SqliteThreadStore.materialize",
|
|
510
|
+
)(function* (request: ThreadMaterialization) {
|
|
511
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadMaterialization))(
|
|
491
512
|
request,
|
|
492
513
|
).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
|
|
514
|
+
|
|
493
515
|
const now = yield* Clock.currentTimeMillis;
|
|
516
|
+
|
|
494
517
|
yield* hitFailpoint("materialize:before");
|
|
495
518
|
yield* journal
|
|
496
519
|
.materialize(
|
|
497
|
-
validated.
|
|
520
|
+
validated.threadId,
|
|
498
521
|
new Date(now).toISOString(),
|
|
499
522
|
EMPTY_TAIL_DIGEST,
|
|
500
523
|
validated.producerEpoch,
|
|
@@ -502,24 +525,28 @@ const makeServices = Effect.fn("SqliteConversationStore.makeServices")(function*
|
|
|
502
525
|
.pipe(
|
|
503
526
|
Effect.mapError((error) =>
|
|
504
527
|
error._tag === "SqliteFenceRejected"
|
|
505
|
-
? mapFence(validated.
|
|
506
|
-
: storeError("materialize
|
|
528
|
+
? mapFence(validated.threadId, error)
|
|
529
|
+
: storeError("materialize thread", error),
|
|
507
530
|
),
|
|
508
531
|
);
|
|
509
532
|
yield* hitFailpoint("materialize:after");
|
|
510
533
|
});
|
|
511
534
|
|
|
512
|
-
const append:
|
|
513
|
-
|
|
514
|
-
)
|
|
535
|
+
const append: ThreadStore["Service"]["append"] = Effect.fn("SqliteThreadStore.append")(function* (
|
|
536
|
+
request: FencedAppendRequest,
|
|
537
|
+
) {
|
|
515
538
|
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(FencedAppendRequest))(
|
|
516
539
|
request,
|
|
517
540
|
).pipe(Effect.mapError((error) => schemaStoreError("validate canonical append", error)));
|
|
518
|
-
|
|
541
|
+
|
|
542
|
+
yield* requireThread(journal, validated.threadId);
|
|
543
|
+
|
|
519
544
|
const tailDigest = yield* provideCrypto(
|
|
520
545
|
digestCanonicalBatch(validated.expectedTailDigest, validated.batch),
|
|
521
546
|
).pipe(Effect.mapError((error) => storeError("digest canonical append", error)));
|
|
547
|
+
|
|
522
548
|
const batchJson = yield* encodeCanonicalBatch(validated.batch);
|
|
549
|
+
|
|
523
550
|
const rawRecords = yield* Effect.forEach(validated.batch.records, (record) =>
|
|
524
551
|
encodeCanonicalRecord(record).pipe(
|
|
525
552
|
Effect.map((recordJson) => ({
|
|
@@ -528,8 +555,9 @@ const makeServices = Effect.fn("SqliteConversationStore.makeServices")(function*
|
|
|
528
555
|
})),
|
|
529
556
|
),
|
|
530
557
|
);
|
|
558
|
+
|
|
531
559
|
const rawRequest = yield* Schema.decodeUnknownEffect(RawAppendRequest)({
|
|
532
|
-
|
|
560
|
+
threadId: validated.threadId,
|
|
533
561
|
batchId: validated.batch.batchId,
|
|
534
562
|
batchDigest: tailDigest,
|
|
535
563
|
batchJson,
|
|
@@ -539,27 +567,30 @@ const makeServices = Effect.fn("SqliteConversationStore.makeServices")(function*
|
|
|
539
567
|
records: rawRecords,
|
|
540
568
|
tailDigest,
|
|
541
569
|
}).pipe(Effect.mapError((error) => schemaStoreError("encode canonical append", error)));
|
|
570
|
+
|
|
542
571
|
yield* hitFailpoint("append:before");
|
|
572
|
+
|
|
543
573
|
const result = yield* journal.append(rawRequest).pipe(
|
|
544
574
|
Effect.mapError((error) => {
|
|
545
|
-
if (error
|
|
546
|
-
return mapFence(validated.
|
|
575
|
+
if (isSqliteFenceRejected(error)) {
|
|
576
|
+
return mapFence(validated.threadId, error);
|
|
547
577
|
}
|
|
548
|
-
if (error
|
|
578
|
+
if (isSqliteAppendConflict(error)) {
|
|
549
579
|
return error.actualTailSequence !== undefined && isDigest(error.actualTailDigest)
|
|
550
580
|
? AppendConflict.make({
|
|
551
|
-
|
|
581
|
+
threadId: validated.threadId,
|
|
552
582
|
batchId: validated.batch.batchId,
|
|
553
583
|
reason: error.reason,
|
|
554
584
|
actualTailSequence: error.actualTailSequence,
|
|
555
585
|
actualTailDigest: error.actualTailDigest,
|
|
556
586
|
})
|
|
557
587
|
: AppendConflict.make({
|
|
558
|
-
|
|
588
|
+
threadId: validated.threadId,
|
|
559
589
|
batchId: validated.batch.batchId,
|
|
560
590
|
reason: error.reason,
|
|
561
591
|
});
|
|
562
592
|
}
|
|
593
|
+
|
|
563
594
|
return storeError("append canonical batch", error);
|
|
564
595
|
}),
|
|
565
596
|
Effect.flatMap((result) =>
|
|
@@ -568,224 +599,264 @@ const makeServices = Effect.fn("SqliteConversationStore.makeServices")(function*
|
|
|
568
599
|
),
|
|
569
600
|
),
|
|
570
601
|
);
|
|
602
|
+
|
|
571
603
|
yield* hitFailpoint("append:after");
|
|
604
|
+
|
|
572
605
|
return result;
|
|
573
606
|
});
|
|
574
607
|
|
|
575
|
-
const loadRecords = Effect.fn("
|
|
608
|
+
const loadRecords = Effect.fn("SqliteThreadStore.loadRecords")(function* (
|
|
576
609
|
request: RawReadRequest,
|
|
577
610
|
) {
|
|
578
611
|
const rows = yield* journal
|
|
579
612
|
.read(request)
|
|
580
613
|
.pipe(Effect.mapError((error) => storeError("read canonical records", error)));
|
|
614
|
+
|
|
581
615
|
return yield* Effect.forEach(rows, decodeEnvelope);
|
|
582
616
|
});
|
|
583
617
|
|
|
584
|
-
const readEffect = Effect.fn("
|
|
585
|
-
request
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
618
|
+
const readEffect = Effect.fn("SqliteThreadStore.read")(function* (request: ThreadRead) {
|
|
619
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadRead))(request).pipe(
|
|
620
|
+
Effect.mapError((error) => schemaStoreError("validate thread read", error)),
|
|
621
|
+
);
|
|
622
|
+
|
|
623
|
+
yield* requireThread(journal, validated.threadId);
|
|
624
|
+
|
|
591
625
|
const records = yield* loadRecords(
|
|
592
626
|
RawReadRequest.make({
|
|
593
|
-
|
|
627
|
+
threadId: validated.threadId,
|
|
594
628
|
fromSequenceExclusive: validated.afterSequence ?? ZERO_CANONICAL_SEQUENCE,
|
|
595
629
|
limit: validated.limit,
|
|
596
630
|
}),
|
|
597
631
|
);
|
|
632
|
+
|
|
598
633
|
return Stream.fromIterable(records);
|
|
599
634
|
});
|
|
600
|
-
const read: ConversationStore["Service"]["read"] = (request) =>
|
|
601
|
-
Stream.unwrap(readEffect(request));
|
|
602
635
|
|
|
603
|
-
const
|
|
604
|
-
|
|
636
|
+
const read: ThreadStore["Service"]["read"] = (request) => Stream.unwrap(readEffect(request));
|
|
637
|
+
|
|
638
|
+
const observeEffect = Effect.fn("SqliteThreadStore.observe")(function* (
|
|
639
|
+
request: ThreadObservation,
|
|
605
640
|
) {
|
|
606
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
641
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadObservation))(
|
|
607
642
|
request,
|
|
608
|
-
).pipe(
|
|
609
|
-
|
|
610
|
-
);
|
|
611
|
-
yield*
|
|
612
|
-
const initialSequence = yield* parseOffset(validated.conversationId, validated.afterOffset);
|
|
643
|
+
).pipe(Effect.mapError((error) => schemaStoreError("validate thread observation", error)));
|
|
644
|
+
|
|
645
|
+
yield* requireThread(journal, validated.threadId);
|
|
646
|
+
const initialSequence = yield* parseOffset(validated.threadId, validated.afterOffset);
|
|
613
647
|
const cursor = yield* Ref.make(initialSequence);
|
|
614
|
-
|
|
648
|
+
|
|
649
|
+
const poll = Effect.fn("SqliteThreadStore.observePoll")(function* () {
|
|
615
650
|
const fromSequenceExclusive = yield* Ref.get(cursor);
|
|
651
|
+
|
|
616
652
|
const records = yield* loadRecords(
|
|
617
653
|
RawReadRequest.make({
|
|
618
|
-
|
|
654
|
+
threadId: validated.threadId,
|
|
619
655
|
fromSequenceExclusive,
|
|
620
656
|
limit: 1_024,
|
|
621
657
|
}),
|
|
622
658
|
);
|
|
659
|
+
|
|
623
660
|
if (records.length === 0) {
|
|
624
661
|
yield* Effect.sleep(config.observationPollInterval);
|
|
662
|
+
|
|
625
663
|
return [];
|
|
626
664
|
}
|
|
627
665
|
yield* Ref.set(cursor, records[records.length - 1].sequence);
|
|
666
|
+
|
|
628
667
|
return records;
|
|
629
668
|
});
|
|
669
|
+
|
|
630
670
|
return Stream.fromIterableEffectRepeat(poll());
|
|
631
671
|
});
|
|
632
|
-
|
|
672
|
+
|
|
673
|
+
const observe: ThreadStore["Service"]["observe"] = (request) =>
|
|
633
674
|
Stream.unwrap(observeEffect(request));
|
|
634
675
|
|
|
635
|
-
const
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
676
|
+
const exportThread: ThreadStore["Service"]["export"] = Effect.fn("SqliteThreadStore.export")(
|
|
677
|
+
function* (request: ThreadExportRequest) {
|
|
678
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadExportRequest))(
|
|
679
|
+
request,
|
|
680
|
+
).pipe(Effect.mapError((error) => schemaStoreError("validate thread export", error)));
|
|
681
|
+
|
|
682
|
+
yield* requireThread(journal, validated.threadId);
|
|
683
|
+
|
|
684
|
+
const exported = yield* journal
|
|
685
|
+
.exportThread(validated.threadId)
|
|
686
|
+
.pipe(Effect.mapError((error) => storeError("export thread", error)));
|
|
687
|
+
|
|
688
|
+
const records = yield* Effect.forEach(exported.records, decodeEnvelope);
|
|
689
|
+
|
|
690
|
+
if (records.length > 65_536) {
|
|
691
|
+
return yield* ThreadStoreError.make({
|
|
692
|
+
operation: "decode thread export",
|
|
693
|
+
message: "The thread exceeds the current export record limit.",
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(
|
|
698
|
+
exported.thread.tail_digest,
|
|
699
|
+
).pipe(Effect.mapError((error) => schemaStoreError("decode export tail digest", error)));
|
|
700
|
+
|
|
701
|
+
return ThreadExport.make({
|
|
702
|
+
format: "effect-agent/thread@1",
|
|
703
|
+
threadId: validated.threadId,
|
|
704
|
+
tailSequence: exported.thread.tail_sequence,
|
|
705
|
+
tailDigest,
|
|
706
|
+
records,
|
|
650
707
|
});
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
exported.conversation.tail_digest,
|
|
654
|
-
).pipe(Effect.mapError((error) => schemaStoreError("decode export tail digest", error)));
|
|
655
|
-
return ConversationExport.make({
|
|
656
|
-
format: "effect-agent/conversation@1",
|
|
657
|
-
conversationId: validated.conversationId,
|
|
658
|
-
tailSequence: exported.conversation.tail_sequence,
|
|
659
|
-
tailDigest,
|
|
660
|
-
records,
|
|
661
|
-
});
|
|
662
|
-
});
|
|
708
|
+
},
|
|
709
|
+
);
|
|
663
710
|
|
|
664
|
-
const inspectTail:
|
|
665
|
-
"
|
|
666
|
-
)(function* (request:
|
|
667
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
711
|
+
const inspectTail: ThreadStore["Service"]["inspectTail"] = Effect.fn(
|
|
712
|
+
"SqliteThreadStore.inspectTail",
|
|
713
|
+
)(function* (request: ThreadTailRequest) {
|
|
714
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadTailRequest))(
|
|
668
715
|
request,
|
|
669
716
|
).pipe(Effect.mapError((error) => schemaStoreError("validate tail inspection", error)));
|
|
670
|
-
|
|
671
|
-
const
|
|
717
|
+
|
|
718
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
719
|
+
|
|
720
|
+
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(thread.tail_digest).pipe(
|
|
672
721
|
Effect.mapError((error) => schemaStoreError("decode tail digest", error)),
|
|
673
722
|
);
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
723
|
+
|
|
724
|
+
return ThreadTail.make({
|
|
725
|
+
threadId: validated.threadId,
|
|
726
|
+
tailSequence: thread.tail_sequence,
|
|
677
727
|
tailDigest,
|
|
678
|
-
producerEpoch:
|
|
728
|
+
producerEpoch: thread.producer_epoch,
|
|
679
729
|
});
|
|
680
730
|
});
|
|
681
731
|
|
|
682
|
-
const saveCheckpoint:
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
|
|
688
|
-
const conversation = yield* requireConversation(journal, validated.checkpoint.conversationId);
|
|
689
|
-
if (validated.checkpoint.throughSequence > conversation.tail_sequence) {
|
|
690
|
-
return yield* CheckpointRejected.make({
|
|
691
|
-
conversationId: validated.checkpoint.conversationId,
|
|
692
|
-
reason: "ahead-of-tail",
|
|
693
|
-
});
|
|
694
|
-
}
|
|
695
|
-
const canonicalDigest = yield* tailDigestAt(
|
|
696
|
-
journal,
|
|
697
|
-
validated.checkpoint.conversationId,
|
|
698
|
-
validated.checkpoint.throughSequence,
|
|
699
|
-
);
|
|
700
|
-
if (canonicalDigest !== validated.checkpoint.tailDigest) {
|
|
701
|
-
return yield* CheckpointRejected.make({
|
|
702
|
-
conversationId: validated.checkpoint.conversationId,
|
|
703
|
-
reason: "digest-mismatch",
|
|
704
|
-
});
|
|
705
|
-
}
|
|
706
|
-
const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
|
|
707
|
-
const raw = RawCheckpoint.make({
|
|
708
|
-
conversationId: validated.checkpoint.conversationId,
|
|
709
|
-
throughSequence: validated.checkpoint.throughSequence,
|
|
710
|
-
tailDigest: validated.checkpoint.tailDigest,
|
|
711
|
-
checkpointJson,
|
|
712
|
-
});
|
|
713
|
-
yield* hitFailpoint("save-checkpoint:before");
|
|
714
|
-
yield* journal.saveCheckpoint(raw).pipe(
|
|
715
|
-
Effect.mapError((error) =>
|
|
716
|
-
error instanceof SqliteCheckpointConflict
|
|
717
|
-
? CheckpointRejected.make({
|
|
718
|
-
conversationId: validated.checkpoint.conversationId,
|
|
719
|
-
reason: "digest-mismatch",
|
|
720
|
-
})
|
|
721
|
-
: storeError("save checkpoint", error),
|
|
722
|
-
),
|
|
723
|
-
);
|
|
724
|
-
yield* hitFailpoint("save-checkpoint:after");
|
|
725
|
-
});
|
|
732
|
+
const saveCheckpoint: ThreadCheckpoints["save"] = Effect.fn("SqliteThreadStore.saveCheckpoint")(
|
|
733
|
+
function* (request: SaveCheckpointRequest) {
|
|
734
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(SaveCheckpointRequest))(
|
|
735
|
+
request,
|
|
736
|
+
).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
|
|
726
737
|
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
conversationId: checkpoint.conversationId,
|
|
756
|
-
reason: "digest-mismatch",
|
|
738
|
+
const thread = yield* requireThread(journal, validated.checkpoint.threadId);
|
|
739
|
+
|
|
740
|
+
if (validated.checkpoint.throughSequence > thread.tail_sequence) {
|
|
741
|
+
return yield* CheckpointRejected.make({
|
|
742
|
+
threadId: validated.checkpoint.threadId,
|
|
743
|
+
reason: "ahead-of-tail",
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
const canonicalDigest = yield* tailDigestAt(
|
|
748
|
+
journal,
|
|
749
|
+
validated.checkpoint.threadId,
|
|
750
|
+
validated.checkpoint.throughSequence,
|
|
751
|
+
);
|
|
752
|
+
|
|
753
|
+
if (canonicalDigest !== validated.checkpoint.tailDigest) {
|
|
754
|
+
return yield* CheckpointRejected.make({
|
|
755
|
+
threadId: validated.checkpoint.threadId,
|
|
756
|
+
reason: "digest-mismatch",
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
|
|
760
|
+
|
|
761
|
+
const raw = RawCheckpoint.make({
|
|
762
|
+
threadId: validated.checkpoint.threadId,
|
|
763
|
+
throughSequence: validated.checkpoint.throughSequence,
|
|
764
|
+
tailDigest: validated.checkpoint.tailDigest,
|
|
765
|
+
checkpointJson,
|
|
757
766
|
});
|
|
758
|
-
}
|
|
759
|
-
return Option.some(checkpoint);
|
|
760
|
-
});
|
|
761
767
|
|
|
762
|
-
|
|
768
|
+
yield* hitFailpoint("save-checkpoint:before");
|
|
769
|
+
yield* journal.saveCheckpoint(raw).pipe(
|
|
770
|
+
Effect.mapError((error) =>
|
|
771
|
+
isSqliteCheckpointConflict(error)
|
|
772
|
+
? CheckpointRejected.make({
|
|
773
|
+
threadId: validated.checkpoint.threadId,
|
|
774
|
+
reason: "digest-mismatch",
|
|
775
|
+
})
|
|
776
|
+
: storeError("save checkpoint", error),
|
|
777
|
+
),
|
|
778
|
+
);
|
|
779
|
+
yield* hitFailpoint("save-checkpoint:after");
|
|
780
|
+
},
|
|
781
|
+
);
|
|
782
|
+
|
|
783
|
+
const loadCheckpoint: ThreadCheckpoints["load"] = Effect.fn("SqliteThreadStore.loadCheckpoint")(
|
|
784
|
+
function* (request: LoadCheckpointRequest) {
|
|
785
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(LoadCheckpointRequest))(
|
|
786
|
+
request,
|
|
787
|
+
).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint lookup", error)));
|
|
788
|
+
|
|
789
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
790
|
+
|
|
791
|
+
const rows = yield* journal
|
|
792
|
+
.loadCheckpoint(validated.threadId, validated.atOrBeforeSequence ?? thread.tail_sequence)
|
|
793
|
+
.pipe(Effect.mapError((error) => storeError("load checkpoint", error)));
|
|
794
|
+
|
|
795
|
+
if (rows.length === 0) return Option.none();
|
|
796
|
+
if (rows.length !== 1) {
|
|
797
|
+
return yield* ThreadStoreError.make({
|
|
798
|
+
operation: "load checkpoint",
|
|
799
|
+
message: `Expected at most one checkpoint row but found ${rows.length}.`,
|
|
800
|
+
});
|
|
801
|
+
}
|
|
802
|
+
const row = rows[0];
|
|
803
|
+
const checkpoint = yield* decodeCheckpoint(row.checkpoint_json);
|
|
804
|
+
|
|
805
|
+
if (
|
|
806
|
+
row.thread_id !== validated.threadId ||
|
|
807
|
+
checkpoint.threadId !== row.thread_id ||
|
|
808
|
+
checkpoint.throughSequence !== row.through_sequence ||
|
|
809
|
+
checkpoint.tailDigest !== row.tail_digest
|
|
810
|
+
) {
|
|
811
|
+
return yield* ThreadStoreError.make({
|
|
812
|
+
operation: "load checkpoint",
|
|
813
|
+
message: "Stored checkpoint metadata does not match its canonical row.",
|
|
814
|
+
});
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
const canonicalDigest = yield* tailDigestAt(
|
|
818
|
+
journal,
|
|
819
|
+
checkpoint.threadId,
|
|
820
|
+
checkpoint.throughSequence,
|
|
821
|
+
);
|
|
822
|
+
|
|
823
|
+
if (canonicalDigest !== checkpoint.tailDigest) {
|
|
824
|
+
return yield* CheckpointRejected.make({
|
|
825
|
+
threadId: checkpoint.threadId,
|
|
826
|
+
reason: "digest-mismatch",
|
|
827
|
+
});
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
return Option.some(checkpoint);
|
|
831
|
+
},
|
|
832
|
+
);
|
|
833
|
+
|
|
834
|
+
const threadStore = ThreadStore.of({
|
|
763
835
|
append,
|
|
764
|
-
export:
|
|
836
|
+
export: exportThread,
|
|
765
837
|
inspectTail,
|
|
766
|
-
loadCheckpoint,
|
|
767
838
|
materialize,
|
|
768
839
|
observe,
|
|
769
840
|
read,
|
|
770
|
-
saveCheckpoint,
|
|
841
|
+
checkpoints: { save: saveCheckpoint, load: loadCheckpoint },
|
|
771
842
|
});
|
|
772
843
|
|
|
773
|
-
return Context.make(
|
|
844
|
+
return Context.make(ThreadStore, threadStore);
|
|
774
845
|
});
|
|
775
846
|
|
|
776
847
|
/**
|
|
777
|
-
* SQLite
|
|
848
|
+
* SQLite Thread Store implementation with configuration, failpoint, SQL, and Crypto
|
|
778
849
|
* authority kept visible in its input channel.
|
|
779
850
|
*/
|
|
780
|
-
export const
|
|
781
|
-
|
|
851
|
+
export const threadStoreLayer: Layer.Layer<
|
|
852
|
+
ThreadStore,
|
|
782
853
|
SqliteStorageInitializationError,
|
|
783
854
|
SqliteStorageConfig | SqliteStorageFailpoint | SqlClientService.SqlClient | Crypto.Crypto
|
|
784
855
|
> = Layer.effectContext(makeServices());
|
|
785
856
|
|
|
786
857
|
/**
|
|
787
858
|
* Validated SQLite storage configuration Layer with the documented defaults applied. Shared
|
|
788
|
-
* by the
|
|
859
|
+
* by the ThreadStore and SubmissionLedger convenience layers so their defaults cannot
|
|
789
860
|
* drift.
|
|
790
861
|
*/
|
|
791
862
|
export const storageConfigLayer = (
|
|
@@ -818,24 +889,23 @@ export const storageFailpointLayer = (
|
|
|
818
889
|
: Layer.succeed(SqliteStorageFailpoint)({ hit: options.failpoint });
|
|
819
890
|
|
|
820
891
|
/**
|
|
821
|
-
* A composition-root convenience Layer for canonical
|
|
892
|
+
* A composition-root convenience Layer for canonical Threads. Durable accepted work is
|
|
822
893
|
* served by the separate SubmissionLedger port.
|
|
823
894
|
*/
|
|
824
895
|
export const layer = (
|
|
825
896
|
options: SqliteStorageOptions,
|
|
826
|
-
): Layer.Layer<
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
897
|
+
): Layer.Layer<ThreadStore, SqliteStorageInitializationError> =>
|
|
898
|
+
Layer.unwrap(
|
|
899
|
+
Effect.map(SqliteStorageConfig, (config) =>
|
|
900
|
+
threadStoreLayer.pipe(
|
|
901
|
+
Layer.provide(
|
|
902
|
+
Layer.mergeAll(
|
|
903
|
+
Layer.succeed(SqliteStorageConfig)(config),
|
|
904
|
+
storageFailpointLayer(options),
|
|
905
|
+
SqliteClient.layer({ filename: options.filename }),
|
|
906
|
+
NodeCrypto.layer,
|
|
907
|
+
),
|
|
908
|
+
),
|
|
835
909
|
),
|
|
836
910
|
),
|
|
837
|
-
);
|
|
838
|
-
};
|
|
839
|
-
|
|
840
|
-
/** Create an adapter-owned resumable observation offset for a known canonical sequence. */
|
|
841
|
-
export const observationOffsetAt = makeOffset;
|
|
911
|
+
).pipe(Layer.provide(storageConfigLayer(options)));
|