@effect-agent/storage-sqlite 0.1.0-beta.8 → 0.1.0-beta.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/SqliteActivityStore.d.mts +14 -0
- package/dist/SqliteActivityStore.mjs +310 -0
- package/dist/SqliteActivityStore.mjs.map +1 -0
- package/dist/SqliteMessageDeliveryStore.d.mts +14 -0
- package/dist/SqliteMessageDeliveryStore.mjs +24 -0
- package/dist/SqliteMessageDeliveryStore.mjs.map +1 -0
- package/dist/SqliteScheduleStore.d.mts +14 -0
- package/dist/SqliteScheduleStore.mjs +255 -0
- package/dist/SqliteScheduleStore.mjs.map +1 -0
- package/dist/SqliteStorageConfig.d.mts +34 -0
- package/dist/SqliteStorageConfig.mjs +39 -0
- package/dist/SqliteStorageConfig.mjs.map +1 -0
- package/dist/SqliteStorageError-DVWeaWLm.d.mts +86 -0
- package/dist/SqliteStorageError.d.mts +2 -0
- package/dist/SqliteStorageError.mjs +150 -0
- package/dist/SqliteStorageError.mjs.map +1 -0
- package/dist/SqliteStorageFailpoint.d.mts +17 -0
- package/dist/SqliteStorageFailpoint.mjs +14 -0
- package/dist/SqliteStorageFailpoint.mjs.map +1 -0
- package/dist/SqliteStorageFailpointTesting.d.mts +15 -0
- package/dist/SqliteStorageFailpointTesting.mjs +19 -0
- package/dist/SqliteStorageFailpointTesting.mjs.map +1 -0
- package/dist/SqliteStorageVersion-BqmqX0CI.d.mts +11 -0
- package/dist/SqliteStorageVersion.d.mts +2 -0
- package/dist/SqliteStorageVersion.mjs +8 -0
- package/dist/SqliteStorageVersion.mjs.map +1 -0
- package/dist/SqliteSubmissionLedger.d.mts +23 -0
- package/dist/SqliteSubmissionLedger.mjs +1788 -0
- package/dist/SqliteSubmissionLedger.mjs.map +1 -0
- package/dist/SqliteSubscriptionStore.d.mts +13 -0
- package/dist/SqliteSubscriptionStore.mjs +28 -0
- package/dist/SqliteSubscriptionStore.mjs.map +1 -0
- package/dist/SqliteThreadStore.d.mts +49 -0
- package/dist/SqliteThreadStore.mjs +460 -0
- package/dist/SqliteThreadStore.mjs.map +1 -0
- package/dist/index.d.mts +11 -193
- package/dist/index.mjs +11 -3082
- package/dist/migrations-Dy5v0HGe.mjs +346 -0
- package/dist/migrations-Dy5v0HGe.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/dist/sqlite-journal-D9xFxCw-.mjs +998 -0
- package/dist/sqlite-journal-D9xFxCw-.mjs.map +1 -0
- package/package.json +1 -41
- package/src/SqliteActivityStore.ts +556 -0
- package/src/SqliteMessageDeliveryStore.ts +42 -0
- package/src/SqliteScheduleStore.ts +404 -0
- package/src/{sqlite-storage-config.ts → SqliteStorageConfig.ts} +1 -1
- package/src/{errors.ts → SqliteStorageError.ts} +10 -3
- package/src/SqliteStorageFailpoint.ts +23 -0
- package/src/{sqlite-storage-failpoint.ts → SqliteStorageFailpointTesting.ts} +7 -18
- package/src/SqliteStorageVersion.ts +2 -0
- package/src/{sqlite-ledger.ts → SqliteSubmissionLedger.ts} +857 -157
- package/src/SqliteSubscriptionStore.ts +59 -0
- package/src/{sqlite-conversation-store.ts → SqliteThreadStore.ts} +448 -288
- package/src/index.ts +10 -6
- package/src/internal/message-delivery-schema.ts +24 -0
- package/src/{migrations.ts → internal/migrations.ts} +156 -79
- package/src/internal/recovery-checkpoint-schema.ts +17 -0
- package/src/{sqlite-journal.ts → internal/sqlite-journal.ts} +752 -239
- package/dist/index.mjs.map +0 -1
|
@@ -1,32 +1,37 @@
|
|
|
1
|
+
import { digestCanonicalBatch, EMPTY_TAIL_DIGEST } from "@effect-agent/thread/Digest";
|
|
1
2
|
import {
|
|
2
|
-
AppendConflict,
|
|
3
|
-
AppendResult,
|
|
4
3
|
CanonicalBatch,
|
|
5
4
|
CanonicalRecord,
|
|
6
5
|
CanonicalRecordEnvelope,
|
|
7
6
|
CanonicalSequence,
|
|
8
|
-
CheckpointRejected,
|
|
9
|
-
ConversationCheckpoint,
|
|
10
|
-
ConversationExport,
|
|
11
|
-
ConversationExportRequest,
|
|
12
|
-
ConversationMaterialization,
|
|
13
|
-
ConversationNotMaterialized,
|
|
14
|
-
ConversationObservation,
|
|
15
|
-
ConversationRead,
|
|
16
|
-
ConversationStore,
|
|
17
|
-
ConversationStoreError,
|
|
18
|
-
ConversationTail,
|
|
19
|
-
ConversationTailRequest,
|
|
20
|
-
DEFAULT_OWNERSHIP_LEASE_DURATION,
|
|
21
|
-
digestCanonicalBatch,
|
|
22
7
|
Digest,
|
|
23
|
-
|
|
8
|
+
ObservationOffset,
|
|
9
|
+
} from "@effect-agent/thread/Records";
|
|
10
|
+
import { DEFAULT_OWNERSHIP_LEASE_DURATION } from "@effect-agent/thread/SubmissionLedger";
|
|
11
|
+
import {
|
|
12
|
+
AppendConflict,
|
|
13
|
+
AppendResult,
|
|
14
|
+
CheckpointRejected,
|
|
15
|
+
ThreadCheckpoint,
|
|
16
|
+
ThreadExport,
|
|
17
|
+
ThreadExportRequest,
|
|
18
|
+
ThreadMaterialization,
|
|
19
|
+
ThreadNotMaterialized,
|
|
20
|
+
ThreadObservation,
|
|
21
|
+
ThreadRead,
|
|
22
|
+
ThreadStore,
|
|
23
|
+
type ThreadCheckpoints,
|
|
24
|
+
ThreadStoreError,
|
|
25
|
+
ThreadTail,
|
|
26
|
+
ThreadTailRequest,
|
|
24
27
|
FenceRejected,
|
|
25
28
|
FencedAppendRequest,
|
|
26
29
|
LoadCheckpointRequest,
|
|
27
|
-
ObservationOffset,
|
|
28
30
|
SaveCheckpointRequest,
|
|
29
|
-
|
|
31
|
+
SaveRecoveryCheckpointRequest,
|
|
32
|
+
MAX_THREAD_EXPORT_RECORDS,
|
|
33
|
+
type ThreadRecoveryCheckpoints,
|
|
34
|
+
} from "@effect-agent/thread/ThreadStore";
|
|
30
35
|
import { NodeCrypto } from "@effect/platform-node";
|
|
31
36
|
import { SqliteClient } from "@effect/sql-sqlite-node";
|
|
32
37
|
import {
|
|
@@ -41,29 +46,29 @@ import {
|
|
|
41
46
|
Schema,
|
|
42
47
|
Stream,
|
|
43
48
|
} from "effect";
|
|
44
|
-
import * as SqlClientService from "effect/unstable/sql/SqlClient";
|
|
49
|
+
import type * as SqlClientService from "effect/unstable/sql/SqlClient";
|
|
45
50
|
|
|
46
51
|
import {
|
|
52
|
+
initializeSqliteJournal,
|
|
53
|
+
RawAppendRequest,
|
|
54
|
+
RawCheckpoint,
|
|
55
|
+
RawReadRequest,
|
|
56
|
+
type SqliteJournal,
|
|
57
|
+
} from "./internal/sqlite-journal.ts";
|
|
58
|
+
import { SqliteStorageConfig, SqliteStorageConfigValue } from "./SqliteStorageConfig.ts";
|
|
59
|
+
import {
|
|
60
|
+
type SqliteStorageCompatibilityError,
|
|
47
61
|
SqliteAppendConflict,
|
|
48
62
|
SqliteCheckpointConflict,
|
|
49
63
|
SqliteFenceRejected,
|
|
50
64
|
type SqliteStorageFailpointLocation,
|
|
51
|
-
SqliteStorageCompatibilityError,
|
|
52
65
|
SqliteStorageCorruptionError,
|
|
53
66
|
SqliteStorageError,
|
|
54
|
-
} from "./
|
|
55
|
-
import {
|
|
56
|
-
initializeSqliteJournal,
|
|
57
|
-
RawAppendRequest,
|
|
58
|
-
RawCheckpoint,
|
|
59
|
-
RawReadRequest,
|
|
60
|
-
type SqliteJournal,
|
|
61
|
-
} from "./sqlite-journal.ts";
|
|
62
|
-
import { SqliteStorageConfig, SqliteStorageConfigValue } from "./sqlite-storage-config.ts";
|
|
67
|
+
} from "./SqliteStorageError.ts";
|
|
63
68
|
import {
|
|
64
69
|
SqliteStorageFailpoint,
|
|
65
70
|
type SqliteStorageFailpointHandler,
|
|
66
|
-
} from "./
|
|
71
|
+
} from "./SqliteStorageFailpoint.ts";
|
|
67
72
|
|
|
68
73
|
export interface SqliteStorageOptions {
|
|
69
74
|
readonly filename: string;
|
|
@@ -72,7 +77,7 @@ export interface SqliteStorageOptions {
|
|
|
72
77
|
readonly busyTimeout?: number | undefined;
|
|
73
78
|
/**
|
|
74
79
|
* Submission ownership lease duration in milliseconds (D5). Defaults to
|
|
75
|
-
* `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@effect-agent/
|
|
80
|
+
* `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@effect-agent/thread`.
|
|
76
81
|
*/
|
|
77
82
|
readonly ownershipLeaseDuration?: number | undefined;
|
|
78
83
|
/**
|
|
@@ -93,100 +98,104 @@ const OffsetText = Schema.String.check(Schema.isMaxLength(4 * 1024));
|
|
|
93
98
|
const SQLITE_OFFSET_PREFIX = "effect-agent-sqlite@1:";
|
|
94
99
|
const ZERO_CANONICAL_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
95
100
|
const isDigest = Schema.is(Digest);
|
|
101
|
+
const isSqliteFenceRejected = Schema.is(SqliteFenceRejected);
|
|
102
|
+
const isSqliteAppendConflict = Schema.is(SqliteAppendConflict);
|
|
103
|
+
const isSqliteCheckpointConflict = Schema.is(SqliteCheckpointConflict);
|
|
96
104
|
|
|
97
105
|
const storeError = (operation: string, error: { readonly message: string }) =>
|
|
98
|
-
|
|
106
|
+
ThreadStoreError.make({
|
|
99
107
|
cause: error,
|
|
100
108
|
operation,
|
|
101
109
|
message: error.message,
|
|
102
110
|
});
|
|
103
111
|
|
|
104
112
|
const schemaStoreError = (operation: string, error: { readonly message: string }) =>
|
|
105
|
-
|
|
113
|
+
ThreadStoreError.make({
|
|
106
114
|
cause: error,
|
|
107
115
|
operation,
|
|
108
116
|
message: error.message,
|
|
109
117
|
});
|
|
110
118
|
|
|
111
|
-
const makeOffset = Effect.fn("
|
|
112
|
-
|
|
119
|
+
const makeOffset = Effect.fn("SqliteThreadStore.makeOffset")(function* (
|
|
120
|
+
threadId: ThreadMaterialization["threadId"],
|
|
113
121
|
sequence: number,
|
|
114
|
-
): Effect.fn.Return<ObservationOffset,
|
|
122
|
+
): Effect.fn.Return<ObservationOffset, ThreadStoreError> {
|
|
115
123
|
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(sequence).pipe(
|
|
116
124
|
Effect.flatMap((validatedSequence) =>
|
|
117
125
|
Schema.decodeUnknownEffect(ObservationOffset)(
|
|
118
|
-
`${SQLITE_OFFSET_PREFIX}${encodeURIComponent(
|
|
126
|
+
`${SQLITE_OFFSET_PREFIX}${encodeURIComponent(threadId)}:${validatedSequence}`,
|
|
119
127
|
),
|
|
120
128
|
),
|
|
121
129
|
Effect.mapError((error) => schemaStoreError("encode observation offset", error)),
|
|
122
130
|
);
|
|
123
131
|
});
|
|
124
132
|
|
|
125
|
-
const parseOffset = Effect.fn("
|
|
126
|
-
|
|
133
|
+
const parseOffset = Effect.fn("SqliteThreadStore.parseOffset")(function* (
|
|
134
|
+
threadId: ThreadMaterialization["threadId"],
|
|
127
135
|
offset: ObservationOffset | undefined,
|
|
128
|
-
): Effect.fn.Return<CanonicalSequence,
|
|
136
|
+
): Effect.fn.Return<CanonicalSequence, ThreadStoreError> {
|
|
129
137
|
if (offset === undefined) return ZERO_CANONICAL_SEQUENCE;
|
|
138
|
+
|
|
130
139
|
const text = yield* Schema.decodeUnknownEffect(OffsetText)(offset).pipe(
|
|
131
140
|
Effect.mapError((error) => schemaStoreError("decode observation offset", error)),
|
|
132
141
|
);
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
142
|
+
|
|
143
|
+
const threadPrefix = `${SQLITE_OFFSET_PREFIX}${encodeURIComponent(threadId)}:`;
|
|
144
|
+
|
|
145
|
+
if (!text.startsWith(threadPrefix)) {
|
|
146
|
+
return yield* ThreadStoreError.make({
|
|
136
147
|
operation: "decode observation offset",
|
|
137
|
-
message:
|
|
138
|
-
"The observation offset belongs to a different adapter, storage version, or Conversation.",
|
|
148
|
+
message: "The observation offset belongs to a different adapter, storage version, or Thread.",
|
|
139
149
|
});
|
|
140
150
|
}
|
|
141
|
-
const sequenceText = text.slice(
|
|
151
|
+
const sequenceText = text.slice(threadPrefix.length);
|
|
152
|
+
|
|
142
153
|
if (!/^(0|[1-9][0-9]*)$/.test(sequenceText)) {
|
|
143
|
-
return yield*
|
|
154
|
+
return yield* ThreadStoreError.make({
|
|
144
155
|
operation: "decode observation offset",
|
|
145
156
|
message: "The observation offset is malformed.",
|
|
146
157
|
});
|
|
147
158
|
}
|
|
159
|
+
|
|
148
160
|
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(Number(sequenceText)).pipe(
|
|
149
161
|
Effect.mapError((error) => schemaStoreError("decode observation offset", error)),
|
|
150
162
|
);
|
|
151
163
|
});
|
|
152
164
|
|
|
153
|
-
const mapFence = (
|
|
154
|
-
conversationId: ConversationMaterialization["conversationId"],
|
|
155
|
-
error: SqliteFenceRejected,
|
|
156
|
-
) =>
|
|
165
|
+
const mapFence = (threadId: ThreadMaterialization["threadId"], error: SqliteFenceRejected) =>
|
|
157
166
|
FenceRejected.make({
|
|
158
|
-
|
|
167
|
+
threadId,
|
|
159
168
|
actualEpoch: error.actualEpoch,
|
|
160
169
|
attemptedEpoch: error.producerEpoch,
|
|
161
170
|
});
|
|
162
171
|
|
|
163
|
-
const encodeCanonicalRecord = Effect.fn("
|
|
172
|
+
const encodeCanonicalRecord = Effect.fn("SqliteThreadStore.encodeCanonicalRecord")(function* (
|
|
164
173
|
record: CanonicalRecord,
|
|
165
|
-
): Effect.fn.Return<string,
|
|
174
|
+
): Effect.fn.Return<string, ThreadStoreError> {
|
|
166
175
|
return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(record).pipe(
|
|
167
176
|
Effect.mapError((error) => schemaStoreError("encode canonical record", error)),
|
|
168
177
|
);
|
|
169
178
|
});
|
|
170
179
|
|
|
171
|
-
const encodeCanonicalBatch = Effect.fn("
|
|
180
|
+
const encodeCanonicalBatch = Effect.fn("SqliteThreadStore.encodeCanonicalBatch")(function* (
|
|
172
181
|
batch: CanonicalBatch,
|
|
173
|
-
): Effect.fn.Return<string,
|
|
182
|
+
): Effect.fn.Return<string, ThreadStoreError> {
|
|
174
183
|
return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalBatch))(batch).pipe(
|
|
175
184
|
Effect.mapError((error) => schemaStoreError("encode canonical batch", error)),
|
|
176
185
|
);
|
|
177
186
|
});
|
|
178
187
|
|
|
179
|
-
const encodeCheckpoint = Effect.fn("
|
|
180
|
-
checkpoint:
|
|
181
|
-
): Effect.fn.Return<string,
|
|
182
|
-
return yield* Schema.encodeEffect(Schema.fromJsonString(
|
|
188
|
+
const encodeCheckpoint = Effect.fn("SqliteThreadStore.encodeCheckpoint")(function* (
|
|
189
|
+
checkpoint: ThreadCheckpoint,
|
|
190
|
+
): Effect.fn.Return<string, ThreadStoreError> {
|
|
191
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint).pipe(
|
|
183
192
|
Effect.mapError((error) => schemaStoreError("encode checkpoint", error)),
|
|
184
193
|
);
|
|
185
194
|
});
|
|
186
195
|
|
|
187
|
-
const decodeEnvelope = Effect.fn("
|
|
196
|
+
const decodeEnvelope = Effect.fn("SqliteThreadStore.decodeEnvelope")(function* (row: {
|
|
188
197
|
readonly batch_id: string;
|
|
189
|
-
readonly
|
|
198
|
+
readonly thread_id: string;
|
|
190
199
|
readonly record_json: string;
|
|
191
200
|
readonly sequence: CanonicalSequence;
|
|
192
201
|
}) {
|
|
@@ -194,23 +203,25 @@ const decodeEnvelope = Effect.fn("SqliteConversationStore.decodeEnvelope")(funct
|
|
|
194
203
|
row.record_json,
|
|
195
204
|
).pipe(
|
|
196
205
|
Effect.mapError((error) =>
|
|
197
|
-
|
|
206
|
+
ThreadStoreError.make({
|
|
198
207
|
operation: "decode canonical record",
|
|
199
208
|
message: error.message,
|
|
200
209
|
}),
|
|
201
210
|
),
|
|
202
211
|
);
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
const offset = yield* makeOffset(
|
|
212
|
+
|
|
213
|
+
const threadId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.threadId)(
|
|
214
|
+
row.thread_id,
|
|
215
|
+
).pipe(Effect.mapError((error) => schemaStoreError("decode thread identity", error)));
|
|
216
|
+
|
|
217
|
+
const offset = yield* makeOffset(threadId, row.sequence);
|
|
218
|
+
|
|
209
219
|
const batchId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.batchId)(
|
|
210
220
|
row.batch_id,
|
|
211
221
|
).pipe(Effect.mapError((error) => schemaStoreError("decode batch identity", error)));
|
|
222
|
+
|
|
212
223
|
return CanonicalRecordEnvelope.make({
|
|
213
|
-
|
|
224
|
+
threadId,
|
|
214
225
|
batchId,
|
|
215
226
|
sequence: row.sequence,
|
|
216
227
|
offset,
|
|
@@ -218,42 +229,47 @@ const decodeEnvelope = Effect.fn("SqliteConversationStore.decodeEnvelope")(funct
|
|
|
218
229
|
});
|
|
219
230
|
});
|
|
220
231
|
|
|
221
|
-
const decodeCheckpoint = Effect.fn("
|
|
232
|
+
const decodeCheckpoint = Effect.fn("SqliteThreadStore.decodeCheckpoint")(function* (
|
|
222
233
|
checkpointJson: string,
|
|
223
|
-
): Effect.fn.Return<
|
|
224
|
-
return yield* Schema.decodeEffect(Schema.fromJsonString(
|
|
225
|
-
|
|
226
|
-
)
|
|
234
|
+
): Effect.fn.Return<ThreadCheckpoint, ThreadStoreError> {
|
|
235
|
+
return yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpointJson).pipe(
|
|
236
|
+
Effect.mapError((error) => schemaStoreError("decode checkpoint", error)),
|
|
237
|
+
);
|
|
227
238
|
});
|
|
228
239
|
|
|
229
|
-
const
|
|
240
|
+
const requireThread = Effect.fn("SqliteThreadStore.requireThread")(function* (
|
|
230
241
|
journal: SqliteJournal,
|
|
231
|
-
|
|
242
|
+
threadId: ThreadMaterialization["threadId"],
|
|
232
243
|
) {
|
|
233
244
|
const rows = yield* journal
|
|
234
|
-
.
|
|
235
|
-
.pipe(Effect.mapError((error) => storeError("read
|
|
245
|
+
.getThread(threadId)
|
|
246
|
+
.pipe(Effect.mapError((error) => storeError("read thread", error)));
|
|
247
|
+
|
|
236
248
|
if (rows.length === 0) {
|
|
237
|
-
return yield*
|
|
249
|
+
return yield* ThreadNotMaterialized.make({ threadId });
|
|
238
250
|
}
|
|
251
|
+
|
|
239
252
|
return rows[0];
|
|
240
253
|
});
|
|
241
254
|
|
|
242
|
-
const tailDigestAt = Effect.fn("
|
|
255
|
+
const tailDigestAt = Effect.fn("SqliteThreadStore.tailDigestAt")(function* (
|
|
243
256
|
journal: SqliteJournal,
|
|
244
|
-
|
|
257
|
+
threadId: ThreadMaterialization["threadId"],
|
|
245
258
|
sequence: CanonicalSequence,
|
|
246
259
|
) {
|
|
247
260
|
if (sequence === 0) return EMPTY_TAIL_DIGEST;
|
|
261
|
+
|
|
248
262
|
const digests = yield* journal
|
|
249
|
-
.getTailDigestAt(
|
|
263
|
+
.getTailDigestAt(threadId, sequence)
|
|
250
264
|
.pipe(Effect.mapError((error) => storeError("read checkpoint digest", error)));
|
|
265
|
+
|
|
251
266
|
if (digests.length !== 1) {
|
|
252
267
|
return yield* CheckpointRejected.make({
|
|
253
|
-
|
|
268
|
+
threadId,
|
|
254
269
|
reason: "digest-mismatch",
|
|
255
270
|
});
|
|
256
271
|
}
|
|
272
|
+
|
|
257
273
|
return yield* Schema.decodeUnknownEffect(Digest)(digests[0]).pipe(
|
|
258
274
|
Effect.mapError((error) => schemaStoreError("decode checkpoint digest", error)),
|
|
259
275
|
);
|
|
@@ -264,85 +280,87 @@ const groupByKey = <A>(
|
|
|
264
280
|
key: (row: A) => string,
|
|
265
281
|
): ReadonlyMap<string, ReadonlyArray<A>> => {
|
|
266
282
|
const grouped = new Map<string, Array<A>>();
|
|
283
|
+
|
|
267
284
|
for (const row of rows) {
|
|
268
285
|
const existing = grouped.get(key(row));
|
|
286
|
+
|
|
269
287
|
if (existing === undefined) {
|
|
270
288
|
grouped.set(key(row), [row]);
|
|
271
289
|
} else {
|
|
272
290
|
existing.push(row);
|
|
273
291
|
}
|
|
274
292
|
}
|
|
293
|
+
|
|
275
294
|
return grouped;
|
|
276
295
|
};
|
|
277
296
|
|
|
278
297
|
/**
|
|
279
|
-
* Opt-in
|
|
280
|
-
*
|
|
281
|
-
*
|
|
298
|
+
* Opt-in integrity audit (`verifyOnOpen`) of canonical payloads, their digest chains and generic
|
|
299
|
+
* projection checkpoints. Disposable recovery checkpoints are validated when loaded. Routine opens
|
|
300
|
+
* skip this scan: per-operation Schema decoding fails clearly on corrupt canonical rows.
|
|
282
301
|
*/
|
|
283
|
-
const decodeStartupPayloads = Effect.fn("
|
|
302
|
+
const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads")(function* (
|
|
284
303
|
journal: SqliteJournal,
|
|
285
304
|
crypto: Crypto.Crypto,
|
|
286
305
|
) {
|
|
287
306
|
const stored = yield* journal.scanStoredPayloads();
|
|
307
|
+
|
|
288
308
|
const batches = yield* Effect.forEach(stored.batches, (batch) =>
|
|
289
309
|
Schema.decodeEffect(Schema.fromJsonString(CanonicalBatch))(batch.batch_json).pipe(
|
|
290
310
|
Effect.map((decoded) => ({ decoded, row: batch })),
|
|
291
311
|
Effect.mapError((error) =>
|
|
292
312
|
SqliteStorageCorruptionError.make({
|
|
293
313
|
table: "effect_agent_canonical_batches",
|
|
294
|
-
rowKey: `${batch.
|
|
314
|
+
rowKey: `${batch.thread_id}/${batch.batch_id}`,
|
|
295
315
|
message: error.message,
|
|
296
316
|
}),
|
|
297
317
|
),
|
|
298
318
|
),
|
|
299
319
|
);
|
|
320
|
+
|
|
300
321
|
const records = yield* Effect.forEach(stored.records, (record) =>
|
|
301
322
|
Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(record.record_json).pipe(
|
|
302
323
|
Effect.map((decoded) => ({ decoded, row: record })),
|
|
303
324
|
Effect.mapError((error) =>
|
|
304
325
|
SqliteStorageCorruptionError.make({
|
|
305
326
|
table: "effect_agent_canonical_records",
|
|
306
|
-
rowKey: `${record.
|
|
327
|
+
rowKey: `${record.thread_id}/${record.sequence}`,
|
|
307
328
|
message: error.message,
|
|
308
329
|
}),
|
|
309
330
|
),
|
|
310
331
|
),
|
|
311
332
|
);
|
|
333
|
+
|
|
312
334
|
const checkpoints = yield* Effect.forEach(stored.checkpoints, (checkpoint) =>
|
|
313
|
-
Schema.decodeEffect(Schema.fromJsonString(
|
|
314
|
-
checkpoint.checkpoint_json,
|
|
315
|
-
).pipe(
|
|
335
|
+
Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint.checkpoint_json).pipe(
|
|
316
336
|
Effect.map((decoded) => ({ decoded, row: checkpoint })),
|
|
317
337
|
Effect.mapError((error) =>
|
|
318
338
|
SqliteStorageCorruptionError.make({
|
|
319
339
|
table: "effect_agent_checkpoints",
|
|
320
|
-
rowKey: `${checkpoint.
|
|
340
|
+
rowKey: `${checkpoint.thread_id}/${checkpoint.through_sequence}`,
|
|
321
341
|
message: error.message,
|
|
322
342
|
}),
|
|
323
343
|
),
|
|
324
344
|
),
|
|
325
345
|
);
|
|
326
346
|
|
|
327
|
-
const
|
|
328
|
-
const
|
|
329
|
-
const
|
|
330
|
-
const materializedIds = new Set(
|
|
331
|
-
stored.conversations.map((conversation) => conversation.conversation_id),
|
|
332
|
-
);
|
|
347
|
+
const batchesByThread = groupByKey(batches, ({ row }) => row.thread_id);
|
|
348
|
+
const recordsByThread = groupByKey(records, ({ row }) => row.thread_id);
|
|
349
|
+
const checkpointsByThread = groupByKey(checkpoints, ({ row }) => row.thread_id);
|
|
350
|
+
const materializedIds = new Set(stored.threads.map((thread) => thread.thread_id));
|
|
333
351
|
|
|
334
|
-
for (const
|
|
335
|
-
const
|
|
336
|
-
const
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
const recordsByBatch = groupByKey(conversationRecords, ({ row }) => row.batch_id);
|
|
352
|
+
for (const thread of stored.threads) {
|
|
353
|
+
const threadBatches = batchesByThread.get(thread.thread_id) ?? [];
|
|
354
|
+
const threadRecords = recordsByThread.get(thread.thread_id) ?? [];
|
|
355
|
+
const threadCheckpoints = checkpointsByThread.get(thread.thread_id) ?? [];
|
|
356
|
+
const recordsByBatch = groupByKey(threadRecords, ({ row }) => row.batch_id);
|
|
340
357
|
let previousDigest = EMPTY_TAIL_DIGEST;
|
|
341
358
|
let expectedSequence = 1;
|
|
342
359
|
const tailDigests = new Map<number, string>([[0, EMPTY_TAIL_DIGEST]]);
|
|
343
360
|
|
|
344
|
-
for (const { decoded: canonicalBatch, row: batchRow } of
|
|
345
|
-
const key = `${batchRow.
|
|
361
|
+
for (const { decoded: canonicalBatch, row: batchRow } of threadBatches) {
|
|
362
|
+
const key = `${batchRow.thread_id}/${batchRow.batch_id}`;
|
|
363
|
+
|
|
346
364
|
if (
|
|
347
365
|
canonicalBatch.batchId !== batchRow.batch_id ||
|
|
348
366
|
batchRow.first_sequence !== expectedSequence ||
|
|
@@ -365,6 +383,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
365
383
|
}),
|
|
366
384
|
),
|
|
367
385
|
);
|
|
386
|
+
|
|
368
387
|
if (batchRow.batch_digest !== digest || batchRow.tail_digest !== digest) {
|
|
369
388
|
return yield* SqliteStorageCorruptionError.make({
|
|
370
389
|
table: "effect_agent_canonical_batches",
|
|
@@ -374,6 +393,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
374
393
|
}
|
|
375
394
|
|
|
376
395
|
const batchRecords = recordsByBatch.get(batchRow.batch_id) ?? [];
|
|
396
|
+
|
|
377
397
|
if (batchRecords.length !== canonicalBatch.records.length) {
|
|
378
398
|
return yield* SqliteStorageCorruptionError.make({
|
|
379
399
|
table: "effect_agent_canonical_records",
|
|
@@ -384,6 +404,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
384
404
|
for (let index = 0; index < canonicalBatch.records.length; index++) {
|
|
385
405
|
const expectedRecord = canonicalBatch.records[index];
|
|
386
406
|
const storedRecord = batchRecords[index];
|
|
407
|
+
|
|
387
408
|
const expectedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(
|
|
388
409
|
expectedRecord,
|
|
389
410
|
).pipe(
|
|
@@ -395,6 +416,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
395
416
|
}),
|
|
396
417
|
),
|
|
397
418
|
);
|
|
419
|
+
|
|
398
420
|
const storedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(
|
|
399
421
|
storedRecord.decoded,
|
|
400
422
|
).pipe(
|
|
@@ -406,6 +428,7 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
406
428
|
}),
|
|
407
429
|
),
|
|
408
430
|
);
|
|
431
|
+
|
|
409
432
|
if (
|
|
410
433
|
storedRecord.row.sequence !== batchRow.first_sequence + index ||
|
|
411
434
|
storedRecord.row.record_id !== expectedRecord.recordId ||
|
|
@@ -425,27 +448,27 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
425
448
|
}
|
|
426
449
|
|
|
427
450
|
if (
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
451
|
+
threadRecords.length !== thread.tail_sequence ||
|
|
452
|
+
thread.tail_sequence !== expectedSequence - 1 ||
|
|
453
|
+
thread.tail_digest !== previousDigest
|
|
431
454
|
) {
|
|
432
455
|
return yield* SqliteStorageCorruptionError.make({
|
|
433
|
-
table: "
|
|
434
|
-
rowKey:
|
|
435
|
-
message: "
|
|
456
|
+
table: "effect_agent_threads",
|
|
457
|
+
rowKey: thread.thread_id,
|
|
458
|
+
message: "Thread tail does not match its canonical batch chain.",
|
|
436
459
|
});
|
|
437
460
|
}
|
|
438
461
|
|
|
439
|
-
for (const checkpoint of
|
|
462
|
+
for (const checkpoint of threadCheckpoints) {
|
|
440
463
|
if (
|
|
441
|
-
checkpoint.decoded.
|
|
464
|
+
checkpoint.decoded.threadId !== thread.thread_id ||
|
|
442
465
|
checkpoint.decoded.throughSequence !== checkpoint.row.through_sequence ||
|
|
443
466
|
checkpoint.decoded.tailDigest !== checkpoint.row.tail_digest ||
|
|
444
467
|
tailDigests.get(checkpoint.row.through_sequence) !== checkpoint.row.tail_digest
|
|
445
468
|
) {
|
|
446
469
|
return yield* SqliteStorageCorruptionError.make({
|
|
447
470
|
table: "effect_agent_checkpoints",
|
|
448
|
-
rowKey: `${
|
|
471
|
+
rowKey: `${thread.thread_id}/${checkpoint.row.through_sequence}`,
|
|
449
472
|
message: "Checkpoint identity or digest is not bound to a canonical batch tail.",
|
|
450
473
|
});
|
|
451
474
|
}
|
|
@@ -453,48 +476,51 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
453
476
|
}
|
|
454
477
|
|
|
455
478
|
if (
|
|
456
|
-
batches.some(({ row }) => !materializedIds.has(row.
|
|
457
|
-
records.some(({ row }) => !materializedIds.has(row.
|
|
458
|
-
checkpoints.some(({ row }) => !materializedIds.has(row.
|
|
479
|
+
batches.some(({ row }) => !materializedIds.has(row.thread_id)) ||
|
|
480
|
+
records.some(({ row }) => !materializedIds.has(row.thread_id)) ||
|
|
481
|
+
checkpoints.some(({ row }) => !materializedIds.has(row.thread_id))
|
|
459
482
|
) {
|
|
460
483
|
return yield* SqliteStorageCorruptionError.make({
|
|
461
|
-
table: "
|
|
484
|
+
table: "effect_agent_threads",
|
|
462
485
|
rowKey: "startup_scan",
|
|
463
|
-
message: "Canonical rows exist without a materialized
|
|
486
|
+
message: "Canonical rows exist without a materialized Thread.",
|
|
464
487
|
});
|
|
465
488
|
}
|
|
466
489
|
});
|
|
467
490
|
|
|
468
|
-
const makeServices = Effect.fn("
|
|
491
|
+
const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
|
|
469
492
|
const config = yield* SqliteStorageConfig;
|
|
470
493
|
const failpoint = yield* SqliteStorageFailpoint;
|
|
471
|
-
const sql = yield* SqlClientService.SqlClient;
|
|
472
494
|
const crypto = yield* Crypto.Crypto;
|
|
473
|
-
const journal = yield* initializeSqliteJournal(
|
|
495
|
+
const journal = yield* initializeSqliteJournal();
|
|
496
|
+
|
|
474
497
|
if (config.verifyOnOpen) {
|
|
475
498
|
yield* decodeStartupPayloads(journal, crypto);
|
|
476
499
|
}
|
|
477
500
|
|
|
478
501
|
const provideCrypto = <A, E>(effect: Effect.Effect<A, E, Crypto.Crypto>) =>
|
|
479
502
|
Effect.provideService(effect, Crypto.Crypto, crypto);
|
|
480
|
-
|
|
481
|
-
|
|
503
|
+
|
|
504
|
+
const hitFailpoint = Effect.fn("SqliteThreadStore.hitFailpoint")(
|
|
505
|
+
(location: SqliteStorageFailpointLocation): Effect.Effect<void, ThreadStoreError> =>
|
|
482
506
|
failpoint
|
|
483
507
|
.hit(location)
|
|
484
508
|
.pipe(Effect.mapError((error) => storeError(`storage failpoint ${location}`, error))),
|
|
485
509
|
);
|
|
486
510
|
|
|
487
|
-
const materialize:
|
|
488
|
-
"
|
|
489
|
-
)(function* (request:
|
|
490
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
511
|
+
const materialize: ThreadStore["Service"]["materialize"] = Effect.fn(
|
|
512
|
+
"SqliteThreadStore.materialize",
|
|
513
|
+
)(function* (request: ThreadMaterialization) {
|
|
514
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadMaterialization))(
|
|
491
515
|
request,
|
|
492
516
|
).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
|
|
517
|
+
|
|
493
518
|
const now = yield* Clock.currentTimeMillis;
|
|
519
|
+
|
|
494
520
|
yield* hitFailpoint("materialize:before");
|
|
495
521
|
yield* journal
|
|
496
522
|
.materialize(
|
|
497
|
-
validated.
|
|
523
|
+
validated.threadId,
|
|
498
524
|
new Date(now).toISOString(),
|
|
499
525
|
EMPTY_TAIL_DIGEST,
|
|
500
526
|
validated.producerEpoch,
|
|
@@ -502,24 +528,28 @@ const makeServices = Effect.fn("SqliteConversationStore.makeServices")(function*
|
|
|
502
528
|
.pipe(
|
|
503
529
|
Effect.mapError((error) =>
|
|
504
530
|
error._tag === "SqliteFenceRejected"
|
|
505
|
-
? mapFence(validated.
|
|
506
|
-
: storeError("materialize
|
|
531
|
+
? mapFence(validated.threadId, error)
|
|
532
|
+
: storeError("materialize thread", error),
|
|
507
533
|
),
|
|
508
534
|
);
|
|
509
535
|
yield* hitFailpoint("materialize:after");
|
|
510
536
|
});
|
|
511
537
|
|
|
512
|
-
const append:
|
|
513
|
-
|
|
514
|
-
)
|
|
538
|
+
const append: ThreadStore["Service"]["append"] = Effect.fn("SqliteThreadStore.append")(function* (
|
|
539
|
+
request: FencedAppendRequest,
|
|
540
|
+
) {
|
|
515
541
|
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(FencedAppendRequest))(
|
|
516
542
|
request,
|
|
517
543
|
).pipe(Effect.mapError((error) => schemaStoreError("validate canonical append", error)));
|
|
518
|
-
|
|
544
|
+
|
|
545
|
+
yield* requireThread(journal, validated.threadId);
|
|
546
|
+
|
|
519
547
|
const tailDigest = yield* provideCrypto(
|
|
520
548
|
digestCanonicalBatch(validated.expectedTailDigest, validated.batch),
|
|
521
549
|
).pipe(Effect.mapError((error) => storeError("digest canonical append", error)));
|
|
550
|
+
|
|
522
551
|
const batchJson = yield* encodeCanonicalBatch(validated.batch);
|
|
552
|
+
|
|
523
553
|
const rawRecords = yield* Effect.forEach(validated.batch.records, (record) =>
|
|
524
554
|
encodeCanonicalRecord(record).pipe(
|
|
525
555
|
Effect.map((recordJson) => ({
|
|
@@ -528,8 +558,9 @@ const makeServices = Effect.fn("SqliteConversationStore.makeServices")(function*
|
|
|
528
558
|
})),
|
|
529
559
|
),
|
|
530
560
|
);
|
|
561
|
+
|
|
531
562
|
const rawRequest = yield* Schema.decodeUnknownEffect(RawAppendRequest)({
|
|
532
|
-
|
|
563
|
+
threadId: validated.threadId,
|
|
533
564
|
batchId: validated.batch.batchId,
|
|
534
565
|
batchDigest: tailDigest,
|
|
535
566
|
batchJson,
|
|
@@ -539,27 +570,30 @@ const makeServices = Effect.fn("SqliteConversationStore.makeServices")(function*
|
|
|
539
570
|
records: rawRecords,
|
|
540
571
|
tailDigest,
|
|
541
572
|
}).pipe(Effect.mapError((error) => schemaStoreError("encode canonical append", error)));
|
|
573
|
+
|
|
542
574
|
yield* hitFailpoint("append:before");
|
|
575
|
+
|
|
543
576
|
const result = yield* journal.append(rawRequest).pipe(
|
|
544
577
|
Effect.mapError((error) => {
|
|
545
|
-
if (error
|
|
546
|
-
return mapFence(validated.
|
|
578
|
+
if (isSqliteFenceRejected(error)) {
|
|
579
|
+
return mapFence(validated.threadId, error);
|
|
547
580
|
}
|
|
548
|
-
if (error
|
|
581
|
+
if (isSqliteAppendConflict(error)) {
|
|
549
582
|
return error.actualTailSequence !== undefined && isDigest(error.actualTailDigest)
|
|
550
583
|
? AppendConflict.make({
|
|
551
|
-
|
|
584
|
+
threadId: validated.threadId,
|
|
552
585
|
batchId: validated.batch.batchId,
|
|
553
586
|
reason: error.reason,
|
|
554
587
|
actualTailSequence: error.actualTailSequence,
|
|
555
588
|
actualTailDigest: error.actualTailDigest,
|
|
556
589
|
})
|
|
557
590
|
: AppendConflict.make({
|
|
558
|
-
|
|
591
|
+
threadId: validated.threadId,
|
|
559
592
|
batchId: validated.batch.batchId,
|
|
560
593
|
reason: error.reason,
|
|
561
594
|
});
|
|
562
595
|
}
|
|
596
|
+
|
|
563
597
|
return storeError("append canonical batch", error);
|
|
564
598
|
}),
|
|
565
599
|
Effect.flatMap((result) =>
|
|
@@ -568,224 +602,351 @@ const makeServices = Effect.fn("SqliteConversationStore.makeServices")(function*
|
|
|
568
602
|
),
|
|
569
603
|
),
|
|
570
604
|
);
|
|
605
|
+
|
|
571
606
|
yield* hitFailpoint("append:after");
|
|
607
|
+
|
|
572
608
|
return result;
|
|
573
609
|
});
|
|
574
610
|
|
|
575
|
-
const loadRecords = Effect.fn("
|
|
611
|
+
const loadRecords = Effect.fn("SqliteThreadStore.loadRecords")(function* (
|
|
576
612
|
request: RawReadRequest,
|
|
577
613
|
) {
|
|
578
614
|
const rows = yield* journal
|
|
579
615
|
.read(request)
|
|
580
616
|
.pipe(Effect.mapError((error) => storeError("read canonical records", error)));
|
|
617
|
+
|
|
581
618
|
return yield* Effect.forEach(rows, decodeEnvelope);
|
|
582
619
|
});
|
|
583
620
|
|
|
584
|
-
const readEffect = Effect.fn("
|
|
585
|
-
request
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
621
|
+
const readEffect = Effect.fn("SqliteThreadStore.read")(function* (request: ThreadRead) {
|
|
622
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadRead))(request).pipe(
|
|
623
|
+
Effect.mapError((error) => schemaStoreError("validate thread read", error)),
|
|
624
|
+
);
|
|
625
|
+
|
|
626
|
+
yield* requireThread(journal, validated.threadId);
|
|
627
|
+
|
|
591
628
|
const records = yield* loadRecords(
|
|
592
629
|
RawReadRequest.make({
|
|
593
|
-
|
|
630
|
+
threadId: validated.threadId,
|
|
594
631
|
fromSequenceExclusive: validated.afterSequence ?? ZERO_CANONICAL_SEQUENCE,
|
|
595
632
|
limit: validated.limit,
|
|
596
633
|
}),
|
|
597
634
|
);
|
|
635
|
+
|
|
598
636
|
return Stream.fromIterable(records);
|
|
599
637
|
});
|
|
600
|
-
const read: ConversationStore["Service"]["read"] = (request) =>
|
|
601
|
-
Stream.unwrap(readEffect(request));
|
|
602
638
|
|
|
603
|
-
const
|
|
604
|
-
|
|
639
|
+
const read: ThreadStore["Service"]["read"] = (request) => Stream.unwrap(readEffect(request));
|
|
640
|
+
|
|
641
|
+
const observeEffect = Effect.fn("SqliteThreadStore.observe")(function* (
|
|
642
|
+
request: ThreadObservation,
|
|
605
643
|
) {
|
|
606
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
644
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadObservation))(
|
|
607
645
|
request,
|
|
608
|
-
).pipe(
|
|
609
|
-
|
|
610
|
-
);
|
|
611
|
-
yield*
|
|
612
|
-
const initialSequence = yield* parseOffset(validated.conversationId, validated.afterOffset);
|
|
646
|
+
).pipe(Effect.mapError((error) => schemaStoreError("validate thread observation", error)));
|
|
647
|
+
|
|
648
|
+
yield* requireThread(journal, validated.threadId);
|
|
649
|
+
const initialSequence = yield* parseOffset(validated.threadId, validated.afterOffset);
|
|
613
650
|
const cursor = yield* Ref.make(initialSequence);
|
|
614
|
-
|
|
651
|
+
|
|
652
|
+
const poll = Effect.fn("SqliteThreadStore.observePoll")(function* () {
|
|
615
653
|
const fromSequenceExclusive = yield* Ref.get(cursor);
|
|
654
|
+
|
|
616
655
|
const records = yield* loadRecords(
|
|
617
656
|
RawReadRequest.make({
|
|
618
|
-
|
|
657
|
+
threadId: validated.threadId,
|
|
619
658
|
fromSequenceExclusive,
|
|
620
659
|
limit: 1_024,
|
|
621
660
|
}),
|
|
622
661
|
);
|
|
662
|
+
|
|
623
663
|
if (records.length === 0) {
|
|
624
664
|
yield* Effect.sleep(config.observationPollInterval);
|
|
665
|
+
|
|
625
666
|
return [];
|
|
626
667
|
}
|
|
627
668
|
yield* Ref.set(cursor, records[records.length - 1].sequence);
|
|
669
|
+
|
|
628
670
|
return records;
|
|
629
671
|
});
|
|
672
|
+
|
|
630
673
|
return Stream.fromIterableEffectRepeat(poll());
|
|
631
674
|
});
|
|
632
|
-
|
|
675
|
+
|
|
676
|
+
const observe: ThreadStore["Service"]["observe"] = (request) =>
|
|
633
677
|
Stream.unwrap(observeEffect(request));
|
|
634
678
|
|
|
635
|
-
const
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
679
|
+
const exportThread: ThreadStore["Service"]["export"] = Effect.fn("SqliteThreadStore.export")(
|
|
680
|
+
function* (request: ThreadExportRequest) {
|
|
681
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadExportRequest))(
|
|
682
|
+
request,
|
|
683
|
+
).pipe(Effect.mapError((error) => schemaStoreError("validate thread export", error)));
|
|
684
|
+
|
|
685
|
+
yield* requireThread(journal, validated.threadId);
|
|
686
|
+
|
|
687
|
+
const exported = yield* journal
|
|
688
|
+
.exportThread(validated.threadId)
|
|
689
|
+
.pipe(Effect.mapError((error) => storeError("export thread", error)));
|
|
690
|
+
|
|
691
|
+
const records = yield* Effect.forEach(exported.records, decodeEnvelope);
|
|
692
|
+
|
|
693
|
+
if (records.length > MAX_THREAD_EXPORT_RECORDS) {
|
|
694
|
+
return yield* ThreadStoreError.make({
|
|
695
|
+
operation: "decode thread export",
|
|
696
|
+
message: "The thread exceeds the current export record limit.",
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(
|
|
701
|
+
exported.thread.tail_digest,
|
|
702
|
+
).pipe(Effect.mapError((error) => schemaStoreError("decode export tail digest", error)));
|
|
703
|
+
|
|
704
|
+
return ThreadExport.make({
|
|
705
|
+
format: "effect-agent/thread@1",
|
|
706
|
+
threadId: validated.threadId,
|
|
707
|
+
tailSequence: exported.thread.tail_sequence,
|
|
708
|
+
tailDigest,
|
|
709
|
+
records,
|
|
650
710
|
});
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
exported.conversation.tail_digest,
|
|
654
|
-
).pipe(Effect.mapError((error) => schemaStoreError("decode export tail digest", error)));
|
|
655
|
-
return ConversationExport.make({
|
|
656
|
-
format: "effect-agent/conversation@1",
|
|
657
|
-
conversationId: validated.conversationId,
|
|
658
|
-
tailSequence: exported.conversation.tail_sequence,
|
|
659
|
-
tailDigest,
|
|
660
|
-
records,
|
|
661
|
-
});
|
|
662
|
-
});
|
|
711
|
+
},
|
|
712
|
+
);
|
|
663
713
|
|
|
664
|
-
const inspectTail:
|
|
665
|
-
"
|
|
666
|
-
)(function* (request:
|
|
667
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
714
|
+
const inspectTail: ThreadStore["Service"]["inspectTail"] = Effect.fn(
|
|
715
|
+
"SqliteThreadStore.inspectTail",
|
|
716
|
+
)(function* (request: ThreadTailRequest) {
|
|
717
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadTailRequest))(
|
|
668
718
|
request,
|
|
669
719
|
).pipe(Effect.mapError((error) => schemaStoreError("validate tail inspection", error)));
|
|
670
|
-
|
|
671
|
-
const
|
|
720
|
+
|
|
721
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
722
|
+
|
|
723
|
+
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(thread.tail_digest).pipe(
|
|
672
724
|
Effect.mapError((error) => schemaStoreError("decode tail digest", error)),
|
|
673
725
|
);
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
726
|
+
|
|
727
|
+
return ThreadTail.make({
|
|
728
|
+
threadId: validated.threadId,
|
|
729
|
+
tailSequence: thread.tail_sequence,
|
|
677
730
|
tailDigest,
|
|
678
|
-
producerEpoch:
|
|
731
|
+
producerEpoch: thread.producer_epoch,
|
|
679
732
|
});
|
|
680
733
|
});
|
|
681
734
|
|
|
682
|
-
const saveCheckpoint:
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
735
|
+
const saveCheckpoint: ThreadCheckpoints["save"] = Effect.fn("SqliteThreadStore.saveCheckpoint")(
|
|
736
|
+
function* (request: SaveCheckpointRequest) {
|
|
737
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(SaveCheckpointRequest))(
|
|
738
|
+
request,
|
|
739
|
+
).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
|
|
740
|
+
|
|
741
|
+
const thread = yield* requireThread(journal, validated.checkpoint.threadId);
|
|
742
|
+
|
|
743
|
+
if (validated.checkpoint.throughSequence > thread.tail_sequence) {
|
|
744
|
+
return yield* CheckpointRejected.make({
|
|
745
|
+
threadId: validated.checkpoint.threadId,
|
|
746
|
+
reason: "ahead-of-tail",
|
|
747
|
+
});
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
const canonicalDigest = yield* tailDigestAt(
|
|
751
|
+
journal,
|
|
752
|
+
validated.checkpoint.threadId,
|
|
753
|
+
validated.checkpoint.throughSequence,
|
|
754
|
+
);
|
|
755
|
+
|
|
756
|
+
if (canonicalDigest !== validated.checkpoint.tailDigest) {
|
|
757
|
+
return yield* CheckpointRejected.make({
|
|
758
|
+
threadId: validated.checkpoint.threadId,
|
|
759
|
+
reason: "digest-mismatch",
|
|
760
|
+
});
|
|
761
|
+
}
|
|
762
|
+
const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
|
|
763
|
+
|
|
764
|
+
const raw = RawCheckpoint.make({
|
|
765
|
+
threadId: validated.checkpoint.threadId,
|
|
766
|
+
throughSequence: validated.checkpoint.throughSequence,
|
|
767
|
+
tailDigest: validated.checkpoint.tailDigest,
|
|
768
|
+
checkpointJson,
|
|
693
769
|
});
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
journal
|
|
697
|
-
|
|
698
|
-
|
|
770
|
+
|
|
771
|
+
yield* hitFailpoint("save-checkpoint:before");
|
|
772
|
+
yield* journal.saveCheckpoint(raw).pipe(
|
|
773
|
+
Effect.mapError((error) =>
|
|
774
|
+
isSqliteCheckpointConflict(error)
|
|
775
|
+
? CheckpointRejected.make({
|
|
776
|
+
threadId: validated.checkpoint.threadId,
|
|
777
|
+
reason: "digest-mismatch",
|
|
778
|
+
})
|
|
779
|
+
: storeError("save checkpoint", error),
|
|
780
|
+
),
|
|
781
|
+
);
|
|
782
|
+
yield* hitFailpoint("save-checkpoint:after");
|
|
783
|
+
},
|
|
784
|
+
);
|
|
785
|
+
|
|
786
|
+
const loadCheckpoint: ThreadCheckpoints["load"] = Effect.fn("SqliteThreadStore.loadCheckpoint")(
|
|
787
|
+
function* (request: LoadCheckpointRequest) {
|
|
788
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(LoadCheckpointRequest))(
|
|
789
|
+
request,
|
|
790
|
+
).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint lookup", error)));
|
|
791
|
+
|
|
792
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
793
|
+
|
|
794
|
+
const rows = yield* journal
|
|
795
|
+
.loadCheckpoint(validated.threadId, validated.atOrBeforeSequence ?? thread.tail_sequence)
|
|
796
|
+
.pipe(Effect.mapError((error) => storeError("load checkpoint", error)));
|
|
797
|
+
|
|
798
|
+
if (rows.length === 0) return Option.none();
|
|
799
|
+
if (rows.length !== 1) {
|
|
800
|
+
return yield* ThreadStoreError.make({
|
|
801
|
+
operation: "load checkpoint",
|
|
802
|
+
message: `Expected at most one checkpoint row but found ${rows.length}.`,
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
const row = rows[0];
|
|
806
|
+
const checkpoint = yield* decodeCheckpoint(row.checkpoint_json);
|
|
807
|
+
|
|
808
|
+
if (
|
|
809
|
+
row.thread_id !== validated.threadId ||
|
|
810
|
+
checkpoint.threadId !== row.thread_id ||
|
|
811
|
+
checkpoint.throughSequence !== row.through_sequence ||
|
|
812
|
+
checkpoint.tailDigest !== row.tail_digest
|
|
813
|
+
) {
|
|
814
|
+
return yield* ThreadStoreError.make({
|
|
815
|
+
operation: "load checkpoint",
|
|
816
|
+
message: "Stored checkpoint metadata does not match its canonical row.",
|
|
817
|
+
});
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
const canonicalDigest = yield* tailDigestAt(
|
|
821
|
+
journal,
|
|
822
|
+
checkpoint.threadId,
|
|
823
|
+
checkpoint.throughSequence,
|
|
824
|
+
);
|
|
825
|
+
|
|
826
|
+
if (canonicalDigest !== checkpoint.tailDigest) {
|
|
827
|
+
return yield* CheckpointRejected.make({
|
|
828
|
+
threadId: checkpoint.threadId,
|
|
829
|
+
reason: "digest-mismatch",
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
return Option.some(checkpoint);
|
|
834
|
+
},
|
|
835
|
+
);
|
|
836
|
+
|
|
837
|
+
const saveRecoveryCheckpoint: ThreadRecoveryCheckpoints["save"] = Effect.fn(
|
|
838
|
+
"SqliteThreadStore.saveRecoveryCheckpoint",
|
|
839
|
+
)(function* (request) {
|
|
840
|
+
const validated = yield* Schema.decodeUnknownEffect(
|
|
841
|
+
Schema.toType(SaveRecoveryCheckpointRequest),
|
|
842
|
+
)(request).pipe(
|
|
843
|
+
Effect.mapError((error) => schemaStoreError("validate recovery checkpoint", error)),
|
|
699
844
|
);
|
|
700
|
-
|
|
701
|
-
return yield* CheckpointRejected.make({
|
|
702
|
-
conversationId: validated.checkpoint.conversationId,
|
|
703
|
-
reason: "digest-mismatch",
|
|
704
|
-
});
|
|
705
|
-
}
|
|
845
|
+
|
|
706
846
|
const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
reason: "digest-mismatch",
|
|
720
|
-
})
|
|
721
|
-
: storeError("save checkpoint", error),
|
|
722
|
-
),
|
|
723
|
-
);
|
|
724
|
-
yield* hitFailpoint("save-checkpoint:after");
|
|
847
|
+
|
|
848
|
+
yield* journal
|
|
849
|
+
.saveRecoveryCheckpoint(validated, checkpointJson)
|
|
850
|
+
.pipe(
|
|
851
|
+
Effect.mapError((error) =>
|
|
852
|
+
error._tag === "CheckpointRejected" ||
|
|
853
|
+
error._tag === "FenceRejected" ||
|
|
854
|
+
error._tag === "ThreadNotMaterialized"
|
|
855
|
+
? error
|
|
856
|
+
: storeError("save recovery checkpoint", error),
|
|
857
|
+
),
|
|
858
|
+
);
|
|
725
859
|
});
|
|
726
860
|
|
|
727
|
-
const
|
|
728
|
-
"
|
|
729
|
-
)(function* (request
|
|
861
|
+
const loadRecoveryCheckpoint: ThreadRecoveryCheckpoints["load"] = Effect.fn(
|
|
862
|
+
"SqliteThreadStore.loadRecoveryCheckpoint",
|
|
863
|
+
)(function* (request) {
|
|
730
864
|
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(LoadCheckpointRequest))(
|
|
731
865
|
request,
|
|
732
|
-
).pipe(
|
|
733
|
-
|
|
866
|
+
).pipe(
|
|
867
|
+
Effect.mapError((error) => schemaStoreError("validate recovery checkpoint lookup", error)),
|
|
868
|
+
);
|
|
869
|
+
|
|
870
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
871
|
+
|
|
872
|
+
const corrupt = () =>
|
|
873
|
+
CheckpointRejected.make({ threadId: validated.threadId, reason: "corrupt" });
|
|
874
|
+
|
|
734
875
|
const rows = yield* journal
|
|
735
|
-
.
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
876
|
+
.loadRecoveryCheckpoint(validated.threadId)
|
|
877
|
+
.pipe(
|
|
878
|
+
Effect.mapError((error) =>
|
|
879
|
+
error._tag === "SqliteStorageCorruptionError"
|
|
880
|
+
? corrupt()
|
|
881
|
+
: storeError("load recovery checkpoint", error),
|
|
882
|
+
),
|
|
883
|
+
);
|
|
884
|
+
|
|
740
885
|
if (rows.length === 0) return Option.none();
|
|
741
|
-
if (rows.length !== 1)
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
886
|
+
if (rows.length !== 1) return yield* corrupt();
|
|
887
|
+
const row = rows[0];
|
|
888
|
+
|
|
889
|
+
const checkpoint = yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(
|
|
890
|
+
row.checkpoint_json,
|
|
891
|
+
).pipe(Effect.mapError(corrupt));
|
|
892
|
+
|
|
893
|
+
if (
|
|
894
|
+
row.thread_id !== validated.threadId ||
|
|
895
|
+
checkpoint.threadId !== row.thread_id ||
|
|
896
|
+
checkpoint.throughSequence !== row.through_sequence ||
|
|
897
|
+
checkpoint.tailDigest !== row.tail_digest
|
|
898
|
+
)
|
|
899
|
+
return yield* corrupt();
|
|
900
|
+
if (checkpoint.throughSequence > thread.tail_sequence)
|
|
901
|
+
return yield* CheckpointRejected.make({
|
|
902
|
+
threadId: validated.threadId,
|
|
903
|
+
reason: "ahead-of-tail",
|
|
745
904
|
});
|
|
746
|
-
|
|
747
|
-
|
|
905
|
+
if (checkpoint.throughSequence > (validated.atOrBeforeSequence ?? thread.tail_sequence))
|
|
906
|
+
return Option.none();
|
|
907
|
+
|
|
748
908
|
const canonicalDigest = yield* tailDigestAt(
|
|
749
909
|
journal,
|
|
750
|
-
checkpoint.
|
|
910
|
+
checkpoint.threadId,
|
|
751
911
|
checkpoint.throughSequence,
|
|
752
912
|
);
|
|
753
|
-
|
|
913
|
+
|
|
914
|
+
if (canonicalDigest !== checkpoint.tailDigest)
|
|
754
915
|
return yield* CheckpointRejected.make({
|
|
755
|
-
|
|
916
|
+
threadId: validated.threadId,
|
|
756
917
|
reason: "digest-mismatch",
|
|
757
918
|
});
|
|
758
|
-
|
|
919
|
+
|
|
759
920
|
return Option.some(checkpoint);
|
|
760
921
|
});
|
|
761
922
|
|
|
762
|
-
const
|
|
923
|
+
const threadStore = ThreadStore.of({
|
|
763
924
|
append,
|
|
764
|
-
export:
|
|
925
|
+
export: exportThread,
|
|
765
926
|
inspectTail,
|
|
766
|
-
loadCheckpoint,
|
|
767
927
|
materialize,
|
|
768
928
|
observe,
|
|
769
929
|
read,
|
|
770
|
-
saveCheckpoint,
|
|
930
|
+
checkpoints: { save: saveCheckpoint, load: loadCheckpoint },
|
|
931
|
+
recoveryCheckpoints: { save: saveRecoveryCheckpoint, load: loadRecoveryCheckpoint },
|
|
771
932
|
});
|
|
772
933
|
|
|
773
|
-
return Context.make(
|
|
934
|
+
return Context.make(ThreadStore, threadStore);
|
|
774
935
|
});
|
|
775
936
|
|
|
776
937
|
/**
|
|
777
|
-
* SQLite
|
|
938
|
+
* SQLite Thread Store implementation with configuration, failpoint, SQL, and Crypto
|
|
778
939
|
* authority kept visible in its input channel.
|
|
779
940
|
*/
|
|
780
|
-
export const
|
|
781
|
-
|
|
941
|
+
export const threadStoreLayer: Layer.Layer<
|
|
942
|
+
ThreadStore,
|
|
782
943
|
SqliteStorageInitializationError,
|
|
783
944
|
SqliteStorageConfig | SqliteStorageFailpoint | SqlClientService.SqlClient | Crypto.Crypto
|
|
784
945
|
> = Layer.effectContext(makeServices());
|
|
785
946
|
|
|
786
947
|
/**
|
|
787
948
|
* Validated SQLite storage configuration Layer with the documented defaults applied. Shared
|
|
788
|
-
* by the
|
|
949
|
+
* by the ThreadStore and SubmissionLedger convenience layers so their defaults cannot
|
|
789
950
|
* drift.
|
|
790
951
|
*/
|
|
791
952
|
export const storageConfigLayer = (
|
|
@@ -818,24 +979,23 @@ export const storageFailpointLayer = (
|
|
|
818
979
|
: Layer.succeed(SqliteStorageFailpoint)({ hit: options.failpoint });
|
|
819
980
|
|
|
820
981
|
/**
|
|
821
|
-
* A composition-root convenience Layer for canonical
|
|
982
|
+
* A composition-root convenience Layer for canonical Threads. Durable accepted work is
|
|
822
983
|
* served by the separate SubmissionLedger port.
|
|
823
984
|
*/
|
|
824
985
|
export const layer = (
|
|
825
986
|
options: SqliteStorageOptions,
|
|
826
|
-
): Layer.Layer<
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
987
|
+
): Layer.Layer<ThreadStore, SqliteStorageInitializationError> =>
|
|
988
|
+
Layer.unwrap(
|
|
989
|
+
Effect.map(SqliteStorageConfig, (config) =>
|
|
990
|
+
threadStoreLayer.pipe(
|
|
991
|
+
Layer.provide(
|
|
992
|
+
Layer.mergeAll(
|
|
993
|
+
Layer.succeed(SqliteStorageConfig)(config),
|
|
994
|
+
storageFailpointLayer(options),
|
|
995
|
+
SqliteClient.layer({ filename: options.filename }),
|
|
996
|
+
NodeCrypto.layer,
|
|
997
|
+
),
|
|
998
|
+
),
|
|
835
999
|
),
|
|
836
1000
|
),
|
|
837
|
-
);
|
|
838
|
-
};
|
|
839
|
-
|
|
840
|
-
/** Create an adapter-owned resumable observation offset for a known canonical sequence. */
|
|
841
|
-
export const observationOffsetAt = makeOffset;
|
|
1001
|
+
).pipe(Layer.provide(storageConfigLayer(options)));
|