@effect-agent/storage-sqlite 0.1.0-beta.38 → 0.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +34 -114
- package/dist/index.mjs +1693 -494
- package/dist/index.mjs.map +1 -1
- package/dist/sqlite-storage-failpoint-Cr0SXflP.d.mts +95 -0
- package/dist/sqlite-storage-failpoint-DIwWk5dw.mjs +12 -0
- package/dist/sqlite-storage-failpoint-DIwWk5dw.mjs.map +1 -0
- package/dist/testing.d.mts +15 -0
- package/dist/testing.mjs +19 -0
- package/dist/testing.mjs.map +1 -0
- package/package.json +14 -6
- package/src/errors.ts +3 -3
- package/src/index.ts +5 -2
- package/src/migrations.ts +106 -84
- package/src/sqlite-activity-store.ts +503 -0
- package/src/sqlite-journal.ts +147 -156
- package/src/sqlite-ledger.ts +60 -60
- package/src/sqlite-memory-store.ts +532 -0
- package/src/sqlite-schedule-store.ts +2 -2
- package/src/sqlite-storage-config.ts +1 -1
- package/src/sqlite-storage-failpoint-testing.ts +38 -0
- package/src/sqlite-storage-failpoint.ts +1 -31
- package/src/sqlite-subscription-store.ts +935 -0
- package/src/{sqlite-conversation-store.ts → sqlite-thread-store.ts} +248 -267
- package/src/testing.ts +2 -0
|
@@ -0,0 +1,532 @@
|
|
|
1
|
+
import {
|
|
2
|
+
applyMemoryWrite,
|
|
3
|
+
MemoryDocument,
|
|
4
|
+
MemoryKey,
|
|
5
|
+
MemoryNamespaceAddress,
|
|
6
|
+
MemoryMutationFailpoint,
|
|
7
|
+
type MemoryMutationFailure,
|
|
8
|
+
MemoryOperationConflict,
|
|
9
|
+
MemoryReader,
|
|
10
|
+
MemoryStorageError,
|
|
11
|
+
MemoryWrite,
|
|
12
|
+
MemoryWriter,
|
|
13
|
+
} from "@effect-agent/core";
|
|
14
|
+
import { Clock, Context, Effect, Layer, Schema } from "effect";
|
|
15
|
+
import * as SqlClientService from "effect/unstable/sql/SqlClient";
|
|
16
|
+
import type { SqlError } from "effect/unstable/sql/SqlError";
|
|
17
|
+
|
|
18
|
+
const STORAGE_VERSION = 2 as const;
|
|
19
|
+
const METADATA_COMPONENT = "memory";
|
|
20
|
+
const DOCUMENT_TABLE = "effect_agent_memory_documents_v1";
|
|
21
|
+
const RECEIPT_TABLE = "effect_agent_memory_receipts_v1";
|
|
22
|
+
const MAX_STORED_JSON_CODE_UNITS = 16 * 1024 * 1024;
|
|
23
|
+
const StoredJson = Schema.String.check(Schema.isMaxLength(MAX_STORED_JSON_CODE_UNITS));
|
|
24
|
+
const EncodedMemoryChange = Schema.Struct({
|
|
25
|
+
commandJson: StoredJson,
|
|
26
|
+
documentJson: StoredJson,
|
|
27
|
+
resultJson: StoredJson,
|
|
28
|
+
});
|
|
29
|
+
const equivalentContent = Schema.toEquivalence(MemoryWrite.Wire.members[0].fields.content);
|
|
30
|
+
const equivalentScopes = Schema.toEquivalence(MemoryWrite.Wire.members[0].fields.scopes);
|
|
31
|
+
|
|
32
|
+
class MemoryMetadataRow extends Schema.Class<MemoryMetadataRow>(
|
|
33
|
+
"@effect-agent/storage-sqlite/MemoryMetadataRow",
|
|
34
|
+
)({
|
|
35
|
+
version: Schema.Int,
|
|
36
|
+
}) {}
|
|
37
|
+
|
|
38
|
+
class MemoryTableRow extends Schema.Class<MemoryTableRow>(
|
|
39
|
+
"@effect-agent/storage-sqlite/MemoryTableRow",
|
|
40
|
+
)({
|
|
41
|
+
name: Schema.NonEmptyString,
|
|
42
|
+
}) {}
|
|
43
|
+
|
|
44
|
+
class MemoryDocumentRow extends Schema.Class<MemoryDocumentRow>(
|
|
45
|
+
"@effect-agent/storage-sqlite/MemoryDocumentRow",
|
|
46
|
+
)({
|
|
47
|
+
namespace: MemoryNamespaceAddress,
|
|
48
|
+
source_id: MemoryKey.Wire.fields.id,
|
|
49
|
+
format_version: Schema.Int,
|
|
50
|
+
generation: Schema.Int,
|
|
51
|
+
revision: Schema.NonEmptyString,
|
|
52
|
+
document_json: StoredJson,
|
|
53
|
+
}) {}
|
|
54
|
+
|
|
55
|
+
class MemoryReceiptRow extends Schema.Class<MemoryReceiptRow>(
|
|
56
|
+
"@effect-agent/storage-sqlite/MemoryReceiptRow",
|
|
57
|
+
)({
|
|
58
|
+
namespace: MemoryNamespaceAddress,
|
|
59
|
+
operation_id: MemoryWrite.Wire.members[0].fields.operationId,
|
|
60
|
+
source_id: MemoryKey.Wire.fields.id,
|
|
61
|
+
format_version: Schema.Int,
|
|
62
|
+
command_json: StoredJson,
|
|
63
|
+
result_json: StoredJson,
|
|
64
|
+
}) {}
|
|
65
|
+
|
|
66
|
+
class MemoryChangeCountRow extends Schema.Class<MemoryChangeCountRow>(
|
|
67
|
+
"@effect-agent/storage-sqlite/MemoryChangeCountRow",
|
|
68
|
+
)({
|
|
69
|
+
changed: Schema.Int,
|
|
70
|
+
}) {}
|
|
71
|
+
|
|
72
|
+
class StoredMemoryCommand extends Schema.Class<StoredMemoryCommand>(
|
|
73
|
+
"@effect-agent/storage-sqlite/StoredMemoryCommand",
|
|
74
|
+
)({
|
|
75
|
+
version: Schema.Literal(STORAGE_VERSION),
|
|
76
|
+
value: MemoryWrite.Wire,
|
|
77
|
+
}) {}
|
|
78
|
+
|
|
79
|
+
class StoredMemoryResult extends Schema.Class<StoredMemoryResult>(
|
|
80
|
+
"@effect-agent/storage-sqlite/StoredMemoryResult",
|
|
81
|
+
)({
|
|
82
|
+
version: Schema.Literal(STORAGE_VERSION),
|
|
83
|
+
value: MemoryDocument.Wire,
|
|
84
|
+
}) {}
|
|
85
|
+
|
|
86
|
+
const StoredVersionHeader = Schema.Struct({ version: Schema.Int });
|
|
87
|
+
|
|
88
|
+
export type SqliteMemoryInitializationError = MemoryStorageError | MemoryMutationFailure;
|
|
89
|
+
|
|
90
|
+
const storageError = (
|
|
91
|
+
operation: string,
|
|
92
|
+
reason: MemoryStorageError["reason"] = "unavailable",
|
|
93
|
+
): MemoryStorageError => MemoryStorageError.make({ operation, reason });
|
|
94
|
+
|
|
95
|
+
const query = <A extends object>(
|
|
96
|
+
effect: Effect.Effect<ReadonlyArray<A>, SqlError>,
|
|
97
|
+
operation: string,
|
|
98
|
+
) => effect.pipe(Effect.mapError(() => storageError(operation)));
|
|
99
|
+
|
|
100
|
+
const decodeRows = Effect.fn("SqliteMemoryStore.decodeRows")(function* <A, I>(
|
|
101
|
+
schema: Schema.Codec<A, I, never>,
|
|
102
|
+
rows: ReadonlyArray<unknown>,
|
|
103
|
+
operation: string,
|
|
104
|
+
): Effect.fn.Return<ReadonlyArray<A>, MemoryStorageError> {
|
|
105
|
+
return yield* Schema.decodeUnknownEffect(Schema.Array(schema))(rows).pipe(
|
|
106
|
+
Effect.mapError(() => storageError(operation, "corrupt")),
|
|
107
|
+
);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
const decodeInput = Effect.fn("SqliteMemoryStore.decodeInput")(function* <A, I>(
|
|
111
|
+
schema: Schema.Codec<A, I, never>,
|
|
112
|
+
value: unknown,
|
|
113
|
+
operation: string,
|
|
114
|
+
): Effect.fn.Return<A, MemoryStorageError> {
|
|
115
|
+
return yield* Schema.decodeUnknownEffect(schema)(value).pipe(
|
|
116
|
+
Effect.mapError(() => storageError(operation, "invalid-input")),
|
|
117
|
+
);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
const encodeJson = Effect.fn("SqliteMemoryStore.encodeJson")(function* <A, I>(
|
|
121
|
+
schema: Schema.Codec<A, I, never>,
|
|
122
|
+
value: A,
|
|
123
|
+
operation: string,
|
|
124
|
+
): Effect.fn.Return<string, MemoryStorageError> {
|
|
125
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(schema))(value).pipe(
|
|
126
|
+
Effect.mapError(() => storageError(operation, "corrupt")),
|
|
127
|
+
);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
const validateEncodedChange = Effect.fn("SqliteMemoryStore.validateEncodedChange")(function* (
|
|
131
|
+
encoded: typeof EncodedMemoryChange.Type,
|
|
132
|
+
operation: string,
|
|
133
|
+
): Effect.fn.Return<void, MemoryStorageError> {
|
|
134
|
+
yield* Schema.decodeEffect(EncodedMemoryChange)(encoded).pipe(
|
|
135
|
+
Effect.mapError(() => storageError(operation, "invalid-input")),
|
|
136
|
+
);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
const decodeVersionedJson = Effect.fn("SqliteMemoryStore.decodeVersionedJson")(function* <A, I>(
|
|
140
|
+
schema: Schema.Codec<A, I, never>,
|
|
141
|
+
value: string,
|
|
142
|
+
operation: string,
|
|
143
|
+
): Effect.fn.Return<A, MemoryStorageError> {
|
|
144
|
+
const header = yield* Schema.decodeEffect(Schema.fromJsonString(StoredVersionHeader))(value).pipe(
|
|
145
|
+
Effect.mapError(() => storageError(operation, "corrupt")),
|
|
146
|
+
);
|
|
147
|
+
if (header.version !== STORAGE_VERSION) {
|
|
148
|
+
return yield* storageError(operation, "incompatible");
|
|
149
|
+
}
|
|
150
|
+
const decoded = yield* Schema.decodeEffect(Schema.fromJsonString(schema))(value).pipe(
|
|
151
|
+
Effect.mapError(() => storageError(operation, "corrupt")),
|
|
152
|
+
);
|
|
153
|
+
const canonical = yield* encodeJson(schema, decoded, operation);
|
|
154
|
+
if (canonical !== value) return yield* storageError(operation, "corrupt");
|
|
155
|
+
return decoded;
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
const validateDocument = Effect.fn("SqliteMemoryStore.validateDocument")(function* (
|
|
159
|
+
document: MemoryDocument,
|
|
160
|
+
key: MemoryKey,
|
|
161
|
+
operation: string,
|
|
162
|
+
): Effect.fn.Return<MemoryDocument, MemoryStorageError> {
|
|
163
|
+
if (
|
|
164
|
+
document.key.namespace.address !== key.namespace.address ||
|
|
165
|
+
document.key.id !== key.id ||
|
|
166
|
+
document.source.id !== key.id ||
|
|
167
|
+
document.source.revision !== String(document.generation) ||
|
|
168
|
+
(document.generation === 1) !== (document.predecessor === null) ||
|
|
169
|
+
(document.predecessor !== null &&
|
|
170
|
+
(document.predecessor.id !== key.id ||
|
|
171
|
+
document.predecessor.revision !== String(document.generation - 1)))
|
|
172
|
+
) {
|
|
173
|
+
return yield* storageError(operation, "corrupt");
|
|
174
|
+
}
|
|
175
|
+
return document;
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
const validateReceiptResult = Effect.fn("SqliteMemoryStore.validateReceiptResult")(function* (
|
|
179
|
+
command: MemoryWrite,
|
|
180
|
+
result: MemoryDocument,
|
|
181
|
+
operation: string,
|
|
182
|
+
): Effect.fn.Return<void, MemoryStorageError> {
|
|
183
|
+
if ((result.predecessor?.revision ?? null) !== command.expectedRevision) {
|
|
184
|
+
return yield* storageError(operation, "corrupt");
|
|
185
|
+
}
|
|
186
|
+
if (command._tag === "Put") {
|
|
187
|
+
if (result._tag !== "ActiveMemoryDocument" || result.source.locator !== command.locator) {
|
|
188
|
+
return yield* storageError(operation, "corrupt");
|
|
189
|
+
}
|
|
190
|
+
if (
|
|
191
|
+
!equivalentContent(command.content, result.content) ||
|
|
192
|
+
!equivalentScopes(command.scopes, result.scopes)
|
|
193
|
+
) {
|
|
194
|
+
return yield* storageError(operation, "corrupt");
|
|
195
|
+
}
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (
|
|
199
|
+
result._tag !== "WithdrawnMemoryDocument" ||
|
|
200
|
+
result.reason !== command.reason ||
|
|
201
|
+
result.predecessor === null ||
|
|
202
|
+
result.source.locator !== result.predecessor.locator
|
|
203
|
+
) {
|
|
204
|
+
return yield* storageError(operation, "corrupt");
|
|
205
|
+
}
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
const initializeMemorySchema = Effect.fn("SqliteMemoryStore.initialize")(function* () {
|
|
209
|
+
const sql = yield* SqlClientService.SqlClient;
|
|
210
|
+
const failpoint = yield* MemoryMutationFailpoint;
|
|
211
|
+
|
|
212
|
+
yield* failpoint.hit("memory:initialize:before");
|
|
213
|
+
yield* sql
|
|
214
|
+
.withTransaction(
|
|
215
|
+
Effect.gen(function* () {
|
|
216
|
+
yield* sql`
|
|
217
|
+
CREATE TABLE IF NOT EXISTS effect_agent_memory_metadata (
|
|
218
|
+
component TEXT PRIMARY KEY NOT NULL,
|
|
219
|
+
version INTEGER NOT NULL
|
|
220
|
+
)
|
|
221
|
+
`;
|
|
222
|
+
const metadataRows = yield* sql<Record<string, unknown>>`
|
|
223
|
+
SELECT version
|
|
224
|
+
FROM effect_agent_memory_metadata
|
|
225
|
+
WHERE component = ${METADATA_COMPONENT}
|
|
226
|
+
`;
|
|
227
|
+
const metadata = yield* decodeRows(
|
|
228
|
+
MemoryMetadataRow,
|
|
229
|
+
metadataRows,
|
|
230
|
+
"decode memory schema version",
|
|
231
|
+
);
|
|
232
|
+
if (metadata.length > 1)
|
|
233
|
+
return yield* storageError("decode memory schema version", "corrupt");
|
|
234
|
+
const currentVersion = metadata[0]?.version;
|
|
235
|
+
if (currentVersion !== undefined && currentVersion !== STORAGE_VERSION) {
|
|
236
|
+
return yield* storageError("initialize memory schema", "incompatible");
|
|
237
|
+
}
|
|
238
|
+
if (currentVersion === undefined) {
|
|
239
|
+
const tableRows = yield* sql<Record<string, unknown>>`
|
|
240
|
+
SELECT name
|
|
241
|
+
FROM sqlite_master
|
|
242
|
+
WHERE type = 'table' AND name IN (${DOCUMENT_TABLE}, ${RECEIPT_TABLE})
|
|
243
|
+
`;
|
|
244
|
+
const existingTables = yield* decodeRows(
|
|
245
|
+
MemoryTableRow,
|
|
246
|
+
tableRows,
|
|
247
|
+
"inspect memory schema",
|
|
248
|
+
);
|
|
249
|
+
if (existingTables.length > 0) {
|
|
250
|
+
return yield* storageError("initialize memory schema", "incompatible");
|
|
251
|
+
}
|
|
252
|
+
yield* sql`
|
|
253
|
+
CREATE TABLE effect_agent_memory_documents_v1 (
|
|
254
|
+
namespace TEXT NOT NULL,
|
|
255
|
+
source_id TEXT NOT NULL,
|
|
256
|
+
format_version INTEGER NOT NULL,
|
|
257
|
+
generation INTEGER NOT NULL,
|
|
258
|
+
revision TEXT NOT NULL,
|
|
259
|
+
document_json TEXT NOT NULL,
|
|
260
|
+
PRIMARY KEY (namespace, source_id)
|
|
261
|
+
)
|
|
262
|
+
`;
|
|
263
|
+
yield* sql`
|
|
264
|
+
CREATE TABLE effect_agent_memory_receipts_v1 (
|
|
265
|
+
namespace TEXT NOT NULL,
|
|
266
|
+
operation_id TEXT NOT NULL,
|
|
267
|
+
source_id TEXT NOT NULL,
|
|
268
|
+
format_version INTEGER NOT NULL,
|
|
269
|
+
command_json TEXT NOT NULL,
|
|
270
|
+
result_json TEXT NOT NULL,
|
|
271
|
+
PRIMARY KEY (namespace, operation_id)
|
|
272
|
+
)
|
|
273
|
+
`;
|
|
274
|
+
yield* sql`
|
|
275
|
+
INSERT INTO effect_agent_memory_metadata (component, version)
|
|
276
|
+
VALUES (${METADATA_COMPONENT}, ${STORAGE_VERSION})
|
|
277
|
+
`;
|
|
278
|
+
}
|
|
279
|
+
yield* sql`
|
|
280
|
+
SELECT namespace, source_id, format_version, generation, revision, document_json
|
|
281
|
+
FROM effect_agent_memory_documents_v1
|
|
282
|
+
LIMIT 0
|
|
283
|
+
`;
|
|
284
|
+
yield* sql`
|
|
285
|
+
SELECT namespace, operation_id, source_id, format_version, command_json, result_json
|
|
286
|
+
FROM effect_agent_memory_receipts_v1
|
|
287
|
+
LIMIT 0
|
|
288
|
+
`;
|
|
289
|
+
}),
|
|
290
|
+
)
|
|
291
|
+
.pipe(Effect.catchTag("SqlError", () => Effect.fail(storageError("initialize memory schema"))));
|
|
292
|
+
yield* failpoint.hit("memory:initialize:after");
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
const makeMemoryReader = Effect.fn("SqliteMemoryStore.makeReader")(function* () {
|
|
296
|
+
const sql = yield* SqlClientService.SqlClient;
|
|
297
|
+
|
|
298
|
+
const readDocument = Effect.fn("SqliteMemoryStore.readDocument")(function* (
|
|
299
|
+
key: MemoryKey,
|
|
300
|
+
operation: string,
|
|
301
|
+
): Effect.fn.Return<MemoryDocument | null, MemoryStorageError> {
|
|
302
|
+
const rawRows = yield* query(
|
|
303
|
+
sql<Record<string, unknown>>`
|
|
304
|
+
SELECT namespace, source_id, format_version, generation, revision, document_json
|
|
305
|
+
FROM effect_agent_memory_documents_v1
|
|
306
|
+
WHERE namespace = ${key.namespace.address} AND source_id = ${key.id}
|
|
307
|
+
`,
|
|
308
|
+
operation,
|
|
309
|
+
);
|
|
310
|
+
const rows = yield* decodeRows(MemoryDocumentRow, rawRows, operation);
|
|
311
|
+
if (rows.length === 0) return null;
|
|
312
|
+
if (rows.length !== 1) return yield* storageError(operation, "corrupt");
|
|
313
|
+
const row = rows[0];
|
|
314
|
+
if (row.format_version !== STORAGE_VERSION) {
|
|
315
|
+
return yield* storageError(operation, "incompatible");
|
|
316
|
+
}
|
|
317
|
+
const stored = yield* decodeVersionedJson(StoredMemoryResult, row.document_json, operation);
|
|
318
|
+
const document = stored.value;
|
|
319
|
+
yield* validateDocument(document, key, operation);
|
|
320
|
+
if (
|
|
321
|
+
row.namespace !== key.namespace.address ||
|
|
322
|
+
row.source_id !== key.id ||
|
|
323
|
+
row.generation !== document.generation ||
|
|
324
|
+
row.revision !== document.source.revision
|
|
325
|
+
) {
|
|
326
|
+
return yield* storageError(operation, "corrupt");
|
|
327
|
+
}
|
|
328
|
+
return document;
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
const get = Effect.fn("SqliteMemoryStore.get")(function* (key: MemoryKey) {
|
|
332
|
+
const decodedKey = yield* decodeInput(MemoryKey.Wire, key, "get memory document");
|
|
333
|
+
return yield* readDocument(decodedKey, "get memory document");
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
return { get, readDocument };
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
const makeMemoryServices = Effect.fn("SqliteMemoryStore.make")(function* () {
|
|
340
|
+
const sql = yield* SqlClientService.SqlClient;
|
|
341
|
+
const failpoint = yield* MemoryMutationFailpoint;
|
|
342
|
+
yield* initializeMemorySchema();
|
|
343
|
+
const { get, readDocument } = yield* makeMemoryReader();
|
|
344
|
+
|
|
345
|
+
const readReceipt = Effect.fn("SqliteMemoryStore.readReceipt")(function* (
|
|
346
|
+
write: MemoryWrite,
|
|
347
|
+
operation: string,
|
|
348
|
+
) {
|
|
349
|
+
const rawRows = yield* query(
|
|
350
|
+
sql<Record<string, unknown>>`
|
|
351
|
+
SELECT namespace, operation_id, source_id, format_version, command_json, result_json
|
|
352
|
+
FROM effect_agent_memory_receipts_v1
|
|
353
|
+
WHERE namespace = ${write.key.namespace.address} AND operation_id = ${write.operationId}
|
|
354
|
+
`,
|
|
355
|
+
operation,
|
|
356
|
+
);
|
|
357
|
+
const rows = yield* decodeRows(MemoryReceiptRow, rawRows, operation);
|
|
358
|
+
if (rows.length === 0) return null;
|
|
359
|
+
if (rows.length !== 1) return yield* storageError(operation, "corrupt");
|
|
360
|
+
const row = rows[0];
|
|
361
|
+
if (row.format_version !== STORAGE_VERSION) {
|
|
362
|
+
return yield* storageError(operation, "incompatible");
|
|
363
|
+
}
|
|
364
|
+
const command = yield* decodeVersionedJson(
|
|
365
|
+
StoredMemoryCommand,
|
|
366
|
+
row.command_json,
|
|
367
|
+
`${operation} command`,
|
|
368
|
+
);
|
|
369
|
+
const result = yield* decodeVersionedJson(
|
|
370
|
+
StoredMemoryResult,
|
|
371
|
+
row.result_json,
|
|
372
|
+
`${operation} result`,
|
|
373
|
+
);
|
|
374
|
+
if (
|
|
375
|
+
row.namespace !== command.value.key.namespace.address ||
|
|
376
|
+
row.operation_id !== command.value.operationId ||
|
|
377
|
+
row.source_id !== command.value.key.id ||
|
|
378
|
+
result.value.key.namespace.address !== command.value.key.namespace.address ||
|
|
379
|
+
result.value.key.id !== command.value.key.id
|
|
380
|
+
) {
|
|
381
|
+
return yield* storageError(operation, "corrupt");
|
|
382
|
+
}
|
|
383
|
+
yield* validateDocument(result.value, command.value.key, operation);
|
|
384
|
+
yield* validateReceiptResult(command.value, result.value, operation);
|
|
385
|
+
return { commandJson: row.command_json, result: result.value };
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
const change = Effect.fn("SqliteMemoryStore.change")(function* (write: MemoryWrite) {
|
|
389
|
+
const operation = "change memory document";
|
|
390
|
+
const decodedWrite = yield* decodeInput(MemoryWrite.Wire, write, operation);
|
|
391
|
+
const commandJson = yield* encodeJson(
|
|
392
|
+
StoredMemoryCommand,
|
|
393
|
+
StoredMemoryCommand.make({ version: STORAGE_VERSION, value: decodedWrite }),
|
|
394
|
+
"encode memory command",
|
|
395
|
+
);
|
|
396
|
+
yield* failpoint.hit("memory:change:before");
|
|
397
|
+
// The pinned Node SQLite client starts writable transactions with BEGIN IMMEDIATE.
|
|
398
|
+
// Acquiring the write lock before the receipt and document reads makes this CAS safe
|
|
399
|
+
// across independent SqlClient connections and avoids deferred-transaction upgrades.
|
|
400
|
+
const transactionResult = yield* sql
|
|
401
|
+
.withTransaction(
|
|
402
|
+
Effect.gen(function* () {
|
|
403
|
+
const receipt = yield* readReceipt(decodedWrite, operation);
|
|
404
|
+
if (receipt !== null) {
|
|
405
|
+
if (receipt.commandJson !== commandJson) {
|
|
406
|
+
return yield* MemoryOperationConflict.make({
|
|
407
|
+
key: decodedWrite.key,
|
|
408
|
+
operationId: decodedWrite.operationId,
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
return { document: receipt.result, changed: false } as const;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
const current = yield* readDocument(decodedWrite.key, operation);
|
|
415
|
+
const modifiedAt = yield* Clock.currentTimeMillis;
|
|
416
|
+
const next = yield* applyMemoryWrite(current, decodedWrite, modifiedAt);
|
|
417
|
+
const resultJson = yield* encodeJson(
|
|
418
|
+
StoredMemoryResult,
|
|
419
|
+
StoredMemoryResult.make({ version: STORAGE_VERSION, value: next }),
|
|
420
|
+
"encode memory result",
|
|
421
|
+
);
|
|
422
|
+
const documentJson = resultJson;
|
|
423
|
+
yield* validateEncodedChange({ commandJson, documentJson, resultJson }, operation);
|
|
424
|
+
|
|
425
|
+
if (current === null) {
|
|
426
|
+
yield* sql`
|
|
427
|
+
INSERT INTO effect_agent_memory_documents_v1 (
|
|
428
|
+
namespace, source_id, format_version, generation, revision, document_json
|
|
429
|
+
) VALUES (
|
|
430
|
+
${next.key.namespace.address}, ${next.key.id}, ${STORAGE_VERSION},
|
|
431
|
+
${next.generation}, ${next.source.revision}, ${documentJson}
|
|
432
|
+
)
|
|
433
|
+
`;
|
|
434
|
+
} else {
|
|
435
|
+
yield* sql`
|
|
436
|
+
UPDATE effect_agent_memory_documents_v1
|
|
437
|
+
SET format_version = ${STORAGE_VERSION},
|
|
438
|
+
generation = ${next.generation},
|
|
439
|
+
revision = ${next.source.revision},
|
|
440
|
+
document_json = ${documentJson}
|
|
441
|
+
WHERE namespace = ${next.key.namespace.address}
|
|
442
|
+
AND source_id = ${next.key.id}
|
|
443
|
+
AND generation = ${current.generation}
|
|
444
|
+
AND revision = ${current.source.revision}
|
|
445
|
+
`;
|
|
446
|
+
}
|
|
447
|
+
const changedRows = yield* sql<Record<string, unknown>>`
|
|
448
|
+
SELECT changes() AS changed
|
|
449
|
+
`;
|
|
450
|
+
const changed = yield* decodeRows(MemoryChangeCountRow, changedRows, operation);
|
|
451
|
+
if (changed.length !== 1 || changed[0].changed !== 1) {
|
|
452
|
+
return yield* storageError(operation, "corrupt");
|
|
453
|
+
}
|
|
454
|
+
yield* failpoint.hit("memory:change:after-state");
|
|
455
|
+
yield* sql`
|
|
456
|
+
INSERT INTO effect_agent_memory_receipts_v1 (
|
|
457
|
+
namespace, operation_id, source_id, format_version, command_json, result_json
|
|
458
|
+
) VALUES (
|
|
459
|
+
${decodedWrite.key.namespace.address}, ${decodedWrite.operationId}, ${decodedWrite.key.id},
|
|
460
|
+
${STORAGE_VERSION}, ${commandJson}, ${resultJson}
|
|
461
|
+
)
|
|
462
|
+
`;
|
|
463
|
+
yield* failpoint.hit("memory:change:after-receipt");
|
|
464
|
+
return { document: next, changed: true } as const;
|
|
465
|
+
}),
|
|
466
|
+
)
|
|
467
|
+
.pipe(Effect.catchTag("SqlError", () => Effect.fail(storageError(operation))));
|
|
468
|
+
if (transactionResult.changed) yield* failpoint.hit("memory:change:after");
|
|
469
|
+
return transactionResult.document;
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
return Context.make(MemoryReader, MemoryReader.fromAdapter({ get })).pipe(
|
|
473
|
+
Context.add(MemoryWriter, MemoryWriter.fromAdapter({ change })),
|
|
474
|
+
);
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
/** SQLite memory ports with the mutation failpoint kept injectable for recovery tests. */
|
|
478
|
+
export const memoryStoreLayerWithFailpoints: Layer.Layer<
|
|
479
|
+
MemoryReader | MemoryWriter,
|
|
480
|
+
SqliteMemoryInitializationError,
|
|
481
|
+
SqlClientService.SqlClient | MemoryMutationFailpoint
|
|
482
|
+
> = Layer.effectContext(makeMemoryServices());
|
|
483
|
+
|
|
484
|
+
/** SQLite memory reader and writer with the production no-op mutation failpoint. */
|
|
485
|
+
export const memoryStoreLayer: Layer.Layer<
|
|
486
|
+
MemoryReader | MemoryWriter,
|
|
487
|
+
SqliteMemoryInitializationError,
|
|
488
|
+
SqlClientService.SqlClient
|
|
489
|
+
> = memoryStoreLayerWithFailpoints.pipe(Layer.provide(MemoryMutationFailpoint.layer));
|
|
490
|
+
|
|
491
|
+
/** Reads an existing memory schema without writes, transactions, or mutation failpoints. */
|
|
492
|
+
export const memoryReaderLayer: Layer.Layer<
|
|
493
|
+
MemoryReader,
|
|
494
|
+
MemoryStorageError,
|
|
495
|
+
SqlClientService.SqlClient
|
|
496
|
+
> = Layer.effect(
|
|
497
|
+
MemoryReader,
|
|
498
|
+
Effect.gen(function* () {
|
|
499
|
+
const sql = yield* SqlClientService.SqlClient;
|
|
500
|
+
const operation = "open memory reader";
|
|
501
|
+
const tables = yield* query(
|
|
502
|
+
sql<Record<string, unknown>>`
|
|
503
|
+
SELECT name FROM sqlite_master
|
|
504
|
+
WHERE type = 'table' AND name IN (
|
|
505
|
+
'effect_agent_memory_metadata', ${DOCUMENT_TABLE}, ${RECEIPT_TABLE}
|
|
506
|
+
)
|
|
507
|
+
`,
|
|
508
|
+
operation,
|
|
509
|
+
).pipe(Effect.flatMap((rows) => decodeRows(MemoryTableRow, rows, operation)));
|
|
510
|
+
if (tables.length !== 3) {
|
|
511
|
+
return yield* storageError(operation, tables.length === 0 ? "unavailable" : "incompatible");
|
|
512
|
+
}
|
|
513
|
+
const metadata = yield* query(
|
|
514
|
+
sql<Record<string, unknown>>`
|
|
515
|
+
SELECT version FROM effect_agent_memory_metadata WHERE component = ${METADATA_COMPONENT}
|
|
516
|
+
`,
|
|
517
|
+
operation,
|
|
518
|
+
).pipe(Effect.flatMap((rows) => decodeRows(MemoryMetadataRow, rows, operation)));
|
|
519
|
+
if (metadata.length !== 1 || metadata[0].version !== STORAGE_VERSION) {
|
|
520
|
+
return yield* storageError(operation, "incompatible");
|
|
521
|
+
}
|
|
522
|
+
yield* query(
|
|
523
|
+
sql`
|
|
524
|
+
SELECT namespace, source_id, format_version, generation, revision, document_json
|
|
525
|
+
FROM effect_agent_memory_documents_v1 LIMIT 0
|
|
526
|
+
`,
|
|
527
|
+
operation,
|
|
528
|
+
);
|
|
529
|
+
const { get } = yield* makeMemoryReader();
|
|
530
|
+
return MemoryReader.fromAdapter({ get });
|
|
531
|
+
}),
|
|
532
|
+
);
|
|
@@ -18,14 +18,14 @@ import {
|
|
|
18
18
|
ScheduleStorageError,
|
|
19
19
|
ScheduleStore,
|
|
20
20
|
scheduleDeadline,
|
|
21
|
-
} from "@effect-agent/
|
|
21
|
+
} from "@effect-agent/thread";
|
|
22
22
|
import { Effect, Layer, Result, Schema } from "effect";
|
|
23
23
|
import * as SqlClientService from "effect/unstable/sql/SqlClient";
|
|
24
24
|
|
|
25
|
-
import type { SqliteStorageInitializationError } from "./sqlite-conversation-store.ts";
|
|
26
25
|
import { initializeSqliteJournal } from "./sqlite-journal.ts";
|
|
27
26
|
import type { SqliteStorageConfig } from "./sqlite-storage-config.ts";
|
|
28
27
|
import type { SqliteStorageFailpoint } from "./sqlite-storage-failpoint.ts";
|
|
28
|
+
import type { SqliteStorageInitializationError } from "./sqlite-thread-store.ts";
|
|
29
29
|
|
|
30
30
|
// Configuration and the immutable pending envelope may each carry the canonical input. Leave
|
|
31
31
|
// room for JSON escaping and bounded status while rejecting an unreadable oversized row.
|
|
@@ -19,7 +19,7 @@ export class SqliteStorageConfigValue extends Schema.Class<SqliteStorageConfigVa
|
|
|
19
19
|
* Submission ownership lease duration in milliseconds (D5). The lease is a liveness hint
|
|
20
20
|
* that makes an abandoned claim reclaimable; correctness never depends on it because every
|
|
21
21
|
* canonical append is fenced by producer epoch. Convenience layers default this to
|
|
22
|
-
* `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@effect-agent/
|
|
22
|
+
* `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@effect-agent/thread`.
|
|
23
23
|
*/
|
|
24
24
|
ownershipLeaseDuration: OwnershipLeaseMillis,
|
|
25
25
|
/**
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { Context, Effect, Layer, Ref } from "effect";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
SqliteStorageFailpoint,
|
|
5
|
+
type SqliteStorageFailpointHandler,
|
|
6
|
+
} from "./sqlite-storage-failpoint.ts";
|
|
7
|
+
|
|
8
|
+
const noFailpoint: SqliteStorageFailpointHandler = () => Effect.void;
|
|
9
|
+
|
|
10
|
+
/** Test-only control for replacing the active SQLite failpoint handler. */
|
|
11
|
+
export class SqliteStorageFailpointTestControl extends Context.Service<
|
|
12
|
+
SqliteStorageFailpointTestControl,
|
|
13
|
+
{
|
|
14
|
+
readonly clear: Effect.Effect<void>;
|
|
15
|
+
readonly setHandler: (handler: SqliteStorageFailpointHandler) => Effect.Effect<void>;
|
|
16
|
+
}
|
|
17
|
+
>()("@effect-agent/storage-sqlite/SqliteStorageFailpointTestControl") {
|
|
18
|
+
/** Reusable test Layer with a control service backed by the same handler Ref. */
|
|
19
|
+
static readonly layer = Layer.effectContext(
|
|
20
|
+
Effect.gen(function* () {
|
|
21
|
+
const handler = yield* Ref.make<SqliteStorageFailpointHandler>(noFailpoint);
|
|
22
|
+
return Context.make(
|
|
23
|
+
SqliteStorageFailpoint,
|
|
24
|
+
SqliteStorageFailpoint.of({
|
|
25
|
+
hit: (location) => Ref.get(handler).pipe(Effect.flatMap((current) => current(location))),
|
|
26
|
+
}),
|
|
27
|
+
).pipe(
|
|
28
|
+
Context.add(
|
|
29
|
+
SqliteStorageFailpointTestControl,
|
|
30
|
+
SqliteStorageFailpointTestControl.of({
|
|
31
|
+
clear: Ref.set(handler, noFailpoint),
|
|
32
|
+
setHandler: (next) => Ref.set(handler, next),
|
|
33
|
+
}),
|
|
34
|
+
),
|
|
35
|
+
);
|
|
36
|
+
}),
|
|
37
|
+
);
|
|
38
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Context, Effect, Layer
|
|
1
|
+
import { Context, Effect, Layer } from "effect";
|
|
2
2
|
|
|
3
3
|
import type { SqliteStorageFailpointError, SqliteStorageFailpointLocation } from "./errors.ts";
|
|
4
4
|
|
|
@@ -8,15 +8,6 @@ export type SqliteStorageFailpointHandler = (
|
|
|
8
8
|
|
|
9
9
|
const noFailpoint: SqliteStorageFailpointHandler = () => Effect.void;
|
|
10
10
|
|
|
11
|
-
/** Test-only control for replacing the active SQLite failpoint handler. */
|
|
12
|
-
export class SqliteStorageFailpointTestControl extends Context.Service<
|
|
13
|
-
SqliteStorageFailpointTestControl,
|
|
14
|
-
{
|
|
15
|
-
readonly clear: Effect.Effect<void>;
|
|
16
|
-
readonly setHandler: (handler: SqliteStorageFailpointHandler) => Effect.Effect<void>;
|
|
17
|
-
}
|
|
18
|
-
>()("@effect-agent/storage-sqlite/SqliteStorageFailpointTestControl") {}
|
|
19
|
-
|
|
20
11
|
/** Explicit fault-injection authority used at SQLite operation boundaries. */
|
|
21
12
|
export class SqliteStorageFailpoint extends Context.Service<
|
|
22
13
|
SqliteStorageFailpoint,
|
|
@@ -26,25 +17,4 @@ export class SqliteStorageFailpoint extends Context.Service<
|
|
|
26
17
|
>()("@effect-agent/storage-sqlite/SqliteStorageFailpoint") {
|
|
27
18
|
/** Production default: no fault injection. */
|
|
28
19
|
static readonly layer = Layer.succeed(this)({ hit: noFailpoint });
|
|
29
|
-
|
|
30
|
-
/** Reusable test Layer with a control service backed by the same handler Ref. */
|
|
31
|
-
static readonly layerTest = Layer.effectContext(
|
|
32
|
-
Effect.gen(function* () {
|
|
33
|
-
const handler = yield* Ref.make<SqliteStorageFailpointHandler>(noFailpoint);
|
|
34
|
-
return Context.make(
|
|
35
|
-
SqliteStorageFailpoint,
|
|
36
|
-
SqliteStorageFailpoint.of({
|
|
37
|
-
hit: (location) => Ref.get(handler).pipe(Effect.flatMap((current) => current(location))),
|
|
38
|
-
}),
|
|
39
|
-
).pipe(
|
|
40
|
-
Context.add(
|
|
41
|
-
SqliteStorageFailpointTestControl,
|
|
42
|
-
SqliteStorageFailpointTestControl.of({
|
|
43
|
-
clear: Ref.set(handler, noFailpoint),
|
|
44
|
-
setHandler: (next) => Ref.set(handler, next),
|
|
45
|
-
}),
|
|
46
|
-
),
|
|
47
|
-
);
|
|
48
|
-
}),
|
|
49
|
-
);
|
|
50
20
|
}
|