@effect-agent/storage-sqlite 0.1.0-beta.9 → 0.1.0-beta.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/SqliteActivityStore.d.mts +14 -0
- package/dist/SqliteActivityStore.mjs +310 -0
- package/dist/SqliteActivityStore.mjs.map +1 -0
- package/dist/SqliteMessageDeliveryStore.d.mts +14 -0
- package/dist/SqliteMessageDeliveryStore.mjs +24 -0
- package/dist/SqliteMessageDeliveryStore.mjs.map +1 -0
- package/dist/SqliteScheduleStore.d.mts +14 -0
- package/dist/SqliteScheduleStore.mjs +255 -0
- package/dist/SqliteScheduleStore.mjs.map +1 -0
- package/dist/SqliteStorageConfig.d.mts +34 -0
- package/dist/SqliteStorageConfig.mjs +39 -0
- package/dist/SqliteStorageConfig.mjs.map +1 -0
- package/dist/SqliteStorageError-DVWeaWLm.d.mts +86 -0
- package/dist/SqliteStorageError.d.mts +2 -0
- package/dist/SqliteStorageError.mjs +150 -0
- package/dist/SqliteStorageError.mjs.map +1 -0
- package/dist/SqliteStorageFailpoint.d.mts +17 -0
- package/dist/SqliteStorageFailpoint.mjs +14 -0
- package/dist/SqliteStorageFailpoint.mjs.map +1 -0
- package/dist/SqliteStorageFailpointTesting.d.mts +15 -0
- package/dist/SqliteStorageFailpointTesting.mjs +19 -0
- package/dist/SqliteStorageFailpointTesting.mjs.map +1 -0
- package/dist/SqliteStorageVersion-BqmqX0CI.d.mts +11 -0
- package/dist/SqliteStorageVersion.d.mts +2 -0
- package/dist/SqliteStorageVersion.mjs +8 -0
- package/dist/SqliteStorageVersion.mjs.map +1 -0
- package/dist/SqliteSubmissionLedger.d.mts +23 -0
- package/dist/SqliteSubmissionLedger.mjs +1788 -0
- package/dist/SqliteSubmissionLedger.mjs.map +1 -0
- package/dist/SqliteSubscriptionStore.d.mts +13 -0
- package/dist/SqliteSubscriptionStore.mjs +28 -0
- package/dist/SqliteSubscriptionStore.mjs.map +1 -0
- package/dist/SqliteThreadStore.d.mts +49 -0
- package/dist/SqliteThreadStore.mjs +460 -0
- package/dist/SqliteThreadStore.mjs.map +1 -0
- package/dist/index.d.mts +11 -193
- package/dist/index.mjs +11 -3082
- package/dist/migrations-Dy5v0HGe.mjs +346 -0
- package/dist/migrations-Dy5v0HGe.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/dist/sqlite-journal-CBOQDySe.mjs +998 -0
- package/dist/sqlite-journal-CBOQDySe.mjs.map +1 -0
- package/package.json +1 -41
- package/src/SqliteActivityStore.ts +556 -0
- package/src/SqliteMessageDeliveryStore.ts +42 -0
- package/src/SqliteScheduleStore.ts +404 -0
- package/src/{sqlite-storage-config.ts → SqliteStorageConfig.ts} +1 -1
- package/src/{errors.ts → SqliteStorageError.ts} +10 -3
- package/src/SqliteStorageFailpoint.ts +23 -0
- package/src/{sqlite-storage-failpoint.ts → SqliteStorageFailpointTesting.ts} +7 -18
- package/src/SqliteStorageVersion.ts +2 -0
- package/src/{sqlite-ledger.ts → SqliteSubmissionLedger.ts} +912 -212
- package/src/SqliteSubscriptionStore.ts +55 -0
- package/src/SqliteThreadStore.ts +999 -0
- package/src/index.ts +10 -6
- package/src/internal/message-delivery-schema.ts +24 -0
- package/src/{migrations.ts → internal/migrations.ts} +156 -79
- package/src/internal/recovery-checkpoint-schema.ts +17 -0
- package/src/{sqlite-journal.ts → internal/sqlite-journal.ts} +751 -241
- package/dist/index.mjs.map +0 -1
- package/src/sqlite-conversation-store.ts +0 -841
|
@@ -1,26 +1,51 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { NodeCrypto } from "@effect/platform-node";
|
|
2
2
|
import { SqliteMigrator } from "@effect/sql-sqlite-node";
|
|
3
3
|
import { Effect, Exit, Schema } from "effect";
|
|
4
|
+
import { EMPTY_TAIL_DIGEST } from "effect-agent/digest";
|
|
5
|
+
import { CanonicalSequence, ProducerEpoch } from "effect-agent/records";
|
|
6
|
+
import { ScheduleFailpoint, ScheduleFailpointError } from "effect-agent/schedule";
|
|
7
|
+
import {
|
|
8
|
+
checkV2ThreadLayout,
|
|
9
|
+
upgradeV2Schedules,
|
|
10
|
+
upgradeV2Subscriptions,
|
|
11
|
+
} from "effect-agent/sql-storage-v2-upgrade";
|
|
12
|
+
import { SubscriptionFailpoint, SubscriptionFailpointError } from "effect-agent/subscription";
|
|
13
|
+
import {
|
|
14
|
+
MAX_THREAD_EXPORT_RECORDS,
|
|
15
|
+
CheckpointRejected,
|
|
16
|
+
FenceRejected,
|
|
17
|
+
ThreadNotMaterialized,
|
|
18
|
+
type SaveRecoveryCheckpointRequest,
|
|
19
|
+
} from "effect-agent/thread-store";
|
|
4
20
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
5
21
|
import type { SqlError } from "effect/unstable/sql/SqlError";
|
|
6
22
|
|
|
23
|
+
import { SqliteStorageConfig } from "../SqliteStorageConfig.ts";
|
|
24
|
+
import type { SqliteStorageFailpointError } from "../SqliteStorageError.ts";
|
|
7
25
|
import {
|
|
8
26
|
SqliteAppendConflict,
|
|
9
27
|
SqliteCheckpointConflict,
|
|
10
28
|
SqliteFenceRejected,
|
|
11
29
|
SqliteStorageCompatibilityError,
|
|
30
|
+
SqliteStorageFailpointLocation,
|
|
12
31
|
SqliteStorageCorruptionError,
|
|
13
32
|
SqliteStorageError,
|
|
14
|
-
SqliteStorageFailpointError,
|
|
15
33
|
SqliteWriteContention,
|
|
16
|
-
|
|
17
|
-
} from "
|
|
18
|
-
import {
|
|
34
|
+
} from "../SqliteStorageError.ts";
|
|
35
|
+
import { SqliteStorageFailpoint } from "../SqliteStorageFailpoint.ts";
|
|
36
|
+
import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
|
|
37
|
+
import {
|
|
38
|
+
CurrentSqliteStorageVersion,
|
|
39
|
+
createNonterminalIndex,
|
|
40
|
+
sqliteMigrations,
|
|
41
|
+
} from "./migrations.ts";
|
|
42
|
+
import { createRecoveryCheckpointTable } from "./recovery-checkpoint-schema.ts";
|
|
19
43
|
|
|
20
44
|
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(16 * 1024 * 1024));
|
|
21
45
|
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
22
46
|
const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0));
|
|
23
|
-
const
|
|
47
|
+
const MAX_RECORDS_PER_THREAD = MAX_THREAD_EXPORT_RECORDS;
|
|
48
|
+
const ZERO_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
24
49
|
const MAX_STORED_TEXT_BYTES = 16 * 1024 * 1024;
|
|
25
50
|
const MAX_IDENTIFIER_LENGTH = 1_024;
|
|
26
51
|
|
|
@@ -38,8 +63,8 @@ class SqliteNameRow extends Schema.Class<SqliteNameRow>("SqliteNameRow")({
|
|
|
38
63
|
name: BoundedIdentifier,
|
|
39
64
|
}) {}
|
|
40
65
|
|
|
41
|
-
class
|
|
42
|
-
|
|
66
|
+
class ThreadRow extends Schema.Class<ThreadRow>("ThreadRow")({
|
|
67
|
+
thread_id: BoundedIdentifier,
|
|
43
68
|
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
44
69
|
producer_epoch: ProducerEpoch,
|
|
45
70
|
tail_digest: BoundedStoredText,
|
|
@@ -50,7 +75,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
50
75
|
batch_digest: BoundedStoredText,
|
|
51
76
|
batch_id: BoundedIdentifier,
|
|
52
77
|
batch_json: BoundedStoredText,
|
|
53
|
-
|
|
78
|
+
thread_id: BoundedIdentifier,
|
|
54
79
|
first_sequence: CanonicalSequence,
|
|
55
80
|
last_sequence: CanonicalSequence,
|
|
56
81
|
tail_digest: BoundedStoredText,
|
|
@@ -58,7 +83,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
58
83
|
|
|
59
84
|
class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
60
85
|
batch_id: BoundedIdentifier,
|
|
61
|
-
|
|
86
|
+
thread_id: BoundedIdentifier,
|
|
62
87
|
record_id: BoundedIdentifier,
|
|
63
88
|
record_json: BoundedStoredText,
|
|
64
89
|
sequence: CanonicalSequence,
|
|
@@ -66,7 +91,7 @@ class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
|
66
91
|
|
|
67
92
|
class CheckpointRow extends Schema.Class<CheckpointRow>("CheckpointRow")({
|
|
68
93
|
checkpoint_json: BoundedStoredText,
|
|
69
|
-
|
|
94
|
+
thread_id: BoundedIdentifier,
|
|
70
95
|
tail_digest: BoundedStoredText,
|
|
71
96
|
through_sequence: CanonicalSequence,
|
|
72
97
|
}) {}
|
|
@@ -82,7 +107,7 @@ export class RawAppendRequest extends Schema.Class<RawAppendRequest>(
|
|
|
82
107
|
batchDigest: BoundedStoredText,
|
|
83
108
|
batchId: BoundedIdentifier,
|
|
84
109
|
batchJson: BoundedStoredText,
|
|
85
|
-
|
|
110
|
+
threadId: BoundedIdentifier,
|
|
86
111
|
expectedTailDigest: BoundedStoredText,
|
|
87
112
|
expectedTailSequence: CanonicalSequence,
|
|
88
113
|
producerEpoch: ProducerEpoch,
|
|
@@ -102,7 +127,7 @@ export class RawAppendResult extends Schema.Class<RawAppendResult>(
|
|
|
102
127
|
export class RawReadRequest extends Schema.Class<RawReadRequest>(
|
|
103
128
|
"@effect-agent/storage-sqlite/RawReadRequest",
|
|
104
129
|
)({
|
|
105
|
-
|
|
130
|
+
threadId: BoundedIdentifier,
|
|
106
131
|
fromSequenceExclusive: CanonicalSequence,
|
|
107
132
|
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
|
|
108
133
|
}) {}
|
|
@@ -111,17 +136,15 @@ export class RawCheckpoint extends Schema.Class<RawCheckpoint>(
|
|
|
111
136
|
"@effect-agent/storage-sqlite/RawCheckpoint",
|
|
112
137
|
)({
|
|
113
138
|
checkpointJson: BoundedStoredText,
|
|
114
|
-
|
|
139
|
+
threadId: BoundedIdentifier,
|
|
115
140
|
tailDigest: BoundedStoredText,
|
|
116
141
|
throughSequence: CanonicalSequence,
|
|
117
142
|
}) {}
|
|
118
143
|
|
|
119
|
-
export class
|
|
120
|
-
"@effect-agent/storage-sqlite/
|
|
144
|
+
export class RawThreadExport extends Schema.Class<RawThreadExport>(
|
|
145
|
+
"@effect-agent/storage-sqlite/RawThreadExport",
|
|
121
146
|
)({
|
|
122
|
-
|
|
123
|
-
checkpoints: Schema.Array(CheckpointRow),
|
|
124
|
-
conversation: ConversationRow,
|
|
147
|
+
thread: ThreadRow,
|
|
125
148
|
records: Schema.Array(RecordRow),
|
|
126
149
|
}) {}
|
|
127
150
|
|
|
@@ -138,11 +161,6 @@ type CheckpointError =
|
|
|
138
161
|
| SqliteStorageCorruptionError
|
|
139
162
|
| SqliteStorageError
|
|
140
163
|
| SqliteWriteContention;
|
|
141
|
-
type SqliteJournalFailpoint = (
|
|
142
|
-
location: SqliteStorageFailpointLocation,
|
|
143
|
-
) => Effect.Effect<void, SqliteStorageFailpointError>;
|
|
144
|
-
|
|
145
|
-
const noFailpoint: SqliteJournalFailpoint = () => Effect.void;
|
|
146
164
|
|
|
147
165
|
const storageError =
|
|
148
166
|
(operation: string) =>
|
|
@@ -195,26 +213,255 @@ export const decodeSingleRow = Effect.fn("SqliteJournal.decodeSingleRow")(
|
|
|
195
213
|
),
|
|
196
214
|
);
|
|
197
215
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
216
|
+
/** Column inventory of the supported v8 predecessor, independent of physical column order. */
|
|
217
|
+
const predecessorColumns = {
|
|
218
|
+
effect_agent_threads: [
|
|
219
|
+
"thread_id",
|
|
220
|
+
"created_at",
|
|
221
|
+
"tail_sequence",
|
|
222
|
+
"tail_digest",
|
|
223
|
+
"producer_epoch",
|
|
224
|
+
],
|
|
225
|
+
effect_agent_canonical_batches: [
|
|
226
|
+
"thread_id",
|
|
227
|
+
"batch_id",
|
|
228
|
+
"first_sequence",
|
|
229
|
+
"last_sequence",
|
|
230
|
+
"batch_digest",
|
|
231
|
+
"tail_digest",
|
|
232
|
+
"batch_json",
|
|
233
|
+
],
|
|
234
|
+
effect_agent_canonical_records: ["thread_id", "sequence", "record_id", "batch_id", "record_json"],
|
|
235
|
+
effect_agent_checkpoints: ["thread_id", "through_sequence", "tail_digest", "checkpoint_json"],
|
|
236
|
+
effect_agent_submissions: [
|
|
237
|
+
"submission_id",
|
|
238
|
+
"thread_id",
|
|
239
|
+
"queue_sequence",
|
|
240
|
+
"principal",
|
|
241
|
+
"idempotency_key",
|
|
242
|
+
"agent_id",
|
|
243
|
+
"agent_digests_json",
|
|
244
|
+
"deployment_id",
|
|
245
|
+
"input_json",
|
|
246
|
+
"input_digest",
|
|
247
|
+
"receipt_id",
|
|
248
|
+
"state",
|
|
249
|
+
"settled_outcome",
|
|
250
|
+
"created_at",
|
|
251
|
+
"ready_at",
|
|
252
|
+
"input_applied_record_id",
|
|
253
|
+
"input_applied_sequence",
|
|
254
|
+
"joined_host_submission_id",
|
|
255
|
+
"suspended_reason_json",
|
|
256
|
+
"suspended_at",
|
|
257
|
+
"unknown_reason",
|
|
258
|
+
"unknown_tool_call_ids_json",
|
|
259
|
+
"parent_submission_id",
|
|
260
|
+
"parent_tool_call_id",
|
|
261
|
+
"admission_group",
|
|
262
|
+
"admission_fence_json",
|
|
263
|
+
],
|
|
264
|
+
effect_agent_submission_ownership: [
|
|
265
|
+
"submission_id",
|
|
266
|
+
"attempt_id",
|
|
267
|
+
"ownership_token",
|
|
268
|
+
"producer_epoch",
|
|
269
|
+
"owner_producer_id",
|
|
270
|
+
"lease_expires_at",
|
|
271
|
+
],
|
|
272
|
+
effect_agent_attempts: [
|
|
273
|
+
"attempt_id",
|
|
274
|
+
"submission_id",
|
|
275
|
+
"thread_id",
|
|
276
|
+
"owner_producer_id",
|
|
277
|
+
"producer_epoch",
|
|
278
|
+
"claimed_at",
|
|
279
|
+
],
|
|
280
|
+
effect_agent_settlement_reservations: [
|
|
281
|
+
"submission_id",
|
|
282
|
+
"settlement_id",
|
|
283
|
+
"outcome",
|
|
284
|
+
"record_id",
|
|
285
|
+
"record_json",
|
|
286
|
+
"record_digest",
|
|
287
|
+
"reserved_at",
|
|
288
|
+
"finalized_at",
|
|
289
|
+
],
|
|
290
|
+
effect_agent_abort_intents: [
|
|
291
|
+
"submission_id",
|
|
292
|
+
"author",
|
|
293
|
+
"reason",
|
|
294
|
+
"requested_at",
|
|
295
|
+
"canonical_record_id",
|
|
296
|
+
],
|
|
297
|
+
effect_agent_approval_decisions: [
|
|
298
|
+
"submission_id",
|
|
299
|
+
"tool_call_id",
|
|
300
|
+
"decision",
|
|
301
|
+
"resolver",
|
|
302
|
+
"reason",
|
|
303
|
+
"decided_at",
|
|
304
|
+
],
|
|
305
|
+
effect_agent_unknown_resolutions: [
|
|
306
|
+
"submission_id",
|
|
307
|
+
"tool_call_id",
|
|
308
|
+
"author",
|
|
309
|
+
"reason",
|
|
310
|
+
"resolution_json",
|
|
311
|
+
"resolved_at",
|
|
312
|
+
],
|
|
313
|
+
effect_agent_child_reservations: [
|
|
314
|
+
"reservation_id",
|
|
315
|
+
"parent_submission_id",
|
|
316
|
+
"parent_tool_call_id",
|
|
317
|
+
"child_submission_id",
|
|
318
|
+
"status",
|
|
319
|
+
"allocation_json",
|
|
320
|
+
"allocation_digest",
|
|
321
|
+
"accounting_json",
|
|
322
|
+
"reserved_at",
|
|
323
|
+
"release_began_at",
|
|
324
|
+
"released_at",
|
|
325
|
+
],
|
|
326
|
+
effect_agent_schedules: [
|
|
327
|
+
"tenant_id",
|
|
328
|
+
"owner_id",
|
|
329
|
+
"schedule_id",
|
|
330
|
+
"deadline_at_millis",
|
|
331
|
+
"record_json",
|
|
332
|
+
],
|
|
333
|
+
effect_agent_subscription_sequences: [
|
|
334
|
+
"tenant_id",
|
|
335
|
+
"source_address",
|
|
336
|
+
"sequence",
|
|
337
|
+
"event_scan_cursor",
|
|
338
|
+
"delivery_scan_cursor",
|
|
339
|
+
"recovery_scan_cursor",
|
|
340
|
+
],
|
|
341
|
+
effect_agent_subscriptions: [
|
|
342
|
+
"tenant_id",
|
|
343
|
+
"source_address",
|
|
344
|
+
"owner_id",
|
|
345
|
+
"subscription_id",
|
|
346
|
+
"ordinal",
|
|
347
|
+
"source_name",
|
|
348
|
+
"source_version",
|
|
349
|
+
"matching_key",
|
|
350
|
+
"state",
|
|
351
|
+
"expires_at_millis",
|
|
352
|
+
"recovery_at_millis",
|
|
353
|
+
"recovery_present",
|
|
354
|
+
"record_json",
|
|
355
|
+
],
|
|
356
|
+
effect_agent_subscription_events: [
|
|
357
|
+
"tenant_id",
|
|
358
|
+
"source_address",
|
|
359
|
+
"event_id",
|
|
360
|
+
"source_name",
|
|
361
|
+
"source_version",
|
|
362
|
+
"matching_key",
|
|
363
|
+
"payload_digest",
|
|
364
|
+
"cutoff",
|
|
365
|
+
"cursor",
|
|
366
|
+
"routing_complete",
|
|
367
|
+
"next_attempt_at_millis",
|
|
368
|
+
"record_json",
|
|
369
|
+
"tombstone",
|
|
370
|
+
],
|
|
371
|
+
effect_agent_subscription_deliveries: [
|
|
372
|
+
"tenant_id",
|
|
373
|
+
"source_address",
|
|
374
|
+
"owner_id",
|
|
375
|
+
"subscription_id",
|
|
376
|
+
"event_id",
|
|
377
|
+
"delivery_key",
|
|
378
|
+
"state",
|
|
379
|
+
"next_attempt_at_millis",
|
|
380
|
+
"record_json",
|
|
381
|
+
],
|
|
382
|
+
} as const;
|
|
383
|
+
|
|
384
|
+
const checkPredecessorLayout = Effect.fn("SqliteJournal.checkPredecessorLayout")(function* (
|
|
385
|
+
version: 8 | 9 | 10,
|
|
202
386
|
) {
|
|
387
|
+
const sql = yield* SqlClient.SqlClient;
|
|
388
|
+
|
|
389
|
+
const messageColumns =
|
|
390
|
+
version === 8
|
|
391
|
+
? predecessorColumns
|
|
392
|
+
: {
|
|
393
|
+
...predecessorColumns,
|
|
394
|
+
effect_agent_submissions: [
|
|
395
|
+
...predecessorColumns.effect_agent_submissions,
|
|
396
|
+
"worker_admission_json",
|
|
397
|
+
"message_admission_json",
|
|
398
|
+
],
|
|
399
|
+
effect_agent_message_deliveries: [
|
|
400
|
+
"owner_thread_id",
|
|
401
|
+
"message_id",
|
|
402
|
+
"version",
|
|
403
|
+
"state",
|
|
404
|
+
"deadline_at_millis",
|
|
405
|
+
"record_json",
|
|
406
|
+
],
|
|
407
|
+
};
|
|
408
|
+
|
|
409
|
+
const expectedColumns = {
|
|
410
|
+
...messageColumns,
|
|
411
|
+
...(version === 10
|
|
412
|
+
? {
|
|
413
|
+
effect_agent_recovery_checkpoints: [
|
|
414
|
+
"thread_id",
|
|
415
|
+
"through_sequence",
|
|
416
|
+
"tail_digest",
|
|
417
|
+
"checkpoint_json",
|
|
418
|
+
],
|
|
419
|
+
}
|
|
420
|
+
: {}),
|
|
421
|
+
};
|
|
422
|
+
|
|
423
|
+
for (const [table, expected] of Object.entries(expectedColumns)) {
|
|
424
|
+
const columns = yield* decodeRows(
|
|
425
|
+
Schema.Array(Schema.Struct({ name: BoundedIdentifier })),
|
|
426
|
+
table,
|
|
427
|
+
"schema",
|
|
428
|
+
yield* sql.unsafe(`PRAGMA table_info(${table})`),
|
|
429
|
+
);
|
|
430
|
+
|
|
431
|
+
const names = new Set<string>(expected);
|
|
432
|
+
|
|
433
|
+
if (columns.length !== names.size || columns.some((column) => !names.has(column.name)))
|
|
434
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
435
|
+
actualVersion: version,
|
|
436
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
437
|
+
message: `The v${version} ${table} columns do not match the supported predecessor; no upgrade was committed.`,
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function* () {
|
|
443
|
+
const sql = yield* SqlClient.SqlClient;
|
|
444
|
+
const { hit: failpoint } = yield* SqliteStorageFailpoint;
|
|
445
|
+
const { busyTimeout } = yield* SqliteStorageConfig;
|
|
446
|
+
|
|
203
447
|
yield* sql`PRAGMA foreign_keys = ON`.pipe(Effect.mapError(storageError("enable foreign keys")));
|
|
204
448
|
// PRAGMA statements do not accept bound parameters; the value is a schema-validated
|
|
205
449
|
// non-negative integer, never caller-controlled text.
|
|
206
450
|
yield* sql
|
|
207
|
-
.unsafe(`PRAGMA busy_timeout = ${
|
|
451
|
+
.unsafe(`PRAGMA busy_timeout = ${busyTimeout}`)
|
|
208
452
|
.pipe(Effect.mapError(storageError("configure busy timeout")));
|
|
453
|
+
|
|
209
454
|
const journalModeRows = yield* sql<Record<string, unknown>>`PRAGMA journal_mode`.pipe(
|
|
210
455
|
Effect.mapError(storageError("read journal mode")),
|
|
211
456
|
);
|
|
457
|
+
|
|
212
458
|
const journalMode = yield* decodeSingleRow(
|
|
213
459
|
Schema.Array(SqliteJournalModeRow),
|
|
214
460
|
"pragma_journal_mode",
|
|
215
461
|
"singleton",
|
|
216
462
|
journalModeRows,
|
|
217
463
|
);
|
|
464
|
+
|
|
218
465
|
if (journalMode.journal_mode.toLowerCase() !== "wal") {
|
|
219
466
|
return yield* SqliteStorageCompatibilityError.make({
|
|
220
467
|
actualVersion: 0,
|
|
@@ -226,6 +473,7 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
226
473
|
const versionRows = yield* sql<Record<string, unknown>>`PRAGMA user_version`.pipe(
|
|
227
474
|
Effect.mapError(storageError("read storage version")),
|
|
228
475
|
);
|
|
476
|
+
|
|
229
477
|
const version = yield* decodeSingleRow(
|
|
230
478
|
Schema.Array(SqliteVersionRow),
|
|
231
479
|
"pragma_user_version",
|
|
@@ -233,20 +481,183 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
233
481
|
versionRows,
|
|
234
482
|
);
|
|
235
483
|
|
|
236
|
-
//
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
484
|
+
// Support the known beta49/beta50 and immediate predecessor formats atomically.
|
|
485
|
+
if (
|
|
486
|
+
version.user_version !== 0 &&
|
|
487
|
+
version.user_version !== 7 &&
|
|
488
|
+
version.user_version !== 8 &&
|
|
489
|
+
version.user_version !== 9 &&
|
|
490
|
+
version.user_version !== 10 &&
|
|
491
|
+
version.user_version !== CurrentSqliteStorageVersion
|
|
492
|
+
) {
|
|
240
493
|
return yield* SqliteStorageCompatibilityError.make({
|
|
241
494
|
actualVersion: version.user_version,
|
|
242
495
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
243
496
|
message:
|
|
244
|
-
`The SQLite file uses
|
|
497
|
+
`The SQLite file uses unsupported storage version ${version.user_version}; ` +
|
|
245
498
|
`this build supports exactly version ${CurrentSqliteStorageVersion}. ` +
|
|
246
|
-
"
|
|
499
|
+
"Only supported v7, v8, v9 and v10 can be upgraded automatically. Keep the original file and use a compatible library version.",
|
|
247
500
|
});
|
|
248
501
|
}
|
|
249
502
|
|
|
503
|
+
if (
|
|
504
|
+
version.user_version === 7 ||
|
|
505
|
+
version.user_version === 8 ||
|
|
506
|
+
version.user_version === 9 ||
|
|
507
|
+
version.user_version === 10
|
|
508
|
+
) {
|
|
509
|
+
yield* sql
|
|
510
|
+
.withTransaction(
|
|
511
|
+
Effect.gen(function* () {
|
|
512
|
+
const current = yield* sql<{ user_version: number }>`PRAGMA user_version`;
|
|
513
|
+
|
|
514
|
+
if (current.length === 1 && current[0].user_version === CurrentSqliteStorageVersion)
|
|
515
|
+
return;
|
|
516
|
+
if (
|
|
517
|
+
current.length !== 1 ||
|
|
518
|
+
(current[0].user_version !== 7 &&
|
|
519
|
+
current[0].user_version !== 8 &&
|
|
520
|
+
current[0].user_version !== 9 &&
|
|
521
|
+
current[0].user_version !== 10)
|
|
522
|
+
)
|
|
523
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
524
|
+
actualVersion: -1,
|
|
525
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
526
|
+
message: "Storage version changed while acquiring the upgrade transaction.",
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
const required = yield* sql<{
|
|
530
|
+
name: string;
|
|
531
|
+
}>`SELECT name FROM sqlite_master WHERE type='table' AND name IN (
|
|
532
|
+
'effect_agent_threads', 'effect_agent_canonical_batches', 'effect_agent_canonical_records',
|
|
533
|
+
'effect_agent_checkpoints', 'effect_agent_submissions', 'effect_agent_submission_ownership',
|
|
534
|
+
'effect_agent_attempts', 'effect_agent_settlement_reservations', 'effect_agent_abort_intents',
|
|
535
|
+
'effect_agent_approval_decisions', 'effect_agent_unknown_resolutions', 'effect_agent_schedules'
|
|
536
|
+
)`;
|
|
537
|
+
|
|
538
|
+
if (required.length !== 12)
|
|
539
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
540
|
+
actualVersion: current[0].user_version,
|
|
541
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
542
|
+
message:
|
|
543
|
+
"The predecessor store is missing required tables; no upgrade was committed.",
|
|
544
|
+
});
|
|
545
|
+
|
|
546
|
+
const recoveryTables =
|
|
547
|
+
yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name='effect_agent_recovery_checkpoints'`;
|
|
548
|
+
|
|
549
|
+
if (recoveryTables.length !== (current[0].user_version === 10 ? 1 : 0))
|
|
550
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
551
|
+
actualVersion: current[0].user_version,
|
|
552
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
553
|
+
message:
|
|
554
|
+
"The predecessor recovery checkpoint storage does not match its version; refusing ambiguous data without mutation.",
|
|
555
|
+
});
|
|
556
|
+
|
|
557
|
+
const indexes =
|
|
558
|
+
yield* sql`SELECT name FROM sqlite_master WHERE name='effect_agent_submissions_nonterminal'`;
|
|
559
|
+
|
|
560
|
+
if (indexes.length !== 0)
|
|
561
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
562
|
+
actualVersion: current[0].user_version,
|
|
563
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
564
|
+
message:
|
|
565
|
+
"The predecessor already contains the nonterminal index; refusing ambiguous storage without mutation.",
|
|
566
|
+
});
|
|
567
|
+
if (current[0].user_version === 7) {
|
|
568
|
+
yield* checkV2ThreadLayout();
|
|
569
|
+
for (const statement of [
|
|
570
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
|
|
571
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
|
|
572
|
+
sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`,
|
|
573
|
+
]) {
|
|
574
|
+
yield* failpoint("upgrade:before-mutation");
|
|
575
|
+
yield* statement;
|
|
576
|
+
yield* failpoint("upgrade:after-mutation");
|
|
577
|
+
}
|
|
578
|
+
yield* upgradeV2Schedules(16 * 1024 * 1024).pipe(
|
|
579
|
+
Effect.provideService(ScheduleFailpoint, {
|
|
580
|
+
hit: (point) =>
|
|
581
|
+
Schema.decodeUnknownEffect(SqliteStorageFailpointLocation)(point).pipe(
|
|
582
|
+
Effect.flatMap(failpoint),
|
|
583
|
+
Effect.mapError(() => ScheduleFailpointError.make({ point })),
|
|
584
|
+
),
|
|
585
|
+
}),
|
|
586
|
+
);
|
|
587
|
+
yield* upgradeV2Subscriptions(16 * 1024 * 1024).pipe(
|
|
588
|
+
Effect.provideService(SubscriptionFailpoint, {
|
|
589
|
+
hit: (point) =>
|
|
590
|
+
Schema.decodeUnknownEffect(SqliteStorageFailpointLocation)(point).pipe(
|
|
591
|
+
Effect.flatMap(failpoint),
|
|
592
|
+
Effect.mapError(() => SubscriptionFailpointError.make({ point })),
|
|
593
|
+
),
|
|
594
|
+
}),
|
|
595
|
+
);
|
|
596
|
+
}
|
|
597
|
+
if (
|
|
598
|
+
current[0].user_version === 8 ||
|
|
599
|
+
current[0].user_version === 9 ||
|
|
600
|
+
current[0].user_version === 10
|
|
601
|
+
)
|
|
602
|
+
yield* checkPredecessorLayout(current[0].user_version);
|
|
603
|
+
if (current[0].user_version === 7 || current[0].user_version === 8) {
|
|
604
|
+
yield* failpoint("upgrade:before-mutation");
|
|
605
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
|
|
606
|
+
yield* failpoint("upgrade:after-mutation");
|
|
607
|
+
yield* failpoint("upgrade:before-mutation");
|
|
608
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
|
|
609
|
+
yield* failpoint("upgrade:after-mutation");
|
|
610
|
+
yield* failpoint("upgrade:before-mutation");
|
|
611
|
+
yield* createMessageDeliveryTables;
|
|
612
|
+
yield* failpoint("upgrade:after-mutation");
|
|
613
|
+
}
|
|
614
|
+
if (current[0].user_version !== 10) {
|
|
615
|
+
yield* failpoint("upgrade:before-mutation");
|
|
616
|
+
yield* createRecoveryCheckpointTable;
|
|
617
|
+
yield* failpoint("upgrade:after-mutation");
|
|
618
|
+
}
|
|
619
|
+
yield* failpoint("upgrade:before-mutation");
|
|
620
|
+
yield* createNonterminalIndex;
|
|
621
|
+
yield* failpoint("upgrade:after-mutation");
|
|
622
|
+
yield* failpoint("upgrade:before-version");
|
|
623
|
+
yield* sql`PRAGMA user_version = 11`;
|
|
624
|
+
yield* failpoint("upgrade:after-version");
|
|
625
|
+
}),
|
|
626
|
+
)
|
|
627
|
+
.pipe(
|
|
628
|
+
Effect.provide(NodeCrypto.layer),
|
|
629
|
+
Effect.catchTag("SqliteStorageFailpointError", (error) =>
|
|
630
|
+
SqliteStorageError.make({
|
|
631
|
+
cause: error,
|
|
632
|
+
operation: "upgrade storage",
|
|
633
|
+
message: error.message,
|
|
634
|
+
}),
|
|
635
|
+
),
|
|
636
|
+
Effect.catchTag(["ScheduleFailpointError", "SubscriptionFailpointError"], (error) =>
|
|
637
|
+
SqliteStorageError.make({
|
|
638
|
+
cause: error,
|
|
639
|
+
operation: "upgrade storage",
|
|
640
|
+
message: "Injected storage upgrade failure",
|
|
641
|
+
}),
|
|
642
|
+
),
|
|
643
|
+
Effect.catchTag("StorageUpgradeError", (error) =>
|
|
644
|
+
SqliteStorageCorruptionError.make({
|
|
645
|
+
table: error.table,
|
|
646
|
+
rowKey: error.rowKey,
|
|
647
|
+
message: error.message,
|
|
648
|
+
}),
|
|
649
|
+
),
|
|
650
|
+
Effect.catchTag("SqlError", storageError("upgrade supported storage")),
|
|
651
|
+
Effect.catchTag("SchemaError", (error) =>
|
|
652
|
+
SqliteStorageCorruptionError.make({
|
|
653
|
+
table: "upgrade",
|
|
654
|
+
rowKey: "v7",
|
|
655
|
+
message: error.message,
|
|
656
|
+
}),
|
|
657
|
+
),
|
|
658
|
+
);
|
|
659
|
+
}
|
|
660
|
+
|
|
250
661
|
if (version.user_version === 0) {
|
|
251
662
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
252
663
|
SELECT name
|
|
@@ -255,6 +666,7 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
255
666
|
AND name LIKE 'effect_agent_%'
|
|
256
667
|
ORDER BY name
|
|
257
668
|
`.pipe(Effect.mapError(storageError("inspect unversioned storage")));
|
|
669
|
+
|
|
258
670
|
const existing = yield* decodeRows(
|
|
259
671
|
Schema.Array(SqliteNameRow),
|
|
260
672
|
"sqlite_master",
|
|
@@ -267,14 +679,11 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
267
679
|
actualVersion: 0,
|
|
268
680
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
269
681
|
message:
|
|
270
|
-
"The SQLite file contains unversioned Effect Agent tables.
|
|
682
|
+
"The SQLite file contains unversioned Effect Agent tables. Refusing to mutate ambiguous stored data; retain it for inspection with its original writer.",
|
|
271
683
|
});
|
|
272
684
|
}
|
|
273
685
|
|
|
274
686
|
yield* SqliteMigrator.run({ loader: sqliteMigrations }).pipe(
|
|
275
|
-
// SqliteMigrator depends on the generic client supplied by this adapter.
|
|
276
|
-
// The concrete Node client is kept at the outer Layer boundary.
|
|
277
|
-
Effect.provideService(SqlClient.SqlClient, sql),
|
|
278
687
|
Effect.mapError((error) =>
|
|
279
688
|
SqliteStorageError.make({
|
|
280
689
|
cause: error,
|
|
@@ -288,9 +697,9 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
288
697
|
const requiredRows = yield* sql<Record<string, unknown>>`
|
|
289
698
|
SELECT name
|
|
290
699
|
FROM sqlite_master
|
|
291
|
-
WHERE type = 'table'
|
|
700
|
+
WHERE (type = 'table'
|
|
292
701
|
AND name IN (
|
|
293
|
-
'
|
|
702
|
+
'effect_agent_threads',
|
|
294
703
|
'effect_agent_canonical_batches',
|
|
295
704
|
'effect_agent_canonical_records',
|
|
296
705
|
'effect_agent_checkpoints',
|
|
@@ -300,29 +709,30 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
300
709
|
'effect_agent_settlement_reservations',
|
|
301
710
|
'effect_agent_abort_intents',
|
|
302
711
|
'effect_agent_approval_decisions',
|
|
303
|
-
'effect_agent_unknown_resolutions'
|
|
304
|
-
|
|
712
|
+
'effect_agent_unknown_resolutions',
|
|
713
|
+
'effect_agent_schedules',
|
|
714
|
+
'effect_agent_message_deliveries',
|
|
715
|
+
'effect_agent_recovery_checkpoints'
|
|
716
|
+
)) OR (type = 'index' AND name = 'effect_agent_submissions_nonterminal')
|
|
305
717
|
ORDER BY name
|
|
306
718
|
`.pipe(Effect.mapError(storageError("verify storage tables")));
|
|
719
|
+
|
|
307
720
|
const required = yield* decodeRows(
|
|
308
721
|
Schema.Array(SqliteNameRow),
|
|
309
722
|
"sqlite_master",
|
|
310
723
|
"required_tables",
|
|
311
724
|
requiredRows,
|
|
312
725
|
);
|
|
313
|
-
|
|
726
|
+
|
|
727
|
+
if (required.length !== 15) {
|
|
314
728
|
return yield* SqliteStorageCompatibilityError.make({
|
|
315
729
|
actualVersion: CurrentSqliteStorageVersion,
|
|
316
730
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
317
731
|
message:
|
|
318
|
-
"The SQLite file claims the current format but is missing required tables.
|
|
732
|
+
"The SQLite file claims the current format but is missing required tables or its nonterminal index. Retain the original store for inspection.",
|
|
319
733
|
});
|
|
320
734
|
}
|
|
321
735
|
|
|
322
|
-
return makeJournal(sql, failpoint);
|
|
323
|
-
});
|
|
324
|
-
|
|
325
|
-
const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint) => {
|
|
326
736
|
const classifyWriteFailure =
|
|
327
737
|
(operation: string) =>
|
|
328
738
|
(error: SqlError): SqliteStorageError | SqliteWriteContention =>
|
|
@@ -357,19 +767,24 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
357
767
|
const connection = yield* sql.reserve.pipe(
|
|
358
768
|
Effect.mapError(classifyWriteFailure(operation)),
|
|
359
769
|
);
|
|
770
|
+
|
|
360
771
|
yield* connection
|
|
361
772
|
.executeUnprepared("BEGIN IMMEDIATE", [], undefined)
|
|
362
773
|
.pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
774
|
+
|
|
363
775
|
const exit = yield* restore(
|
|
364
776
|
Effect.provideService(effect, sql.transactionService, [connection, 0] as const),
|
|
365
777
|
).pipe(Effect.exit);
|
|
778
|
+
|
|
366
779
|
if (Exit.isSuccess(exit)) {
|
|
367
780
|
yield* connection
|
|
368
781
|
.executeUnprepared("COMMIT", [], undefined)
|
|
369
782
|
.pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
783
|
+
|
|
370
784
|
return exit.value;
|
|
371
785
|
}
|
|
372
786
|
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], undefined));
|
|
787
|
+
|
|
373
788
|
return yield* exit;
|
|
374
789
|
}),
|
|
375
790
|
).pipe(
|
|
@@ -392,26 +807,31 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
392
807
|
Effect.scoped(
|
|
393
808
|
Effect.gen(function* () {
|
|
394
809
|
const connection = yield* sql.reserve.pipe(Effect.mapError(storageError(operation)));
|
|
810
|
+
|
|
395
811
|
yield* connection
|
|
396
812
|
.executeUnprepared("BEGIN", [], undefined)
|
|
397
813
|
.pipe(Effect.mapError(storageError(operation)));
|
|
814
|
+
|
|
398
815
|
const exit = yield* restore(
|
|
399
816
|
Effect.provideService(effect, sql.transactionService, [connection, 0] as const),
|
|
400
817
|
).pipe(Effect.exit);
|
|
818
|
+
|
|
401
819
|
if (Exit.isSuccess(exit)) {
|
|
402
820
|
yield* connection
|
|
403
821
|
.executeUnprepared("COMMIT", [], undefined)
|
|
404
822
|
.pipe(Effect.mapError(storageError(operation)));
|
|
823
|
+
|
|
405
824
|
return exit.value;
|
|
406
825
|
}
|
|
407
826
|
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], undefined));
|
|
827
|
+
|
|
408
828
|
return yield* exit;
|
|
409
829
|
}),
|
|
410
830
|
).pipe(Effect.withSpan("SqliteJournal.withReadTransaction", { attributes: { operation } })),
|
|
411
831
|
);
|
|
412
832
|
|
|
413
833
|
const materialize = Effect.fn("SqliteJournal.materialize")(function* (
|
|
414
|
-
|
|
834
|
+
threadId: string,
|
|
415
835
|
createdAt: string,
|
|
416
836
|
emptyTailDigest: string,
|
|
417
837
|
producerEpoch: ProducerEpoch,
|
|
@@ -420,55 +840,58 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
420
840
|
SqliteFenceRejected | SqliteStorageCorruptionError | SqliteStorageError | SqliteWriteContention
|
|
421
841
|
> {
|
|
422
842
|
if (
|
|
423
|
-
|
|
843
|
+
threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
424
844
|
storedTextBytes(emptyTailDigest) > MAX_STORED_TEXT_BYTES
|
|
425
845
|
) {
|
|
426
846
|
return yield* SqliteStorageError.make({
|
|
427
|
-
operation: "materialize
|
|
428
|
-
message: "
|
|
847
|
+
operation: "materialize thread",
|
|
848
|
+
message: "Thread identity or initial digest exceeds the SQLite storage bounds.",
|
|
429
849
|
});
|
|
430
850
|
}
|
|
431
851
|
yield* withWriteTransaction("materialize transaction")(
|
|
432
852
|
Effect.gen(function* () {
|
|
433
853
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
434
854
|
SELECT
|
|
435
|
-
|
|
855
|
+
thread_id,
|
|
436
856
|
created_at,
|
|
437
857
|
tail_sequence,
|
|
438
858
|
tail_digest,
|
|
439
859
|
producer_epoch
|
|
440
|
-
FROM
|
|
441
|
-
WHERE
|
|
442
|
-
`.pipe(Effect.mapError(storageError("read materialized
|
|
860
|
+
FROM effect_agent_threads
|
|
861
|
+
WHERE thread_id = ${threadId}
|
|
862
|
+
`.pipe(Effect.mapError(storageError("read materialized thread")));
|
|
863
|
+
|
|
443
864
|
const existing = yield* decodeRows(
|
|
444
|
-
Schema.Array(
|
|
445
|
-
"
|
|
446
|
-
|
|
865
|
+
Schema.Array(ThreadRow),
|
|
866
|
+
"effect_agent_threads",
|
|
867
|
+
threadId,
|
|
447
868
|
existingRows,
|
|
448
869
|
);
|
|
870
|
+
|
|
449
871
|
if (existing.length > 1) {
|
|
450
872
|
return yield* SqliteStorageCorruptionError.make({
|
|
451
|
-
table: "
|
|
452
|
-
rowKey:
|
|
453
|
-
message: "A
|
|
873
|
+
table: "effect_agent_threads",
|
|
874
|
+
rowKey: threadId,
|
|
875
|
+
message: "A thread primary key returned more than one row.",
|
|
454
876
|
});
|
|
455
877
|
}
|
|
456
878
|
if (existing.length === 0) {
|
|
457
879
|
yield* sql`
|
|
458
|
-
INSERT INTO
|
|
459
|
-
|
|
880
|
+
INSERT INTO effect_agent_threads (
|
|
881
|
+
thread_id,
|
|
460
882
|
created_at,
|
|
461
883
|
tail_sequence,
|
|
462
884
|
tail_digest,
|
|
463
885
|
producer_epoch
|
|
464
886
|
) VALUES (
|
|
465
|
-
${
|
|
887
|
+
${threadId},
|
|
466
888
|
${createdAt},
|
|
467
889
|
0,
|
|
468
890
|
${emptyTailDigest},
|
|
469
891
|
${producerEpoch}
|
|
470
892
|
)
|
|
471
|
-
`.pipe(Effect.mapError(storageError("materialize
|
|
893
|
+
`.pipe(Effect.mapError(storageError("materialize thread")));
|
|
894
|
+
|
|
472
895
|
return;
|
|
473
896
|
}
|
|
474
897
|
if (producerEpoch < existing[0].producer_epoch) {
|
|
@@ -480,41 +903,35 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
480
903
|
}
|
|
481
904
|
if (producerEpoch > existing[0].producer_epoch) {
|
|
482
905
|
yield* sql`
|
|
483
|
-
UPDATE
|
|
906
|
+
UPDATE effect_agent_threads
|
|
484
907
|
SET producer_epoch = ${producerEpoch}
|
|
485
|
-
WHERE
|
|
908
|
+
WHERE thread_id = ${threadId}
|
|
486
909
|
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
487
910
|
}
|
|
488
911
|
}),
|
|
489
912
|
);
|
|
490
913
|
});
|
|
491
914
|
|
|
492
|
-
const
|
|
493
|
-
conversationId: string,
|
|
494
|
-
) {
|
|
915
|
+
const getThread = Effect.fn("SqliteJournal.getThread")(function* (threadId: string) {
|
|
495
916
|
const rows = yield* sql<Record<string, unknown>>`
|
|
496
917
|
SELECT
|
|
497
|
-
|
|
918
|
+
thread_id,
|
|
498
919
|
created_at,
|
|
499
920
|
tail_sequence,
|
|
500
921
|
tail_digest,
|
|
501
922
|
producer_epoch
|
|
502
|
-
FROM
|
|
503
|
-
WHERE
|
|
504
|
-
`.pipe(Effect.mapError(storageError("read
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
"effect_agent_conversations",
|
|
508
|
-
conversationId,
|
|
509
|
-
rows,
|
|
510
|
-
);
|
|
923
|
+
FROM effect_agent_threads
|
|
924
|
+
WHERE thread_id = ${threadId}
|
|
925
|
+
`.pipe(Effect.mapError(storageError("read thread")));
|
|
926
|
+
|
|
927
|
+
return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
511
928
|
});
|
|
512
929
|
|
|
513
930
|
const append = Effect.fn("SqliteJournal.append")(function* (
|
|
514
931
|
request: RawAppendRequest,
|
|
515
932
|
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
516
933
|
if (
|
|
517
|
-
request.
|
|
934
|
+
request.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
518
935
|
request.batchId.length > MAX_IDENTIFIER_LENGTH ||
|
|
519
936
|
storedTextBytes(request.batchJson) > MAX_STORED_TEXT_BYTES ||
|
|
520
937
|
storedTextBytes(request.batchDigest) > MAX_STORED_TEXT_BYTES ||
|
|
@@ -530,9 +947,11 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
530
947
|
message: "Canonical identifiers or encoded JSON exceed the SQLite storage bounds.",
|
|
531
948
|
});
|
|
532
949
|
}
|
|
950
|
+
|
|
533
951
|
return yield* withWriteTransaction("append transaction")(
|
|
534
952
|
Effect.gen(function* () {
|
|
535
953
|
const recordIds = request.records.map((record) => record.recordId);
|
|
954
|
+
|
|
536
955
|
if (new Set(recordIds).size !== recordIds.length) {
|
|
537
956
|
return yield* SqliteAppendConflict.make({
|
|
538
957
|
message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
|
|
@@ -540,34 +959,35 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
540
959
|
});
|
|
541
960
|
}
|
|
542
961
|
|
|
543
|
-
const
|
|
962
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
544
963
|
SELECT
|
|
545
|
-
|
|
964
|
+
thread_id,
|
|
546
965
|
created_at,
|
|
547
966
|
tail_sequence,
|
|
548
967
|
tail_digest,
|
|
549
968
|
producer_epoch
|
|
550
|
-
FROM
|
|
551
|
-
WHERE
|
|
969
|
+
FROM effect_agent_threads
|
|
970
|
+
WHERE thread_id = ${request.threadId}
|
|
552
971
|
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
972
|
+
|
|
973
|
+
const thread = yield* decodeSingleRow(
|
|
974
|
+
Schema.Array(ThreadRow),
|
|
975
|
+
"effect_agent_threads",
|
|
976
|
+
request.threadId,
|
|
977
|
+
threadRows,
|
|
558
978
|
);
|
|
559
979
|
|
|
560
|
-
if (request.producerEpoch !==
|
|
980
|
+
if (request.producerEpoch !== thread.producer_epoch) {
|
|
561
981
|
return yield* SqliteFenceRejected.make({
|
|
562
982
|
producerEpoch: request.producerEpoch,
|
|
563
|
-
actualEpoch:
|
|
564
|
-
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${
|
|
983
|
+
actualEpoch: thread.producer_epoch,
|
|
984
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`,
|
|
565
985
|
});
|
|
566
986
|
}
|
|
567
987
|
|
|
568
988
|
const batchRows = yield* sql<Record<string, unknown>>`
|
|
569
989
|
SELECT
|
|
570
|
-
|
|
990
|
+
thread_id,
|
|
571
991
|
batch_id,
|
|
572
992
|
first_sequence,
|
|
573
993
|
last_sequence,
|
|
@@ -575,31 +995,34 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
575
995
|
tail_digest,
|
|
576
996
|
batch_json
|
|
577
997
|
FROM effect_agent_canonical_batches
|
|
578
|
-
WHERE
|
|
998
|
+
WHERE thread_id = ${request.threadId}
|
|
579
999
|
AND batch_id = ${request.batchId}
|
|
580
1000
|
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
1001
|
+
|
|
581
1002
|
const batches = yield* decodeRows(
|
|
582
1003
|
Schema.Array(BatchRow),
|
|
583
1004
|
"effect_agent_canonical_batches",
|
|
584
|
-
`${request.
|
|
1005
|
+
`${request.threadId}/${request.batchId}`,
|
|
585
1006
|
batchRows,
|
|
586
1007
|
);
|
|
587
1008
|
|
|
588
1009
|
if (batches.length > 1) {
|
|
589
1010
|
return yield* SqliteStorageCorruptionError.make({
|
|
590
1011
|
table: "effect_agent_canonical_batches",
|
|
591
|
-
rowKey: `${request.
|
|
1012
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
592
1013
|
message: "A canonical batch primary key returned more than one row.",
|
|
593
1014
|
});
|
|
594
1015
|
}
|
|
595
1016
|
if (batches.length === 1) {
|
|
596
1017
|
const existing = batches[0];
|
|
1018
|
+
|
|
597
1019
|
if (existing.batch_digest !== request.batchDigest) {
|
|
598
1020
|
return yield* SqliteAppendConflict.make({
|
|
599
1021
|
message: `Batch ${request.batchId} already exists with different canonical content.`,
|
|
600
1022
|
reason: "batch-digest",
|
|
601
1023
|
});
|
|
602
1024
|
}
|
|
1025
|
+
|
|
603
1026
|
return RawAppendResult.make({
|
|
604
1027
|
firstSequence: existing.first_sequence,
|
|
605
1028
|
lastSequence: existing.last_sequence,
|
|
@@ -609,43 +1032,45 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
609
1032
|
}
|
|
610
1033
|
|
|
611
1034
|
if (
|
|
612
|
-
request.expectedTailSequence !==
|
|
613
|
-
request.expectedTailDigest !==
|
|
1035
|
+
request.expectedTailSequence !== thread.tail_sequence ||
|
|
1036
|
+
request.expectedTailDigest !== thread.tail_digest
|
|
614
1037
|
) {
|
|
615
1038
|
return yield* SqliteAppendConflict.make({
|
|
616
1039
|
message:
|
|
617
1040
|
`Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} ` +
|
|
618
|
-
`but found ${
|
|
1041
|
+
`but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
619
1042
|
reason: "tail",
|
|
620
|
-
actualTailSequence:
|
|
621
|
-
actualTailDigest:
|
|
1043
|
+
actualTailSequence: thread.tail_sequence,
|
|
1044
|
+
actualTailDigest: thread.tail_digest,
|
|
622
1045
|
});
|
|
623
1046
|
}
|
|
624
|
-
if (
|
|
1047
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) {
|
|
625
1048
|
return yield* SqliteStorageError.make({
|
|
626
1049
|
operation: "append canonical batch",
|
|
627
|
-
message: `
|
|
1050
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`,
|
|
628
1051
|
});
|
|
629
1052
|
}
|
|
630
1053
|
|
|
631
1054
|
const existingRecordRows = yield* sql<Record<string, unknown>>`
|
|
632
1055
|
SELECT
|
|
633
|
-
|
|
1056
|
+
thread_id,
|
|
634
1057
|
sequence,
|
|
635
1058
|
record_id,
|
|
636
1059
|
batch_id,
|
|
637
1060
|
record_json
|
|
638
1061
|
FROM effect_agent_canonical_records
|
|
639
|
-
WHERE
|
|
1062
|
+
WHERE thread_id = ${request.threadId}
|
|
640
1063
|
AND record_id IN ${sql.in(recordIds)}
|
|
641
1064
|
ORDER BY sequence
|
|
642
1065
|
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
1066
|
+
|
|
643
1067
|
const existingRecords = yield* decodeRows(
|
|
644
1068
|
Schema.Array(RecordRow),
|
|
645
1069
|
"effect_agent_canonical_records",
|
|
646
|
-
`${request.
|
|
1070
|
+
`${request.threadId}/record_ids`,
|
|
647
1071
|
existingRecordRows,
|
|
648
1072
|
);
|
|
1073
|
+
|
|
649
1074
|
if (existingRecords.length > 0) {
|
|
650
1075
|
return yield* SqliteAppendConflict.make({
|
|
651
1076
|
message: `Canonical record ID ${existingRecords[0].record_id} already exists.`,
|
|
@@ -653,8 +1078,8 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
653
1078
|
});
|
|
654
1079
|
}
|
|
655
1080
|
|
|
656
|
-
const firstSequence = yield* Schema.
|
|
657
|
-
|
|
1081
|
+
const firstSequence = yield* Schema.decodeEffect(CanonicalSequence)(
|
|
1082
|
+
thread.tail_sequence + 1,
|
|
658
1083
|
).pipe(
|
|
659
1084
|
Effect.mapError((error) =>
|
|
660
1085
|
SqliteStorageError.make({
|
|
@@ -664,7 +1089,8 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
664
1089
|
}),
|
|
665
1090
|
),
|
|
666
1091
|
);
|
|
667
|
-
|
|
1092
|
+
|
|
1093
|
+
const lastSequence = yield* Schema.decodeEffect(CanonicalSequence)(
|
|
668
1094
|
firstSequence + request.records.length - 1,
|
|
669
1095
|
).pipe(
|
|
670
1096
|
Effect.mapError((error) =>
|
|
@@ -678,7 +1104,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
678
1104
|
|
|
679
1105
|
yield* sql`
|
|
680
1106
|
INSERT INTO effect_agent_canonical_batches (
|
|
681
|
-
|
|
1107
|
+
thread_id,
|
|
682
1108
|
batch_id,
|
|
683
1109
|
first_sequence,
|
|
684
1110
|
last_sequence,
|
|
@@ -686,7 +1112,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
686
1112
|
tail_digest,
|
|
687
1113
|
batch_json
|
|
688
1114
|
) VALUES (
|
|
689
|
-
${request.
|
|
1115
|
+
${request.threadId},
|
|
690
1116
|
${request.batchId},
|
|
691
1117
|
${firstSequence},
|
|
692
1118
|
${lastSequence},
|
|
@@ -703,13 +1129,13 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
703
1129
|
Effect.gen(function* () {
|
|
704
1130
|
yield* sql`
|
|
705
1131
|
INSERT INTO effect_agent_canonical_records (
|
|
706
|
-
|
|
1132
|
+
thread_id,
|
|
707
1133
|
sequence,
|
|
708
1134
|
record_id,
|
|
709
1135
|
batch_id,
|
|
710
1136
|
record_json
|
|
711
1137
|
) VALUES (
|
|
712
|
-
${request.
|
|
1138
|
+
${request.threadId},
|
|
713
1139
|
${firstSequence + index},
|
|
714
1140
|
${record.recordId},
|
|
715
1141
|
${request.batchId},
|
|
@@ -722,13 +1148,13 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
722
1148
|
);
|
|
723
1149
|
|
|
724
1150
|
yield* sql`
|
|
725
|
-
UPDATE
|
|
1151
|
+
UPDATE effect_agent_threads
|
|
726
1152
|
SET
|
|
727
1153
|
tail_sequence = ${lastSequence},
|
|
728
1154
|
tail_digest = ${request.tailDigest},
|
|
729
1155
|
producer_epoch = ${request.producerEpoch}
|
|
730
|
-
WHERE
|
|
731
|
-
`.pipe(Effect.mapError(storageError("advance
|
|
1156
|
+
WHERE thread_id = ${request.threadId}
|
|
1157
|
+
`.pipe(Effect.mapError(storageError("advance thread tail")));
|
|
732
1158
|
yield* failpoint("append:after-tail-update");
|
|
733
1159
|
|
|
734
1160
|
return RawAppendResult.make({
|
|
@@ -744,103 +1170,95 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
744
1170
|
const read = Effect.fn("SqliteJournal.read")(function* (request: RawReadRequest) {
|
|
745
1171
|
const rows = yield* sql<Record<string, unknown>>`
|
|
746
1172
|
SELECT
|
|
747
|
-
|
|
1173
|
+
thread_id,
|
|
748
1174
|
sequence,
|
|
749
1175
|
record_id,
|
|
750
1176
|
batch_id,
|
|
751
1177
|
record_json
|
|
752
1178
|
FROM effect_agent_canonical_records
|
|
753
|
-
WHERE
|
|
1179
|
+
WHERE thread_id = ${request.threadId}
|
|
754
1180
|
AND sequence > ${request.fromSequenceExclusive}
|
|
755
1181
|
ORDER BY sequence
|
|
756
1182
|
LIMIT ${request.limit}
|
|
757
1183
|
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
1184
|
+
|
|
758
1185
|
return yield* decodeRows(
|
|
759
1186
|
Schema.Array(RecordRow),
|
|
760
1187
|
"effect_agent_canonical_records",
|
|
761
|
-
`${request.
|
|
1188
|
+
`${request.threadId}>${request.fromSequenceExclusive}`,
|
|
762
1189
|
rows,
|
|
763
1190
|
);
|
|
764
1191
|
});
|
|
765
1192
|
|
|
766
|
-
const
|
|
767
|
-
conversationId: string,
|
|
768
|
-
) {
|
|
1193
|
+
const exportThread = Effect.fn("SqliteJournal.exportThread")(function* (threadId: string) {
|
|
769
1194
|
return yield* withReadTransaction("export transaction")(
|
|
770
1195
|
Effect.gen(function* () {
|
|
771
|
-
const
|
|
1196
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
772
1197
|
SELECT
|
|
773
|
-
|
|
1198
|
+
thread_id,
|
|
774
1199
|
created_at,
|
|
775
1200
|
tail_sequence,
|
|
776
1201
|
tail_digest,
|
|
777
1202
|
producer_epoch
|
|
778
|
-
FROM
|
|
779
|
-
WHERE
|
|
780
|
-
`.pipe(Effect.mapError(storageError("export
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
1203
|
+
FROM effect_agent_threads
|
|
1204
|
+
WHERE thread_id = ${threadId}
|
|
1205
|
+
`.pipe(Effect.mapError(storageError("export thread")));
|
|
1206
|
+
|
|
1207
|
+
const thread = yield* decodeSingleRow(
|
|
1208
|
+
Schema.Array(ThreadRow),
|
|
1209
|
+
"effect_agent_threads",
|
|
1210
|
+
threadId,
|
|
1211
|
+
threadRows,
|
|
786
1212
|
);
|
|
787
|
-
yield* failpoint("export:after-conversation-read");
|
|
788
|
-
const batchRows = yield* sql<Record<string, unknown>>`
|
|
789
|
-
SELECT
|
|
790
|
-
conversation_id,
|
|
791
|
-
batch_id,
|
|
792
|
-
first_sequence,
|
|
793
|
-
last_sequence,
|
|
794
|
-
batch_digest,
|
|
795
|
-
tail_digest,
|
|
796
|
-
batch_json
|
|
797
|
-
FROM effect_agent_canonical_batches
|
|
798
|
-
WHERE conversation_id = ${conversationId}
|
|
799
|
-
ORDER BY first_sequence
|
|
800
|
-
`.pipe(Effect.mapError(storageError("export canonical batches")));
|
|
801
|
-
const recordRows = yield* sql<Record<string, unknown>>`
|
|
802
|
-
SELECT
|
|
803
|
-
conversation_id,
|
|
804
|
-
sequence,
|
|
805
|
-
record_id,
|
|
806
|
-
batch_id,
|
|
807
|
-
record_json
|
|
808
|
-
FROM effect_agent_canonical_records
|
|
809
|
-
WHERE conversation_id = ${conversationId}
|
|
810
|
-
ORDER BY sequence
|
|
811
|
-
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
812
|
-
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
813
|
-
SELECT
|
|
814
|
-
conversation_id,
|
|
815
|
-
through_sequence,
|
|
816
|
-
tail_digest,
|
|
817
|
-
checkpoint_json
|
|
818
|
-
FROM effect_agent_checkpoints
|
|
819
|
-
WHERE conversation_id = ${conversationId}
|
|
820
|
-
ORDER BY through_sequence
|
|
821
|
-
`.pipe(Effect.mapError(storageError("export checkpoints")));
|
|
822
1213
|
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
"
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
)
|
|
843
|
-
|
|
1214
|
+
yield* failpoint("export:after-thread-read");
|
|
1215
|
+
|
|
1216
|
+
if (thread.tail_sequence > MAX_RECORDS_PER_THREAD)
|
|
1217
|
+
return yield* SqliteStorageError.make({
|
|
1218
|
+
operation: "export thread",
|
|
1219
|
+
message: "The thread exceeds the current export record limit.",
|
|
1220
|
+
});
|
|
1221
|
+
const records: Array<RecordRow> = [];
|
|
1222
|
+
let afterSequence = ZERO_SEQUENCE;
|
|
1223
|
+
|
|
1224
|
+
while (afterSequence < thread.tail_sequence) {
|
|
1225
|
+
const limit = Math.min(1_024, thread.tail_sequence - afterSequence);
|
|
1226
|
+
|
|
1227
|
+
const request = RawReadRequest.make({
|
|
1228
|
+
threadId,
|
|
1229
|
+
fromSequenceExclusive: afterSequence,
|
|
1230
|
+
limit,
|
|
1231
|
+
});
|
|
1232
|
+
|
|
1233
|
+
const page = yield* read(request);
|
|
1234
|
+
|
|
1235
|
+
if (
|
|
1236
|
+
page.length !== limit ||
|
|
1237
|
+
page.some((record, index) => record.sequence !== afterSequence + index + 1)
|
|
1238
|
+
) {
|
|
1239
|
+
return yield* SqliteStorageCorruptionError.make({
|
|
1240
|
+
table: "effect_agent_canonical_records",
|
|
1241
|
+
rowKey: threadId,
|
|
1242
|
+
message: "The exported canonical prefix is not contiguous through its captured tail.",
|
|
1243
|
+
});
|
|
1244
|
+
}
|
|
1245
|
+
records.push(...page);
|
|
1246
|
+
afterSequence = page[page.length - 1].sequence;
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
const beyondTail =
|
|
1250
|
+
yield* sql`SELECT sequence FROM effect_agent_canonical_records WHERE thread_id=${threadId} AND sequence > ${thread.tail_sequence} LIMIT 1`.pipe(
|
|
1251
|
+
Effect.mapError(storageError("verify export tail")),
|
|
1252
|
+
);
|
|
1253
|
+
|
|
1254
|
+
if (beyondTail.length !== 0)
|
|
1255
|
+
return yield* SqliteStorageCorruptionError.make({
|
|
1256
|
+
table: "effect_agent_canonical_records",
|
|
1257
|
+
rowKey: threadId,
|
|
1258
|
+
message: "Canonical records exist beyond the captured thread tail.",
|
|
1259
|
+
});
|
|
1260
|
+
|
|
1261
|
+
return RawThreadExport.make({ thread, records });
|
|
844
1262
|
}),
|
|
845
1263
|
);
|
|
846
1264
|
});
|
|
@@ -849,7 +1267,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
849
1267
|
checkpoint: RawCheckpoint,
|
|
850
1268
|
): Effect.fn.Return<void, CheckpointError> {
|
|
851
1269
|
if (
|
|
852
|
-
checkpoint.
|
|
1270
|
+
checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
853
1271
|
storedTextBytes(checkpoint.checkpointJson) > MAX_STORED_TEXT_BYTES
|
|
854
1272
|
) {
|
|
855
1273
|
return yield* SqliteStorageError.make({
|
|
@@ -859,50 +1277,54 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
859
1277
|
}
|
|
860
1278
|
yield* withWriteTransaction("checkpoint transaction")(
|
|
861
1279
|
Effect.gen(function* () {
|
|
862
|
-
const
|
|
1280
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
863
1281
|
SELECT
|
|
864
|
-
|
|
1282
|
+
thread_id,
|
|
865
1283
|
created_at,
|
|
866
1284
|
tail_sequence,
|
|
867
1285
|
tail_digest,
|
|
868
1286
|
producer_epoch
|
|
869
|
-
FROM
|
|
870
|
-
WHERE
|
|
1287
|
+
FROM effect_agent_threads
|
|
1288
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
871
1289
|
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
1290
|
+
|
|
1291
|
+
const thread = yield* decodeSingleRow(
|
|
1292
|
+
Schema.Array(ThreadRow),
|
|
1293
|
+
"effect_agent_threads",
|
|
1294
|
+
checkpoint.threadId,
|
|
1295
|
+
threadRows,
|
|
877
1296
|
);
|
|
878
|
-
|
|
1297
|
+
|
|
1298
|
+
if (checkpoint.throughSequence > thread.tail_sequence) {
|
|
879
1299
|
return yield* SqliteCheckpointConflict.make({
|
|
880
1300
|
message:
|
|
881
1301
|
`Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
|
|
882
|
-
`${
|
|
1302
|
+
`${thread.tail_sequence}.`,
|
|
883
1303
|
});
|
|
884
1304
|
}
|
|
885
1305
|
|
|
886
1306
|
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
887
1307
|
SELECT
|
|
888
|
-
|
|
1308
|
+
thread_id,
|
|
889
1309
|
through_sequence,
|
|
890
1310
|
tail_digest,
|
|
891
1311
|
checkpoint_json
|
|
892
1312
|
FROM effect_agent_checkpoints
|
|
893
|
-
WHERE
|
|
1313
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
894
1314
|
AND through_sequence = ${checkpoint.throughSequence}
|
|
895
1315
|
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
1316
|
+
|
|
896
1317
|
const existing = yield* decodeRows(
|
|
897
1318
|
Schema.Array(CheckpointRow),
|
|
898
1319
|
"effect_agent_checkpoints",
|
|
899
|
-
`${checkpoint.
|
|
1320
|
+
`${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
900
1321
|
checkpointRows,
|
|
901
1322
|
);
|
|
1323
|
+
|
|
902
1324
|
if (existing.length > 1) {
|
|
903
1325
|
return yield* SqliteStorageCorruptionError.make({
|
|
904
1326
|
table: "effect_agent_checkpoints",
|
|
905
|
-
rowKey: `${checkpoint.
|
|
1327
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
906
1328
|
message: "A checkpoint primary key returned more than one row.",
|
|
907
1329
|
});
|
|
908
1330
|
}
|
|
@@ -915,17 +1337,18 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
915
1337
|
message: "A different checkpoint already exists at this canonical sequence.",
|
|
916
1338
|
});
|
|
917
1339
|
}
|
|
1340
|
+
|
|
918
1341
|
return;
|
|
919
1342
|
}
|
|
920
1343
|
|
|
921
1344
|
yield* sql`
|
|
922
1345
|
INSERT INTO effect_agent_checkpoints (
|
|
923
|
-
|
|
1346
|
+
thread_id,
|
|
924
1347
|
through_sequence,
|
|
925
1348
|
tail_digest,
|
|
926
1349
|
checkpoint_json
|
|
927
1350
|
) VALUES (
|
|
928
|
-
${checkpoint.
|
|
1351
|
+
${checkpoint.threadId},
|
|
929
1352
|
${checkpoint.throughSequence},
|
|
930
1353
|
${checkpoint.tailDigest},
|
|
931
1354
|
${checkpoint.checkpointJson}
|
|
@@ -935,45 +1358,125 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
935
1358
|
);
|
|
936
1359
|
});
|
|
937
1360
|
|
|
1361
|
+
const saveRecoveryCheckpoint = Effect.fn("SqliteJournal.saveRecoveryCheckpoint")(function* (
|
|
1362
|
+
request: SaveRecoveryCheckpointRequest,
|
|
1363
|
+
checkpointJson: string,
|
|
1364
|
+
) {
|
|
1365
|
+
const { checkpoint } = request;
|
|
1366
|
+
|
|
1367
|
+
if (
|
|
1368
|
+
checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
1369
|
+
storedTextBytes(checkpointJson) > MAX_STORED_TEXT_BYTES
|
|
1370
|
+
) {
|
|
1371
|
+
return yield* SqliteStorageError.make({
|
|
1372
|
+
operation: "save recovery checkpoint",
|
|
1373
|
+
message: "Checkpoint identity or encoded JSON exceeds the SQLite storage bounds.",
|
|
1374
|
+
});
|
|
1375
|
+
}
|
|
1376
|
+
yield* withWriteTransaction("recovery checkpoint transaction")(
|
|
1377
|
+
Effect.gen(function* () {
|
|
1378
|
+
const threads = yield* getThread(checkpoint.threadId);
|
|
1379
|
+
const thread = threads[0];
|
|
1380
|
+
|
|
1381
|
+
if (thread === undefined)
|
|
1382
|
+
return yield* ThreadNotMaterialized.make({ threadId: checkpoint.threadId });
|
|
1383
|
+
if (request.producerEpoch !== thread.producer_epoch)
|
|
1384
|
+
return yield* FenceRejected.make({
|
|
1385
|
+
threadId: checkpoint.threadId,
|
|
1386
|
+
actualEpoch: thread.producer_epoch,
|
|
1387
|
+
attemptedEpoch: request.producerEpoch,
|
|
1388
|
+
});
|
|
1389
|
+
if (checkpoint.throughSequence > thread.tail_sequence)
|
|
1390
|
+
return yield* CheckpointRejected.make({
|
|
1391
|
+
threadId: checkpoint.threadId,
|
|
1392
|
+
reason: "ahead-of-tail",
|
|
1393
|
+
});
|
|
1394
|
+
|
|
1395
|
+
const digests =
|
|
1396
|
+
checkpoint.throughSequence === 0
|
|
1397
|
+
? [EMPTY_TAIL_DIGEST]
|
|
1398
|
+
: yield* getTailDigestAt(checkpoint.threadId, checkpoint.throughSequence);
|
|
1399
|
+
|
|
1400
|
+
if (digests.length !== 1 || digests[0] !== checkpoint.tailDigest)
|
|
1401
|
+
return yield* CheckpointRejected.make({
|
|
1402
|
+
threadId: checkpoint.threadId,
|
|
1403
|
+
reason: "digest-mismatch",
|
|
1404
|
+
});
|
|
1405
|
+
|
|
1406
|
+
yield* failpoint("save-recovery-checkpoint:before");
|
|
1407
|
+
yield* sql`
|
|
1408
|
+
INSERT INTO effect_agent_recovery_checkpoints (thread_id, through_sequence, tail_digest, checkpoint_json)
|
|
1409
|
+
VALUES (${checkpoint.threadId}, ${checkpoint.throughSequence}, ${checkpoint.tailDigest}, ${checkpointJson})
|
|
1410
|
+
ON CONFLICT (thread_id) DO UPDATE SET
|
|
1411
|
+
through_sequence = excluded.through_sequence,
|
|
1412
|
+
tail_digest = excluded.tail_digest,
|
|
1413
|
+
checkpoint_json = excluded.checkpoint_json
|
|
1414
|
+
WHERE excluded.through_sequence >= effect_agent_recovery_checkpoints.through_sequence
|
|
1415
|
+
`.pipe(Effect.mapError(storageError("save recovery checkpoint")));
|
|
1416
|
+
}),
|
|
1417
|
+
);
|
|
1418
|
+
yield* failpoint("save-recovery-checkpoint:after");
|
|
1419
|
+
});
|
|
1420
|
+
|
|
1421
|
+
const loadRecoveryCheckpoint = Effect.fn("SqliteJournal.loadRecoveryCheckpoint")(function* (
|
|
1422
|
+
threadId: string,
|
|
1423
|
+
) {
|
|
1424
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
1425
|
+
SELECT thread_id, through_sequence, tail_digest, checkpoint_json
|
|
1426
|
+
FROM effect_agent_recovery_checkpoints
|
|
1427
|
+
WHERE thread_id = ${threadId}
|
|
1428
|
+
`.pipe(Effect.mapError(storageError("load recovery checkpoint")));
|
|
1429
|
+
|
|
1430
|
+
return yield* decodeRows(
|
|
1431
|
+
Schema.Array(CheckpointRow),
|
|
1432
|
+
"effect_agent_recovery_checkpoints",
|
|
1433
|
+
threadId,
|
|
1434
|
+
rows,
|
|
1435
|
+
);
|
|
1436
|
+
});
|
|
1437
|
+
|
|
938
1438
|
const loadCheckpoint = Effect.fn("SqliteJournal.loadCheckpoint")(function* (
|
|
939
|
-
|
|
1439
|
+
threadId: string,
|
|
940
1440
|
atOrBeforeSequence: CanonicalSequence,
|
|
941
1441
|
) {
|
|
942
1442
|
const rows = yield* sql<Record<string, unknown>>`
|
|
943
1443
|
SELECT
|
|
944
|
-
|
|
1444
|
+
thread_id,
|
|
945
1445
|
through_sequence,
|
|
946
1446
|
tail_digest,
|
|
947
1447
|
checkpoint_json
|
|
948
1448
|
FROM effect_agent_checkpoints
|
|
949
|
-
WHERE
|
|
1449
|
+
WHERE thread_id = ${threadId}
|
|
950
1450
|
AND through_sequence <= ${atOrBeforeSequence}
|
|
951
1451
|
ORDER BY through_sequence DESC
|
|
952
1452
|
LIMIT 1
|
|
953
1453
|
`.pipe(Effect.mapError(storageError("load checkpoint")));
|
|
1454
|
+
|
|
954
1455
|
return yield* decodeRows(
|
|
955
1456
|
Schema.Array(CheckpointRow),
|
|
956
1457
|
"effect_agent_checkpoints",
|
|
957
|
-
`${
|
|
1458
|
+
`${threadId}<=${atOrBeforeSequence}`,
|
|
958
1459
|
rows,
|
|
959
1460
|
);
|
|
960
1461
|
});
|
|
961
1462
|
|
|
962
1463
|
const getTailDigestAt = Effect.fn("SqliteJournal.getTailDigestAt")(function* (
|
|
963
|
-
|
|
1464
|
+
threadId: string,
|
|
964
1465
|
sequence: CanonicalSequence,
|
|
965
1466
|
) {
|
|
966
1467
|
if (sequence === 0) {
|
|
967
|
-
const
|
|
968
|
-
|
|
1468
|
+
const threads = yield* getThread(threadId);
|
|
1469
|
+
|
|
1470
|
+
return threads.length === 0
|
|
969
1471
|
? []
|
|
970
|
-
: [
|
|
1472
|
+
: [threads[0].tail_sequence === 0 ? threads[0].tail_digest : undefined].filter(
|
|
971
1473
|
(value): value is string => value !== undefined,
|
|
972
1474
|
);
|
|
973
1475
|
}
|
|
1476
|
+
|
|
974
1477
|
const rows = yield* sql<Record<string, unknown>>`
|
|
975
1478
|
SELECT
|
|
976
|
-
|
|
1479
|
+
thread_id,
|
|
977
1480
|
batch_id,
|
|
978
1481
|
first_sequence,
|
|
979
1482
|
last_sequence,
|
|
@@ -981,34 +1484,37 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
981
1484
|
tail_digest,
|
|
982
1485
|
batch_json
|
|
983
1486
|
FROM effect_agent_canonical_batches
|
|
984
|
-
WHERE
|
|
1487
|
+
WHERE thread_id = ${threadId}
|
|
985
1488
|
AND last_sequence = ${sequence}
|
|
986
1489
|
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
1490
|
+
|
|
987
1491
|
const batches = yield* decodeRows(
|
|
988
1492
|
Schema.Array(BatchRow),
|
|
989
1493
|
"effect_agent_canonical_batches",
|
|
990
|
-
`${
|
|
1494
|
+
`${threadId}/${sequence}`,
|
|
991
1495
|
rows,
|
|
992
1496
|
);
|
|
1497
|
+
|
|
993
1498
|
return batches.map((batch) => batch.tail_digest);
|
|
994
1499
|
});
|
|
995
1500
|
|
|
996
1501
|
const scanStoredPayloads = Effect.fn("SqliteJournal.scanStoredPayloads")(function* () {
|
|
997
1502
|
return yield* withReadTransaction("startup scan transaction")(
|
|
998
1503
|
Effect.gen(function* () {
|
|
999
|
-
const
|
|
1504
|
+
const threads = yield* sql<Record<string, unknown>>`
|
|
1000
1505
|
SELECT
|
|
1001
|
-
|
|
1506
|
+
thread_id,
|
|
1002
1507
|
created_at,
|
|
1003
1508
|
tail_sequence,
|
|
1004
1509
|
tail_digest,
|
|
1005
1510
|
producer_epoch
|
|
1006
|
-
FROM
|
|
1007
|
-
ORDER BY
|
|
1008
|
-
`.pipe(Effect.mapError(storageError("scan
|
|
1511
|
+
FROM effect_agent_threads
|
|
1512
|
+
ORDER BY thread_id
|
|
1513
|
+
`.pipe(Effect.mapError(storageError("scan threads")));
|
|
1514
|
+
|
|
1009
1515
|
const batches = yield* sql<Record<string, unknown>>`
|
|
1010
1516
|
SELECT
|
|
1011
|
-
|
|
1517
|
+
thread_id,
|
|
1012
1518
|
batch_id,
|
|
1013
1519
|
first_sequence,
|
|
1014
1520
|
last_sequence,
|
|
@@ -1016,33 +1522,36 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
1016
1522
|
tail_digest,
|
|
1017
1523
|
batch_json
|
|
1018
1524
|
FROM effect_agent_canonical_batches
|
|
1019
|
-
ORDER BY
|
|
1525
|
+
ORDER BY thread_id, first_sequence
|
|
1020
1526
|
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
1527
|
+
|
|
1021
1528
|
const records = yield* sql<Record<string, unknown>>`
|
|
1022
1529
|
SELECT
|
|
1023
|
-
|
|
1530
|
+
thread_id,
|
|
1024
1531
|
sequence,
|
|
1025
1532
|
record_id,
|
|
1026
1533
|
batch_id,
|
|
1027
1534
|
record_json
|
|
1028
1535
|
FROM effect_agent_canonical_records
|
|
1029
|
-
ORDER BY
|
|
1536
|
+
ORDER BY thread_id, sequence
|
|
1030
1537
|
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
1538
|
+
|
|
1031
1539
|
const checkpoints = yield* sql<Record<string, unknown>>`
|
|
1032
1540
|
SELECT
|
|
1033
|
-
|
|
1541
|
+
thread_id,
|
|
1034
1542
|
through_sequence,
|
|
1035
1543
|
tail_digest,
|
|
1036
1544
|
checkpoint_json
|
|
1037
1545
|
FROM effect_agent_checkpoints
|
|
1038
|
-
ORDER BY
|
|
1546
|
+
ORDER BY thread_id, through_sequence
|
|
1039
1547
|
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
1548
|
+
|
|
1040
1549
|
return {
|
|
1041
|
-
|
|
1042
|
-
Schema.Array(
|
|
1043
|
-
"
|
|
1550
|
+
threads: yield* decodeRows(
|
|
1551
|
+
Schema.Array(ThreadRow),
|
|
1552
|
+
"effect_agent_threads",
|
|
1044
1553
|
"startup_scan",
|
|
1045
|
-
|
|
1554
|
+
threads,
|
|
1046
1555
|
),
|
|
1047
1556
|
batches: yield* decodeRows(
|
|
1048
1557
|
Schema.Array(BatchRow),
|
|
@@ -1069,18 +1578,19 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
1069
1578
|
|
|
1070
1579
|
return {
|
|
1071
1580
|
append,
|
|
1072
|
-
|
|
1073
|
-
|
|
1581
|
+
exportThread,
|
|
1582
|
+
getThread,
|
|
1074
1583
|
getTailDigestAt,
|
|
1075
1584
|
loadCheckpoint,
|
|
1585
|
+
loadRecoveryCheckpoint,
|
|
1586
|
+
saveRecoveryCheckpoint,
|
|
1076
1587
|
materialize,
|
|
1077
1588
|
read,
|
|
1078
1589
|
saveCheckpoint,
|
|
1079
1590
|
scanStoredPayloads,
|
|
1080
1591
|
withWriteTransaction,
|
|
1592
|
+
withReadTransaction,
|
|
1081
1593
|
} as const;
|
|
1082
|
-
};
|
|
1083
|
-
|
|
1084
|
-
export type SqliteJournal = ReturnType<typeof makeJournal>;
|
|
1594
|
+
});
|
|
1085
1595
|
|
|
1086
|
-
export
|
|
1596
|
+
export type SqliteJournal = Effect.Success<ReturnType<typeof initializeSqliteJournal>>;
|