@effect-agent/storage-sqlite 0.1.0-beta.8 → 0.1.0-beta.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/SqliteActivityStore.d.mts +14 -0
- package/dist/SqliteActivityStore.mjs +310 -0
- package/dist/SqliteActivityStore.mjs.map +1 -0
- package/dist/SqliteMessageDeliveryStore.d.mts +14 -0
- package/dist/SqliteMessageDeliveryStore.mjs +24 -0
- package/dist/SqliteMessageDeliveryStore.mjs.map +1 -0
- package/dist/SqliteScheduleStore.d.mts +14 -0
- package/dist/SqliteScheduleStore.mjs +255 -0
- package/dist/SqliteScheduleStore.mjs.map +1 -0
- package/dist/SqliteStorageConfig.d.mts +34 -0
- package/dist/SqliteStorageConfig.mjs +39 -0
- package/dist/SqliteStorageConfig.mjs.map +1 -0
- package/dist/SqliteStorageError-DVWeaWLm.d.mts +86 -0
- package/dist/SqliteStorageError.d.mts +2 -0
- package/dist/SqliteStorageError.mjs +150 -0
- package/dist/SqliteStorageError.mjs.map +1 -0
- package/dist/SqliteStorageFailpoint.d.mts +17 -0
- package/dist/SqliteStorageFailpoint.mjs +14 -0
- package/dist/SqliteStorageFailpoint.mjs.map +1 -0
- package/dist/SqliteStorageFailpointTesting.d.mts +15 -0
- package/dist/SqliteStorageFailpointTesting.mjs +19 -0
- package/dist/SqliteStorageFailpointTesting.mjs.map +1 -0
- package/dist/SqliteStorageVersion-BqmqX0CI.d.mts +11 -0
- package/dist/SqliteStorageVersion.d.mts +2 -0
- package/dist/SqliteStorageVersion.mjs +8 -0
- package/dist/SqliteStorageVersion.mjs.map +1 -0
- package/dist/SqliteSubmissionLedger.d.mts +23 -0
- package/dist/SqliteSubmissionLedger.mjs +1788 -0
- package/dist/SqliteSubmissionLedger.mjs.map +1 -0
- package/dist/SqliteSubscriptionStore.d.mts +13 -0
- package/dist/SqliteSubscriptionStore.mjs +28 -0
- package/dist/SqliteSubscriptionStore.mjs.map +1 -0
- package/dist/SqliteThreadStore.d.mts +49 -0
- package/dist/SqliteThreadStore.mjs +460 -0
- package/dist/SqliteThreadStore.mjs.map +1 -0
- package/dist/index.d.mts +11 -193
- package/dist/index.mjs +11 -3082
- package/dist/migrations-Dy5v0HGe.mjs +346 -0
- package/dist/migrations-Dy5v0HGe.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/dist/sqlite-journal-D9xFxCw-.mjs +998 -0
- package/dist/sqlite-journal-D9xFxCw-.mjs.map +1 -0
- package/package.json +1 -41
- package/src/SqliteActivityStore.ts +556 -0
- package/src/SqliteMessageDeliveryStore.ts +42 -0
- package/src/SqliteScheduleStore.ts +404 -0
- package/src/{sqlite-storage-config.ts → SqliteStorageConfig.ts} +1 -1
- package/src/{errors.ts → SqliteStorageError.ts} +10 -3
- package/src/SqliteStorageFailpoint.ts +23 -0
- package/src/{sqlite-storage-failpoint.ts → SqliteStorageFailpointTesting.ts} +7 -18
- package/src/SqliteStorageVersion.ts +2 -0
- package/src/{sqlite-ledger.ts → SqliteSubmissionLedger.ts} +857 -157
- package/src/SqliteSubscriptionStore.ts +59 -0
- package/src/{sqlite-conversation-store.ts → SqliteThreadStore.ts} +448 -288
- package/src/index.ts +10 -6
- package/src/internal/message-delivery-schema.ts +24 -0
- package/src/{migrations.ts → internal/migrations.ts} +156 -79
- package/src/internal/recovery-checkpoint-schema.ts +17 -0
- package/src/{sqlite-journal.ts → internal/sqlite-journal.ts} +752 -239
- package/dist/index.mjs.map +0 -1
|
@@ -1,26 +1,54 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { EMPTY_TAIL_DIGEST } from "@effect-agent/thread/Digest";
|
|
2
|
+
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread/Records";
|
|
3
|
+
import { ScheduleFailpoint, ScheduleFailpointError } from "@effect-agent/thread/Schedule";
|
|
4
|
+
import {
|
|
5
|
+
checkV2ThreadLayout,
|
|
6
|
+
upgradeV2Schedules,
|
|
7
|
+
upgradeV2Subscriptions,
|
|
8
|
+
} from "@effect-agent/thread/SqlStorageV2Upgrade";
|
|
9
|
+
import {
|
|
10
|
+
SubscriptionFailpoint,
|
|
11
|
+
SubscriptionFailpointError,
|
|
12
|
+
} from "@effect-agent/thread/Subscription";
|
|
13
|
+
import {
|
|
14
|
+
MAX_THREAD_EXPORT_RECORDS,
|
|
15
|
+
CheckpointRejected,
|
|
16
|
+
FenceRejected,
|
|
17
|
+
ThreadNotMaterialized,
|
|
18
|
+
type SaveRecoveryCheckpointRequest,
|
|
19
|
+
} from "@effect-agent/thread/ThreadStore";
|
|
20
|
+
import { NodeCrypto } from "@effect/platform-node";
|
|
2
21
|
import { SqliteMigrator } from "@effect/sql-sqlite-node";
|
|
3
22
|
import { Effect, Exit, Schema } from "effect";
|
|
4
23
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
5
24
|
import type { SqlError } from "effect/unstable/sql/SqlError";
|
|
6
25
|
|
|
26
|
+
import { SqliteStorageConfig } from "../SqliteStorageConfig.ts";
|
|
27
|
+
import type { SqliteStorageFailpointError } from "../SqliteStorageError.ts";
|
|
7
28
|
import {
|
|
8
29
|
SqliteAppendConflict,
|
|
9
30
|
SqliteCheckpointConflict,
|
|
10
31
|
SqliteFenceRejected,
|
|
11
32
|
SqliteStorageCompatibilityError,
|
|
33
|
+
SqliteStorageFailpointLocation,
|
|
12
34
|
SqliteStorageCorruptionError,
|
|
13
35
|
SqliteStorageError,
|
|
14
|
-
SqliteStorageFailpointError,
|
|
15
36
|
SqliteWriteContention,
|
|
16
|
-
|
|
17
|
-
} from "
|
|
18
|
-
import {
|
|
37
|
+
} from "../SqliteStorageError.ts";
|
|
38
|
+
import { SqliteStorageFailpoint } from "../SqliteStorageFailpoint.ts";
|
|
39
|
+
import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
|
|
40
|
+
import {
|
|
41
|
+
CurrentSqliteStorageVersion,
|
|
42
|
+
createNonterminalIndex,
|
|
43
|
+
sqliteMigrations,
|
|
44
|
+
} from "./migrations.ts";
|
|
45
|
+
import { createRecoveryCheckpointTable } from "./recovery-checkpoint-schema.ts";
|
|
19
46
|
|
|
20
47
|
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(16 * 1024 * 1024));
|
|
21
48
|
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
22
49
|
const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0));
|
|
23
|
-
const
|
|
50
|
+
const MAX_RECORDS_PER_THREAD = MAX_THREAD_EXPORT_RECORDS;
|
|
51
|
+
const ZERO_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
24
52
|
const MAX_STORED_TEXT_BYTES = 16 * 1024 * 1024;
|
|
25
53
|
const MAX_IDENTIFIER_LENGTH = 1_024;
|
|
26
54
|
|
|
@@ -38,8 +66,8 @@ class SqliteNameRow extends Schema.Class<SqliteNameRow>("SqliteNameRow")({
|
|
|
38
66
|
name: BoundedIdentifier,
|
|
39
67
|
}) {}
|
|
40
68
|
|
|
41
|
-
class
|
|
42
|
-
|
|
69
|
+
class ThreadRow extends Schema.Class<ThreadRow>("ThreadRow")({
|
|
70
|
+
thread_id: BoundedIdentifier,
|
|
43
71
|
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
44
72
|
producer_epoch: ProducerEpoch,
|
|
45
73
|
tail_digest: BoundedStoredText,
|
|
@@ -50,7 +78,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
50
78
|
batch_digest: BoundedStoredText,
|
|
51
79
|
batch_id: BoundedIdentifier,
|
|
52
80
|
batch_json: BoundedStoredText,
|
|
53
|
-
|
|
81
|
+
thread_id: BoundedIdentifier,
|
|
54
82
|
first_sequence: CanonicalSequence,
|
|
55
83
|
last_sequence: CanonicalSequence,
|
|
56
84
|
tail_digest: BoundedStoredText,
|
|
@@ -58,7 +86,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
58
86
|
|
|
59
87
|
class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
60
88
|
batch_id: BoundedIdentifier,
|
|
61
|
-
|
|
89
|
+
thread_id: BoundedIdentifier,
|
|
62
90
|
record_id: BoundedIdentifier,
|
|
63
91
|
record_json: BoundedStoredText,
|
|
64
92
|
sequence: CanonicalSequence,
|
|
@@ -66,7 +94,7 @@ class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
|
66
94
|
|
|
67
95
|
class CheckpointRow extends Schema.Class<CheckpointRow>("CheckpointRow")({
|
|
68
96
|
checkpoint_json: BoundedStoredText,
|
|
69
|
-
|
|
97
|
+
thread_id: BoundedIdentifier,
|
|
70
98
|
tail_digest: BoundedStoredText,
|
|
71
99
|
through_sequence: CanonicalSequence,
|
|
72
100
|
}) {}
|
|
@@ -82,7 +110,7 @@ export class RawAppendRequest extends Schema.Class<RawAppendRequest>(
|
|
|
82
110
|
batchDigest: BoundedStoredText,
|
|
83
111
|
batchId: BoundedIdentifier,
|
|
84
112
|
batchJson: BoundedStoredText,
|
|
85
|
-
|
|
113
|
+
threadId: BoundedIdentifier,
|
|
86
114
|
expectedTailDigest: BoundedStoredText,
|
|
87
115
|
expectedTailSequence: CanonicalSequence,
|
|
88
116
|
producerEpoch: ProducerEpoch,
|
|
@@ -102,7 +130,7 @@ export class RawAppendResult extends Schema.Class<RawAppendResult>(
|
|
|
102
130
|
export class RawReadRequest extends Schema.Class<RawReadRequest>(
|
|
103
131
|
"@effect-agent/storage-sqlite/RawReadRequest",
|
|
104
132
|
)({
|
|
105
|
-
|
|
133
|
+
threadId: BoundedIdentifier,
|
|
106
134
|
fromSequenceExclusive: CanonicalSequence,
|
|
107
135
|
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
|
|
108
136
|
}) {}
|
|
@@ -111,17 +139,15 @@ export class RawCheckpoint extends Schema.Class<RawCheckpoint>(
|
|
|
111
139
|
"@effect-agent/storage-sqlite/RawCheckpoint",
|
|
112
140
|
)({
|
|
113
141
|
checkpointJson: BoundedStoredText,
|
|
114
|
-
|
|
142
|
+
threadId: BoundedIdentifier,
|
|
115
143
|
tailDigest: BoundedStoredText,
|
|
116
144
|
throughSequence: CanonicalSequence,
|
|
117
145
|
}) {}
|
|
118
146
|
|
|
119
|
-
export class
|
|
120
|
-
"@effect-agent/storage-sqlite/
|
|
147
|
+
export class RawThreadExport extends Schema.Class<RawThreadExport>(
|
|
148
|
+
"@effect-agent/storage-sqlite/RawThreadExport",
|
|
121
149
|
)({
|
|
122
|
-
|
|
123
|
-
checkpoints: Schema.Array(CheckpointRow),
|
|
124
|
-
conversation: ConversationRow,
|
|
150
|
+
thread: ThreadRow,
|
|
125
151
|
records: Schema.Array(RecordRow),
|
|
126
152
|
}) {}
|
|
127
153
|
|
|
@@ -138,11 +164,6 @@ type CheckpointError =
|
|
|
138
164
|
| SqliteStorageCorruptionError
|
|
139
165
|
| SqliteStorageError
|
|
140
166
|
| SqliteWriteContention;
|
|
141
|
-
type SqliteJournalFailpoint = (
|
|
142
|
-
location: SqliteStorageFailpointLocation,
|
|
143
|
-
) => Effect.Effect<void, SqliteStorageFailpointError>;
|
|
144
|
-
|
|
145
|
-
const noFailpoint: SqliteJournalFailpoint = () => Effect.void;
|
|
146
167
|
|
|
147
168
|
const storageError =
|
|
148
169
|
(operation: string) =>
|
|
@@ -195,26 +216,255 @@ export const decodeSingleRow = Effect.fn("SqliteJournal.decodeSingleRow")(
|
|
|
195
216
|
),
|
|
196
217
|
);
|
|
197
218
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
219
|
+
/** Column inventory of the supported v8 predecessor, independent of physical column order. */
|
|
220
|
+
const predecessorColumns = {
|
|
221
|
+
effect_agent_threads: [
|
|
222
|
+
"thread_id",
|
|
223
|
+
"created_at",
|
|
224
|
+
"tail_sequence",
|
|
225
|
+
"tail_digest",
|
|
226
|
+
"producer_epoch",
|
|
227
|
+
],
|
|
228
|
+
effect_agent_canonical_batches: [
|
|
229
|
+
"thread_id",
|
|
230
|
+
"batch_id",
|
|
231
|
+
"first_sequence",
|
|
232
|
+
"last_sequence",
|
|
233
|
+
"batch_digest",
|
|
234
|
+
"tail_digest",
|
|
235
|
+
"batch_json",
|
|
236
|
+
],
|
|
237
|
+
effect_agent_canonical_records: ["thread_id", "sequence", "record_id", "batch_id", "record_json"],
|
|
238
|
+
effect_agent_checkpoints: ["thread_id", "through_sequence", "tail_digest", "checkpoint_json"],
|
|
239
|
+
effect_agent_submissions: [
|
|
240
|
+
"submission_id",
|
|
241
|
+
"thread_id",
|
|
242
|
+
"queue_sequence",
|
|
243
|
+
"principal",
|
|
244
|
+
"idempotency_key",
|
|
245
|
+
"agent_id",
|
|
246
|
+
"agent_digests_json",
|
|
247
|
+
"deployment_id",
|
|
248
|
+
"input_json",
|
|
249
|
+
"input_digest",
|
|
250
|
+
"receipt_id",
|
|
251
|
+
"state",
|
|
252
|
+
"settled_outcome",
|
|
253
|
+
"created_at",
|
|
254
|
+
"ready_at",
|
|
255
|
+
"input_applied_record_id",
|
|
256
|
+
"input_applied_sequence",
|
|
257
|
+
"joined_host_submission_id",
|
|
258
|
+
"suspended_reason_json",
|
|
259
|
+
"suspended_at",
|
|
260
|
+
"unknown_reason",
|
|
261
|
+
"unknown_tool_call_ids_json",
|
|
262
|
+
"parent_submission_id",
|
|
263
|
+
"parent_tool_call_id",
|
|
264
|
+
"admission_group",
|
|
265
|
+
"admission_fence_json",
|
|
266
|
+
],
|
|
267
|
+
effect_agent_submission_ownership: [
|
|
268
|
+
"submission_id",
|
|
269
|
+
"attempt_id",
|
|
270
|
+
"ownership_token",
|
|
271
|
+
"producer_epoch",
|
|
272
|
+
"owner_producer_id",
|
|
273
|
+
"lease_expires_at",
|
|
274
|
+
],
|
|
275
|
+
effect_agent_attempts: [
|
|
276
|
+
"attempt_id",
|
|
277
|
+
"submission_id",
|
|
278
|
+
"thread_id",
|
|
279
|
+
"owner_producer_id",
|
|
280
|
+
"producer_epoch",
|
|
281
|
+
"claimed_at",
|
|
282
|
+
],
|
|
283
|
+
effect_agent_settlement_reservations: [
|
|
284
|
+
"submission_id",
|
|
285
|
+
"settlement_id",
|
|
286
|
+
"outcome",
|
|
287
|
+
"record_id",
|
|
288
|
+
"record_json",
|
|
289
|
+
"record_digest",
|
|
290
|
+
"reserved_at",
|
|
291
|
+
"finalized_at",
|
|
292
|
+
],
|
|
293
|
+
effect_agent_abort_intents: [
|
|
294
|
+
"submission_id",
|
|
295
|
+
"author",
|
|
296
|
+
"reason",
|
|
297
|
+
"requested_at",
|
|
298
|
+
"canonical_record_id",
|
|
299
|
+
],
|
|
300
|
+
effect_agent_approval_decisions: [
|
|
301
|
+
"submission_id",
|
|
302
|
+
"tool_call_id",
|
|
303
|
+
"decision",
|
|
304
|
+
"resolver",
|
|
305
|
+
"reason",
|
|
306
|
+
"decided_at",
|
|
307
|
+
],
|
|
308
|
+
effect_agent_unknown_resolutions: [
|
|
309
|
+
"submission_id",
|
|
310
|
+
"tool_call_id",
|
|
311
|
+
"author",
|
|
312
|
+
"reason",
|
|
313
|
+
"resolution_json",
|
|
314
|
+
"resolved_at",
|
|
315
|
+
],
|
|
316
|
+
effect_agent_child_reservations: [
|
|
317
|
+
"reservation_id",
|
|
318
|
+
"parent_submission_id",
|
|
319
|
+
"parent_tool_call_id",
|
|
320
|
+
"child_submission_id",
|
|
321
|
+
"status",
|
|
322
|
+
"allocation_json",
|
|
323
|
+
"allocation_digest",
|
|
324
|
+
"accounting_json",
|
|
325
|
+
"reserved_at",
|
|
326
|
+
"release_began_at",
|
|
327
|
+
"released_at",
|
|
328
|
+
],
|
|
329
|
+
effect_agent_schedules: [
|
|
330
|
+
"tenant_id",
|
|
331
|
+
"owner_id",
|
|
332
|
+
"schedule_id",
|
|
333
|
+
"deadline_at_millis",
|
|
334
|
+
"record_json",
|
|
335
|
+
],
|
|
336
|
+
effect_agent_subscription_sequences: [
|
|
337
|
+
"tenant_id",
|
|
338
|
+
"source_address",
|
|
339
|
+
"sequence",
|
|
340
|
+
"event_scan_cursor",
|
|
341
|
+
"delivery_scan_cursor",
|
|
342
|
+
"recovery_scan_cursor",
|
|
343
|
+
],
|
|
344
|
+
effect_agent_subscriptions: [
|
|
345
|
+
"tenant_id",
|
|
346
|
+
"source_address",
|
|
347
|
+
"owner_id",
|
|
348
|
+
"subscription_id",
|
|
349
|
+
"ordinal",
|
|
350
|
+
"source_name",
|
|
351
|
+
"source_version",
|
|
352
|
+
"matching_key",
|
|
353
|
+
"state",
|
|
354
|
+
"expires_at_millis",
|
|
355
|
+
"recovery_at_millis",
|
|
356
|
+
"recovery_present",
|
|
357
|
+
"record_json",
|
|
358
|
+
],
|
|
359
|
+
effect_agent_subscription_events: [
|
|
360
|
+
"tenant_id",
|
|
361
|
+
"source_address",
|
|
362
|
+
"event_id",
|
|
363
|
+
"source_name",
|
|
364
|
+
"source_version",
|
|
365
|
+
"matching_key",
|
|
366
|
+
"payload_digest",
|
|
367
|
+
"cutoff",
|
|
368
|
+
"cursor",
|
|
369
|
+
"routing_complete",
|
|
370
|
+
"next_attempt_at_millis",
|
|
371
|
+
"record_json",
|
|
372
|
+
"tombstone",
|
|
373
|
+
],
|
|
374
|
+
effect_agent_subscription_deliveries: [
|
|
375
|
+
"tenant_id",
|
|
376
|
+
"source_address",
|
|
377
|
+
"owner_id",
|
|
378
|
+
"subscription_id",
|
|
379
|
+
"event_id",
|
|
380
|
+
"delivery_key",
|
|
381
|
+
"state",
|
|
382
|
+
"next_attempt_at_millis",
|
|
383
|
+
"record_json",
|
|
384
|
+
],
|
|
385
|
+
} as const;
|
|
386
|
+
|
|
387
|
+
const checkPredecessorLayout = Effect.fn("SqliteJournal.checkPredecessorLayout")(function* (
|
|
388
|
+
version: 8 | 9 | 10,
|
|
202
389
|
) {
|
|
390
|
+
const sql = yield* SqlClient.SqlClient;
|
|
391
|
+
|
|
392
|
+
const messageColumns =
|
|
393
|
+
version === 8
|
|
394
|
+
? predecessorColumns
|
|
395
|
+
: {
|
|
396
|
+
...predecessorColumns,
|
|
397
|
+
effect_agent_submissions: [
|
|
398
|
+
...predecessorColumns.effect_agent_submissions,
|
|
399
|
+
"worker_admission_json",
|
|
400
|
+
"message_admission_json",
|
|
401
|
+
],
|
|
402
|
+
effect_agent_message_deliveries: [
|
|
403
|
+
"owner_thread_id",
|
|
404
|
+
"message_id",
|
|
405
|
+
"version",
|
|
406
|
+
"state",
|
|
407
|
+
"deadline_at_millis",
|
|
408
|
+
"record_json",
|
|
409
|
+
],
|
|
410
|
+
};
|
|
411
|
+
|
|
412
|
+
const expectedColumns = {
|
|
413
|
+
...messageColumns,
|
|
414
|
+
...(version === 10
|
|
415
|
+
? {
|
|
416
|
+
effect_agent_recovery_checkpoints: [
|
|
417
|
+
"thread_id",
|
|
418
|
+
"through_sequence",
|
|
419
|
+
"tail_digest",
|
|
420
|
+
"checkpoint_json",
|
|
421
|
+
],
|
|
422
|
+
}
|
|
423
|
+
: {}),
|
|
424
|
+
};
|
|
425
|
+
|
|
426
|
+
for (const [table, expected] of Object.entries(expectedColumns)) {
|
|
427
|
+
const columns = yield* decodeRows(
|
|
428
|
+
Schema.Array(Schema.Struct({ name: BoundedIdentifier })),
|
|
429
|
+
table,
|
|
430
|
+
"schema",
|
|
431
|
+
yield* sql.unsafe(`PRAGMA table_info(${table})`),
|
|
432
|
+
);
|
|
433
|
+
|
|
434
|
+
const names = new Set<string>(expected);
|
|
435
|
+
|
|
436
|
+
if (columns.length !== names.size || columns.some((column) => !names.has(column.name)))
|
|
437
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
438
|
+
actualVersion: version,
|
|
439
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
440
|
+
message: `The v${version} ${table} columns do not match the supported predecessor; no upgrade was committed.`,
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function* () {
|
|
446
|
+
const sql = yield* SqlClient.SqlClient;
|
|
447
|
+
const { hit: failpoint } = yield* SqliteStorageFailpoint;
|
|
448
|
+
const { busyTimeout } = yield* SqliteStorageConfig;
|
|
449
|
+
|
|
203
450
|
yield* sql`PRAGMA foreign_keys = ON`.pipe(Effect.mapError(storageError("enable foreign keys")));
|
|
204
451
|
// PRAGMA statements do not accept bound parameters; the value is a schema-validated
|
|
205
452
|
// non-negative integer, never caller-controlled text.
|
|
206
453
|
yield* sql
|
|
207
|
-
.unsafe(`PRAGMA busy_timeout = ${
|
|
454
|
+
.unsafe(`PRAGMA busy_timeout = ${busyTimeout}`)
|
|
208
455
|
.pipe(Effect.mapError(storageError("configure busy timeout")));
|
|
456
|
+
|
|
209
457
|
const journalModeRows = yield* sql<Record<string, unknown>>`PRAGMA journal_mode`.pipe(
|
|
210
458
|
Effect.mapError(storageError("read journal mode")),
|
|
211
459
|
);
|
|
460
|
+
|
|
212
461
|
const journalMode = yield* decodeSingleRow(
|
|
213
462
|
Schema.Array(SqliteJournalModeRow),
|
|
214
463
|
"pragma_journal_mode",
|
|
215
464
|
"singleton",
|
|
216
465
|
journalModeRows,
|
|
217
466
|
);
|
|
467
|
+
|
|
218
468
|
if (journalMode.journal_mode.toLowerCase() !== "wal") {
|
|
219
469
|
return yield* SqliteStorageCompatibilityError.make({
|
|
220
470
|
actualVersion: 0,
|
|
@@ -226,6 +476,7 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
226
476
|
const versionRows = yield* sql<Record<string, unknown>>`PRAGMA user_version`.pipe(
|
|
227
477
|
Effect.mapError(storageError("read storage version")),
|
|
228
478
|
);
|
|
479
|
+
|
|
229
480
|
const version = yield* decodeSingleRow(
|
|
230
481
|
Schema.Array(SqliteVersionRow),
|
|
231
482
|
"pragma_user_version",
|
|
@@ -233,20 +484,183 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
233
484
|
versionRows,
|
|
234
485
|
);
|
|
235
486
|
|
|
236
|
-
//
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
487
|
+
// Support the known beta49/beta50 and immediate predecessor formats atomically.
|
|
488
|
+
if (
|
|
489
|
+
version.user_version !== 0 &&
|
|
490
|
+
version.user_version !== 7 &&
|
|
491
|
+
version.user_version !== 8 &&
|
|
492
|
+
version.user_version !== 9 &&
|
|
493
|
+
version.user_version !== 10 &&
|
|
494
|
+
version.user_version !== CurrentSqliteStorageVersion
|
|
495
|
+
) {
|
|
240
496
|
return yield* SqliteStorageCompatibilityError.make({
|
|
241
497
|
actualVersion: version.user_version,
|
|
242
498
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
243
499
|
message:
|
|
244
|
-
`The SQLite file uses
|
|
500
|
+
`The SQLite file uses unsupported storage version ${version.user_version}; ` +
|
|
245
501
|
`this build supports exactly version ${CurrentSqliteStorageVersion}. ` +
|
|
246
|
-
"
|
|
502
|
+
"Only supported v7, v8, v9 and v10 can be upgraded automatically. Keep the original file and use a compatible library version.",
|
|
247
503
|
});
|
|
248
504
|
}
|
|
249
505
|
|
|
506
|
+
if (
|
|
507
|
+
version.user_version === 7 ||
|
|
508
|
+
version.user_version === 8 ||
|
|
509
|
+
version.user_version === 9 ||
|
|
510
|
+
version.user_version === 10
|
|
511
|
+
) {
|
|
512
|
+
yield* sql
|
|
513
|
+
.withTransaction(
|
|
514
|
+
Effect.gen(function* () {
|
|
515
|
+
const current = yield* sql<{ user_version: number }>`PRAGMA user_version`;
|
|
516
|
+
|
|
517
|
+
if (current.length === 1 && current[0].user_version === CurrentSqliteStorageVersion)
|
|
518
|
+
return;
|
|
519
|
+
if (
|
|
520
|
+
current.length !== 1 ||
|
|
521
|
+
(current[0].user_version !== 7 &&
|
|
522
|
+
current[0].user_version !== 8 &&
|
|
523
|
+
current[0].user_version !== 9 &&
|
|
524
|
+
current[0].user_version !== 10)
|
|
525
|
+
)
|
|
526
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
527
|
+
actualVersion: -1,
|
|
528
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
529
|
+
message: "Storage version changed while acquiring the upgrade transaction.",
|
|
530
|
+
});
|
|
531
|
+
|
|
532
|
+
const required = yield* sql<{
|
|
533
|
+
name: string;
|
|
534
|
+
}>`SELECT name FROM sqlite_master WHERE type='table' AND name IN (
|
|
535
|
+
'effect_agent_threads', 'effect_agent_canonical_batches', 'effect_agent_canonical_records',
|
|
536
|
+
'effect_agent_checkpoints', 'effect_agent_submissions', 'effect_agent_submission_ownership',
|
|
537
|
+
'effect_agent_attempts', 'effect_agent_settlement_reservations', 'effect_agent_abort_intents',
|
|
538
|
+
'effect_agent_approval_decisions', 'effect_agent_unknown_resolutions', 'effect_agent_schedules'
|
|
539
|
+
)`;
|
|
540
|
+
|
|
541
|
+
if (required.length !== 12)
|
|
542
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
543
|
+
actualVersion: current[0].user_version,
|
|
544
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
545
|
+
message:
|
|
546
|
+
"The predecessor store is missing required tables; no upgrade was committed.",
|
|
547
|
+
});
|
|
548
|
+
|
|
549
|
+
const recoveryTables =
|
|
550
|
+
yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name='effect_agent_recovery_checkpoints'`;
|
|
551
|
+
|
|
552
|
+
if (recoveryTables.length !== (current[0].user_version === 10 ? 1 : 0))
|
|
553
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
554
|
+
actualVersion: current[0].user_version,
|
|
555
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
556
|
+
message:
|
|
557
|
+
"The predecessor recovery checkpoint storage does not match its version; refusing ambiguous data without mutation.",
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
const indexes =
|
|
561
|
+
yield* sql`SELECT name FROM sqlite_master WHERE name='effect_agent_submissions_nonterminal'`;
|
|
562
|
+
|
|
563
|
+
if (indexes.length !== 0)
|
|
564
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
565
|
+
actualVersion: current[0].user_version,
|
|
566
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
567
|
+
message:
|
|
568
|
+
"The predecessor already contains the nonterminal index; refusing ambiguous storage without mutation.",
|
|
569
|
+
});
|
|
570
|
+
if (current[0].user_version === 7) {
|
|
571
|
+
yield* checkV2ThreadLayout();
|
|
572
|
+
for (const statement of [
|
|
573
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
|
|
574
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
|
|
575
|
+
sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`,
|
|
576
|
+
]) {
|
|
577
|
+
yield* failpoint("upgrade:before-mutation");
|
|
578
|
+
yield* statement;
|
|
579
|
+
yield* failpoint("upgrade:after-mutation");
|
|
580
|
+
}
|
|
581
|
+
yield* upgradeV2Schedules(16 * 1024 * 1024).pipe(
|
|
582
|
+
Effect.provideService(ScheduleFailpoint, {
|
|
583
|
+
hit: (point) =>
|
|
584
|
+
Schema.decodeUnknownEffect(SqliteStorageFailpointLocation)(point).pipe(
|
|
585
|
+
Effect.flatMap(failpoint),
|
|
586
|
+
Effect.mapError(() => ScheduleFailpointError.make({ point })),
|
|
587
|
+
),
|
|
588
|
+
}),
|
|
589
|
+
);
|
|
590
|
+
yield* upgradeV2Subscriptions(16 * 1024 * 1024).pipe(
|
|
591
|
+
Effect.provideService(SubscriptionFailpoint, {
|
|
592
|
+
hit: (point) =>
|
|
593
|
+
Schema.decodeUnknownEffect(SqliteStorageFailpointLocation)(point).pipe(
|
|
594
|
+
Effect.flatMap(failpoint),
|
|
595
|
+
Effect.mapError(() => SubscriptionFailpointError.make({ point })),
|
|
596
|
+
),
|
|
597
|
+
}),
|
|
598
|
+
);
|
|
599
|
+
}
|
|
600
|
+
if (
|
|
601
|
+
current[0].user_version === 8 ||
|
|
602
|
+
current[0].user_version === 9 ||
|
|
603
|
+
current[0].user_version === 10
|
|
604
|
+
)
|
|
605
|
+
yield* checkPredecessorLayout(current[0].user_version);
|
|
606
|
+
if (current[0].user_version === 7 || current[0].user_version === 8) {
|
|
607
|
+
yield* failpoint("upgrade:before-mutation");
|
|
608
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
|
|
609
|
+
yield* failpoint("upgrade:after-mutation");
|
|
610
|
+
yield* failpoint("upgrade:before-mutation");
|
|
611
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
|
|
612
|
+
yield* failpoint("upgrade:after-mutation");
|
|
613
|
+
yield* failpoint("upgrade:before-mutation");
|
|
614
|
+
yield* createMessageDeliveryTables;
|
|
615
|
+
yield* failpoint("upgrade:after-mutation");
|
|
616
|
+
}
|
|
617
|
+
if (current[0].user_version !== 10) {
|
|
618
|
+
yield* failpoint("upgrade:before-mutation");
|
|
619
|
+
yield* createRecoveryCheckpointTable;
|
|
620
|
+
yield* failpoint("upgrade:after-mutation");
|
|
621
|
+
}
|
|
622
|
+
yield* failpoint("upgrade:before-mutation");
|
|
623
|
+
yield* createNonterminalIndex;
|
|
624
|
+
yield* failpoint("upgrade:after-mutation");
|
|
625
|
+
yield* failpoint("upgrade:before-version");
|
|
626
|
+
yield* sql`PRAGMA user_version = 11`;
|
|
627
|
+
yield* failpoint("upgrade:after-version");
|
|
628
|
+
}),
|
|
629
|
+
)
|
|
630
|
+
.pipe(
|
|
631
|
+
Effect.provide(NodeCrypto.layer),
|
|
632
|
+
Effect.catchTag("SqliteStorageFailpointError", (error) =>
|
|
633
|
+
SqliteStorageError.make({
|
|
634
|
+
cause: error,
|
|
635
|
+
operation: "upgrade storage",
|
|
636
|
+
message: error.message,
|
|
637
|
+
}),
|
|
638
|
+
),
|
|
639
|
+
Effect.catchTag(["ScheduleFailpointError", "SubscriptionFailpointError"], (error) =>
|
|
640
|
+
SqliteStorageError.make({
|
|
641
|
+
cause: error,
|
|
642
|
+
operation: "upgrade storage",
|
|
643
|
+
message: "Injected storage upgrade failure",
|
|
644
|
+
}),
|
|
645
|
+
),
|
|
646
|
+
Effect.catchTag("StorageUpgradeError", (error) =>
|
|
647
|
+
SqliteStorageCorruptionError.make({
|
|
648
|
+
table: error.table,
|
|
649
|
+
rowKey: error.rowKey,
|
|
650
|
+
message: error.message,
|
|
651
|
+
}),
|
|
652
|
+
),
|
|
653
|
+
Effect.catchTag("SqlError", storageError("upgrade supported storage")),
|
|
654
|
+
Effect.catchTag("SchemaError", (error) =>
|
|
655
|
+
SqliteStorageCorruptionError.make({
|
|
656
|
+
table: "upgrade",
|
|
657
|
+
rowKey: "v7",
|
|
658
|
+
message: error.message,
|
|
659
|
+
}),
|
|
660
|
+
),
|
|
661
|
+
);
|
|
662
|
+
}
|
|
663
|
+
|
|
250
664
|
if (version.user_version === 0) {
|
|
251
665
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
252
666
|
SELECT name
|
|
@@ -255,6 +669,7 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
255
669
|
AND name LIKE 'effect_agent_%'
|
|
256
670
|
ORDER BY name
|
|
257
671
|
`.pipe(Effect.mapError(storageError("inspect unversioned storage")));
|
|
672
|
+
|
|
258
673
|
const existing = yield* decodeRows(
|
|
259
674
|
Schema.Array(SqliteNameRow),
|
|
260
675
|
"sqlite_master",
|
|
@@ -267,14 +682,11 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
267
682
|
actualVersion: 0,
|
|
268
683
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
269
684
|
message:
|
|
270
|
-
"The SQLite file contains unversioned Effect Agent tables.
|
|
685
|
+
"The SQLite file contains unversioned Effect Agent tables. Refusing to mutate ambiguous stored data; retain it for inspection with its original writer.",
|
|
271
686
|
});
|
|
272
687
|
}
|
|
273
688
|
|
|
274
689
|
yield* SqliteMigrator.run({ loader: sqliteMigrations }).pipe(
|
|
275
|
-
// SqliteMigrator depends on the generic client supplied by this adapter.
|
|
276
|
-
// The concrete Node client is kept at the outer Layer boundary.
|
|
277
|
-
Effect.provideService(SqlClient.SqlClient, sql),
|
|
278
690
|
Effect.mapError((error) =>
|
|
279
691
|
SqliteStorageError.make({
|
|
280
692
|
cause: error,
|
|
@@ -288,9 +700,9 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
288
700
|
const requiredRows = yield* sql<Record<string, unknown>>`
|
|
289
701
|
SELECT name
|
|
290
702
|
FROM sqlite_master
|
|
291
|
-
WHERE type = 'table'
|
|
703
|
+
WHERE (type = 'table'
|
|
292
704
|
AND name IN (
|
|
293
|
-
'
|
|
705
|
+
'effect_agent_threads',
|
|
294
706
|
'effect_agent_canonical_batches',
|
|
295
707
|
'effect_agent_canonical_records',
|
|
296
708
|
'effect_agent_checkpoints',
|
|
@@ -300,29 +712,30 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
300
712
|
'effect_agent_settlement_reservations',
|
|
301
713
|
'effect_agent_abort_intents',
|
|
302
714
|
'effect_agent_approval_decisions',
|
|
303
|
-
'effect_agent_unknown_resolutions'
|
|
304
|
-
|
|
715
|
+
'effect_agent_unknown_resolutions',
|
|
716
|
+
'effect_agent_schedules',
|
|
717
|
+
'effect_agent_message_deliveries',
|
|
718
|
+
'effect_agent_recovery_checkpoints'
|
|
719
|
+
)) OR (type = 'index' AND name = 'effect_agent_submissions_nonterminal')
|
|
305
720
|
ORDER BY name
|
|
306
721
|
`.pipe(Effect.mapError(storageError("verify storage tables")));
|
|
722
|
+
|
|
307
723
|
const required = yield* decodeRows(
|
|
308
724
|
Schema.Array(SqliteNameRow),
|
|
309
725
|
"sqlite_master",
|
|
310
726
|
"required_tables",
|
|
311
727
|
requiredRows,
|
|
312
728
|
);
|
|
313
|
-
|
|
729
|
+
|
|
730
|
+
if (required.length !== 15) {
|
|
314
731
|
return yield* SqliteStorageCompatibilityError.make({
|
|
315
732
|
actualVersion: CurrentSqliteStorageVersion,
|
|
316
733
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
317
734
|
message:
|
|
318
|
-
"The SQLite file claims the current format but is missing required tables.
|
|
735
|
+
"The SQLite file claims the current format but is missing required tables or its nonterminal index. Retain the original store for inspection.",
|
|
319
736
|
});
|
|
320
737
|
}
|
|
321
738
|
|
|
322
|
-
return makeJournal(sql, failpoint);
|
|
323
|
-
});
|
|
324
|
-
|
|
325
|
-
const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint) => {
|
|
326
739
|
const classifyWriteFailure =
|
|
327
740
|
(operation: string) =>
|
|
328
741
|
(error: SqlError): SqliteStorageError | SqliteWriteContention =>
|
|
@@ -357,19 +770,24 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
357
770
|
const connection = yield* sql.reserve.pipe(
|
|
358
771
|
Effect.mapError(classifyWriteFailure(operation)),
|
|
359
772
|
);
|
|
773
|
+
|
|
360
774
|
yield* connection
|
|
361
775
|
.executeUnprepared("BEGIN IMMEDIATE", [], undefined)
|
|
362
776
|
.pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
777
|
+
|
|
363
778
|
const exit = yield* restore(
|
|
364
779
|
Effect.provideService(effect, sql.transactionService, [connection, 0] as const),
|
|
365
780
|
).pipe(Effect.exit);
|
|
781
|
+
|
|
366
782
|
if (Exit.isSuccess(exit)) {
|
|
367
783
|
yield* connection
|
|
368
784
|
.executeUnprepared("COMMIT", [], undefined)
|
|
369
785
|
.pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
786
|
+
|
|
370
787
|
return exit.value;
|
|
371
788
|
}
|
|
372
789
|
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], undefined));
|
|
790
|
+
|
|
373
791
|
return yield* exit;
|
|
374
792
|
}),
|
|
375
793
|
).pipe(
|
|
@@ -392,26 +810,31 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
392
810
|
Effect.scoped(
|
|
393
811
|
Effect.gen(function* () {
|
|
394
812
|
const connection = yield* sql.reserve.pipe(Effect.mapError(storageError(operation)));
|
|
813
|
+
|
|
395
814
|
yield* connection
|
|
396
815
|
.executeUnprepared("BEGIN", [], undefined)
|
|
397
816
|
.pipe(Effect.mapError(storageError(operation)));
|
|
817
|
+
|
|
398
818
|
const exit = yield* restore(
|
|
399
819
|
Effect.provideService(effect, sql.transactionService, [connection, 0] as const),
|
|
400
820
|
).pipe(Effect.exit);
|
|
821
|
+
|
|
401
822
|
if (Exit.isSuccess(exit)) {
|
|
402
823
|
yield* connection
|
|
403
824
|
.executeUnprepared("COMMIT", [], undefined)
|
|
404
825
|
.pipe(Effect.mapError(storageError(operation)));
|
|
826
|
+
|
|
405
827
|
return exit.value;
|
|
406
828
|
}
|
|
407
829
|
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], undefined));
|
|
830
|
+
|
|
408
831
|
return yield* exit;
|
|
409
832
|
}),
|
|
410
833
|
).pipe(Effect.withSpan("SqliteJournal.withReadTransaction", { attributes: { operation } })),
|
|
411
834
|
);
|
|
412
835
|
|
|
413
836
|
const materialize = Effect.fn("SqliteJournal.materialize")(function* (
|
|
414
|
-
|
|
837
|
+
threadId: string,
|
|
415
838
|
createdAt: string,
|
|
416
839
|
emptyTailDigest: string,
|
|
417
840
|
producerEpoch: ProducerEpoch,
|
|
@@ -420,55 +843,58 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
420
843
|
SqliteFenceRejected | SqliteStorageCorruptionError | SqliteStorageError | SqliteWriteContention
|
|
421
844
|
> {
|
|
422
845
|
if (
|
|
423
|
-
|
|
846
|
+
threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
424
847
|
storedTextBytes(emptyTailDigest) > MAX_STORED_TEXT_BYTES
|
|
425
848
|
) {
|
|
426
849
|
return yield* SqliteStorageError.make({
|
|
427
|
-
operation: "materialize
|
|
428
|
-
message: "
|
|
850
|
+
operation: "materialize thread",
|
|
851
|
+
message: "Thread identity or initial digest exceeds the SQLite storage bounds.",
|
|
429
852
|
});
|
|
430
853
|
}
|
|
431
854
|
yield* withWriteTransaction("materialize transaction")(
|
|
432
855
|
Effect.gen(function* () {
|
|
433
856
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
434
857
|
SELECT
|
|
435
|
-
|
|
858
|
+
thread_id,
|
|
436
859
|
created_at,
|
|
437
860
|
tail_sequence,
|
|
438
861
|
tail_digest,
|
|
439
862
|
producer_epoch
|
|
440
|
-
FROM
|
|
441
|
-
WHERE
|
|
442
|
-
`.pipe(Effect.mapError(storageError("read materialized
|
|
863
|
+
FROM effect_agent_threads
|
|
864
|
+
WHERE thread_id = ${threadId}
|
|
865
|
+
`.pipe(Effect.mapError(storageError("read materialized thread")));
|
|
866
|
+
|
|
443
867
|
const existing = yield* decodeRows(
|
|
444
|
-
Schema.Array(
|
|
445
|
-
"
|
|
446
|
-
|
|
868
|
+
Schema.Array(ThreadRow),
|
|
869
|
+
"effect_agent_threads",
|
|
870
|
+
threadId,
|
|
447
871
|
existingRows,
|
|
448
872
|
);
|
|
873
|
+
|
|
449
874
|
if (existing.length > 1) {
|
|
450
875
|
return yield* SqliteStorageCorruptionError.make({
|
|
451
|
-
table: "
|
|
452
|
-
rowKey:
|
|
453
|
-
message: "A
|
|
876
|
+
table: "effect_agent_threads",
|
|
877
|
+
rowKey: threadId,
|
|
878
|
+
message: "A thread primary key returned more than one row.",
|
|
454
879
|
});
|
|
455
880
|
}
|
|
456
881
|
if (existing.length === 0) {
|
|
457
882
|
yield* sql`
|
|
458
|
-
INSERT INTO
|
|
459
|
-
|
|
883
|
+
INSERT INTO effect_agent_threads (
|
|
884
|
+
thread_id,
|
|
460
885
|
created_at,
|
|
461
886
|
tail_sequence,
|
|
462
887
|
tail_digest,
|
|
463
888
|
producer_epoch
|
|
464
889
|
) VALUES (
|
|
465
|
-
${
|
|
890
|
+
${threadId},
|
|
466
891
|
${createdAt},
|
|
467
892
|
0,
|
|
468
893
|
${emptyTailDigest},
|
|
469
894
|
${producerEpoch}
|
|
470
895
|
)
|
|
471
|
-
`.pipe(Effect.mapError(storageError("materialize
|
|
896
|
+
`.pipe(Effect.mapError(storageError("materialize thread")));
|
|
897
|
+
|
|
472
898
|
return;
|
|
473
899
|
}
|
|
474
900
|
if (producerEpoch < existing[0].producer_epoch) {
|
|
@@ -480,41 +906,35 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
480
906
|
}
|
|
481
907
|
if (producerEpoch > existing[0].producer_epoch) {
|
|
482
908
|
yield* sql`
|
|
483
|
-
UPDATE
|
|
909
|
+
UPDATE effect_agent_threads
|
|
484
910
|
SET producer_epoch = ${producerEpoch}
|
|
485
|
-
WHERE
|
|
911
|
+
WHERE thread_id = ${threadId}
|
|
486
912
|
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
487
913
|
}
|
|
488
914
|
}),
|
|
489
915
|
);
|
|
490
916
|
});
|
|
491
917
|
|
|
492
|
-
const
|
|
493
|
-
conversationId: string,
|
|
494
|
-
) {
|
|
918
|
+
const getThread = Effect.fn("SqliteJournal.getThread")(function* (threadId: string) {
|
|
495
919
|
const rows = yield* sql<Record<string, unknown>>`
|
|
496
920
|
SELECT
|
|
497
|
-
|
|
921
|
+
thread_id,
|
|
498
922
|
created_at,
|
|
499
923
|
tail_sequence,
|
|
500
924
|
tail_digest,
|
|
501
925
|
producer_epoch
|
|
502
|
-
FROM
|
|
503
|
-
WHERE
|
|
504
|
-
`.pipe(Effect.mapError(storageError("read
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
"effect_agent_conversations",
|
|
508
|
-
conversationId,
|
|
509
|
-
rows,
|
|
510
|
-
);
|
|
926
|
+
FROM effect_agent_threads
|
|
927
|
+
WHERE thread_id = ${threadId}
|
|
928
|
+
`.pipe(Effect.mapError(storageError("read thread")));
|
|
929
|
+
|
|
930
|
+
return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
511
931
|
});
|
|
512
932
|
|
|
513
933
|
const append = Effect.fn("SqliteJournal.append")(function* (
|
|
514
934
|
request: RawAppendRequest,
|
|
515
935
|
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
516
936
|
if (
|
|
517
|
-
request.
|
|
937
|
+
request.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
518
938
|
request.batchId.length > MAX_IDENTIFIER_LENGTH ||
|
|
519
939
|
storedTextBytes(request.batchJson) > MAX_STORED_TEXT_BYTES ||
|
|
520
940
|
storedTextBytes(request.batchDigest) > MAX_STORED_TEXT_BYTES ||
|
|
@@ -530,9 +950,11 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
530
950
|
message: "Canonical identifiers or encoded JSON exceed the SQLite storage bounds.",
|
|
531
951
|
});
|
|
532
952
|
}
|
|
953
|
+
|
|
533
954
|
return yield* withWriteTransaction("append transaction")(
|
|
534
955
|
Effect.gen(function* () {
|
|
535
956
|
const recordIds = request.records.map((record) => record.recordId);
|
|
957
|
+
|
|
536
958
|
if (new Set(recordIds).size !== recordIds.length) {
|
|
537
959
|
return yield* SqliteAppendConflict.make({
|
|
538
960
|
message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
|
|
@@ -540,34 +962,35 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
540
962
|
});
|
|
541
963
|
}
|
|
542
964
|
|
|
543
|
-
const
|
|
965
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
544
966
|
SELECT
|
|
545
|
-
|
|
967
|
+
thread_id,
|
|
546
968
|
created_at,
|
|
547
969
|
tail_sequence,
|
|
548
970
|
tail_digest,
|
|
549
971
|
producer_epoch
|
|
550
|
-
FROM
|
|
551
|
-
WHERE
|
|
972
|
+
FROM effect_agent_threads
|
|
973
|
+
WHERE thread_id = ${request.threadId}
|
|
552
974
|
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
975
|
+
|
|
976
|
+
const thread = yield* decodeSingleRow(
|
|
977
|
+
Schema.Array(ThreadRow),
|
|
978
|
+
"effect_agent_threads",
|
|
979
|
+
request.threadId,
|
|
980
|
+
threadRows,
|
|
558
981
|
);
|
|
559
982
|
|
|
560
|
-
if (request.producerEpoch !==
|
|
983
|
+
if (request.producerEpoch !== thread.producer_epoch) {
|
|
561
984
|
return yield* SqliteFenceRejected.make({
|
|
562
985
|
producerEpoch: request.producerEpoch,
|
|
563
|
-
actualEpoch:
|
|
564
|
-
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${
|
|
986
|
+
actualEpoch: thread.producer_epoch,
|
|
987
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`,
|
|
565
988
|
});
|
|
566
989
|
}
|
|
567
990
|
|
|
568
991
|
const batchRows = yield* sql<Record<string, unknown>>`
|
|
569
992
|
SELECT
|
|
570
|
-
|
|
993
|
+
thread_id,
|
|
571
994
|
batch_id,
|
|
572
995
|
first_sequence,
|
|
573
996
|
last_sequence,
|
|
@@ -575,31 +998,34 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
575
998
|
tail_digest,
|
|
576
999
|
batch_json
|
|
577
1000
|
FROM effect_agent_canonical_batches
|
|
578
|
-
WHERE
|
|
1001
|
+
WHERE thread_id = ${request.threadId}
|
|
579
1002
|
AND batch_id = ${request.batchId}
|
|
580
1003
|
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
1004
|
+
|
|
581
1005
|
const batches = yield* decodeRows(
|
|
582
1006
|
Schema.Array(BatchRow),
|
|
583
1007
|
"effect_agent_canonical_batches",
|
|
584
|
-
`${request.
|
|
1008
|
+
`${request.threadId}/${request.batchId}`,
|
|
585
1009
|
batchRows,
|
|
586
1010
|
);
|
|
587
1011
|
|
|
588
1012
|
if (batches.length > 1) {
|
|
589
1013
|
return yield* SqliteStorageCorruptionError.make({
|
|
590
1014
|
table: "effect_agent_canonical_batches",
|
|
591
|
-
rowKey: `${request.
|
|
1015
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
592
1016
|
message: "A canonical batch primary key returned more than one row.",
|
|
593
1017
|
});
|
|
594
1018
|
}
|
|
595
1019
|
if (batches.length === 1) {
|
|
596
1020
|
const existing = batches[0];
|
|
1021
|
+
|
|
597
1022
|
if (existing.batch_digest !== request.batchDigest) {
|
|
598
1023
|
return yield* SqliteAppendConflict.make({
|
|
599
1024
|
message: `Batch ${request.batchId} already exists with different canonical content.`,
|
|
600
1025
|
reason: "batch-digest",
|
|
601
1026
|
});
|
|
602
1027
|
}
|
|
1028
|
+
|
|
603
1029
|
return RawAppendResult.make({
|
|
604
1030
|
firstSequence: existing.first_sequence,
|
|
605
1031
|
lastSequence: existing.last_sequence,
|
|
@@ -609,43 +1035,45 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
609
1035
|
}
|
|
610
1036
|
|
|
611
1037
|
if (
|
|
612
|
-
request.expectedTailSequence !==
|
|
613
|
-
request.expectedTailDigest !==
|
|
1038
|
+
request.expectedTailSequence !== thread.tail_sequence ||
|
|
1039
|
+
request.expectedTailDigest !== thread.tail_digest
|
|
614
1040
|
) {
|
|
615
1041
|
return yield* SqliteAppendConflict.make({
|
|
616
1042
|
message:
|
|
617
1043
|
`Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} ` +
|
|
618
|
-
`but found ${
|
|
1044
|
+
`but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
619
1045
|
reason: "tail",
|
|
620
|
-
actualTailSequence:
|
|
621
|
-
actualTailDigest:
|
|
1046
|
+
actualTailSequence: thread.tail_sequence,
|
|
1047
|
+
actualTailDigest: thread.tail_digest,
|
|
622
1048
|
});
|
|
623
1049
|
}
|
|
624
|
-
if (
|
|
1050
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) {
|
|
625
1051
|
return yield* SqliteStorageError.make({
|
|
626
1052
|
operation: "append canonical batch",
|
|
627
|
-
message: `
|
|
1053
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`,
|
|
628
1054
|
});
|
|
629
1055
|
}
|
|
630
1056
|
|
|
631
1057
|
const existingRecordRows = yield* sql<Record<string, unknown>>`
|
|
632
1058
|
SELECT
|
|
633
|
-
|
|
1059
|
+
thread_id,
|
|
634
1060
|
sequence,
|
|
635
1061
|
record_id,
|
|
636
1062
|
batch_id,
|
|
637
1063
|
record_json
|
|
638
1064
|
FROM effect_agent_canonical_records
|
|
639
|
-
WHERE
|
|
1065
|
+
WHERE thread_id = ${request.threadId}
|
|
640
1066
|
AND record_id IN ${sql.in(recordIds)}
|
|
641
1067
|
ORDER BY sequence
|
|
642
1068
|
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
1069
|
+
|
|
643
1070
|
const existingRecords = yield* decodeRows(
|
|
644
1071
|
Schema.Array(RecordRow),
|
|
645
1072
|
"effect_agent_canonical_records",
|
|
646
|
-
`${request.
|
|
1073
|
+
`${request.threadId}/record_ids`,
|
|
647
1074
|
existingRecordRows,
|
|
648
1075
|
);
|
|
1076
|
+
|
|
649
1077
|
if (existingRecords.length > 0) {
|
|
650
1078
|
return yield* SqliteAppendConflict.make({
|
|
651
1079
|
message: `Canonical record ID ${existingRecords[0].record_id} already exists.`,
|
|
@@ -654,7 +1082,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
654
1082
|
}
|
|
655
1083
|
|
|
656
1084
|
const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
657
|
-
|
|
1085
|
+
thread.tail_sequence + 1,
|
|
658
1086
|
).pipe(
|
|
659
1087
|
Effect.mapError((error) =>
|
|
660
1088
|
SqliteStorageError.make({
|
|
@@ -664,6 +1092,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
664
1092
|
}),
|
|
665
1093
|
),
|
|
666
1094
|
);
|
|
1095
|
+
|
|
667
1096
|
const lastSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
668
1097
|
firstSequence + request.records.length - 1,
|
|
669
1098
|
).pipe(
|
|
@@ -678,7 +1107,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
678
1107
|
|
|
679
1108
|
yield* sql`
|
|
680
1109
|
INSERT INTO effect_agent_canonical_batches (
|
|
681
|
-
|
|
1110
|
+
thread_id,
|
|
682
1111
|
batch_id,
|
|
683
1112
|
first_sequence,
|
|
684
1113
|
last_sequence,
|
|
@@ -686,7 +1115,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
686
1115
|
tail_digest,
|
|
687
1116
|
batch_json
|
|
688
1117
|
) VALUES (
|
|
689
|
-
${request.
|
|
1118
|
+
${request.threadId},
|
|
690
1119
|
${request.batchId},
|
|
691
1120
|
${firstSequence},
|
|
692
1121
|
${lastSequence},
|
|
@@ -703,13 +1132,13 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
703
1132
|
Effect.gen(function* () {
|
|
704
1133
|
yield* sql`
|
|
705
1134
|
INSERT INTO effect_agent_canonical_records (
|
|
706
|
-
|
|
1135
|
+
thread_id,
|
|
707
1136
|
sequence,
|
|
708
1137
|
record_id,
|
|
709
1138
|
batch_id,
|
|
710
1139
|
record_json
|
|
711
1140
|
) VALUES (
|
|
712
|
-
${request.
|
|
1141
|
+
${request.threadId},
|
|
713
1142
|
${firstSequence + index},
|
|
714
1143
|
${record.recordId},
|
|
715
1144
|
${request.batchId},
|
|
@@ -722,13 +1151,13 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
722
1151
|
);
|
|
723
1152
|
|
|
724
1153
|
yield* sql`
|
|
725
|
-
UPDATE
|
|
1154
|
+
UPDATE effect_agent_threads
|
|
726
1155
|
SET
|
|
727
1156
|
tail_sequence = ${lastSequence},
|
|
728
1157
|
tail_digest = ${request.tailDigest},
|
|
729
1158
|
producer_epoch = ${request.producerEpoch}
|
|
730
|
-
WHERE
|
|
731
|
-
`.pipe(Effect.mapError(storageError("advance
|
|
1159
|
+
WHERE thread_id = ${request.threadId}
|
|
1160
|
+
`.pipe(Effect.mapError(storageError("advance thread tail")));
|
|
732
1161
|
yield* failpoint("append:after-tail-update");
|
|
733
1162
|
|
|
734
1163
|
return RawAppendResult.make({
|
|
@@ -744,103 +1173,95 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
744
1173
|
const read = Effect.fn("SqliteJournal.read")(function* (request: RawReadRequest) {
|
|
745
1174
|
const rows = yield* sql<Record<string, unknown>>`
|
|
746
1175
|
SELECT
|
|
747
|
-
|
|
1176
|
+
thread_id,
|
|
748
1177
|
sequence,
|
|
749
1178
|
record_id,
|
|
750
1179
|
batch_id,
|
|
751
1180
|
record_json
|
|
752
1181
|
FROM effect_agent_canonical_records
|
|
753
|
-
WHERE
|
|
1182
|
+
WHERE thread_id = ${request.threadId}
|
|
754
1183
|
AND sequence > ${request.fromSequenceExclusive}
|
|
755
1184
|
ORDER BY sequence
|
|
756
1185
|
LIMIT ${request.limit}
|
|
757
1186
|
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
1187
|
+
|
|
758
1188
|
return yield* decodeRows(
|
|
759
1189
|
Schema.Array(RecordRow),
|
|
760
1190
|
"effect_agent_canonical_records",
|
|
761
|
-
`${request.
|
|
1191
|
+
`${request.threadId}>${request.fromSequenceExclusive}`,
|
|
762
1192
|
rows,
|
|
763
1193
|
);
|
|
764
1194
|
});
|
|
765
1195
|
|
|
766
|
-
const
|
|
767
|
-
conversationId: string,
|
|
768
|
-
) {
|
|
1196
|
+
const exportThread = Effect.fn("SqliteJournal.exportThread")(function* (threadId: string) {
|
|
769
1197
|
return yield* withReadTransaction("export transaction")(
|
|
770
1198
|
Effect.gen(function* () {
|
|
771
|
-
const
|
|
1199
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
772
1200
|
SELECT
|
|
773
|
-
|
|
1201
|
+
thread_id,
|
|
774
1202
|
created_at,
|
|
775
1203
|
tail_sequence,
|
|
776
1204
|
tail_digest,
|
|
777
1205
|
producer_epoch
|
|
778
|
-
FROM
|
|
779
|
-
WHERE
|
|
780
|
-
`.pipe(Effect.mapError(storageError("export
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
1206
|
+
FROM effect_agent_threads
|
|
1207
|
+
WHERE thread_id = ${threadId}
|
|
1208
|
+
`.pipe(Effect.mapError(storageError("export thread")));
|
|
1209
|
+
|
|
1210
|
+
const thread = yield* decodeSingleRow(
|
|
1211
|
+
Schema.Array(ThreadRow),
|
|
1212
|
+
"effect_agent_threads",
|
|
1213
|
+
threadId,
|
|
1214
|
+
threadRows,
|
|
786
1215
|
);
|
|
787
|
-
yield* failpoint("export:after-conversation-read");
|
|
788
|
-
const batchRows = yield* sql<Record<string, unknown>>`
|
|
789
|
-
SELECT
|
|
790
|
-
conversation_id,
|
|
791
|
-
batch_id,
|
|
792
|
-
first_sequence,
|
|
793
|
-
last_sequence,
|
|
794
|
-
batch_digest,
|
|
795
|
-
tail_digest,
|
|
796
|
-
batch_json
|
|
797
|
-
FROM effect_agent_canonical_batches
|
|
798
|
-
WHERE conversation_id = ${conversationId}
|
|
799
|
-
ORDER BY first_sequence
|
|
800
|
-
`.pipe(Effect.mapError(storageError("export canonical batches")));
|
|
801
|
-
const recordRows = yield* sql<Record<string, unknown>>`
|
|
802
|
-
SELECT
|
|
803
|
-
conversation_id,
|
|
804
|
-
sequence,
|
|
805
|
-
record_id,
|
|
806
|
-
batch_id,
|
|
807
|
-
record_json
|
|
808
|
-
FROM effect_agent_canonical_records
|
|
809
|
-
WHERE conversation_id = ${conversationId}
|
|
810
|
-
ORDER BY sequence
|
|
811
|
-
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
812
|
-
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
813
|
-
SELECT
|
|
814
|
-
conversation_id,
|
|
815
|
-
through_sequence,
|
|
816
|
-
tail_digest,
|
|
817
|
-
checkpoint_json
|
|
818
|
-
FROM effect_agent_checkpoints
|
|
819
|
-
WHERE conversation_id = ${conversationId}
|
|
820
|
-
ORDER BY through_sequence
|
|
821
|
-
`.pipe(Effect.mapError(storageError("export checkpoints")));
|
|
822
1216
|
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
"
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
)
|
|
843
|
-
|
|
1217
|
+
yield* failpoint("export:after-thread-read");
|
|
1218
|
+
|
|
1219
|
+
if (thread.tail_sequence > MAX_RECORDS_PER_THREAD)
|
|
1220
|
+
return yield* SqliteStorageError.make({
|
|
1221
|
+
operation: "export thread",
|
|
1222
|
+
message: "The thread exceeds the current export record limit.",
|
|
1223
|
+
});
|
|
1224
|
+
const records: Array<RecordRow> = [];
|
|
1225
|
+
let afterSequence = ZERO_SEQUENCE;
|
|
1226
|
+
|
|
1227
|
+
while (afterSequence < thread.tail_sequence) {
|
|
1228
|
+
const limit = Math.min(1_024, thread.tail_sequence - afterSequence);
|
|
1229
|
+
|
|
1230
|
+
const request = RawReadRequest.make({
|
|
1231
|
+
threadId,
|
|
1232
|
+
fromSequenceExclusive: afterSequence,
|
|
1233
|
+
limit,
|
|
1234
|
+
});
|
|
1235
|
+
|
|
1236
|
+
const page = yield* read(request);
|
|
1237
|
+
|
|
1238
|
+
if (
|
|
1239
|
+
page.length !== limit ||
|
|
1240
|
+
page.some((record, index) => record.sequence !== afterSequence + index + 1)
|
|
1241
|
+
) {
|
|
1242
|
+
return yield* SqliteStorageCorruptionError.make({
|
|
1243
|
+
table: "effect_agent_canonical_records",
|
|
1244
|
+
rowKey: threadId,
|
|
1245
|
+
message: "The exported canonical prefix is not contiguous through its captured tail.",
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
records.push(...page);
|
|
1249
|
+
afterSequence = page[page.length - 1].sequence;
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
const beyondTail =
|
|
1253
|
+
yield* sql`SELECT sequence FROM effect_agent_canonical_records WHERE thread_id=${threadId} AND sequence > ${thread.tail_sequence} LIMIT 1`.pipe(
|
|
1254
|
+
Effect.mapError(storageError("verify export tail")),
|
|
1255
|
+
);
|
|
1256
|
+
|
|
1257
|
+
if (beyondTail.length !== 0)
|
|
1258
|
+
return yield* SqliteStorageCorruptionError.make({
|
|
1259
|
+
table: "effect_agent_canonical_records",
|
|
1260
|
+
rowKey: threadId,
|
|
1261
|
+
message: "Canonical records exist beyond the captured thread tail.",
|
|
1262
|
+
});
|
|
1263
|
+
|
|
1264
|
+
return RawThreadExport.make({ thread, records });
|
|
844
1265
|
}),
|
|
845
1266
|
);
|
|
846
1267
|
});
|
|
@@ -849,7 +1270,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
849
1270
|
checkpoint: RawCheckpoint,
|
|
850
1271
|
): Effect.fn.Return<void, CheckpointError> {
|
|
851
1272
|
if (
|
|
852
|
-
checkpoint.
|
|
1273
|
+
checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
853
1274
|
storedTextBytes(checkpoint.checkpointJson) > MAX_STORED_TEXT_BYTES
|
|
854
1275
|
) {
|
|
855
1276
|
return yield* SqliteStorageError.make({
|
|
@@ -859,50 +1280,54 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
859
1280
|
}
|
|
860
1281
|
yield* withWriteTransaction("checkpoint transaction")(
|
|
861
1282
|
Effect.gen(function* () {
|
|
862
|
-
const
|
|
1283
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
863
1284
|
SELECT
|
|
864
|
-
|
|
1285
|
+
thread_id,
|
|
865
1286
|
created_at,
|
|
866
1287
|
tail_sequence,
|
|
867
1288
|
tail_digest,
|
|
868
1289
|
producer_epoch
|
|
869
|
-
FROM
|
|
870
|
-
WHERE
|
|
1290
|
+
FROM effect_agent_threads
|
|
1291
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
871
1292
|
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
1293
|
+
|
|
1294
|
+
const thread = yield* decodeSingleRow(
|
|
1295
|
+
Schema.Array(ThreadRow),
|
|
1296
|
+
"effect_agent_threads",
|
|
1297
|
+
checkpoint.threadId,
|
|
1298
|
+
threadRows,
|
|
877
1299
|
);
|
|
878
|
-
|
|
1300
|
+
|
|
1301
|
+
if (checkpoint.throughSequence > thread.tail_sequence) {
|
|
879
1302
|
return yield* SqliteCheckpointConflict.make({
|
|
880
1303
|
message:
|
|
881
1304
|
`Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
|
|
882
|
-
`${
|
|
1305
|
+
`${thread.tail_sequence}.`,
|
|
883
1306
|
});
|
|
884
1307
|
}
|
|
885
1308
|
|
|
886
1309
|
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
887
1310
|
SELECT
|
|
888
|
-
|
|
1311
|
+
thread_id,
|
|
889
1312
|
through_sequence,
|
|
890
1313
|
tail_digest,
|
|
891
1314
|
checkpoint_json
|
|
892
1315
|
FROM effect_agent_checkpoints
|
|
893
|
-
WHERE
|
|
1316
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
894
1317
|
AND through_sequence = ${checkpoint.throughSequence}
|
|
895
1318
|
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
1319
|
+
|
|
896
1320
|
const existing = yield* decodeRows(
|
|
897
1321
|
Schema.Array(CheckpointRow),
|
|
898
1322
|
"effect_agent_checkpoints",
|
|
899
|
-
`${checkpoint.
|
|
1323
|
+
`${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
900
1324
|
checkpointRows,
|
|
901
1325
|
);
|
|
1326
|
+
|
|
902
1327
|
if (existing.length > 1) {
|
|
903
1328
|
return yield* SqliteStorageCorruptionError.make({
|
|
904
1329
|
table: "effect_agent_checkpoints",
|
|
905
|
-
rowKey: `${checkpoint.
|
|
1330
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
906
1331
|
message: "A checkpoint primary key returned more than one row.",
|
|
907
1332
|
});
|
|
908
1333
|
}
|
|
@@ -915,17 +1340,18 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
915
1340
|
message: "A different checkpoint already exists at this canonical sequence.",
|
|
916
1341
|
});
|
|
917
1342
|
}
|
|
1343
|
+
|
|
918
1344
|
return;
|
|
919
1345
|
}
|
|
920
1346
|
|
|
921
1347
|
yield* sql`
|
|
922
1348
|
INSERT INTO effect_agent_checkpoints (
|
|
923
|
-
|
|
1349
|
+
thread_id,
|
|
924
1350
|
through_sequence,
|
|
925
1351
|
tail_digest,
|
|
926
1352
|
checkpoint_json
|
|
927
1353
|
) VALUES (
|
|
928
|
-
${checkpoint.
|
|
1354
|
+
${checkpoint.threadId},
|
|
929
1355
|
${checkpoint.throughSequence},
|
|
930
1356
|
${checkpoint.tailDigest},
|
|
931
1357
|
${checkpoint.checkpointJson}
|
|
@@ -935,45 +1361,125 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
935
1361
|
);
|
|
936
1362
|
});
|
|
937
1363
|
|
|
1364
|
+
const saveRecoveryCheckpoint = Effect.fn("SqliteJournal.saveRecoveryCheckpoint")(function* (
|
|
1365
|
+
request: SaveRecoveryCheckpointRequest,
|
|
1366
|
+
checkpointJson: string,
|
|
1367
|
+
) {
|
|
1368
|
+
const { checkpoint } = request;
|
|
1369
|
+
|
|
1370
|
+
if (
|
|
1371
|
+
checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
1372
|
+
storedTextBytes(checkpointJson) > MAX_STORED_TEXT_BYTES
|
|
1373
|
+
) {
|
|
1374
|
+
return yield* SqliteStorageError.make({
|
|
1375
|
+
operation: "save recovery checkpoint",
|
|
1376
|
+
message: "Checkpoint identity or encoded JSON exceeds the SQLite storage bounds.",
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1379
|
+
yield* withWriteTransaction("recovery checkpoint transaction")(
|
|
1380
|
+
Effect.gen(function* () {
|
|
1381
|
+
const threads = yield* getThread(checkpoint.threadId);
|
|
1382
|
+
const thread = threads[0];
|
|
1383
|
+
|
|
1384
|
+
if (thread === undefined)
|
|
1385
|
+
return yield* ThreadNotMaterialized.make({ threadId: checkpoint.threadId });
|
|
1386
|
+
if (request.producerEpoch !== thread.producer_epoch)
|
|
1387
|
+
return yield* FenceRejected.make({
|
|
1388
|
+
threadId: checkpoint.threadId,
|
|
1389
|
+
actualEpoch: thread.producer_epoch,
|
|
1390
|
+
attemptedEpoch: request.producerEpoch,
|
|
1391
|
+
});
|
|
1392
|
+
if (checkpoint.throughSequence > thread.tail_sequence)
|
|
1393
|
+
return yield* CheckpointRejected.make({
|
|
1394
|
+
threadId: checkpoint.threadId,
|
|
1395
|
+
reason: "ahead-of-tail",
|
|
1396
|
+
});
|
|
1397
|
+
|
|
1398
|
+
const digests =
|
|
1399
|
+
checkpoint.throughSequence === 0
|
|
1400
|
+
? [EMPTY_TAIL_DIGEST]
|
|
1401
|
+
: yield* getTailDigestAt(checkpoint.threadId, checkpoint.throughSequence);
|
|
1402
|
+
|
|
1403
|
+
if (digests.length !== 1 || digests[0] !== checkpoint.tailDigest)
|
|
1404
|
+
return yield* CheckpointRejected.make({
|
|
1405
|
+
threadId: checkpoint.threadId,
|
|
1406
|
+
reason: "digest-mismatch",
|
|
1407
|
+
});
|
|
1408
|
+
|
|
1409
|
+
yield* failpoint("save-recovery-checkpoint:before");
|
|
1410
|
+
yield* sql`
|
|
1411
|
+
INSERT INTO effect_agent_recovery_checkpoints (thread_id, through_sequence, tail_digest, checkpoint_json)
|
|
1412
|
+
VALUES (${checkpoint.threadId}, ${checkpoint.throughSequence}, ${checkpoint.tailDigest}, ${checkpointJson})
|
|
1413
|
+
ON CONFLICT (thread_id) DO UPDATE SET
|
|
1414
|
+
through_sequence = excluded.through_sequence,
|
|
1415
|
+
tail_digest = excluded.tail_digest,
|
|
1416
|
+
checkpoint_json = excluded.checkpoint_json
|
|
1417
|
+
WHERE excluded.through_sequence >= effect_agent_recovery_checkpoints.through_sequence
|
|
1418
|
+
`.pipe(Effect.mapError(storageError("save recovery checkpoint")));
|
|
1419
|
+
}),
|
|
1420
|
+
);
|
|
1421
|
+
yield* failpoint("save-recovery-checkpoint:after");
|
|
1422
|
+
});
|
|
1423
|
+
|
|
1424
|
+
const loadRecoveryCheckpoint = Effect.fn("SqliteJournal.loadRecoveryCheckpoint")(function* (
|
|
1425
|
+
threadId: string,
|
|
1426
|
+
) {
|
|
1427
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
1428
|
+
SELECT thread_id, through_sequence, tail_digest, checkpoint_json
|
|
1429
|
+
FROM effect_agent_recovery_checkpoints
|
|
1430
|
+
WHERE thread_id = ${threadId}
|
|
1431
|
+
`.pipe(Effect.mapError(storageError("load recovery checkpoint")));
|
|
1432
|
+
|
|
1433
|
+
return yield* decodeRows(
|
|
1434
|
+
Schema.Array(CheckpointRow),
|
|
1435
|
+
"effect_agent_recovery_checkpoints",
|
|
1436
|
+
threadId,
|
|
1437
|
+
rows,
|
|
1438
|
+
);
|
|
1439
|
+
});
|
|
1440
|
+
|
|
938
1441
|
const loadCheckpoint = Effect.fn("SqliteJournal.loadCheckpoint")(function* (
|
|
939
|
-
|
|
1442
|
+
threadId: string,
|
|
940
1443
|
atOrBeforeSequence: CanonicalSequence,
|
|
941
1444
|
) {
|
|
942
1445
|
const rows = yield* sql<Record<string, unknown>>`
|
|
943
1446
|
SELECT
|
|
944
|
-
|
|
1447
|
+
thread_id,
|
|
945
1448
|
through_sequence,
|
|
946
1449
|
tail_digest,
|
|
947
1450
|
checkpoint_json
|
|
948
1451
|
FROM effect_agent_checkpoints
|
|
949
|
-
WHERE
|
|
1452
|
+
WHERE thread_id = ${threadId}
|
|
950
1453
|
AND through_sequence <= ${atOrBeforeSequence}
|
|
951
1454
|
ORDER BY through_sequence DESC
|
|
952
1455
|
LIMIT 1
|
|
953
1456
|
`.pipe(Effect.mapError(storageError("load checkpoint")));
|
|
1457
|
+
|
|
954
1458
|
return yield* decodeRows(
|
|
955
1459
|
Schema.Array(CheckpointRow),
|
|
956
1460
|
"effect_agent_checkpoints",
|
|
957
|
-
`${
|
|
1461
|
+
`${threadId}<=${atOrBeforeSequence}`,
|
|
958
1462
|
rows,
|
|
959
1463
|
);
|
|
960
1464
|
});
|
|
961
1465
|
|
|
962
1466
|
const getTailDigestAt = Effect.fn("SqliteJournal.getTailDigestAt")(function* (
|
|
963
|
-
|
|
1467
|
+
threadId: string,
|
|
964
1468
|
sequence: CanonicalSequence,
|
|
965
1469
|
) {
|
|
966
1470
|
if (sequence === 0) {
|
|
967
|
-
const
|
|
968
|
-
|
|
1471
|
+
const threads = yield* getThread(threadId);
|
|
1472
|
+
|
|
1473
|
+
return threads.length === 0
|
|
969
1474
|
? []
|
|
970
|
-
: [
|
|
1475
|
+
: [threads[0].tail_sequence === 0 ? threads[0].tail_digest : undefined].filter(
|
|
971
1476
|
(value): value is string => value !== undefined,
|
|
972
1477
|
);
|
|
973
1478
|
}
|
|
1479
|
+
|
|
974
1480
|
const rows = yield* sql<Record<string, unknown>>`
|
|
975
1481
|
SELECT
|
|
976
|
-
|
|
1482
|
+
thread_id,
|
|
977
1483
|
batch_id,
|
|
978
1484
|
first_sequence,
|
|
979
1485
|
last_sequence,
|
|
@@ -981,34 +1487,37 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
981
1487
|
tail_digest,
|
|
982
1488
|
batch_json
|
|
983
1489
|
FROM effect_agent_canonical_batches
|
|
984
|
-
WHERE
|
|
1490
|
+
WHERE thread_id = ${threadId}
|
|
985
1491
|
AND last_sequence = ${sequence}
|
|
986
1492
|
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
1493
|
+
|
|
987
1494
|
const batches = yield* decodeRows(
|
|
988
1495
|
Schema.Array(BatchRow),
|
|
989
1496
|
"effect_agent_canonical_batches",
|
|
990
|
-
`${
|
|
1497
|
+
`${threadId}/${sequence}`,
|
|
991
1498
|
rows,
|
|
992
1499
|
);
|
|
1500
|
+
|
|
993
1501
|
return batches.map((batch) => batch.tail_digest);
|
|
994
1502
|
});
|
|
995
1503
|
|
|
996
1504
|
const scanStoredPayloads = Effect.fn("SqliteJournal.scanStoredPayloads")(function* () {
|
|
997
1505
|
return yield* withReadTransaction("startup scan transaction")(
|
|
998
1506
|
Effect.gen(function* () {
|
|
999
|
-
const
|
|
1507
|
+
const threads = yield* sql<Record<string, unknown>>`
|
|
1000
1508
|
SELECT
|
|
1001
|
-
|
|
1509
|
+
thread_id,
|
|
1002
1510
|
created_at,
|
|
1003
1511
|
tail_sequence,
|
|
1004
1512
|
tail_digest,
|
|
1005
1513
|
producer_epoch
|
|
1006
|
-
FROM
|
|
1007
|
-
ORDER BY
|
|
1008
|
-
`.pipe(Effect.mapError(storageError("scan
|
|
1514
|
+
FROM effect_agent_threads
|
|
1515
|
+
ORDER BY thread_id
|
|
1516
|
+
`.pipe(Effect.mapError(storageError("scan threads")));
|
|
1517
|
+
|
|
1009
1518
|
const batches = yield* sql<Record<string, unknown>>`
|
|
1010
1519
|
SELECT
|
|
1011
|
-
|
|
1520
|
+
thread_id,
|
|
1012
1521
|
batch_id,
|
|
1013
1522
|
first_sequence,
|
|
1014
1523
|
last_sequence,
|
|
@@ -1016,33 +1525,36 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
1016
1525
|
tail_digest,
|
|
1017
1526
|
batch_json
|
|
1018
1527
|
FROM effect_agent_canonical_batches
|
|
1019
|
-
ORDER BY
|
|
1528
|
+
ORDER BY thread_id, first_sequence
|
|
1020
1529
|
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
1530
|
+
|
|
1021
1531
|
const records = yield* sql<Record<string, unknown>>`
|
|
1022
1532
|
SELECT
|
|
1023
|
-
|
|
1533
|
+
thread_id,
|
|
1024
1534
|
sequence,
|
|
1025
1535
|
record_id,
|
|
1026
1536
|
batch_id,
|
|
1027
1537
|
record_json
|
|
1028
1538
|
FROM effect_agent_canonical_records
|
|
1029
|
-
ORDER BY
|
|
1539
|
+
ORDER BY thread_id, sequence
|
|
1030
1540
|
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
1541
|
+
|
|
1031
1542
|
const checkpoints = yield* sql<Record<string, unknown>>`
|
|
1032
1543
|
SELECT
|
|
1033
|
-
|
|
1544
|
+
thread_id,
|
|
1034
1545
|
through_sequence,
|
|
1035
1546
|
tail_digest,
|
|
1036
1547
|
checkpoint_json
|
|
1037
1548
|
FROM effect_agent_checkpoints
|
|
1038
|
-
ORDER BY
|
|
1549
|
+
ORDER BY thread_id, through_sequence
|
|
1039
1550
|
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
1551
|
+
|
|
1040
1552
|
return {
|
|
1041
|
-
|
|
1042
|
-
Schema.Array(
|
|
1043
|
-
"
|
|
1553
|
+
threads: yield* decodeRows(
|
|
1554
|
+
Schema.Array(ThreadRow),
|
|
1555
|
+
"effect_agent_threads",
|
|
1044
1556
|
"startup_scan",
|
|
1045
|
-
|
|
1557
|
+
threads,
|
|
1046
1558
|
),
|
|
1047
1559
|
batches: yield* decodeRows(
|
|
1048
1560
|
Schema.Array(BatchRow),
|
|
@@ -1069,18 +1581,19 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
1069
1581
|
|
|
1070
1582
|
return {
|
|
1071
1583
|
append,
|
|
1072
|
-
|
|
1073
|
-
|
|
1584
|
+
exportThread,
|
|
1585
|
+
getThread,
|
|
1074
1586
|
getTailDigestAt,
|
|
1075
1587
|
loadCheckpoint,
|
|
1588
|
+
loadRecoveryCheckpoint,
|
|
1589
|
+
saveRecoveryCheckpoint,
|
|
1076
1590
|
materialize,
|
|
1077
1591
|
read,
|
|
1078
1592
|
saveCheckpoint,
|
|
1079
1593
|
scanStoredPayloads,
|
|
1080
1594
|
withWriteTransaction,
|
|
1595
|
+
withReadTransaction,
|
|
1081
1596
|
} as const;
|
|
1082
|
-
};
|
|
1083
|
-
|
|
1084
|
-
export type SqliteJournal = ReturnType<typeof makeJournal>;
|
|
1597
|
+
});
|
|
1085
1598
|
|
|
1086
|
-
export
|
|
1599
|
+
export type SqliteJournal = Effect.Success<ReturnType<typeof initializeSqliteJournal>>;
|