@effect-agent/storage-sqlite 0.1.0-beta.38 → 0.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +34 -114
- package/dist/index.mjs +1693 -494
- package/dist/index.mjs.map +1 -1
- package/dist/sqlite-storage-failpoint-Cr0SXflP.d.mts +95 -0
- package/dist/sqlite-storage-failpoint-DIwWk5dw.mjs +12 -0
- package/dist/sqlite-storage-failpoint-DIwWk5dw.mjs.map +1 -0
- package/dist/testing.d.mts +15 -0
- package/dist/testing.mjs +19 -0
- package/dist/testing.mjs.map +1 -0
- package/package.json +14 -6
- package/src/errors.ts +3 -3
- package/src/index.ts +5 -2
- package/src/migrations.ts +106 -84
- package/src/sqlite-activity-store.ts +503 -0
- package/src/sqlite-journal.ts +147 -156
- package/src/sqlite-ledger.ts +60 -60
- package/src/sqlite-memory-store.ts +532 -0
- package/src/sqlite-schedule-store.ts +2 -2
- package/src/sqlite-storage-config.ts +1 -1
- package/src/sqlite-storage-failpoint-testing.ts +38 -0
- package/src/sqlite-storage-failpoint.ts +1 -31
- package/src/sqlite-subscription-store.ts +935 -0
- package/src/{sqlite-conversation-store.ts → sqlite-thread-store.ts} +248 -267
- package/src/testing.ts +2 -0
package/src/sqlite-journal.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/
|
|
1
|
+
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread";
|
|
2
2
|
import { SqliteMigrator } from "@effect/sql-sqlite-node";
|
|
3
3
|
import { Effect, Exit, Schema } from "effect";
|
|
4
4
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
@@ -21,7 +21,7 @@ import { SqliteStorageFailpoint } from "./sqlite-storage-failpoint.ts";
|
|
|
21
21
|
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(16 * 1024 * 1024));
|
|
22
22
|
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
23
23
|
const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0));
|
|
24
|
-
const
|
|
24
|
+
const MAX_RECORDS_PER_THREAD = 65_536;
|
|
25
25
|
const MAX_STORED_TEXT_BYTES = 16 * 1024 * 1024;
|
|
26
26
|
const MAX_IDENTIFIER_LENGTH = 1_024;
|
|
27
27
|
|
|
@@ -39,8 +39,8 @@ class SqliteNameRow extends Schema.Class<SqliteNameRow>("SqliteNameRow")({
|
|
|
39
39
|
name: BoundedIdentifier,
|
|
40
40
|
}) {}
|
|
41
41
|
|
|
42
|
-
class
|
|
43
|
-
|
|
42
|
+
class ThreadRow extends Schema.Class<ThreadRow>("ThreadRow")({
|
|
43
|
+
thread_id: BoundedIdentifier,
|
|
44
44
|
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
45
45
|
producer_epoch: ProducerEpoch,
|
|
46
46
|
tail_digest: BoundedStoredText,
|
|
@@ -51,7 +51,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
51
51
|
batch_digest: BoundedStoredText,
|
|
52
52
|
batch_id: BoundedIdentifier,
|
|
53
53
|
batch_json: BoundedStoredText,
|
|
54
|
-
|
|
54
|
+
thread_id: BoundedIdentifier,
|
|
55
55
|
first_sequence: CanonicalSequence,
|
|
56
56
|
last_sequence: CanonicalSequence,
|
|
57
57
|
tail_digest: BoundedStoredText,
|
|
@@ -59,7 +59,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
59
59
|
|
|
60
60
|
class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
61
61
|
batch_id: BoundedIdentifier,
|
|
62
|
-
|
|
62
|
+
thread_id: BoundedIdentifier,
|
|
63
63
|
record_id: BoundedIdentifier,
|
|
64
64
|
record_json: BoundedStoredText,
|
|
65
65
|
sequence: CanonicalSequence,
|
|
@@ -67,7 +67,7 @@ class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
|
67
67
|
|
|
68
68
|
class CheckpointRow extends Schema.Class<CheckpointRow>("CheckpointRow")({
|
|
69
69
|
checkpoint_json: BoundedStoredText,
|
|
70
|
-
|
|
70
|
+
thread_id: BoundedIdentifier,
|
|
71
71
|
tail_digest: BoundedStoredText,
|
|
72
72
|
through_sequence: CanonicalSequence,
|
|
73
73
|
}) {}
|
|
@@ -83,7 +83,7 @@ export class RawAppendRequest extends Schema.Class<RawAppendRequest>(
|
|
|
83
83
|
batchDigest: BoundedStoredText,
|
|
84
84
|
batchId: BoundedIdentifier,
|
|
85
85
|
batchJson: BoundedStoredText,
|
|
86
|
-
|
|
86
|
+
threadId: BoundedIdentifier,
|
|
87
87
|
expectedTailDigest: BoundedStoredText,
|
|
88
88
|
expectedTailSequence: CanonicalSequence,
|
|
89
89
|
producerEpoch: ProducerEpoch,
|
|
@@ -103,7 +103,7 @@ export class RawAppendResult extends Schema.Class<RawAppendResult>(
|
|
|
103
103
|
export class RawReadRequest extends Schema.Class<RawReadRequest>(
|
|
104
104
|
"@effect-agent/storage-sqlite/RawReadRequest",
|
|
105
105
|
)({
|
|
106
|
-
|
|
106
|
+
threadId: BoundedIdentifier,
|
|
107
107
|
fromSequenceExclusive: CanonicalSequence,
|
|
108
108
|
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
|
|
109
109
|
}) {}
|
|
@@ -112,17 +112,17 @@ export class RawCheckpoint extends Schema.Class<RawCheckpoint>(
|
|
|
112
112
|
"@effect-agent/storage-sqlite/RawCheckpoint",
|
|
113
113
|
)({
|
|
114
114
|
checkpointJson: BoundedStoredText,
|
|
115
|
-
|
|
115
|
+
threadId: BoundedIdentifier,
|
|
116
116
|
tailDigest: BoundedStoredText,
|
|
117
117
|
throughSequence: CanonicalSequence,
|
|
118
118
|
}) {}
|
|
119
119
|
|
|
120
|
-
export class
|
|
121
|
-
"@effect-agent/storage-sqlite/
|
|
120
|
+
export class RawThreadExport extends Schema.Class<RawThreadExport>(
|
|
121
|
+
"@effect-agent/storage-sqlite/RawThreadExport",
|
|
122
122
|
)({
|
|
123
123
|
batches: Schema.Array(BatchRow),
|
|
124
124
|
checkpoints: Schema.Array(CheckpointRow),
|
|
125
|
-
|
|
125
|
+
thread: ThreadRow,
|
|
126
126
|
records: Schema.Array(RecordRow),
|
|
127
127
|
}) {}
|
|
128
128
|
|
|
@@ -281,7 +281,7 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
281
281
|
FROM sqlite_master
|
|
282
282
|
WHERE type = 'table'
|
|
283
283
|
AND name IN (
|
|
284
|
-
'
|
|
284
|
+
'effect_agent_threads',
|
|
285
285
|
'effect_agent_canonical_batches',
|
|
286
286
|
'effect_agent_canonical_records',
|
|
287
287
|
'effect_agent_checkpoints',
|
|
@@ -399,7 +399,7 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
399
399
|
);
|
|
400
400
|
|
|
401
401
|
const materialize = Effect.fn("SqliteJournal.materialize")(function* (
|
|
402
|
-
|
|
402
|
+
threadId: string,
|
|
403
403
|
createdAt: string,
|
|
404
404
|
emptyTailDigest: string,
|
|
405
405
|
producerEpoch: ProducerEpoch,
|
|
@@ -408,55 +408,55 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
408
408
|
SqliteFenceRejected | SqliteStorageCorruptionError | SqliteStorageError | SqliteWriteContention
|
|
409
409
|
> {
|
|
410
410
|
if (
|
|
411
|
-
|
|
411
|
+
threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
412
412
|
storedTextBytes(emptyTailDigest) > MAX_STORED_TEXT_BYTES
|
|
413
413
|
) {
|
|
414
414
|
return yield* SqliteStorageError.make({
|
|
415
|
-
operation: "materialize
|
|
416
|
-
message: "
|
|
415
|
+
operation: "materialize thread",
|
|
416
|
+
message: "Thread identity or initial digest exceeds the SQLite storage bounds.",
|
|
417
417
|
});
|
|
418
418
|
}
|
|
419
419
|
yield* withWriteTransaction("materialize transaction")(
|
|
420
420
|
Effect.gen(function* () {
|
|
421
421
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
422
422
|
SELECT
|
|
423
|
-
|
|
423
|
+
thread_id,
|
|
424
424
|
created_at,
|
|
425
425
|
tail_sequence,
|
|
426
426
|
tail_digest,
|
|
427
427
|
producer_epoch
|
|
428
|
-
FROM
|
|
429
|
-
WHERE
|
|
430
|
-
`.pipe(Effect.mapError(storageError("read materialized
|
|
428
|
+
FROM effect_agent_threads
|
|
429
|
+
WHERE thread_id = ${threadId}
|
|
430
|
+
`.pipe(Effect.mapError(storageError("read materialized thread")));
|
|
431
431
|
const existing = yield* decodeRows(
|
|
432
|
-
Schema.Array(
|
|
433
|
-
"
|
|
434
|
-
|
|
432
|
+
Schema.Array(ThreadRow),
|
|
433
|
+
"effect_agent_threads",
|
|
434
|
+
threadId,
|
|
435
435
|
existingRows,
|
|
436
436
|
);
|
|
437
437
|
if (existing.length > 1) {
|
|
438
438
|
return yield* SqliteStorageCorruptionError.make({
|
|
439
|
-
table: "
|
|
440
|
-
rowKey:
|
|
441
|
-
message: "A
|
|
439
|
+
table: "effect_agent_threads",
|
|
440
|
+
rowKey: threadId,
|
|
441
|
+
message: "A thread primary key returned more than one row.",
|
|
442
442
|
});
|
|
443
443
|
}
|
|
444
444
|
if (existing.length === 0) {
|
|
445
445
|
yield* sql`
|
|
446
|
-
INSERT INTO
|
|
447
|
-
|
|
446
|
+
INSERT INTO effect_agent_threads (
|
|
447
|
+
thread_id,
|
|
448
448
|
created_at,
|
|
449
449
|
tail_sequence,
|
|
450
450
|
tail_digest,
|
|
451
451
|
producer_epoch
|
|
452
452
|
) VALUES (
|
|
453
|
-
${
|
|
453
|
+
${threadId},
|
|
454
454
|
${createdAt},
|
|
455
455
|
0,
|
|
456
456
|
${emptyTailDigest},
|
|
457
457
|
${producerEpoch}
|
|
458
458
|
)
|
|
459
|
-
`.pipe(Effect.mapError(storageError("materialize
|
|
459
|
+
`.pipe(Effect.mapError(storageError("materialize thread")));
|
|
460
460
|
return;
|
|
461
461
|
}
|
|
462
462
|
if (producerEpoch < existing[0].producer_epoch) {
|
|
@@ -468,41 +468,34 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
468
468
|
}
|
|
469
469
|
if (producerEpoch > existing[0].producer_epoch) {
|
|
470
470
|
yield* sql`
|
|
471
|
-
UPDATE
|
|
471
|
+
UPDATE effect_agent_threads
|
|
472
472
|
SET producer_epoch = ${producerEpoch}
|
|
473
|
-
WHERE
|
|
473
|
+
WHERE thread_id = ${threadId}
|
|
474
474
|
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
475
475
|
}
|
|
476
476
|
}),
|
|
477
477
|
);
|
|
478
478
|
});
|
|
479
479
|
|
|
480
|
-
const
|
|
481
|
-
conversationId: string,
|
|
482
|
-
) {
|
|
480
|
+
const getThread = Effect.fn("SqliteJournal.getThread")(function* (threadId: string) {
|
|
483
481
|
const rows = yield* sql<Record<string, unknown>>`
|
|
484
482
|
SELECT
|
|
485
|
-
|
|
483
|
+
thread_id,
|
|
486
484
|
created_at,
|
|
487
485
|
tail_sequence,
|
|
488
486
|
tail_digest,
|
|
489
487
|
producer_epoch
|
|
490
|
-
FROM
|
|
491
|
-
WHERE
|
|
492
|
-
`.pipe(Effect.mapError(storageError("read
|
|
493
|
-
return yield* decodeRows(
|
|
494
|
-
Schema.Array(ConversationRow),
|
|
495
|
-
"effect_agent_conversations",
|
|
496
|
-
conversationId,
|
|
497
|
-
rows,
|
|
498
|
-
);
|
|
488
|
+
FROM effect_agent_threads
|
|
489
|
+
WHERE thread_id = ${threadId}
|
|
490
|
+
`.pipe(Effect.mapError(storageError("read thread")));
|
|
491
|
+
return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
499
492
|
});
|
|
500
493
|
|
|
501
494
|
const append = Effect.fn("SqliteJournal.append")(function* (
|
|
502
495
|
request: RawAppendRequest,
|
|
503
496
|
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
504
497
|
if (
|
|
505
|
-
request.
|
|
498
|
+
request.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
506
499
|
request.batchId.length > MAX_IDENTIFIER_LENGTH ||
|
|
507
500
|
storedTextBytes(request.batchJson) > MAX_STORED_TEXT_BYTES ||
|
|
508
501
|
storedTextBytes(request.batchDigest) > MAX_STORED_TEXT_BYTES ||
|
|
@@ -528,34 +521,34 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
528
521
|
});
|
|
529
522
|
}
|
|
530
523
|
|
|
531
|
-
const
|
|
524
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
532
525
|
SELECT
|
|
533
|
-
|
|
526
|
+
thread_id,
|
|
534
527
|
created_at,
|
|
535
528
|
tail_sequence,
|
|
536
529
|
tail_digest,
|
|
537
530
|
producer_epoch
|
|
538
|
-
FROM
|
|
539
|
-
WHERE
|
|
531
|
+
FROM effect_agent_threads
|
|
532
|
+
WHERE thread_id = ${request.threadId}
|
|
540
533
|
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
541
|
-
const
|
|
542
|
-
Schema.Array(
|
|
543
|
-
"
|
|
544
|
-
request.
|
|
545
|
-
|
|
534
|
+
const thread = yield* decodeSingleRow(
|
|
535
|
+
Schema.Array(ThreadRow),
|
|
536
|
+
"effect_agent_threads",
|
|
537
|
+
request.threadId,
|
|
538
|
+
threadRows,
|
|
546
539
|
);
|
|
547
540
|
|
|
548
|
-
if (request.producerEpoch !==
|
|
541
|
+
if (request.producerEpoch !== thread.producer_epoch) {
|
|
549
542
|
return yield* SqliteFenceRejected.make({
|
|
550
543
|
producerEpoch: request.producerEpoch,
|
|
551
|
-
actualEpoch:
|
|
552
|
-
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${
|
|
544
|
+
actualEpoch: thread.producer_epoch,
|
|
545
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`,
|
|
553
546
|
});
|
|
554
547
|
}
|
|
555
548
|
|
|
556
549
|
const batchRows = yield* sql<Record<string, unknown>>`
|
|
557
550
|
SELECT
|
|
558
|
-
|
|
551
|
+
thread_id,
|
|
559
552
|
batch_id,
|
|
560
553
|
first_sequence,
|
|
561
554
|
last_sequence,
|
|
@@ -563,20 +556,20 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
563
556
|
tail_digest,
|
|
564
557
|
batch_json
|
|
565
558
|
FROM effect_agent_canonical_batches
|
|
566
|
-
WHERE
|
|
559
|
+
WHERE thread_id = ${request.threadId}
|
|
567
560
|
AND batch_id = ${request.batchId}
|
|
568
561
|
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
569
562
|
const batches = yield* decodeRows(
|
|
570
563
|
Schema.Array(BatchRow),
|
|
571
564
|
"effect_agent_canonical_batches",
|
|
572
|
-
`${request.
|
|
565
|
+
`${request.threadId}/${request.batchId}`,
|
|
573
566
|
batchRows,
|
|
574
567
|
);
|
|
575
568
|
|
|
576
569
|
if (batches.length > 1) {
|
|
577
570
|
return yield* SqliteStorageCorruptionError.make({
|
|
578
571
|
table: "effect_agent_canonical_batches",
|
|
579
|
-
rowKey: `${request.
|
|
572
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
580
573
|
message: "A canonical batch primary key returned more than one row.",
|
|
581
574
|
});
|
|
582
575
|
}
|
|
@@ -597,41 +590,41 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
597
590
|
}
|
|
598
591
|
|
|
599
592
|
if (
|
|
600
|
-
request.expectedTailSequence !==
|
|
601
|
-
request.expectedTailDigest !==
|
|
593
|
+
request.expectedTailSequence !== thread.tail_sequence ||
|
|
594
|
+
request.expectedTailDigest !== thread.tail_digest
|
|
602
595
|
) {
|
|
603
596
|
return yield* SqliteAppendConflict.make({
|
|
604
597
|
message:
|
|
605
598
|
`Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} ` +
|
|
606
|
-
`but found ${
|
|
599
|
+
`but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
607
600
|
reason: "tail",
|
|
608
|
-
actualTailSequence:
|
|
609
|
-
actualTailDigest:
|
|
601
|
+
actualTailSequence: thread.tail_sequence,
|
|
602
|
+
actualTailDigest: thread.tail_digest,
|
|
610
603
|
});
|
|
611
604
|
}
|
|
612
|
-
if (
|
|
605
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) {
|
|
613
606
|
return yield* SqliteStorageError.make({
|
|
614
607
|
operation: "append canonical batch",
|
|
615
|
-
message: `
|
|
608
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`,
|
|
616
609
|
});
|
|
617
610
|
}
|
|
618
611
|
|
|
619
612
|
const existingRecordRows = yield* sql<Record<string, unknown>>`
|
|
620
613
|
SELECT
|
|
621
|
-
|
|
614
|
+
thread_id,
|
|
622
615
|
sequence,
|
|
623
616
|
record_id,
|
|
624
617
|
batch_id,
|
|
625
618
|
record_json
|
|
626
619
|
FROM effect_agent_canonical_records
|
|
627
|
-
WHERE
|
|
620
|
+
WHERE thread_id = ${request.threadId}
|
|
628
621
|
AND record_id IN ${sql.in(recordIds)}
|
|
629
622
|
ORDER BY sequence
|
|
630
623
|
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
631
624
|
const existingRecords = yield* decodeRows(
|
|
632
625
|
Schema.Array(RecordRow),
|
|
633
626
|
"effect_agent_canonical_records",
|
|
634
|
-
`${request.
|
|
627
|
+
`${request.threadId}/record_ids`,
|
|
635
628
|
existingRecordRows,
|
|
636
629
|
);
|
|
637
630
|
if (existingRecords.length > 0) {
|
|
@@ -642,7 +635,7 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
642
635
|
}
|
|
643
636
|
|
|
644
637
|
const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
645
|
-
|
|
638
|
+
thread.tail_sequence + 1,
|
|
646
639
|
).pipe(
|
|
647
640
|
Effect.mapError((error) =>
|
|
648
641
|
SqliteStorageError.make({
|
|
@@ -666,7 +659,7 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
666
659
|
|
|
667
660
|
yield* sql`
|
|
668
661
|
INSERT INTO effect_agent_canonical_batches (
|
|
669
|
-
|
|
662
|
+
thread_id,
|
|
670
663
|
batch_id,
|
|
671
664
|
first_sequence,
|
|
672
665
|
last_sequence,
|
|
@@ -674,7 +667,7 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
674
667
|
tail_digest,
|
|
675
668
|
batch_json
|
|
676
669
|
) VALUES (
|
|
677
|
-
${request.
|
|
670
|
+
${request.threadId},
|
|
678
671
|
${request.batchId},
|
|
679
672
|
${firstSequence},
|
|
680
673
|
${lastSequence},
|
|
@@ -691,13 +684,13 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
691
684
|
Effect.gen(function* () {
|
|
692
685
|
yield* sql`
|
|
693
686
|
INSERT INTO effect_agent_canonical_records (
|
|
694
|
-
|
|
687
|
+
thread_id,
|
|
695
688
|
sequence,
|
|
696
689
|
record_id,
|
|
697
690
|
batch_id,
|
|
698
691
|
record_json
|
|
699
692
|
) VALUES (
|
|
700
|
-
${request.
|
|
693
|
+
${request.threadId},
|
|
701
694
|
${firstSequence + index},
|
|
702
695
|
${record.recordId},
|
|
703
696
|
${request.batchId},
|
|
@@ -710,13 +703,13 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
710
703
|
);
|
|
711
704
|
|
|
712
705
|
yield* sql`
|
|
713
|
-
UPDATE
|
|
706
|
+
UPDATE effect_agent_threads
|
|
714
707
|
SET
|
|
715
708
|
tail_sequence = ${lastSequence},
|
|
716
709
|
tail_digest = ${request.tailDigest},
|
|
717
710
|
producer_epoch = ${request.producerEpoch}
|
|
718
|
-
WHERE
|
|
719
|
-
`.pipe(Effect.mapError(storageError("advance
|
|
711
|
+
WHERE thread_id = ${request.threadId}
|
|
712
|
+
`.pipe(Effect.mapError(storageError("advance thread tail")));
|
|
720
713
|
yield* failpoint("append:after-tail-update");
|
|
721
714
|
|
|
722
715
|
return RawAppendResult.make({
|
|
@@ -732,13 +725,13 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
732
725
|
const read = Effect.fn("SqliteJournal.read")(function* (request: RawReadRequest) {
|
|
733
726
|
const rows = yield* sql<Record<string, unknown>>`
|
|
734
727
|
SELECT
|
|
735
|
-
|
|
728
|
+
thread_id,
|
|
736
729
|
sequence,
|
|
737
730
|
record_id,
|
|
738
731
|
batch_id,
|
|
739
732
|
record_json
|
|
740
733
|
FROM effect_agent_canonical_records
|
|
741
|
-
WHERE
|
|
734
|
+
WHERE thread_id = ${request.threadId}
|
|
742
735
|
AND sequence > ${request.fromSequenceExclusive}
|
|
743
736
|
ORDER BY sequence
|
|
744
737
|
LIMIT ${request.limit}
|
|
@@ -746,36 +739,34 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
746
739
|
return yield* decodeRows(
|
|
747
740
|
Schema.Array(RecordRow),
|
|
748
741
|
"effect_agent_canonical_records",
|
|
749
|
-
`${request.
|
|
742
|
+
`${request.threadId}>${request.fromSequenceExclusive}`,
|
|
750
743
|
rows,
|
|
751
744
|
);
|
|
752
745
|
});
|
|
753
746
|
|
|
754
|
-
const
|
|
755
|
-
conversationId: string,
|
|
756
|
-
) {
|
|
747
|
+
const exportThread = Effect.fn("SqliteJournal.exportThread")(function* (threadId: string) {
|
|
757
748
|
return yield* withReadTransaction("export transaction")(
|
|
758
749
|
Effect.gen(function* () {
|
|
759
|
-
const
|
|
750
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
760
751
|
SELECT
|
|
761
|
-
|
|
752
|
+
thread_id,
|
|
762
753
|
created_at,
|
|
763
754
|
tail_sequence,
|
|
764
755
|
tail_digest,
|
|
765
756
|
producer_epoch
|
|
766
|
-
FROM
|
|
767
|
-
WHERE
|
|
768
|
-
`.pipe(Effect.mapError(storageError("export
|
|
769
|
-
const
|
|
770
|
-
Schema.Array(
|
|
771
|
-
"
|
|
772
|
-
|
|
773
|
-
|
|
757
|
+
FROM effect_agent_threads
|
|
758
|
+
WHERE thread_id = ${threadId}
|
|
759
|
+
`.pipe(Effect.mapError(storageError("export thread")));
|
|
760
|
+
const thread = yield* decodeSingleRow(
|
|
761
|
+
Schema.Array(ThreadRow),
|
|
762
|
+
"effect_agent_threads",
|
|
763
|
+
threadId,
|
|
764
|
+
threadRows,
|
|
774
765
|
);
|
|
775
|
-
yield* failpoint("export:after-
|
|
766
|
+
yield* failpoint("export:after-thread-read");
|
|
776
767
|
const batchRows = yield* sql<Record<string, unknown>>`
|
|
777
768
|
SELECT
|
|
778
|
-
|
|
769
|
+
thread_id,
|
|
779
770
|
batch_id,
|
|
780
771
|
first_sequence,
|
|
781
772
|
last_sequence,
|
|
@@ -783,49 +774,49 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
783
774
|
tail_digest,
|
|
784
775
|
batch_json
|
|
785
776
|
FROM effect_agent_canonical_batches
|
|
786
|
-
WHERE
|
|
777
|
+
WHERE thread_id = ${threadId}
|
|
787
778
|
ORDER BY first_sequence
|
|
788
779
|
`.pipe(Effect.mapError(storageError("export canonical batches")));
|
|
789
780
|
const recordRows = yield* sql<Record<string, unknown>>`
|
|
790
781
|
SELECT
|
|
791
|
-
|
|
782
|
+
thread_id,
|
|
792
783
|
sequence,
|
|
793
784
|
record_id,
|
|
794
785
|
batch_id,
|
|
795
786
|
record_json
|
|
796
787
|
FROM effect_agent_canonical_records
|
|
797
|
-
WHERE
|
|
788
|
+
WHERE thread_id = ${threadId}
|
|
798
789
|
ORDER BY sequence
|
|
799
790
|
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
800
791
|
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
801
792
|
SELECT
|
|
802
|
-
|
|
793
|
+
thread_id,
|
|
803
794
|
through_sequence,
|
|
804
795
|
tail_digest,
|
|
805
796
|
checkpoint_json
|
|
806
797
|
FROM effect_agent_checkpoints
|
|
807
|
-
WHERE
|
|
798
|
+
WHERE thread_id = ${threadId}
|
|
808
799
|
ORDER BY through_sequence
|
|
809
800
|
`.pipe(Effect.mapError(storageError("export checkpoints")));
|
|
810
801
|
|
|
811
|
-
return
|
|
812
|
-
|
|
802
|
+
return RawThreadExport.make({
|
|
803
|
+
thread,
|
|
813
804
|
batches: yield* decodeRows(
|
|
814
805
|
Schema.Array(BatchRow),
|
|
815
806
|
"effect_agent_canonical_batches",
|
|
816
|
-
|
|
807
|
+
threadId,
|
|
817
808
|
batchRows,
|
|
818
809
|
),
|
|
819
810
|
records: yield* decodeRows(
|
|
820
811
|
Schema.Array(RecordRow),
|
|
821
812
|
"effect_agent_canonical_records",
|
|
822
|
-
|
|
813
|
+
threadId,
|
|
823
814
|
recordRows,
|
|
824
815
|
),
|
|
825
816
|
checkpoints: yield* decodeRows(
|
|
826
817
|
Schema.Array(CheckpointRow),
|
|
827
818
|
"effect_agent_checkpoints",
|
|
828
|
-
|
|
819
|
+
threadId,
|
|
829
820
|
checkpointRows,
|
|
830
821
|
),
|
|
831
822
|
});
|
|
@@ -837,7 +828,7 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
837
828
|
checkpoint: RawCheckpoint,
|
|
838
829
|
): Effect.fn.Return<void, CheckpointError> {
|
|
839
830
|
if (
|
|
840
|
-
checkpoint.
|
|
831
|
+
checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
841
832
|
storedTextBytes(checkpoint.checkpointJson) > MAX_STORED_TEXT_BYTES
|
|
842
833
|
) {
|
|
843
834
|
return yield* SqliteStorageError.make({
|
|
@@ -847,50 +838,50 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
847
838
|
}
|
|
848
839
|
yield* withWriteTransaction("checkpoint transaction")(
|
|
849
840
|
Effect.gen(function* () {
|
|
850
|
-
const
|
|
841
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
851
842
|
SELECT
|
|
852
|
-
|
|
843
|
+
thread_id,
|
|
853
844
|
created_at,
|
|
854
845
|
tail_sequence,
|
|
855
846
|
tail_digest,
|
|
856
847
|
producer_epoch
|
|
857
|
-
FROM
|
|
858
|
-
WHERE
|
|
848
|
+
FROM effect_agent_threads
|
|
849
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
859
850
|
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
860
|
-
const
|
|
861
|
-
Schema.Array(
|
|
862
|
-
"
|
|
863
|
-
checkpoint.
|
|
864
|
-
|
|
851
|
+
const thread = yield* decodeSingleRow(
|
|
852
|
+
Schema.Array(ThreadRow),
|
|
853
|
+
"effect_agent_threads",
|
|
854
|
+
checkpoint.threadId,
|
|
855
|
+
threadRows,
|
|
865
856
|
);
|
|
866
|
-
if (checkpoint.throughSequence >
|
|
857
|
+
if (checkpoint.throughSequence > thread.tail_sequence) {
|
|
867
858
|
return yield* SqliteCheckpointConflict.make({
|
|
868
859
|
message:
|
|
869
860
|
`Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
|
|
870
|
-
`${
|
|
861
|
+
`${thread.tail_sequence}.`,
|
|
871
862
|
});
|
|
872
863
|
}
|
|
873
864
|
|
|
874
865
|
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
875
866
|
SELECT
|
|
876
|
-
|
|
867
|
+
thread_id,
|
|
877
868
|
through_sequence,
|
|
878
869
|
tail_digest,
|
|
879
870
|
checkpoint_json
|
|
880
871
|
FROM effect_agent_checkpoints
|
|
881
|
-
WHERE
|
|
872
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
882
873
|
AND through_sequence = ${checkpoint.throughSequence}
|
|
883
874
|
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
884
875
|
const existing = yield* decodeRows(
|
|
885
876
|
Schema.Array(CheckpointRow),
|
|
886
877
|
"effect_agent_checkpoints",
|
|
887
|
-
`${checkpoint.
|
|
878
|
+
`${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
888
879
|
checkpointRows,
|
|
889
880
|
);
|
|
890
881
|
if (existing.length > 1) {
|
|
891
882
|
return yield* SqliteStorageCorruptionError.make({
|
|
892
883
|
table: "effect_agent_checkpoints",
|
|
893
|
-
rowKey: `${checkpoint.
|
|
884
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
894
885
|
message: "A checkpoint primary key returned more than one row.",
|
|
895
886
|
});
|
|
896
887
|
}
|
|
@@ -908,12 +899,12 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
908
899
|
|
|
909
900
|
yield* sql`
|
|
910
901
|
INSERT INTO effect_agent_checkpoints (
|
|
911
|
-
|
|
902
|
+
thread_id,
|
|
912
903
|
through_sequence,
|
|
913
904
|
tail_digest,
|
|
914
905
|
checkpoint_json
|
|
915
906
|
) VALUES (
|
|
916
|
-
${checkpoint.
|
|
907
|
+
${checkpoint.threadId},
|
|
917
908
|
${checkpoint.throughSequence},
|
|
918
909
|
${checkpoint.tailDigest},
|
|
919
910
|
${checkpoint.checkpointJson}
|
|
@@ -924,17 +915,17 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
924
915
|
});
|
|
925
916
|
|
|
926
917
|
const loadCheckpoint = Effect.fn("SqliteJournal.loadCheckpoint")(function* (
|
|
927
|
-
|
|
918
|
+
threadId: string,
|
|
928
919
|
atOrBeforeSequence: CanonicalSequence,
|
|
929
920
|
) {
|
|
930
921
|
const rows = yield* sql<Record<string, unknown>>`
|
|
931
922
|
SELECT
|
|
932
|
-
|
|
923
|
+
thread_id,
|
|
933
924
|
through_sequence,
|
|
934
925
|
tail_digest,
|
|
935
926
|
checkpoint_json
|
|
936
927
|
FROM effect_agent_checkpoints
|
|
937
|
-
WHERE
|
|
928
|
+
WHERE thread_id = ${threadId}
|
|
938
929
|
AND through_sequence <= ${atOrBeforeSequence}
|
|
939
930
|
ORDER BY through_sequence DESC
|
|
940
931
|
LIMIT 1
|
|
@@ -942,26 +933,26 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
942
933
|
return yield* decodeRows(
|
|
943
934
|
Schema.Array(CheckpointRow),
|
|
944
935
|
"effect_agent_checkpoints",
|
|
945
|
-
`${
|
|
936
|
+
`${threadId}<=${atOrBeforeSequence}`,
|
|
946
937
|
rows,
|
|
947
938
|
);
|
|
948
939
|
});
|
|
949
940
|
|
|
950
941
|
const getTailDigestAt = Effect.fn("SqliteJournal.getTailDigestAt")(function* (
|
|
951
|
-
|
|
942
|
+
threadId: string,
|
|
952
943
|
sequence: CanonicalSequence,
|
|
953
944
|
) {
|
|
954
945
|
if (sequence === 0) {
|
|
955
|
-
const
|
|
956
|
-
return
|
|
946
|
+
const threads = yield* getThread(threadId);
|
|
947
|
+
return threads.length === 0
|
|
957
948
|
? []
|
|
958
|
-
: [
|
|
949
|
+
: [threads[0].tail_sequence === 0 ? threads[0].tail_digest : undefined].filter(
|
|
959
950
|
(value): value is string => value !== undefined,
|
|
960
951
|
);
|
|
961
952
|
}
|
|
962
953
|
const rows = yield* sql<Record<string, unknown>>`
|
|
963
954
|
SELECT
|
|
964
|
-
|
|
955
|
+
thread_id,
|
|
965
956
|
batch_id,
|
|
966
957
|
first_sequence,
|
|
967
958
|
last_sequence,
|
|
@@ -969,13 +960,13 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
969
960
|
tail_digest,
|
|
970
961
|
batch_json
|
|
971
962
|
FROM effect_agent_canonical_batches
|
|
972
|
-
WHERE
|
|
963
|
+
WHERE thread_id = ${threadId}
|
|
973
964
|
AND last_sequence = ${sequence}
|
|
974
965
|
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
975
966
|
const batches = yield* decodeRows(
|
|
976
967
|
Schema.Array(BatchRow),
|
|
977
968
|
"effect_agent_canonical_batches",
|
|
978
|
-
`${
|
|
969
|
+
`${threadId}/${sequence}`,
|
|
979
970
|
rows,
|
|
980
971
|
);
|
|
981
972
|
return batches.map((batch) => batch.tail_digest);
|
|
@@ -984,19 +975,19 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
984
975
|
const scanStoredPayloads = Effect.fn("SqliteJournal.scanStoredPayloads")(function* () {
|
|
985
976
|
return yield* withReadTransaction("startup scan transaction")(
|
|
986
977
|
Effect.gen(function* () {
|
|
987
|
-
const
|
|
978
|
+
const threads = yield* sql<Record<string, unknown>>`
|
|
988
979
|
SELECT
|
|
989
|
-
|
|
980
|
+
thread_id,
|
|
990
981
|
created_at,
|
|
991
982
|
tail_sequence,
|
|
992
983
|
tail_digest,
|
|
993
984
|
producer_epoch
|
|
994
|
-
FROM
|
|
995
|
-
ORDER BY
|
|
996
|
-
`.pipe(Effect.mapError(storageError("scan
|
|
985
|
+
FROM effect_agent_threads
|
|
986
|
+
ORDER BY thread_id
|
|
987
|
+
`.pipe(Effect.mapError(storageError("scan threads")));
|
|
997
988
|
const batches = yield* sql<Record<string, unknown>>`
|
|
998
989
|
SELECT
|
|
999
|
-
|
|
990
|
+
thread_id,
|
|
1000
991
|
batch_id,
|
|
1001
992
|
first_sequence,
|
|
1002
993
|
last_sequence,
|
|
@@ -1004,33 +995,33 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
1004
995
|
tail_digest,
|
|
1005
996
|
batch_json
|
|
1006
997
|
FROM effect_agent_canonical_batches
|
|
1007
|
-
ORDER BY
|
|
998
|
+
ORDER BY thread_id, first_sequence
|
|
1008
999
|
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
1009
1000
|
const records = yield* sql<Record<string, unknown>>`
|
|
1010
1001
|
SELECT
|
|
1011
|
-
|
|
1002
|
+
thread_id,
|
|
1012
1003
|
sequence,
|
|
1013
1004
|
record_id,
|
|
1014
1005
|
batch_id,
|
|
1015
1006
|
record_json
|
|
1016
1007
|
FROM effect_agent_canonical_records
|
|
1017
|
-
ORDER BY
|
|
1008
|
+
ORDER BY thread_id, sequence
|
|
1018
1009
|
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
1019
1010
|
const checkpoints = yield* sql<Record<string, unknown>>`
|
|
1020
1011
|
SELECT
|
|
1021
|
-
|
|
1012
|
+
thread_id,
|
|
1022
1013
|
through_sequence,
|
|
1023
1014
|
tail_digest,
|
|
1024
1015
|
checkpoint_json
|
|
1025
1016
|
FROM effect_agent_checkpoints
|
|
1026
|
-
ORDER BY
|
|
1017
|
+
ORDER BY thread_id, through_sequence
|
|
1027
1018
|
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
1028
1019
|
return {
|
|
1029
|
-
|
|
1030
|
-
Schema.Array(
|
|
1031
|
-
"
|
|
1020
|
+
threads: yield* decodeRows(
|
|
1021
|
+
Schema.Array(ThreadRow),
|
|
1022
|
+
"effect_agent_threads",
|
|
1032
1023
|
"startup_scan",
|
|
1033
|
-
|
|
1024
|
+
threads,
|
|
1034
1025
|
),
|
|
1035
1026
|
batches: yield* decodeRows(
|
|
1036
1027
|
Schema.Array(BatchRow),
|
|
@@ -1057,8 +1048,8 @@ export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(fun
|
|
|
1057
1048
|
|
|
1058
1049
|
return {
|
|
1059
1050
|
append,
|
|
1060
|
-
|
|
1061
|
-
|
|
1051
|
+
exportThread,
|
|
1052
|
+
getThread,
|
|
1062
1053
|
getTailDigestAt,
|
|
1063
1054
|
loadCheckpoint,
|
|
1064
1055
|
materialize,
|