@effect-agent/storage-cloudflare 0.1.0-beta.38 → 0.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/do-storage-failpoint-9XB9tuif.d.mts +107 -0
- package/dist/do-storage-failpoint-C-Qtchs7.mjs +12 -0
- package/dist/do-storage-failpoint-C-Qtchs7.mjs.map +1 -0
- package/dist/index.d.mts +161 -248
- package/dist/index.mjs +1211 -539
- package/dist/index.mjs.map +1 -1
- package/dist/testing.d.mts +33 -0
- package/dist/testing.mjs +35 -0
- package/dist/testing.mjs.map +1 -0
- package/package.json +14 -7
- package/src/do-journal.ts +148 -157
- package/src/do-ledger.ts +66 -66
- package/src/do-schedule-store.ts +7 -3
- package/src/do-storage-failpoint-testing.ts +73 -0
- package/src/do-storage-failpoint.ts +1 -68
- package/src/do-subscription-store.ts +1122 -0
- package/src/{do-conversation-store.ts → do-thread-store.ts} +313 -334
- package/src/errors.ts +3 -3
- package/src/index.ts +9 -8
- package/src/migrations.ts +28 -35
- package/src/port-protocol.ts +29 -29
- package/src/routing.ts +146 -188
- package/src/testing.ts +2 -0
package/src/do-journal.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/
|
|
1
|
+
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread";
|
|
2
2
|
import { SqliteMigrator } from "@effect/sql-sqlite-do";
|
|
3
3
|
import { Effect, Schema } from "effect";
|
|
4
4
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
@@ -25,7 +25,7 @@ import { CurrentDoStorageVersion, doMigrations } from "./migrations.ts";
|
|
|
25
25
|
*/
|
|
26
26
|
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(2_000_000));
|
|
27
27
|
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
28
|
-
const
|
|
28
|
+
const MAX_RECORDS_PER_THREAD = 65_536;
|
|
29
29
|
const MAX_IDENTIFIER_LENGTH = 1_024;
|
|
30
30
|
/** Durable Object SQL storage allows at most 100 bound parameters per statement. */
|
|
31
31
|
const MAX_BOUND_PARAMETERS = 100;
|
|
@@ -49,8 +49,8 @@ class DoNameRow extends Schema.Class<DoNameRow>("DoNameRow")({
|
|
|
49
49
|
name: BoundedIdentifier,
|
|
50
50
|
}) {}
|
|
51
51
|
|
|
52
|
-
class
|
|
53
|
-
|
|
52
|
+
class ThreadRow extends Schema.Class<ThreadRow>("ThreadRow")({
|
|
53
|
+
thread_id: BoundedIdentifier,
|
|
54
54
|
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
55
55
|
producer_epoch: ProducerEpoch,
|
|
56
56
|
tail_digest: BoundedStoredText,
|
|
@@ -61,7 +61,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
61
61
|
batch_digest: BoundedStoredText,
|
|
62
62
|
batch_id: BoundedIdentifier,
|
|
63
63
|
batch_json: BoundedStoredText,
|
|
64
|
-
|
|
64
|
+
thread_id: BoundedIdentifier,
|
|
65
65
|
first_sequence: CanonicalSequence,
|
|
66
66
|
last_sequence: CanonicalSequence,
|
|
67
67
|
tail_digest: BoundedStoredText,
|
|
@@ -69,7 +69,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
69
69
|
|
|
70
70
|
class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
71
71
|
batch_id: BoundedIdentifier,
|
|
72
|
-
|
|
72
|
+
thread_id: BoundedIdentifier,
|
|
73
73
|
record_id: BoundedIdentifier,
|
|
74
74
|
record_json: BoundedStoredText,
|
|
75
75
|
sequence: CanonicalSequence,
|
|
@@ -77,7 +77,7 @@ class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
|
77
77
|
|
|
78
78
|
class CheckpointRow extends Schema.Class<CheckpointRow>("CheckpointRow")({
|
|
79
79
|
checkpoint_json: BoundedStoredText,
|
|
80
|
-
|
|
80
|
+
thread_id: BoundedIdentifier,
|
|
81
81
|
tail_digest: BoundedStoredText,
|
|
82
82
|
through_sequence: CanonicalSequence,
|
|
83
83
|
}) {}
|
|
@@ -95,7 +95,7 @@ export class RawAppendRequest extends Schema.Class<RawAppendRequest>(
|
|
|
95
95
|
batchDigest: BoundedStoredText,
|
|
96
96
|
batchId: BoundedIdentifier,
|
|
97
97
|
batchJson: BoundedStoredText,
|
|
98
|
-
|
|
98
|
+
threadId: BoundedIdentifier,
|
|
99
99
|
expectedTailDigest: BoundedStoredText,
|
|
100
100
|
expectedTailSequence: CanonicalSequence,
|
|
101
101
|
producerEpoch: ProducerEpoch,
|
|
@@ -115,7 +115,7 @@ export class RawAppendResult extends Schema.Class<RawAppendResult>(
|
|
|
115
115
|
export class RawReadRequest extends Schema.Class<RawReadRequest>(
|
|
116
116
|
"@effect-agent/storage-cloudflare/RawReadRequest",
|
|
117
117
|
)({
|
|
118
|
-
|
|
118
|
+
threadId: BoundedIdentifier,
|
|
119
119
|
fromSequenceExclusive: CanonicalSequence,
|
|
120
120
|
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
|
|
121
121
|
}) {}
|
|
@@ -124,17 +124,17 @@ export class RawCheckpoint extends Schema.Class<RawCheckpoint>(
|
|
|
124
124
|
"@effect-agent/storage-cloudflare/RawCheckpoint",
|
|
125
125
|
)({
|
|
126
126
|
checkpointJson: BoundedStoredText,
|
|
127
|
-
|
|
127
|
+
threadId: BoundedIdentifier,
|
|
128
128
|
tailDigest: BoundedStoredText,
|
|
129
129
|
throughSequence: CanonicalSequence,
|
|
130
130
|
}) {}
|
|
131
131
|
|
|
132
|
-
export class
|
|
133
|
-
"@effect-agent/storage-cloudflare/
|
|
132
|
+
export class RawThreadExport extends Schema.Class<RawThreadExport>(
|
|
133
|
+
"@effect-agent/storage-cloudflare/RawThreadExport",
|
|
134
134
|
)({
|
|
135
135
|
batches: Schema.Array(BatchRow),
|
|
136
136
|
checkpoints: Schema.Array(CheckpointRow),
|
|
137
|
-
|
|
137
|
+
thread: ThreadRow,
|
|
138
138
|
records: Schema.Array(RecordRow),
|
|
139
139
|
}) {}
|
|
140
140
|
|
|
@@ -218,7 +218,7 @@ const REQUIRED_TABLES = [
|
|
|
218
218
|
"effect_agent_checkpoints",
|
|
219
219
|
"effect_agent_child_reservations",
|
|
220
220
|
"effect_agent_child_settlements",
|
|
221
|
-
"
|
|
221
|
+
"effect_agent_threads",
|
|
222
222
|
"effect_agent_meta",
|
|
223
223
|
"effect_agent_settlement_reservations",
|
|
224
224
|
"effect_agent_submission_ownership",
|
|
@@ -384,7 +384,7 @@ const makeJournal = (
|
|
|
384
384
|
);
|
|
385
385
|
|
|
386
386
|
const materialize = Effect.fn("DoJournal.materialize")(function* (
|
|
387
|
-
|
|
387
|
+
threadId: string,
|
|
388
388
|
createdAt: string,
|
|
389
389
|
emptyTailDigest: string,
|
|
390
390
|
producerEpoch: ProducerEpoch,
|
|
@@ -392,54 +392,54 @@ const makeJournal = (
|
|
|
392
392
|
void,
|
|
393
393
|
DoFenceRejected | DoStorageCorruptionError | DoStorageError | DoValueBoundExceeded
|
|
394
394
|
> {
|
|
395
|
-
if (
|
|
395
|
+
if (threadId.length > MAX_IDENTIFIER_LENGTH) {
|
|
396
396
|
return yield* DoStorageError.make({
|
|
397
|
-
operation: "materialize
|
|
398
|
-
message: "
|
|
397
|
+
operation: "materialize thread",
|
|
398
|
+
message: "Thread identity exceeds the Durable Object storage bounds.",
|
|
399
399
|
});
|
|
400
400
|
}
|
|
401
|
-
yield* checkValueBound("materialize
|
|
401
|
+
yield* checkValueBound("materialize thread", emptyTailDigest);
|
|
402
402
|
yield* withWriteTransaction("materialize transaction")(
|
|
403
403
|
Effect.gen(function* () {
|
|
404
404
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
405
405
|
SELECT
|
|
406
|
-
|
|
406
|
+
thread_id,
|
|
407
407
|
created_at,
|
|
408
408
|
tail_sequence,
|
|
409
409
|
tail_digest,
|
|
410
410
|
producer_epoch
|
|
411
|
-
FROM
|
|
412
|
-
WHERE
|
|
413
|
-
`.pipe(Effect.mapError(storageError("read materialized
|
|
411
|
+
FROM effect_agent_threads
|
|
412
|
+
WHERE thread_id = ${threadId}
|
|
413
|
+
`.pipe(Effect.mapError(storageError("read materialized thread")));
|
|
414
414
|
const existing = yield* decodeRows(
|
|
415
|
-
Schema.Array(
|
|
416
|
-
"
|
|
417
|
-
|
|
415
|
+
Schema.Array(ThreadRow),
|
|
416
|
+
"effect_agent_threads",
|
|
417
|
+
threadId,
|
|
418
418
|
existingRows,
|
|
419
419
|
);
|
|
420
420
|
if (existing.length > 1) {
|
|
421
421
|
return yield* DoStorageCorruptionError.make({
|
|
422
|
-
table: "
|
|
423
|
-
rowKey:
|
|
424
|
-
message: "A
|
|
422
|
+
table: "effect_agent_threads",
|
|
423
|
+
rowKey: threadId,
|
|
424
|
+
message: "A thread primary key returned more than one row.",
|
|
425
425
|
});
|
|
426
426
|
}
|
|
427
427
|
if (existing.length === 0) {
|
|
428
428
|
yield* sql`
|
|
429
|
-
INSERT INTO
|
|
430
|
-
|
|
429
|
+
INSERT INTO effect_agent_threads (
|
|
430
|
+
thread_id,
|
|
431
431
|
created_at,
|
|
432
432
|
tail_sequence,
|
|
433
433
|
tail_digest,
|
|
434
434
|
producer_epoch
|
|
435
435
|
) VALUES (
|
|
436
|
-
${
|
|
436
|
+
${threadId},
|
|
437
437
|
${createdAt},
|
|
438
438
|
0,
|
|
439
439
|
${emptyTailDigest},
|
|
440
440
|
${producerEpoch}
|
|
441
441
|
)
|
|
442
|
-
`.pipe(Effect.mapError(storageError("materialize
|
|
442
|
+
`.pipe(Effect.mapError(storageError("materialize thread")));
|
|
443
443
|
return;
|
|
444
444
|
}
|
|
445
445
|
if (producerEpoch < existing[0].producer_epoch) {
|
|
@@ -451,41 +451,34 @@ const makeJournal = (
|
|
|
451
451
|
}
|
|
452
452
|
if (producerEpoch > existing[0].producer_epoch) {
|
|
453
453
|
yield* sql`
|
|
454
|
-
UPDATE
|
|
454
|
+
UPDATE effect_agent_threads
|
|
455
455
|
SET producer_epoch = ${producerEpoch}
|
|
456
|
-
WHERE
|
|
456
|
+
WHERE thread_id = ${threadId}
|
|
457
457
|
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
458
458
|
}
|
|
459
459
|
}),
|
|
460
460
|
);
|
|
461
461
|
});
|
|
462
462
|
|
|
463
|
-
const
|
|
464
|
-
conversationId: string,
|
|
465
|
-
) {
|
|
463
|
+
const getThread = Effect.fn("DoJournal.getThread")(function* (threadId: string) {
|
|
466
464
|
const rows = yield* sql<Record<string, unknown>>`
|
|
467
465
|
SELECT
|
|
468
|
-
|
|
466
|
+
thread_id,
|
|
469
467
|
created_at,
|
|
470
468
|
tail_sequence,
|
|
471
469
|
tail_digest,
|
|
472
470
|
producer_epoch
|
|
473
|
-
FROM
|
|
474
|
-
WHERE
|
|
475
|
-
`.pipe(Effect.mapError(storageError("read
|
|
476
|
-
return yield* decodeRows(
|
|
477
|
-
Schema.Array(ConversationRow),
|
|
478
|
-
"effect_agent_conversations",
|
|
479
|
-
conversationId,
|
|
480
|
-
rows,
|
|
481
|
-
);
|
|
471
|
+
FROM effect_agent_threads
|
|
472
|
+
WHERE thread_id = ${threadId}
|
|
473
|
+
`.pipe(Effect.mapError(storageError("read thread")));
|
|
474
|
+
return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
482
475
|
});
|
|
483
476
|
|
|
484
477
|
const append = Effect.fn("DoJournal.append")(function* (
|
|
485
478
|
request: RawAppendRequest,
|
|
486
479
|
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
487
480
|
if (
|
|
488
|
-
request.
|
|
481
|
+
request.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
489
482
|
request.batchId.length > MAX_IDENTIFIER_LENGTH ||
|
|
490
483
|
request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)
|
|
491
484
|
) {
|
|
@@ -513,34 +506,34 @@ const makeJournal = (
|
|
|
513
506
|
});
|
|
514
507
|
}
|
|
515
508
|
|
|
516
|
-
const
|
|
509
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
517
510
|
SELECT
|
|
518
|
-
|
|
511
|
+
thread_id,
|
|
519
512
|
created_at,
|
|
520
513
|
tail_sequence,
|
|
521
514
|
tail_digest,
|
|
522
515
|
producer_epoch
|
|
523
|
-
FROM
|
|
524
|
-
WHERE
|
|
516
|
+
FROM effect_agent_threads
|
|
517
|
+
WHERE thread_id = ${request.threadId}
|
|
525
518
|
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
526
|
-
const
|
|
527
|
-
Schema.Array(
|
|
528
|
-
"
|
|
529
|
-
request.
|
|
530
|
-
|
|
519
|
+
const thread = yield* decodeSingleRow(
|
|
520
|
+
Schema.Array(ThreadRow),
|
|
521
|
+
"effect_agent_threads",
|
|
522
|
+
request.threadId,
|
|
523
|
+
threadRows,
|
|
531
524
|
);
|
|
532
525
|
|
|
533
|
-
if (request.producerEpoch !==
|
|
526
|
+
if (request.producerEpoch !== thread.producer_epoch) {
|
|
534
527
|
return yield* DoFenceRejected.make({
|
|
535
528
|
producerEpoch: request.producerEpoch,
|
|
536
|
-
actualEpoch:
|
|
537
|
-
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${
|
|
529
|
+
actualEpoch: thread.producer_epoch,
|
|
530
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`,
|
|
538
531
|
});
|
|
539
532
|
}
|
|
540
533
|
|
|
541
534
|
const batchRows = yield* sql<Record<string, unknown>>`
|
|
542
535
|
SELECT
|
|
543
|
-
|
|
536
|
+
thread_id,
|
|
544
537
|
batch_id,
|
|
545
538
|
first_sequence,
|
|
546
539
|
last_sequence,
|
|
@@ -548,20 +541,20 @@ const makeJournal = (
|
|
|
548
541
|
tail_digest,
|
|
549
542
|
batch_json
|
|
550
543
|
FROM effect_agent_canonical_batches
|
|
551
|
-
WHERE
|
|
544
|
+
WHERE thread_id = ${request.threadId}
|
|
552
545
|
AND batch_id = ${request.batchId}
|
|
553
546
|
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
554
547
|
const batches = yield* decodeRows(
|
|
555
548
|
Schema.Array(BatchRow),
|
|
556
549
|
"effect_agent_canonical_batches",
|
|
557
|
-
`${request.
|
|
550
|
+
`${request.threadId}/${request.batchId}`,
|
|
558
551
|
batchRows,
|
|
559
552
|
);
|
|
560
553
|
|
|
561
554
|
if (batches.length > 1) {
|
|
562
555
|
return yield* DoStorageCorruptionError.make({
|
|
563
556
|
table: "effect_agent_canonical_batches",
|
|
564
|
-
rowKey: `${request.
|
|
557
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
565
558
|
message: "A canonical batch primary key returned more than one row.",
|
|
566
559
|
});
|
|
567
560
|
}
|
|
@@ -582,22 +575,22 @@ const makeJournal = (
|
|
|
582
575
|
}
|
|
583
576
|
|
|
584
577
|
if (
|
|
585
|
-
request.expectedTailSequence !==
|
|
586
|
-
request.expectedTailDigest !==
|
|
578
|
+
request.expectedTailSequence !== thread.tail_sequence ||
|
|
579
|
+
request.expectedTailDigest !== thread.tail_digest
|
|
587
580
|
) {
|
|
588
581
|
return yield* DoAppendConflict.make({
|
|
589
582
|
message:
|
|
590
583
|
`Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} ` +
|
|
591
|
-
`but found ${
|
|
584
|
+
`but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
592
585
|
reason: "tail",
|
|
593
|
-
actualTailSequence:
|
|
594
|
-
actualTailDigest:
|
|
586
|
+
actualTailSequence: thread.tail_sequence,
|
|
587
|
+
actualTailDigest: thread.tail_digest,
|
|
595
588
|
});
|
|
596
589
|
}
|
|
597
|
-
if (
|
|
590
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) {
|
|
598
591
|
return yield* DoStorageError.make({
|
|
599
592
|
operation: "append canonical batch",
|
|
600
|
-
message: `
|
|
593
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`,
|
|
601
594
|
});
|
|
602
595
|
}
|
|
603
596
|
|
|
@@ -607,13 +600,13 @@ const makeJournal = (
|
|
|
607
600
|
for (const chunk of chunked(recordIds, MAX_BOUND_PARAMETERS - 10)) {
|
|
608
601
|
const existingRecordRows = yield* sql<Record<string, unknown>>`
|
|
609
602
|
SELECT
|
|
610
|
-
|
|
603
|
+
thread_id,
|
|
611
604
|
sequence,
|
|
612
605
|
record_id,
|
|
613
606
|
batch_id,
|
|
614
607
|
record_json
|
|
615
608
|
FROM effect_agent_canonical_records
|
|
616
|
-
WHERE
|
|
609
|
+
WHERE thread_id = ${request.threadId}
|
|
617
610
|
AND record_id IN ${sql.in([...chunk])}
|
|
618
611
|
ORDER BY sequence
|
|
619
612
|
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
@@ -621,7 +614,7 @@ const makeJournal = (
|
|
|
621
614
|
...(yield* decodeRows(
|
|
622
615
|
Schema.Array(RecordRow),
|
|
623
616
|
"effect_agent_canonical_records",
|
|
624
|
-
`${request.
|
|
617
|
+
`${request.threadId}/record_ids`,
|
|
625
618
|
existingRecordRows,
|
|
626
619
|
)),
|
|
627
620
|
);
|
|
@@ -634,7 +627,7 @@ const makeJournal = (
|
|
|
634
627
|
}
|
|
635
628
|
|
|
636
629
|
const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
637
|
-
|
|
630
|
+
thread.tail_sequence + 1,
|
|
638
631
|
).pipe(
|
|
639
632
|
Effect.mapError((error) =>
|
|
640
633
|
DoStorageError.make({
|
|
@@ -658,7 +651,7 @@ const makeJournal = (
|
|
|
658
651
|
|
|
659
652
|
yield* sql`
|
|
660
653
|
INSERT INTO effect_agent_canonical_batches (
|
|
661
|
-
|
|
654
|
+
thread_id,
|
|
662
655
|
batch_id,
|
|
663
656
|
first_sequence,
|
|
664
657
|
last_sequence,
|
|
@@ -666,7 +659,7 @@ const makeJournal = (
|
|
|
666
659
|
tail_digest,
|
|
667
660
|
batch_json
|
|
668
661
|
) VALUES (
|
|
669
|
-
${request.
|
|
662
|
+
${request.threadId},
|
|
670
663
|
${request.batchId},
|
|
671
664
|
${firstSequence},
|
|
672
665
|
${lastSequence},
|
|
@@ -683,13 +676,13 @@ const makeJournal = (
|
|
|
683
676
|
Effect.gen(function* () {
|
|
684
677
|
yield* sql`
|
|
685
678
|
INSERT INTO effect_agent_canonical_records (
|
|
686
|
-
|
|
679
|
+
thread_id,
|
|
687
680
|
sequence,
|
|
688
681
|
record_id,
|
|
689
682
|
batch_id,
|
|
690
683
|
record_json
|
|
691
684
|
) VALUES (
|
|
692
|
-
${request.
|
|
685
|
+
${request.threadId},
|
|
693
686
|
${firstSequence + index},
|
|
694
687
|
${record.recordId},
|
|
695
688
|
${request.batchId},
|
|
@@ -702,13 +695,13 @@ const makeJournal = (
|
|
|
702
695
|
);
|
|
703
696
|
|
|
704
697
|
yield* sql`
|
|
705
|
-
UPDATE
|
|
698
|
+
UPDATE effect_agent_threads
|
|
706
699
|
SET
|
|
707
700
|
tail_sequence = ${lastSequence},
|
|
708
701
|
tail_digest = ${request.tailDigest},
|
|
709
702
|
producer_epoch = ${request.producerEpoch}
|
|
710
|
-
WHERE
|
|
711
|
-
`.pipe(Effect.mapError(storageError("advance
|
|
703
|
+
WHERE thread_id = ${request.threadId}
|
|
704
|
+
`.pipe(Effect.mapError(storageError("advance thread tail")));
|
|
712
705
|
yield* failpoint("append:after-tail-update");
|
|
713
706
|
|
|
714
707
|
return RawAppendResult.make({
|
|
@@ -724,13 +717,13 @@ const makeJournal = (
|
|
|
724
717
|
const read = Effect.fn("DoJournal.read")(function* (request: RawReadRequest) {
|
|
725
718
|
const rows = yield* sql<Record<string, unknown>>`
|
|
726
719
|
SELECT
|
|
727
|
-
|
|
720
|
+
thread_id,
|
|
728
721
|
sequence,
|
|
729
722
|
record_id,
|
|
730
723
|
batch_id,
|
|
731
724
|
record_json
|
|
732
725
|
FROM effect_agent_canonical_records
|
|
733
|
-
WHERE
|
|
726
|
+
WHERE thread_id = ${request.threadId}
|
|
734
727
|
AND sequence > ${request.fromSequenceExclusive}
|
|
735
728
|
ORDER BY sequence
|
|
736
729
|
LIMIT ${request.limit}
|
|
@@ -738,37 +731,35 @@ const makeJournal = (
|
|
|
738
731
|
return yield* decodeRows(
|
|
739
732
|
Schema.Array(RecordRow),
|
|
740
733
|
"effect_agent_canonical_records",
|
|
741
|
-
`${request.
|
|
734
|
+
`${request.threadId}>${request.fromSequenceExclusive}`,
|
|
742
735
|
rows,
|
|
743
736
|
);
|
|
744
737
|
});
|
|
745
738
|
|
|
746
|
-
const
|
|
747
|
-
conversationId: string,
|
|
748
|
-
) {
|
|
739
|
+
const exportThread = Effect.fn("DoJournal.exportThread")(function* (threadId: string) {
|
|
749
740
|
return yield* sql
|
|
750
741
|
.withTransaction(
|
|
751
742
|
Effect.gen(function* () {
|
|
752
|
-
const
|
|
743
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
753
744
|
SELECT
|
|
754
|
-
|
|
745
|
+
thread_id,
|
|
755
746
|
created_at,
|
|
756
747
|
tail_sequence,
|
|
757
748
|
tail_digest,
|
|
758
749
|
producer_epoch
|
|
759
|
-
FROM
|
|
760
|
-
WHERE
|
|
761
|
-
`.pipe(Effect.mapError(storageError("export
|
|
762
|
-
const
|
|
763
|
-
Schema.Array(
|
|
764
|
-
"
|
|
765
|
-
|
|
766
|
-
|
|
750
|
+
FROM effect_agent_threads
|
|
751
|
+
WHERE thread_id = ${threadId}
|
|
752
|
+
`.pipe(Effect.mapError(storageError("export thread")));
|
|
753
|
+
const thread = yield* decodeSingleRow(
|
|
754
|
+
Schema.Array(ThreadRow),
|
|
755
|
+
"effect_agent_threads",
|
|
756
|
+
threadId,
|
|
757
|
+
threadRows,
|
|
767
758
|
);
|
|
768
|
-
yield* failpoint("export:after-
|
|
759
|
+
yield* failpoint("export:after-thread-read");
|
|
769
760
|
const batchRows = yield* sql<Record<string, unknown>>`
|
|
770
761
|
SELECT
|
|
771
|
-
|
|
762
|
+
thread_id,
|
|
772
763
|
batch_id,
|
|
773
764
|
first_sequence,
|
|
774
765
|
last_sequence,
|
|
@@ -776,49 +767,49 @@ const makeJournal = (
|
|
|
776
767
|
tail_digest,
|
|
777
768
|
batch_json
|
|
778
769
|
FROM effect_agent_canonical_batches
|
|
779
|
-
WHERE
|
|
770
|
+
WHERE thread_id = ${threadId}
|
|
780
771
|
ORDER BY first_sequence
|
|
781
772
|
`.pipe(Effect.mapError(storageError("export canonical batches")));
|
|
782
773
|
const recordRows = yield* sql<Record<string, unknown>>`
|
|
783
774
|
SELECT
|
|
784
|
-
|
|
775
|
+
thread_id,
|
|
785
776
|
sequence,
|
|
786
777
|
record_id,
|
|
787
778
|
batch_id,
|
|
788
779
|
record_json
|
|
789
780
|
FROM effect_agent_canonical_records
|
|
790
|
-
WHERE
|
|
781
|
+
WHERE thread_id = ${threadId}
|
|
791
782
|
ORDER BY sequence
|
|
792
783
|
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
793
784
|
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
794
785
|
SELECT
|
|
795
|
-
|
|
786
|
+
thread_id,
|
|
796
787
|
through_sequence,
|
|
797
788
|
tail_digest,
|
|
798
789
|
checkpoint_json
|
|
799
790
|
FROM effect_agent_checkpoints
|
|
800
|
-
WHERE
|
|
791
|
+
WHERE thread_id = ${threadId}
|
|
801
792
|
ORDER BY through_sequence
|
|
802
793
|
`.pipe(Effect.mapError(storageError("export checkpoints")));
|
|
803
794
|
|
|
804
|
-
return
|
|
805
|
-
|
|
795
|
+
return RawThreadExport.make({
|
|
796
|
+
thread,
|
|
806
797
|
batches: yield* decodeRows(
|
|
807
798
|
Schema.Array(BatchRow),
|
|
808
799
|
"effect_agent_canonical_batches",
|
|
809
|
-
|
|
800
|
+
threadId,
|
|
810
801
|
batchRows,
|
|
811
802
|
),
|
|
812
803
|
records: yield* decodeRows(
|
|
813
804
|
Schema.Array(RecordRow),
|
|
814
805
|
"effect_agent_canonical_records",
|
|
815
|
-
|
|
806
|
+
threadId,
|
|
816
807
|
recordRows,
|
|
817
808
|
),
|
|
818
809
|
checkpoints: yield* decodeRows(
|
|
819
810
|
Schema.Array(CheckpointRow),
|
|
820
811
|
"effect_agent_checkpoints",
|
|
821
|
-
|
|
812
|
+
threadId,
|
|
822
813
|
checkpointRows,
|
|
823
814
|
),
|
|
824
815
|
});
|
|
@@ -834,7 +825,7 @@ const makeJournal = (
|
|
|
834
825
|
const saveCheckpoint = Effect.fn("DoJournal.saveCheckpoint")(function* (
|
|
835
826
|
checkpoint: RawCheckpoint,
|
|
836
827
|
): Effect.fn.Return<void, CheckpointError> {
|
|
837
|
-
if (checkpoint.
|
|
828
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
|
|
838
829
|
return yield* DoStorageError.make({
|
|
839
830
|
operation: "save checkpoint",
|
|
840
831
|
message: "Checkpoint identity exceeds the Durable Object storage bounds.",
|
|
@@ -843,50 +834,50 @@ const makeJournal = (
|
|
|
843
834
|
yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
|
|
844
835
|
yield* withWriteTransaction("checkpoint transaction")(
|
|
845
836
|
Effect.gen(function* () {
|
|
846
|
-
const
|
|
837
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
847
838
|
SELECT
|
|
848
|
-
|
|
839
|
+
thread_id,
|
|
849
840
|
created_at,
|
|
850
841
|
tail_sequence,
|
|
851
842
|
tail_digest,
|
|
852
843
|
producer_epoch
|
|
853
|
-
FROM
|
|
854
|
-
WHERE
|
|
844
|
+
FROM effect_agent_threads
|
|
845
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
855
846
|
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
856
|
-
const
|
|
857
|
-
Schema.Array(
|
|
858
|
-
"
|
|
859
|
-
checkpoint.
|
|
860
|
-
|
|
847
|
+
const thread = yield* decodeSingleRow(
|
|
848
|
+
Schema.Array(ThreadRow),
|
|
849
|
+
"effect_agent_threads",
|
|
850
|
+
checkpoint.threadId,
|
|
851
|
+
threadRows,
|
|
861
852
|
);
|
|
862
|
-
if (checkpoint.throughSequence >
|
|
853
|
+
if (checkpoint.throughSequence > thread.tail_sequence) {
|
|
863
854
|
return yield* DoCheckpointConflict.make({
|
|
864
855
|
message:
|
|
865
856
|
`Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
|
|
866
|
-
`${
|
|
857
|
+
`${thread.tail_sequence}.`,
|
|
867
858
|
});
|
|
868
859
|
}
|
|
869
860
|
|
|
870
861
|
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
871
862
|
SELECT
|
|
872
|
-
|
|
863
|
+
thread_id,
|
|
873
864
|
through_sequence,
|
|
874
865
|
tail_digest,
|
|
875
866
|
checkpoint_json
|
|
876
867
|
FROM effect_agent_checkpoints
|
|
877
|
-
WHERE
|
|
868
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
878
869
|
AND through_sequence = ${checkpoint.throughSequence}
|
|
879
870
|
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
880
871
|
const existing = yield* decodeRows(
|
|
881
872
|
Schema.Array(CheckpointRow),
|
|
882
873
|
"effect_agent_checkpoints",
|
|
883
|
-
`${checkpoint.
|
|
874
|
+
`${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
884
875
|
checkpointRows,
|
|
885
876
|
);
|
|
886
877
|
if (existing.length > 1) {
|
|
887
878
|
return yield* DoStorageCorruptionError.make({
|
|
888
879
|
table: "effect_agent_checkpoints",
|
|
889
|
-
rowKey: `${checkpoint.
|
|
880
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
890
881
|
message: "A checkpoint primary key returned more than one row.",
|
|
891
882
|
});
|
|
892
883
|
}
|
|
@@ -904,12 +895,12 @@ const makeJournal = (
|
|
|
904
895
|
|
|
905
896
|
yield* sql`
|
|
906
897
|
INSERT INTO effect_agent_checkpoints (
|
|
907
|
-
|
|
898
|
+
thread_id,
|
|
908
899
|
through_sequence,
|
|
909
900
|
tail_digest,
|
|
910
901
|
checkpoint_json
|
|
911
902
|
) VALUES (
|
|
912
|
-
${checkpoint.
|
|
903
|
+
${checkpoint.threadId},
|
|
913
904
|
${checkpoint.throughSequence},
|
|
914
905
|
${checkpoint.tailDigest},
|
|
915
906
|
${checkpoint.checkpointJson}
|
|
@@ -920,17 +911,17 @@ const makeJournal = (
|
|
|
920
911
|
});
|
|
921
912
|
|
|
922
913
|
const loadCheckpoint = Effect.fn("DoJournal.loadCheckpoint")(function* (
|
|
923
|
-
|
|
914
|
+
threadId: string,
|
|
924
915
|
atOrBeforeSequence: CanonicalSequence,
|
|
925
916
|
) {
|
|
926
917
|
const rows = yield* sql<Record<string, unknown>>`
|
|
927
918
|
SELECT
|
|
928
|
-
|
|
919
|
+
thread_id,
|
|
929
920
|
through_sequence,
|
|
930
921
|
tail_digest,
|
|
931
922
|
checkpoint_json
|
|
932
923
|
FROM effect_agent_checkpoints
|
|
933
|
-
WHERE
|
|
924
|
+
WHERE thread_id = ${threadId}
|
|
934
925
|
AND through_sequence <= ${atOrBeforeSequence}
|
|
935
926
|
ORDER BY through_sequence DESC
|
|
936
927
|
LIMIT 1
|
|
@@ -938,26 +929,26 @@ const makeJournal = (
|
|
|
938
929
|
return yield* decodeRows(
|
|
939
930
|
Schema.Array(CheckpointRow),
|
|
940
931
|
"effect_agent_checkpoints",
|
|
941
|
-
`${
|
|
932
|
+
`${threadId}<=${atOrBeforeSequence}`,
|
|
942
933
|
rows,
|
|
943
934
|
);
|
|
944
935
|
});
|
|
945
936
|
|
|
946
937
|
const getTailDigestAt = Effect.fn("DoJournal.getTailDigestAt")(function* (
|
|
947
|
-
|
|
938
|
+
threadId: string,
|
|
948
939
|
sequence: CanonicalSequence,
|
|
949
940
|
) {
|
|
950
941
|
if (sequence === 0) {
|
|
951
|
-
const
|
|
952
|
-
return
|
|
942
|
+
const threads = yield* getThread(threadId);
|
|
943
|
+
return threads.length === 0
|
|
953
944
|
? []
|
|
954
|
-
: [
|
|
945
|
+
: [threads[0].tail_sequence === 0 ? threads[0].tail_digest : undefined].filter(
|
|
955
946
|
(value): value is string => value !== undefined,
|
|
956
947
|
);
|
|
957
948
|
}
|
|
958
949
|
const rows = yield* sql<Record<string, unknown>>`
|
|
959
950
|
SELECT
|
|
960
|
-
|
|
951
|
+
thread_id,
|
|
961
952
|
batch_id,
|
|
962
953
|
first_sequence,
|
|
963
954
|
last_sequence,
|
|
@@ -965,13 +956,13 @@ const makeJournal = (
|
|
|
965
956
|
tail_digest,
|
|
966
957
|
batch_json
|
|
967
958
|
FROM effect_agent_canonical_batches
|
|
968
|
-
WHERE
|
|
959
|
+
WHERE thread_id = ${threadId}
|
|
969
960
|
AND last_sequence = ${sequence}
|
|
970
961
|
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
971
962
|
const batches = yield* decodeRows(
|
|
972
963
|
Schema.Array(BatchRow),
|
|
973
964
|
"effect_agent_canonical_batches",
|
|
974
|
-
`${
|
|
965
|
+
`${threadId}/${sequence}`,
|
|
975
966
|
rows,
|
|
976
967
|
);
|
|
977
968
|
return batches.map((batch) => batch.tail_digest);
|
|
@@ -981,19 +972,19 @@ const makeJournal = (
|
|
|
981
972
|
return yield* sql
|
|
982
973
|
.withTransaction(
|
|
983
974
|
Effect.gen(function* () {
|
|
984
|
-
const
|
|
975
|
+
const threads = yield* sql<Record<string, unknown>>`
|
|
985
976
|
SELECT
|
|
986
|
-
|
|
977
|
+
thread_id,
|
|
987
978
|
created_at,
|
|
988
979
|
tail_sequence,
|
|
989
980
|
tail_digest,
|
|
990
981
|
producer_epoch
|
|
991
|
-
FROM
|
|
992
|
-
ORDER BY
|
|
993
|
-
`.pipe(Effect.mapError(storageError("scan
|
|
982
|
+
FROM effect_agent_threads
|
|
983
|
+
ORDER BY thread_id
|
|
984
|
+
`.pipe(Effect.mapError(storageError("scan threads")));
|
|
994
985
|
const batches = yield* sql<Record<string, unknown>>`
|
|
995
986
|
SELECT
|
|
996
|
-
|
|
987
|
+
thread_id,
|
|
997
988
|
batch_id,
|
|
998
989
|
first_sequence,
|
|
999
990
|
last_sequence,
|
|
@@ -1001,33 +992,33 @@ const makeJournal = (
|
|
|
1001
992
|
tail_digest,
|
|
1002
993
|
batch_json
|
|
1003
994
|
FROM effect_agent_canonical_batches
|
|
1004
|
-
ORDER BY
|
|
995
|
+
ORDER BY thread_id, first_sequence
|
|
1005
996
|
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
1006
997
|
const records = yield* sql<Record<string, unknown>>`
|
|
1007
998
|
SELECT
|
|
1008
|
-
|
|
999
|
+
thread_id,
|
|
1009
1000
|
sequence,
|
|
1010
1001
|
record_id,
|
|
1011
1002
|
batch_id,
|
|
1012
1003
|
record_json
|
|
1013
1004
|
FROM effect_agent_canonical_records
|
|
1014
|
-
ORDER BY
|
|
1005
|
+
ORDER BY thread_id, sequence
|
|
1015
1006
|
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
1016
1007
|
const checkpoints = yield* sql<Record<string, unknown>>`
|
|
1017
1008
|
SELECT
|
|
1018
|
-
|
|
1009
|
+
thread_id,
|
|
1019
1010
|
through_sequence,
|
|
1020
1011
|
tail_digest,
|
|
1021
1012
|
checkpoint_json
|
|
1022
1013
|
FROM effect_agent_checkpoints
|
|
1023
|
-
ORDER BY
|
|
1014
|
+
ORDER BY thread_id, through_sequence
|
|
1024
1015
|
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
1025
1016
|
return {
|
|
1026
|
-
|
|
1027
|
-
Schema.Array(
|
|
1028
|
-
"
|
|
1017
|
+
threads: yield* decodeRows(
|
|
1018
|
+
Schema.Array(ThreadRow),
|
|
1019
|
+
"effect_agent_threads",
|
|
1029
1020
|
"startup_scan",
|
|
1030
|
-
|
|
1021
|
+
threads,
|
|
1031
1022
|
),
|
|
1032
1023
|
batches: yield* decodeRows(
|
|
1033
1024
|
Schema.Array(BatchRow),
|
|
@@ -1060,8 +1051,8 @@ const makeJournal = (
|
|
|
1060
1051
|
return {
|
|
1061
1052
|
append,
|
|
1062
1053
|
checkValueBound,
|
|
1063
|
-
|
|
1064
|
-
|
|
1054
|
+
exportThread,
|
|
1055
|
+
getThread,
|
|
1065
1056
|
getTailDigestAt,
|
|
1066
1057
|
loadCheckpoint,
|
|
1067
1058
|
materialize,
|