@effect-agent/storage-cloudflare 0.1.0-beta.62 → 0.1.0-beta.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DoMessageDeliveryStore.d.mts +1 -1
- package/dist/DoMessageDeliveryStore.mjs +1 -1
- package/dist/{DoStorageError-UrxrvbnF.d.mts → DoStorageError-Cmhvl4TQ.d.mts} +3 -3
- package/dist/DoStorageError.d.mts +1 -1
- package/dist/DoStorageError.mjs +2 -0
- package/dist/DoStorageError.mjs.map +1 -1
- package/dist/DoStorageFailpoint.d.mts +1 -1
- package/dist/DoStorageFailpointTesting.d.mts +1 -1
- package/dist/{DoStorageVersion-Pt3jEHdW.d.mts → DoStorageVersion-C05M2uDX.d.mts} +2 -2
- package/dist/DoStorageVersion.d.mts +1 -1
- package/dist/DoStorageVersion.mjs +2 -2
- package/dist/DoSubmissionLedger.mjs +2 -2
- package/dist/DoThreadStore.d.mts +1 -1
- package/dist/DoThreadStore.mjs +40 -7
- package/dist/DoThreadStore.mjs.map +1 -1
- package/dist/PortProtocol.d.mts +12 -12
- package/dist/PortProtocol.mjs +4 -4
- package/dist/PortProtocol.mjs.map +1 -1
- package/dist/PortRouting.d.mts +3 -2
- package/dist/PortRouting.mjs +8 -2
- package/dist/PortRouting.mjs.map +1 -1
- package/dist/{do-journal-Brv7xSH5.mjs → do-journal-CllwJfgW.mjs} +154 -55
- package/dist/do-journal-CllwJfgW.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/{migrations-soC86CQA.mjs → migrations-BK3kMBdI.mjs} +19 -4
- package/dist/migrations-BK3kMBdI.mjs.map +1 -0
- package/package.json +1 -1
- package/src/DoStorageError.ts +2 -0
- package/src/DoThreadStore.ts +94 -4
- package/src/PortProtocol.ts +3 -3
- package/src/PortRouting.ts +25 -4
- package/src/internal/do-journal.ts +195 -38
- package/src/internal/migrations.ts +3 -1
- package/src/internal/recovery-checkpoint-schema.ts +17 -0
- package/dist/do-journal-Brv7xSH5.mjs.map +0 -1
- package/dist/migrations-soC86CQA.mjs.map +0 -1
package/src/DoThreadStore.ts
CHANGED
|
@@ -28,6 +28,9 @@ import {
|
|
|
28
28
|
FencedAppendRequest,
|
|
29
29
|
LoadCheckpointRequest,
|
|
30
30
|
SaveCheckpointRequest,
|
|
31
|
+
SaveRecoveryCheckpointRequest,
|
|
32
|
+
MAX_THREAD_EXPORT_RECORDS,
|
|
33
|
+
type ThreadRecoveryCheckpoints,
|
|
31
34
|
} from "@effect-agent/thread/ThreadStore";
|
|
32
35
|
import { BrowserCrypto } from "@effect/platform-browser";
|
|
33
36
|
import { SqliteClient } from "@effect/sql-sqlite-do";
|
|
@@ -301,9 +304,9 @@ const groupByKey = <A>(
|
|
|
301
304
|
};
|
|
302
305
|
|
|
303
306
|
/**
|
|
304
|
-
* Opt-in
|
|
305
|
-
*
|
|
306
|
-
*
|
|
307
|
+
* Opt-in integrity audit (`verifyOnOpen`) of canonical payloads, their digest chains and generic
|
|
308
|
+
* projection checkpoints. Disposable recovery checkpoints are validated when loaded. Routine opens
|
|
309
|
+
* skip this scan: per-operation Schema decoding fails clearly on corrupt canonical rows.
|
|
307
310
|
*/
|
|
308
311
|
const decodeStartupPayloads = Effect.fn("DoThreadStore.decodeStartupPayloads")(function* (
|
|
309
312
|
journal: DoJournal,
|
|
@@ -698,7 +701,7 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
|
|
|
698
701
|
|
|
699
702
|
const records = yield* Effect.forEach(exported.records, decodeEnvelope);
|
|
700
703
|
|
|
701
|
-
if (records.length >
|
|
704
|
+
if (records.length > MAX_THREAD_EXPORT_RECORDS) {
|
|
702
705
|
return yield* ThreadStoreError.make({
|
|
703
706
|
operation: "decode thread export",
|
|
704
707
|
message: "The thread exceeds the current export record limit.",
|
|
@@ -842,6 +845,92 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
|
|
|
842
845
|
},
|
|
843
846
|
);
|
|
844
847
|
|
|
848
|
+
const saveRecoveryCheckpoint: ThreadRecoveryCheckpoints["save"] = Effect.fn(
|
|
849
|
+
"DoThreadStore.saveRecoveryCheckpoint",
|
|
850
|
+
)(function* (request) {
|
|
851
|
+
const validated = yield* Schema.decodeUnknownEffect(
|
|
852
|
+
Schema.toType(SaveRecoveryCheckpointRequest),
|
|
853
|
+
)(request).pipe(
|
|
854
|
+
Effect.mapError((error) => schemaStoreError("validate recovery checkpoint", error)),
|
|
855
|
+
);
|
|
856
|
+
|
|
857
|
+
const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
|
|
858
|
+
|
|
859
|
+
yield* journal
|
|
860
|
+
.saveRecoveryCheckpoint(validated, checkpointJson)
|
|
861
|
+
.pipe(
|
|
862
|
+
Effect.mapError((error) =>
|
|
863
|
+
error._tag === "CheckpointRejected" ||
|
|
864
|
+
error._tag === "FenceRejected" ||
|
|
865
|
+
error._tag === "ThreadNotMaterialized"
|
|
866
|
+
? error
|
|
867
|
+
: storeError("save recovery checkpoint", error),
|
|
868
|
+
),
|
|
869
|
+
);
|
|
870
|
+
});
|
|
871
|
+
|
|
872
|
+
const loadRecoveryCheckpoint: ThreadRecoveryCheckpoints["load"] = Effect.fn(
|
|
873
|
+
"DoThreadStore.loadRecoveryCheckpoint",
|
|
874
|
+
)(function* (request) {
|
|
875
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(LoadCheckpointRequest))(
|
|
876
|
+
request,
|
|
877
|
+
).pipe(
|
|
878
|
+
Effect.mapError((error) => schemaStoreError("validate recovery checkpoint lookup", error)),
|
|
879
|
+
);
|
|
880
|
+
|
|
881
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
882
|
+
|
|
883
|
+
const corrupt = () =>
|
|
884
|
+
CheckpointRejected.make({ threadId: validated.threadId, reason: "corrupt" });
|
|
885
|
+
|
|
886
|
+
const rows = yield* journal
|
|
887
|
+
.loadRecoveryCheckpoint(validated.threadId)
|
|
888
|
+
.pipe(
|
|
889
|
+
Effect.mapError((error) =>
|
|
890
|
+
error._tag === "DoStorageCorruptionError"
|
|
891
|
+
? corrupt()
|
|
892
|
+
: storeError("load recovery checkpoint", error),
|
|
893
|
+
),
|
|
894
|
+
);
|
|
895
|
+
|
|
896
|
+
if (rows.length === 0) return Option.none();
|
|
897
|
+
if (rows.length !== 1) return yield* corrupt();
|
|
898
|
+
const row = rows[0];
|
|
899
|
+
|
|
900
|
+
const checkpoint = yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(
|
|
901
|
+
row.checkpoint_json,
|
|
902
|
+
).pipe(Effect.mapError(corrupt));
|
|
903
|
+
|
|
904
|
+
if (
|
|
905
|
+
row.thread_id !== validated.threadId ||
|
|
906
|
+
checkpoint.threadId !== row.thread_id ||
|
|
907
|
+
checkpoint.throughSequence !== row.through_sequence ||
|
|
908
|
+
checkpoint.tailDigest !== row.tail_digest
|
|
909
|
+
)
|
|
910
|
+
return yield* corrupt();
|
|
911
|
+
if (checkpoint.throughSequence > thread.tail_sequence)
|
|
912
|
+
return yield* CheckpointRejected.make({
|
|
913
|
+
threadId: validated.threadId,
|
|
914
|
+
reason: "ahead-of-tail",
|
|
915
|
+
});
|
|
916
|
+
if (checkpoint.throughSequence > (validated.atOrBeforeSequence ?? thread.tail_sequence))
|
|
917
|
+
return Option.none();
|
|
918
|
+
|
|
919
|
+
const canonicalDigest = yield* tailDigestAt(
|
|
920
|
+
journal,
|
|
921
|
+
checkpoint.threadId,
|
|
922
|
+
checkpoint.throughSequence,
|
|
923
|
+
);
|
|
924
|
+
|
|
925
|
+
if (canonicalDigest !== checkpoint.tailDigest)
|
|
926
|
+
return yield* CheckpointRejected.make({
|
|
927
|
+
threadId: validated.threadId,
|
|
928
|
+
reason: "digest-mismatch",
|
|
929
|
+
});
|
|
930
|
+
|
|
931
|
+
return Option.some(checkpoint);
|
|
932
|
+
});
|
|
933
|
+
|
|
845
934
|
const threadStore = ThreadStore.of({
|
|
846
935
|
append,
|
|
847
936
|
export: exportThread,
|
|
@@ -850,6 +939,7 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
|
|
|
850
939
|
observe,
|
|
851
940
|
read,
|
|
852
941
|
checkpoints: { save: saveCheckpoint, load: loadCheckpoint },
|
|
942
|
+
recoveryCheckpoints: { save: saveRecoveryCheckpoint, load: loadRecoveryCheckpoint },
|
|
853
943
|
});
|
|
854
944
|
|
|
855
945
|
return Context.make(ThreadStore, threadStore);
|
package/src/PortProtocol.ts
CHANGED
|
@@ -50,9 +50,9 @@ import { Schema } from "effect";
|
|
|
50
50
|
* `recordChildSettled`;
|
|
51
51
|
* - store: `materialize`, `append`, `read` (one page), `inspectTail`, `export`.
|
|
52
52
|
*
|
|
53
|
-
* Every other port operation is lane-local
|
|
54
|
-
*
|
|
55
|
-
* instead of
|
|
53
|
+
* Every other port operation is lane-local and has no envelope. A foreign disposable
|
|
54
|
+
* recovery-cache load returns a miss so the caller can replay canonical history. Other
|
|
55
|
+
* foreign operations fail fast and typed instead of widening the distributed surface.
|
|
56
56
|
*
|
|
57
57
|
* Failures cross the boundary as the `PortFailure` union and re-decode on the caller side to
|
|
58
58
|
* the SAME tagged error types the local facet would have produced, so routed calls keep
|
package/src/PortRouting.ts
CHANGED
|
@@ -720,6 +720,7 @@ const makeRoutedStoreServices = Effect.fn("DoPortRouting.makeRoutedStoreServices
|
|
|
720
720
|
) {
|
|
721
721
|
const local = yield* ThreadStore;
|
|
722
722
|
const checkpoints = local.checkpoints;
|
|
723
|
+
const recoveryCheckpoints = local.recoveryCheckpoints;
|
|
723
724
|
const transport = yield* ThreadPortTransport;
|
|
724
725
|
const transportCall: TransportCall = makeTransportCall(transport);
|
|
725
726
|
|
|
@@ -848,14 +849,33 @@ const makeRoutedStoreServices = Effect.fn("DoPortRouting.makeRoutedStoreServices
|
|
|
848
849
|
ThreadNotMaterialized,
|
|
849
850
|
).pipe(Effect.map((reply) => reply.export)),
|
|
850
851
|
|
|
851
|
-
// Observation and checkpoints are lane-local
|
|
852
|
-
//
|
|
853
|
-
//
|
|
852
|
+
// Observation and checkpoints are lane-local: the closed route-capable store subset
|
|
853
|
+
// is materialize/append/read/inspectTail/export. Recovery cache misses can fall back
|
|
854
|
+
// to those canonical reads, including when the cache belongs to a foreign Object.
|
|
854
855
|
observe: (request) =>
|
|
855
856
|
request.threadId === options.localThreadId
|
|
856
857
|
? local.observe(request)
|
|
857
858
|
: Stream.unwrap(Effect.fail(crossThreadStoreError("thread observe", request.threadId))),
|
|
858
859
|
|
|
860
|
+
...(recoveryCheckpoints === undefined
|
|
861
|
+
? {}
|
|
862
|
+
: {
|
|
863
|
+
recoveryCheckpoints: {
|
|
864
|
+
save: (request) =>
|
|
865
|
+
request.checkpoint.threadId === options.localThreadId
|
|
866
|
+
? recoveryCheckpoints.save(request)
|
|
867
|
+
: Effect.fail(
|
|
868
|
+
crossThreadStoreError(
|
|
869
|
+
"thread save recovery checkpoint",
|
|
870
|
+
request.checkpoint.threadId,
|
|
871
|
+
),
|
|
872
|
+
),
|
|
873
|
+
load: (request) =>
|
|
874
|
+
request.threadId === options.localThreadId
|
|
875
|
+
? recoveryCheckpoints.load(request)
|
|
876
|
+
: Effect.succeed(Option.none()),
|
|
877
|
+
},
|
|
878
|
+
}),
|
|
859
879
|
...(checkpoints === undefined
|
|
860
880
|
? {}
|
|
861
881
|
: {
|
|
@@ -892,7 +912,8 @@ export const routedSubmissionLedgerLayer = (
|
|
|
892
912
|
/**
|
|
893
913
|
* Routing decorator over the LOCAL `ThreadStore` facet (plan §1.3): this-thread
|
|
894
914
|
* requests execute locally; foreign materialize/append/read/inspectTail/export travel the
|
|
895
|
-
* transport
|
|
915
|
+
* transport. Foreign recovery-cache loads return none for canonical replay; other foreign
|
|
916
|
+
* checkpoint operations and observation fail fast typed.
|
|
896
917
|
*/
|
|
897
918
|
export const routedThreadStoreLayer = (
|
|
898
919
|
options: RoutedPortOptions,
|
|
@@ -1,5 +1,13 @@
|
|
|
1
|
+
import { EMPTY_TAIL_DIGEST } from "@effect-agent/thread/Digest";
|
|
1
2
|
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread/Records";
|
|
2
3
|
import { checkV2ThreadLayout } from "@effect-agent/thread/SqlStorageV2Upgrade";
|
|
4
|
+
import {
|
|
5
|
+
MAX_THREAD_EXPORT_RECORDS,
|
|
6
|
+
CheckpointRejected,
|
|
7
|
+
FenceRejected,
|
|
8
|
+
ThreadNotMaterialized,
|
|
9
|
+
type SaveRecoveryCheckpointRequest,
|
|
10
|
+
} from "@effect-agent/thread/ThreadStore";
|
|
3
11
|
import { SqliteMigrator } from "@effect/sql-sqlite-do";
|
|
4
12
|
import { Effect, Schema, Stream } from "effect";
|
|
5
13
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
@@ -18,6 +26,7 @@ import {
|
|
|
18
26
|
} from "../DoStorageError.ts";
|
|
19
27
|
import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
|
|
20
28
|
import { CurrentDoStorageVersion, doMigrations } from "./migrations.ts";
|
|
29
|
+
import { createRecoveryCheckpointTable } from "./recovery-checkpoint-schema.ts";
|
|
21
30
|
|
|
22
31
|
/**
|
|
23
32
|
* Static schema ceiling for stored text columns. Writes are bounded in BYTES by the
|
|
@@ -27,7 +36,8 @@ import { CurrentDoStorageVersion, doMigrations } from "./migrations.ts";
|
|
|
27
36
|
*/
|
|
28
37
|
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(2_000_000));
|
|
29
38
|
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
30
|
-
const MAX_RECORDS_PER_THREAD =
|
|
39
|
+
const MAX_RECORDS_PER_THREAD = MAX_THREAD_EXPORT_RECORDS;
|
|
40
|
+
const ZERO_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
31
41
|
const MAX_IDENTIFIER_LENGTH = 1_024;
|
|
32
42
|
const MAX_READ_PAGE_JSON_BYTES = 4 * 1024 * 1024;
|
|
33
43
|
/** Durable Object SQL storage allows at most 100 bound parameters per statement. */
|
|
@@ -338,10 +348,32 @@ const predecessorColumns = {
|
|
|
338
348
|
],
|
|
339
349
|
} as const;
|
|
340
350
|
|
|
341
|
-
const checkPredecessorLayout = Effect.fn("DoJournal.checkPredecessorLayout")(function* (
|
|
351
|
+
const checkPredecessorLayout = Effect.fn("DoJournal.checkPredecessorLayout")(function* (
|
|
352
|
+
version: 3 | 4,
|
|
353
|
+
) {
|
|
342
354
|
const sql = yield* SqlClient.SqlClient;
|
|
343
355
|
|
|
344
|
-
|
|
356
|
+
const expectedColumns =
|
|
357
|
+
version === 3
|
|
358
|
+
? predecessorColumns
|
|
359
|
+
: {
|
|
360
|
+
...predecessorColumns,
|
|
361
|
+
effect_agent_submissions: [
|
|
362
|
+
...predecessorColumns.effect_agent_submissions,
|
|
363
|
+
"worker_admission_json",
|
|
364
|
+
"message_admission_json",
|
|
365
|
+
],
|
|
366
|
+
effect_agent_message_deliveries: [
|
|
367
|
+
"owner_thread_id",
|
|
368
|
+
"message_id",
|
|
369
|
+
"version",
|
|
370
|
+
"state",
|
|
371
|
+
"deadline_at_millis",
|
|
372
|
+
"record_json",
|
|
373
|
+
],
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
for (const [table, expected] of Object.entries(expectedColumns)) {
|
|
345
377
|
const columns = yield* decodeRows(
|
|
346
378
|
Schema.Array(Schema.Struct({ name: BoundedIdentifier })),
|
|
347
379
|
table,
|
|
@@ -353,9 +385,9 @@ const checkPredecessorLayout = Effect.fn("DoJournal.checkPredecessorLayout")(fun
|
|
|
353
385
|
|
|
354
386
|
if (columns.length !== names.size || columns.some((column) => !names.has(column.name)))
|
|
355
387
|
return yield* DoStorageCompatibilityError.make({
|
|
356
|
-
actualVersion:
|
|
388
|
+
actualVersion: version,
|
|
357
389
|
supportedVersion: CurrentDoStorageVersion,
|
|
358
|
-
message: `The
|
|
390
|
+
message: `The v${version} ${table} columns do not match the supported predecessor; no upgrade was committed.`,
|
|
359
391
|
});
|
|
360
392
|
}
|
|
361
393
|
});
|
|
@@ -454,7 +486,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
454
486
|
versionRows,
|
|
455
487
|
);
|
|
456
488
|
|
|
457
|
-
if (version.value === "2" || version.value === "3") {
|
|
489
|
+
if (version.value === "2" || version.value === "3" || version.value === "4") {
|
|
458
490
|
yield* sql
|
|
459
491
|
.withTransaction(
|
|
460
492
|
Effect.gen(function* () {
|
|
@@ -464,7 +496,10 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
464
496
|
|
|
465
497
|
if (current.length === 1 && current[0].value === String(CurrentDoStorageVersion))
|
|
466
498
|
return;
|
|
467
|
-
if (
|
|
499
|
+
if (
|
|
500
|
+
current.length !== 1 ||
|
|
501
|
+
(current[0].value !== "2" && current[0].value !== "3" && current[0].value !== "4")
|
|
502
|
+
)
|
|
468
503
|
return yield* DoStorageCompatibilityError.make({
|
|
469
504
|
actualVersion: -1,
|
|
470
505
|
supportedVersion: CurrentDoStorageVersion,
|
|
@@ -485,6 +520,17 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
485
520
|
message:
|
|
486
521
|
"The predecessor store is missing required tables. Retain the original store for inspection; no upgrade was committed.",
|
|
487
522
|
});
|
|
523
|
+
|
|
524
|
+
const recoveryTables =
|
|
525
|
+
yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name='effect_agent_recovery_checkpoints'`;
|
|
526
|
+
|
|
527
|
+
if (recoveryTables.length !== 0)
|
|
528
|
+
return yield* DoStorageCompatibilityError.make({
|
|
529
|
+
actualVersion: Number(current[0].value),
|
|
530
|
+
supportedVersion: CurrentDoStorageVersion,
|
|
531
|
+
message:
|
|
532
|
+
"The predecessor already contains recovery checkpoint storage; refusing ambiguous data without mutation.",
|
|
533
|
+
});
|
|
488
534
|
if (current[0].value === "2") {
|
|
489
535
|
yield* checkV2ThreadLayout();
|
|
490
536
|
for (const statement of [
|
|
@@ -497,18 +543,24 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
497
543
|
yield* failpoint("upgrade:after-mutation");
|
|
498
544
|
}
|
|
499
545
|
}
|
|
500
|
-
if (current[0].value === "3") yield* checkPredecessorLayout();
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
546
|
+
if (current[0].value === "3") yield* checkPredecessorLayout(3);
|
|
547
|
+
if (current[0].value === "4") yield* checkPredecessorLayout(4);
|
|
548
|
+
if (current[0].value !== "4") {
|
|
549
|
+
yield* failpoint("upgrade:before-mutation");
|
|
550
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
|
|
551
|
+
yield* failpoint("upgrade:after-mutation");
|
|
552
|
+
yield* failpoint("upgrade:before-mutation");
|
|
553
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
|
|
554
|
+
yield* failpoint("upgrade:after-mutation");
|
|
555
|
+
yield* failpoint("upgrade:before-mutation");
|
|
556
|
+
yield* createMessageDeliveryTables;
|
|
557
|
+
yield* failpoint("upgrade:after-mutation");
|
|
558
|
+
}
|
|
507
559
|
yield* failpoint("upgrade:before-mutation");
|
|
508
|
-
yield*
|
|
560
|
+
yield* createRecoveryCheckpointTable;
|
|
509
561
|
yield* failpoint("upgrade:after-mutation");
|
|
510
562
|
yield* failpoint("upgrade:before-version");
|
|
511
|
-
yield* sql`UPDATE effect_agent_meta SET value='
|
|
563
|
+
yield* sql`UPDATE effect_agent_meta SET value='5' WHERE key='storage_version'`;
|
|
512
564
|
yield* failpoint("upgrade:after-version");
|
|
513
565
|
}),
|
|
514
566
|
)
|
|
@@ -538,7 +590,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
538
590
|
message:
|
|
539
591
|
`The Durable Object uses unsupported storage version ${version.value}; ` +
|
|
540
592
|
`this build supports exactly version ${CurrentDoStorageVersion}. ` +
|
|
541
|
-
"Only supported v2 and
|
|
593
|
+
"Only supported v2, v3 and v4 can be upgraded automatically. Keep the original store and use a compatible library version.",
|
|
542
594
|
});
|
|
543
595
|
}
|
|
544
596
|
}
|
|
@@ -547,7 +599,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
547
599
|
SELECT name
|
|
548
600
|
FROM sqlite_master
|
|
549
601
|
WHERE type = 'table'
|
|
550
|
-
AND name IN ${sql.in([...REQUIRED_TABLES, "effect_agent_message_deliveries"])}
|
|
602
|
+
AND name IN ${sql.in([...REQUIRED_TABLES, "effect_agent_message_deliveries", "effect_agent_recovery_checkpoints"])}
|
|
551
603
|
ORDER BY name
|
|
552
604
|
`.pipe(Effect.mapError(storageError("verify storage tables")));
|
|
553
605
|
|
|
@@ -558,7 +610,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
558
610
|
requiredRows,
|
|
559
611
|
);
|
|
560
612
|
|
|
561
|
-
if (required.length !== REQUIRED_TABLES.length +
|
|
613
|
+
if (required.length !== REQUIRED_TABLES.length + 2) {
|
|
562
614
|
return yield* DoStorageCompatibilityError.make({
|
|
563
615
|
actualVersion: CurrentDoStorageVersion,
|
|
564
616
|
supportedVersion: CurrentDoStorageVersion,
|
|
@@ -1069,27 +1121,54 @@ const makeJournal = (
|
|
|
1069
1121
|
|
|
1070
1122
|
yield* failpoint("export:after-thread-read");
|
|
1071
1123
|
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
FROM effect_agent_canonical_records
|
|
1080
|
-
WHERE thread_id = ${threadId}
|
|
1081
|
-
ORDER BY sequence
|
|
1082
|
-
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
1124
|
+
if (thread.tail_sequence > MAX_RECORDS_PER_THREAD)
|
|
1125
|
+
return yield* DoStorageError.make({
|
|
1126
|
+
operation: "export thread",
|
|
1127
|
+
message: "The thread exceeds the current export record limit.",
|
|
1128
|
+
});
|
|
1129
|
+
const records: Array<RecordRow> = [];
|
|
1130
|
+
let afterSequence = ZERO_SEQUENCE;
|
|
1083
1131
|
|
|
1084
|
-
|
|
1085
|
-
thread
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
"effect_agent_canonical_records",
|
|
1132
|
+
while (afterSequence < thread.tail_sequence) {
|
|
1133
|
+
const limit = Math.min(1_024, thread.tail_sequence - afterSequence);
|
|
1134
|
+
|
|
1135
|
+
const request = RawReadRequest.make({
|
|
1089
1136
|
threadId,
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1137
|
+
fromSequenceExclusive: afterSequence,
|
|
1138
|
+
limit,
|
|
1139
|
+
});
|
|
1140
|
+
|
|
1141
|
+
const plan = yield* read(request);
|
|
1142
|
+
const page = yield* Stream.runCollect(plan.records);
|
|
1143
|
+
|
|
1144
|
+
if (
|
|
1145
|
+
page.length !== limit ||
|
|
1146
|
+
page.some((record, index) => record.sequence !== afterSequence + index + 1)
|
|
1147
|
+
) {
|
|
1148
|
+
return yield* DoStorageCorruptionError.make({
|
|
1149
|
+
table: "effect_agent_canonical_records",
|
|
1150
|
+
rowKey: threadId,
|
|
1151
|
+
message:
|
|
1152
|
+
"The exported canonical prefix is not contiguous through its captured tail.",
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
records.push(...page);
|
|
1156
|
+
afterSequence = page[page.length - 1].sequence;
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
const beyondTail =
|
|
1160
|
+
yield* sql`SELECT sequence FROM effect_agent_canonical_records WHERE thread_id=${threadId} AND sequence > ${thread.tail_sequence} LIMIT 1`.pipe(
|
|
1161
|
+
Effect.mapError(storageError("verify export tail")),
|
|
1162
|
+
);
|
|
1163
|
+
|
|
1164
|
+
if (beyondTail.length !== 0)
|
|
1165
|
+
return yield* DoStorageCorruptionError.make({
|
|
1166
|
+
table: "effect_agent_canonical_records",
|
|
1167
|
+
rowKey: threadId,
|
|
1168
|
+
message: "Canonical records exist beyond the captured thread tail.",
|
|
1169
|
+
});
|
|
1170
|
+
|
|
1171
|
+
return RawThreadExport.make({ thread, records });
|
|
1093
1172
|
}),
|
|
1094
1173
|
)
|
|
1095
1174
|
.pipe(
|
|
@@ -1192,6 +1271,82 @@ const makeJournal = (
|
|
|
1192
1271
|
);
|
|
1193
1272
|
});
|
|
1194
1273
|
|
|
1274
|
+
const saveRecoveryCheckpoint = Effect.fn("DoJournal.saveRecoveryCheckpoint")(function* (
|
|
1275
|
+
request: SaveRecoveryCheckpointRequest,
|
|
1276
|
+
checkpointJson: string,
|
|
1277
|
+
) {
|
|
1278
|
+
const { checkpoint } = request;
|
|
1279
|
+
|
|
1280
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
|
|
1281
|
+
return yield* DoStorageError.make({
|
|
1282
|
+
operation: "save recovery checkpoint",
|
|
1283
|
+
message: "Checkpoint identity exceeds the Durable Object storage bounds.",
|
|
1284
|
+
});
|
|
1285
|
+
}
|
|
1286
|
+
yield* checkValueBound("save recovery checkpoint", checkpointJson);
|
|
1287
|
+
// Keep injected waits outside the storage-backed transaction callback.
|
|
1288
|
+
yield* failpoint("save-recovery-checkpoint:before");
|
|
1289
|
+
yield* withWriteTransaction("recovery checkpoint transaction")(
|
|
1290
|
+
Effect.gen(function* () {
|
|
1291
|
+
const threads = yield* getThread(checkpoint.threadId);
|
|
1292
|
+
const thread = threads[0];
|
|
1293
|
+
|
|
1294
|
+
if (thread === undefined)
|
|
1295
|
+
return yield* ThreadNotMaterialized.make({ threadId: checkpoint.threadId });
|
|
1296
|
+
if (request.producerEpoch !== thread.producer_epoch)
|
|
1297
|
+
return yield* FenceRejected.make({
|
|
1298
|
+
threadId: checkpoint.threadId,
|
|
1299
|
+
actualEpoch: thread.producer_epoch,
|
|
1300
|
+
attemptedEpoch: request.producerEpoch,
|
|
1301
|
+
});
|
|
1302
|
+
if (checkpoint.throughSequence > thread.tail_sequence)
|
|
1303
|
+
return yield* CheckpointRejected.make({
|
|
1304
|
+
threadId: checkpoint.threadId,
|
|
1305
|
+
reason: "ahead-of-tail",
|
|
1306
|
+
});
|
|
1307
|
+
|
|
1308
|
+
const digests =
|
|
1309
|
+
checkpoint.throughSequence === 0
|
|
1310
|
+
? [EMPTY_TAIL_DIGEST]
|
|
1311
|
+
: yield* getTailDigestAt(checkpoint.threadId, checkpoint.throughSequence);
|
|
1312
|
+
|
|
1313
|
+
if (digests.length !== 1 || digests[0] !== checkpoint.tailDigest)
|
|
1314
|
+
return yield* CheckpointRejected.make({
|
|
1315
|
+
threadId: checkpoint.threadId,
|
|
1316
|
+
reason: "digest-mismatch",
|
|
1317
|
+
});
|
|
1318
|
+
|
|
1319
|
+
yield* sql`
|
|
1320
|
+
INSERT INTO effect_agent_recovery_checkpoints (thread_id, through_sequence, tail_digest, checkpoint_json)
|
|
1321
|
+
VALUES (${checkpoint.threadId}, ${checkpoint.throughSequence}, ${checkpoint.tailDigest}, ${checkpointJson})
|
|
1322
|
+
ON CONFLICT (thread_id) DO UPDATE SET
|
|
1323
|
+
through_sequence = excluded.through_sequence,
|
|
1324
|
+
tail_digest = excluded.tail_digest,
|
|
1325
|
+
checkpoint_json = excluded.checkpoint_json
|
|
1326
|
+
WHERE excluded.through_sequence >= effect_agent_recovery_checkpoints.through_sequence
|
|
1327
|
+
`.pipe(Effect.mapError(storageError("save recovery checkpoint")));
|
|
1328
|
+
}),
|
|
1329
|
+
);
|
|
1330
|
+
yield* failpoint("save-recovery-checkpoint:after");
|
|
1331
|
+
});
|
|
1332
|
+
|
|
1333
|
+
const loadRecoveryCheckpoint = Effect.fn("DoJournal.loadRecoveryCheckpoint")(function* (
|
|
1334
|
+
threadId: string,
|
|
1335
|
+
) {
|
|
1336
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
1337
|
+
SELECT thread_id, through_sequence, tail_digest, checkpoint_json
|
|
1338
|
+
FROM effect_agent_recovery_checkpoints
|
|
1339
|
+
WHERE thread_id = ${threadId}
|
|
1340
|
+
`.pipe(Effect.mapError(storageError("load recovery checkpoint")));
|
|
1341
|
+
|
|
1342
|
+
return yield* decodeRows(
|
|
1343
|
+
Schema.Array(CheckpointRow),
|
|
1344
|
+
"effect_agent_recovery_checkpoints",
|
|
1345
|
+
threadId,
|
|
1346
|
+
rows,
|
|
1347
|
+
);
|
|
1348
|
+
});
|
|
1349
|
+
|
|
1195
1350
|
const loadCheckpoint = Effect.fn("DoJournal.loadCheckpoint")(function* (
|
|
1196
1351
|
threadId: string,
|
|
1197
1352
|
atOrBeforeSequence: CanonicalSequence,
|
|
@@ -1346,6 +1501,8 @@ const makeJournal = (
|
|
|
1346
1501
|
getThread,
|
|
1347
1502
|
getTailDigestAt,
|
|
1348
1503
|
loadCheckpoint,
|
|
1504
|
+
loadRecoveryCheckpoint,
|
|
1505
|
+
saveRecoveryCheckpoint,
|
|
1349
1506
|
materialize,
|
|
1350
1507
|
read,
|
|
1351
1508
|
saveCheckpoint,
|
|
@@ -3,9 +3,10 @@ import { Effect } from "effect";
|
|
|
3
3
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
4
4
|
|
|
5
5
|
import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
|
|
6
|
+
import { createRecoveryCheckpointTable } from "./recovery-checkpoint-schema.ts";
|
|
6
7
|
|
|
7
8
|
/** The current storage version recorded in `effect_agent_meta`. */
|
|
8
|
-
export const CurrentDoStorageVersion =
|
|
9
|
+
export const CurrentDoStorageVersion = 5;
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* The Thread Durable Object schema shares its thread and ledger tables with Node/SQLite.
|
|
@@ -257,6 +258,7 @@ export const doMigrations = SqliteMigrator.fromRecord({
|
|
|
257
258
|
`.withoutTransform;
|
|
258
259
|
|
|
259
260
|
yield* createMessageDeliveryTables;
|
|
261
|
+
yield* createRecoveryCheckpointTable;
|
|
260
262
|
yield* sql`
|
|
261
263
|
CREATE TABLE effect_agent_meta (
|
|
262
264
|
key TEXT PRIMARY KEY NOT NULL,
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
3
|
+
|
|
4
|
+
/** One disposable recovery snapshot per Thread, independent of generic projections. */
|
|
5
|
+
export const createRecoveryCheckpointTable = Effect.gen(function* () {
|
|
6
|
+
const sql = yield* SqlClient.SqlClient;
|
|
7
|
+
|
|
8
|
+
yield* sql`
|
|
9
|
+
CREATE TABLE effect_agent_recovery_checkpoints (
|
|
10
|
+
thread_id TEXT PRIMARY KEY NOT NULL,
|
|
11
|
+
through_sequence INTEGER NOT NULL,
|
|
12
|
+
tail_digest TEXT NOT NULL,
|
|
13
|
+
checkpoint_json TEXT NOT NULL,
|
|
14
|
+
FOREIGN KEY (thread_id) REFERENCES effect_agent_threads(thread_id) ON DELETE RESTRICT
|
|
15
|
+
)
|
|
16
|
+
`.withoutTransform;
|
|
17
|
+
});
|