@effect-agent/storage-cloudflare 0.1.0-beta.45 → 0.1.0-beta.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DoMemoryStore.d.mts +24 -0
- package/dist/DoMemoryStore.mjs +50 -0
- package/dist/DoMemoryStore.mjs.map +1 -0
- package/dist/DoScheduleStore.d.mts +36 -0
- package/dist/DoScheduleStore.mjs +369 -0
- package/dist/DoScheduleStore.mjs.map +1 -0
- package/dist/DoStorageConfig.d.mts +45 -0
- package/dist/DoStorageConfig.mjs +52 -0
- package/dist/DoStorageConfig.mjs.map +1 -0
- package/dist/{do-storage-failpoint-9XB9tuif.d.mts → DoStorageError-Qwybzk0O.d.mts} +6 -15
- package/dist/DoStorageError.d.mts +2 -0
- package/dist/DoStorageError.mjs +158 -0
- package/dist/DoStorageError.mjs.map +1 -0
- package/dist/DoStorageFailpoint.d.mts +17 -0
- package/dist/{do-storage-failpoint-C-Qtchs7.mjs → DoStorageFailpoint.mjs} +5 -3
- package/dist/DoStorageFailpoint.mjs.map +1 -0
- package/dist/{testing.d.mts → DoStorageFailpointTesting.d.mts} +4 -3
- package/dist/{testing.mjs → DoStorageFailpointTesting.mjs} +3 -3
- package/dist/DoStorageFailpointTesting.mjs.map +1 -0
- package/dist/DoStorageVersion-KrYUXqeJ.d.mts +10 -0
- package/dist/DoStorageVersion.d.mts +2 -0
- package/dist/DoStorageVersion.mjs +8 -0
- package/dist/DoStorageVersion.mjs.map +1 -0
- package/dist/DoSubmissionLedger.d.mts +23 -0
- package/dist/DoSubmissionLedger.mjs +1716 -0
- package/dist/DoSubmissionLedger.mjs.map +1 -0
- package/dist/DoSubscriptionStore.d.mts +25 -0
- package/dist/DoSubscriptionStore.mjs +720 -0
- package/dist/DoSubscriptionStore.mjs.map +1 -0
- package/dist/DoThreadStore-D9ebIYPv.mjs +1083 -0
- package/dist/DoThreadStore-D9ebIYPv.mjs.map +1 -0
- package/dist/DoThreadStore.d.mts +59 -0
- package/dist/DoThreadStore.mjs +5 -0
- package/dist/MemoryProtocol.d.mts +19796 -0
- package/dist/MemoryProtocol.mjs +182 -0
- package/dist/MemoryProtocol.mjs.map +1 -0
- package/dist/PortProtocol.d.mts +1587 -0
- package/dist/PortProtocol.mjs +175 -0
- package/dist/PortProtocol.mjs.map +1 -0
- package/dist/PortRouting.d.mts +82 -0
- package/dist/PortRouting.mjs +404 -0
- package/dist/PortRouting.mjs.map +1 -0
- package/dist/index.d.mts +13 -21619
- package/dist/index.mjs +13 -4954
- package/dist/migrations-Bo4GU-pp.mjs +236 -0
- package/dist/migrations-Bo4GU-pp.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -1
- package/src/{do-memory-store.ts → DoMemoryStore.ts} +5 -2
- package/src/{do-schedule-store.ts → DoScheduleStore.ts} +6 -4
- package/src/{errors.ts → DoStorageError.ts} +1 -1
- package/src/{do-storage-failpoint.ts → DoStorageFailpoint.ts} +1 -1
- package/src/{do-storage-failpoint-testing.ts → DoStorageFailpointTesting.ts} +2 -2
- package/src/DoStorageVersion.ts +2 -0
- package/src/{do-ledger.ts → DoSubmissionLedger.ts} +22 -20
- package/src/{do-subscription-store.ts → DoSubscriptionStore.ts} +10 -8
- package/src/{do-thread-store.ts → DoThreadStore.ts} +19 -21
- package/src/{memory-protocol.ts → MemoryProtocol.ts} +15 -10
- package/src/{port-protocol.ts → PortProtocol.ts} +13 -11
- package/src/{routing.ts → PortRouting.ts} +10 -8
- package/src/index.ts +12 -36
- package/src/{do-journal.ts → internal/do-journal.ts} +2 -2
- package/dist/do-storage-failpoint-C-Qtchs7.mjs.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/testing.mjs.map +0 -1
- package/src/testing.ts +0 -2
- /package/src/{do-storage-config.ts → DoStorageConfig.ts} +0 -0
- /package/src/{migrations.ts → internal/migrations.ts} +0 -0
|
@@ -0,0 +1,1083 @@
|
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
2
|
+
import { DoStorageConfig, DoStorageConfigValue } from "./DoStorageConfig.mjs";
|
|
3
|
+
import { DoAppendConflict, DoCheckpointConflict, DoFenceRejected, DoStorageCompatibilityError, DoStorageCorruptionError, DoStorageError, DoValueBoundExceeded } from "./DoStorageError.mjs";
|
|
4
|
+
import { DoStorageFailpoint } from "./DoStorageFailpoint.mjs";
|
|
5
|
+
import { n as doMigrations } from "./migrations-Bo4GU-pp.mjs";
|
|
6
|
+
import { SqliteClient, SqliteMigrator } from "@effect/sql-sqlite-do";
|
|
7
|
+
import { Clock, Context, Crypto, Duration, Effect, Layer, Option, Ref, Schema, Stream } from "effect";
|
|
8
|
+
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
9
|
+
import { CanonicalBatch, CanonicalRecord, CanonicalRecordEnvelope, CanonicalSequence, Digest, ObservationOffset, ProducerEpoch } from "@effect-agent/thread/Records";
|
|
10
|
+
import { EMPTY_TAIL_DIGEST, digestCanonicalBatch } from "@effect-agent/thread/Digest";
|
|
11
|
+
import { DEFAULT_OWNERSHIP_LEASE_DURATION } from "@effect-agent/thread/SubmissionLedger";
|
|
12
|
+
import { BrowserCrypto } from "@effect/platform-browser";
|
|
13
|
+
import { AppendConflict, AppendResult, CheckpointRejected, FenceRejected, FencedAppendRequest, LoadCheckpointRequest, SaveCheckpointRequest, ThreadCheckpoint, ThreadExport, ThreadExportRequest, ThreadMaterialization, ThreadNotMaterialized, ThreadObservation, ThreadRead, ThreadStore, ThreadStoreError, ThreadTail, ThreadTailRequest } from "@effect-agent/thread/ThreadStore";
|
|
14
|
+
import { SqlError } from "effect/unstable/sql/SqlError";
|
|
15
|
+
//#region src/internal/do-journal.ts
|
|
16
|
+
/**
|
|
17
|
+
* Static schema ceiling for stored text columns. Writes are bounded in BYTES by the
|
|
18
|
+
* configured `maxStoredValueBytes` (always ≤ 2,000,000); UTF-8 byte length is never smaller
|
|
19
|
+
* than UTF-16 string length, so any value that passed the byte bound also passes this
|
|
20
|
+
* decode-side character ceiling.
|
|
21
|
+
*/
|
|
22
|
+
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(2e6));
|
|
23
|
+
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
24
|
+
const MAX_RECORDS_PER_THREAD = 65536;
|
|
25
|
+
const MAX_IDENTIFIER_LENGTH = 1024;
|
|
26
|
+
const isSqlError = Schema.is(SqlError);
|
|
27
|
+
const storedTextBytes = (value) => new TextEncoder().encode(value).byteLength;
|
|
28
|
+
const chunked = (values, size) => {
|
|
29
|
+
const chunks = [];
|
|
30
|
+
for (let index = 0; index < values.length; index += size) chunks.push(values.slice(index, index + size));
|
|
31
|
+
return chunks;
|
|
32
|
+
};
|
|
33
|
+
var DoMetaRow = class extends Schema.Class("DoMetaRow")({ value: Schema.NonEmptyString.check(Schema.isMaxLength(128)) }) {};
|
|
34
|
+
var DoNameRow = class extends Schema.Class("DoNameRow")({ name: BoundedIdentifier }) {};
|
|
35
|
+
var ThreadRow = class extends Schema.Class("ThreadRow")({
|
|
36
|
+
thread_id: BoundedIdentifier,
|
|
37
|
+
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
38
|
+
producer_epoch: ProducerEpoch,
|
|
39
|
+
tail_digest: BoundedStoredText,
|
|
40
|
+
tail_sequence: CanonicalSequence
|
|
41
|
+
}) {};
|
|
42
|
+
var BatchRow = class extends Schema.Class("BatchRow")({
|
|
43
|
+
batch_digest: BoundedStoredText,
|
|
44
|
+
batch_id: BoundedIdentifier,
|
|
45
|
+
batch_json: BoundedStoredText,
|
|
46
|
+
thread_id: BoundedIdentifier,
|
|
47
|
+
first_sequence: CanonicalSequence,
|
|
48
|
+
last_sequence: CanonicalSequence,
|
|
49
|
+
tail_digest: BoundedStoredText
|
|
50
|
+
}) {};
|
|
51
|
+
var RecordRow = class extends Schema.Class("RecordRow")({
|
|
52
|
+
batch_id: BoundedIdentifier,
|
|
53
|
+
thread_id: BoundedIdentifier,
|
|
54
|
+
record_id: BoundedIdentifier,
|
|
55
|
+
record_json: BoundedStoredText,
|
|
56
|
+
sequence: CanonicalSequence
|
|
57
|
+
}) {};
|
|
58
|
+
var CheckpointRow = class extends Schema.Class("CheckpointRow")({
|
|
59
|
+
checkpoint_json: BoundedStoredText,
|
|
60
|
+
thread_id: BoundedIdentifier,
|
|
61
|
+
tail_digest: BoundedStoredText,
|
|
62
|
+
through_sequence: CanonicalSequence
|
|
63
|
+
}) {};
|
|
64
|
+
var RawRecord = class extends Schema.Class("@effect-agent/storage-cloudflare/RawRecord")({
|
|
65
|
+
recordId: BoundedIdentifier,
|
|
66
|
+
recordJson: BoundedStoredText
|
|
67
|
+
}) {};
|
|
68
|
+
var RawAppendRequest = class extends Schema.Class("@effect-agent/storage-cloudflare/RawAppendRequest")({
|
|
69
|
+
batchDigest: BoundedStoredText,
|
|
70
|
+
batchId: BoundedIdentifier,
|
|
71
|
+
batchJson: BoundedStoredText,
|
|
72
|
+
threadId: BoundedIdentifier,
|
|
73
|
+
expectedTailDigest: BoundedStoredText,
|
|
74
|
+
expectedTailSequence: CanonicalSequence,
|
|
75
|
+
producerEpoch: ProducerEpoch,
|
|
76
|
+
records: Schema.NonEmptyArray(RawRecord).check(Schema.isMaxLength(256)),
|
|
77
|
+
tailDigest: BoundedStoredText
|
|
78
|
+
}) {};
|
|
79
|
+
var RawAppendResult = class extends Schema.Class("@effect-agent/storage-cloudflare/RawAppendResult")({
|
|
80
|
+
firstSequence: CanonicalSequence,
|
|
81
|
+
lastSequence: CanonicalSequence,
|
|
82
|
+
replayed: Schema.Boolean,
|
|
83
|
+
tailDigest: BoundedStoredText
|
|
84
|
+
}) {};
|
|
85
|
+
var RawReadRequest = class extends Schema.Class("@effect-agent/storage-cloudflare/RawReadRequest")({
|
|
86
|
+
threadId: BoundedIdentifier,
|
|
87
|
+
fromSequenceExclusive: CanonicalSequence,
|
|
88
|
+
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1024))
|
|
89
|
+
}) {};
|
|
90
|
+
var RawCheckpoint = class extends Schema.Class("@effect-agent/storage-cloudflare/RawCheckpoint")({
|
|
91
|
+
checkpointJson: BoundedStoredText,
|
|
92
|
+
threadId: BoundedIdentifier,
|
|
93
|
+
tailDigest: BoundedStoredText,
|
|
94
|
+
throughSequence: CanonicalSequence
|
|
95
|
+
}) {};
|
|
96
|
+
var RawThreadExport = class extends Schema.Class("@effect-agent/storage-cloudflare/RawThreadExport")({
|
|
97
|
+
batches: Schema.Array(BatchRow),
|
|
98
|
+
checkpoints: Schema.Array(CheckpointRow),
|
|
99
|
+
thread: ThreadRow,
|
|
100
|
+
records: Schema.Array(RecordRow)
|
|
101
|
+
}) {};
|
|
102
|
+
const noFailpoint = () => Effect.void;
|
|
103
|
+
const storageError = (operation) => (error) => DoStorageError.make({
|
|
104
|
+
cause: error,
|
|
105
|
+
operation,
|
|
106
|
+
message: error.message
|
|
107
|
+
});
|
|
108
|
+
/** Decode raw Durable Object SQLite rows against a Schema, reporting failures as typed corruption. */
|
|
109
|
+
const decodeRows = Effect.fn((schema, table, rowKey, rows) => Schema.decodeUnknownEffect(schema)(rows).pipe(Effect.mapError((error) => DoStorageCorruptionError.make({
|
|
110
|
+
table,
|
|
111
|
+
rowKey,
|
|
112
|
+
message: String(error)
|
|
113
|
+
}))));
|
|
114
|
+
/** Decode exactly one raw row against a Schema, reporting failures as typed corruption. */
|
|
115
|
+
const decodeSingleRow = Effect.fn((schema, table, rowKey, rows) => decodeRows(schema, table, rowKey, rows).pipe(Effect.flatMap((decoded) => decoded.length === 1 ? Effect.succeed(decoded[0]) : Effect.fail(DoStorageCorruptionError.make({
|
|
116
|
+
table,
|
|
117
|
+
rowKey,
|
|
118
|
+
message: `Expected exactly one row but found ${decoded.length}.`
|
|
119
|
+
})))));
|
|
120
|
+
const REQUIRED_TABLES = [
|
|
121
|
+
"effect_agent_abort_intents",
|
|
122
|
+
"effect_agent_approval_decisions",
|
|
123
|
+
"effect_agent_attempts",
|
|
124
|
+
"effect_agent_canonical_batches",
|
|
125
|
+
"effect_agent_canonical_records",
|
|
126
|
+
"effect_agent_checkpoints",
|
|
127
|
+
"effect_agent_child_reservations",
|
|
128
|
+
"effect_agent_child_settlements",
|
|
129
|
+
"effect_agent_threads",
|
|
130
|
+
"effect_agent_meta",
|
|
131
|
+
"effect_agent_settlement_reservations",
|
|
132
|
+
"effect_agent_submission_ownership",
|
|
133
|
+
"effect_agent_submissions",
|
|
134
|
+
"effect_agent_unknown_resolutions"
|
|
135
|
+
];
|
|
136
|
+
/**
|
|
137
|
+
* Exact-or-fresh storage gate (DEPLOY-008) over `effect_agent_meta` instead of
|
|
138
|
+
* `PRAGMA user_version` (unverified on Durable Object SQL storage; a meta table is portable
|
|
139
|
+
* regardless). No WAL check (Durable Object storage owns durability and confirms writes
|
|
140
|
+
* through output gates) and no busy timeout (a Durable Object has exactly one writer): the
|
|
141
|
+
* Node machinery those served has no DC analogue and is deliberately absent.
|
|
142
|
+
*/
|
|
143
|
+
const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(function* (sql, failpoint = noFailpoint, maxStoredValueBytes) {
|
|
144
|
+
const metaTableRows = yield* sql`
|
|
145
|
+
SELECT name
|
|
146
|
+
FROM sqlite_master
|
|
147
|
+
WHERE type = 'table'
|
|
148
|
+
AND name = 'effect_agent_meta'
|
|
149
|
+
`.pipe(Effect.mapError(storageError("read storage version table")));
|
|
150
|
+
if ((yield* decodeRows(Schema.Array(DoNameRow), "sqlite_master", "effect_agent_meta", metaTableRows)).length === 0) {
|
|
151
|
+
const existingRows = yield* sql`
|
|
152
|
+
SELECT name
|
|
153
|
+
FROM sqlite_master
|
|
154
|
+
WHERE type = 'table'
|
|
155
|
+
AND name LIKE 'effect_agent_%'
|
|
156
|
+
ORDER BY name
|
|
157
|
+
`.pipe(Effect.mapError(storageError("inspect unversioned storage")));
|
|
158
|
+
if ((yield* decodeRows(Schema.Array(DoNameRow), "sqlite_master", "effect_agent_%", existingRows)).length > 0) return yield* DoStorageCompatibilityError.make({
|
|
159
|
+
actualVersion: 0,
|
|
160
|
+
supportedVersion: 2,
|
|
161
|
+
message: "The Durable Object contains unversioned Effect Agent tables. Reset the development namespace explicitly; refusing to mutate ambiguous stored data."
|
|
162
|
+
});
|
|
163
|
+
yield* SqliteMigrator.run({ loader: doMigrations }).pipe(Effect.provideService(SqlClient.SqlClient, sql), Effect.mapError((error) => DoStorageError.make({
|
|
164
|
+
cause: error,
|
|
165
|
+
operation: "initialize current storage",
|
|
166
|
+
message: error.message
|
|
167
|
+
})));
|
|
168
|
+
} else {
|
|
169
|
+
const versionRows = yield* sql`
|
|
170
|
+
SELECT value
|
|
171
|
+
FROM effect_agent_meta
|
|
172
|
+
WHERE key = 'storage_version'
|
|
173
|
+
`.pipe(Effect.mapError(storageError("read storage version")));
|
|
174
|
+
const version = yield* decodeSingleRow(Schema.Array(DoMetaRow), "effect_agent_meta", "storage_version", versionRows);
|
|
175
|
+
if (version.value !== String(2)) {
|
|
176
|
+
const actualVersion = Number.parseInt(version.value, 10);
|
|
177
|
+
return yield* DoStorageCompatibilityError.make({
|
|
178
|
+
actualVersion: Number.isSafeInteger(actualVersion) ? actualVersion : -1,
|
|
179
|
+
supportedVersion: 2,
|
|
180
|
+
message: `The Durable Object uses private-development storage version ${version.value}; this build supports exactly version 2. Replace the development namespace explicitly; automatic stored-data migrations are not provided during private development.`
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
const requiredRows = yield* sql`
|
|
185
|
+
SELECT name
|
|
186
|
+
FROM sqlite_master
|
|
187
|
+
WHERE type = 'table'
|
|
188
|
+
AND name IN ${sql.in([...REQUIRED_TABLES])}
|
|
189
|
+
ORDER BY name
|
|
190
|
+
`.pipe(Effect.mapError(storageError("verify storage tables")));
|
|
191
|
+
if ((yield* decodeRows(Schema.Array(DoNameRow), "sqlite_master", "required_tables", requiredRows)).length !== REQUIRED_TABLES.length) return yield* DoStorageCompatibilityError.make({
|
|
192
|
+
actualVersion: 2,
|
|
193
|
+
supportedVersion: 2,
|
|
194
|
+
message: "The Durable Object claims the current format but is missing required tables. Reset the corrupt private-development data."
|
|
195
|
+
});
|
|
196
|
+
return makeJournal(sql, failpoint, maxStoredValueBytes);
|
|
197
|
+
});
|
|
198
|
+
const makeJournal = (sql, failpoint, maxStoredValueBytes) => {
|
|
199
|
+
/** Typed pre-write refusal for any single value over the configured byte bound. */
|
|
200
|
+
const checkValueBound = (operation, value) => {
|
|
201
|
+
const actualBytes = storedTextBytes(value);
|
|
202
|
+
return actualBytes > maxStoredValueBytes ? Effect.fail(DoValueBoundExceeded.make({
|
|
203
|
+
actualBytes,
|
|
204
|
+
maxBytes: maxStoredValueBytes,
|
|
205
|
+
operation
|
|
206
|
+
})) : Effect.void;
|
|
207
|
+
};
|
|
208
|
+
/**
|
|
209
|
+
* Runs one journal write transaction on the Durable Object storage-backed
|
|
210
|
+
* `withTransaction` (`ctx.storage.transaction()` under the hood). Within one Durable
|
|
211
|
+
* Object there is exactly ONE writer, so the Node `BEGIN IMMEDIATE` + busy-retry +
|
|
212
|
+
* `SqliteWriteContention` machinery has no analogue here and is deliberately absent.
|
|
213
|
+
* Ownership-token and epoch checks still run INSIDE the transaction, so fencing atomicity
|
|
214
|
+
* (DUR-006) is preserved identically.
|
|
215
|
+
*
|
|
216
|
+
* Journal write transactions are always top level: the Durable Object client rejects
|
|
217
|
+
* nested transactions, so new journal operations must not wrap this helper inside another
|
|
218
|
+
* transaction.
|
|
219
|
+
*/
|
|
220
|
+
const withWriteTransaction = (operation) => (effect) => sql.withTransaction(effect).pipe(Effect.mapError((error) => isSqlError(error) ? storageError(operation)(error) : error), Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }));
|
|
221
|
+
const materialize = Effect.fn("DoJournal.materialize")(function* (threadId, createdAt, emptyTailDigest, producerEpoch) {
|
|
222
|
+
if (threadId.length > MAX_IDENTIFIER_LENGTH) return yield* DoStorageError.make({
|
|
223
|
+
operation: "materialize thread",
|
|
224
|
+
message: "Thread identity exceeds the Durable Object storage bounds."
|
|
225
|
+
});
|
|
226
|
+
yield* checkValueBound("materialize thread", emptyTailDigest);
|
|
227
|
+
yield* withWriteTransaction("materialize transaction")(Effect.gen(function* () {
|
|
228
|
+
const existingRows = yield* sql`
|
|
229
|
+
SELECT
|
|
230
|
+
thread_id,
|
|
231
|
+
created_at,
|
|
232
|
+
tail_sequence,
|
|
233
|
+
tail_digest,
|
|
234
|
+
producer_epoch
|
|
235
|
+
FROM effect_agent_threads
|
|
236
|
+
WHERE thread_id = ${threadId}
|
|
237
|
+
`.pipe(Effect.mapError(storageError("read materialized thread")));
|
|
238
|
+
const existing = yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, existingRows);
|
|
239
|
+
if (existing.length > 1) return yield* DoStorageCorruptionError.make({
|
|
240
|
+
table: "effect_agent_threads",
|
|
241
|
+
rowKey: threadId,
|
|
242
|
+
message: "A thread primary key returned more than one row."
|
|
243
|
+
});
|
|
244
|
+
if (existing.length === 0) {
|
|
245
|
+
yield* sql`
|
|
246
|
+
INSERT INTO effect_agent_threads (
|
|
247
|
+
thread_id,
|
|
248
|
+
created_at,
|
|
249
|
+
tail_sequence,
|
|
250
|
+
tail_digest,
|
|
251
|
+
producer_epoch
|
|
252
|
+
) VALUES (
|
|
253
|
+
${threadId},
|
|
254
|
+
${createdAt},
|
|
255
|
+
0,
|
|
256
|
+
${emptyTailDigest},
|
|
257
|
+
${producerEpoch}
|
|
258
|
+
)
|
|
259
|
+
`.pipe(Effect.mapError(storageError("materialize thread")));
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
if (producerEpoch < existing[0].producer_epoch) return yield* DoFenceRejected.make({
|
|
263
|
+
producerEpoch,
|
|
264
|
+
actualEpoch: existing[0].producer_epoch,
|
|
265
|
+
message: `Producer epoch ${producerEpoch} is stale; current epoch is ${existing[0].producer_epoch}.`
|
|
266
|
+
});
|
|
267
|
+
if (producerEpoch > existing[0].producer_epoch) yield* sql`
|
|
268
|
+
UPDATE effect_agent_threads
|
|
269
|
+
SET producer_epoch = ${producerEpoch}
|
|
270
|
+
WHERE thread_id = ${threadId}
|
|
271
|
+
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
272
|
+
}));
|
|
273
|
+
});
|
|
274
|
+
const getThread = Effect.fn("DoJournal.getThread")(function* (threadId) {
|
|
275
|
+
const rows = yield* sql`
|
|
276
|
+
SELECT
|
|
277
|
+
thread_id,
|
|
278
|
+
created_at,
|
|
279
|
+
tail_sequence,
|
|
280
|
+
tail_digest,
|
|
281
|
+
producer_epoch
|
|
282
|
+
FROM effect_agent_threads
|
|
283
|
+
WHERE thread_id = ${threadId}
|
|
284
|
+
`.pipe(Effect.mapError(storageError("read thread")));
|
|
285
|
+
return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
286
|
+
});
|
|
287
|
+
const append = Effect.fn("DoJournal.append")(function* (request) {
|
|
288
|
+
if (request.threadId.length > MAX_IDENTIFIER_LENGTH || request.batchId.length > MAX_IDENTIFIER_LENGTH || request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)) return yield* DoStorageError.make({
|
|
289
|
+
operation: "append canonical batch",
|
|
290
|
+
message: "Canonical identifiers exceed the Durable Object storage bounds."
|
|
291
|
+
});
|
|
292
|
+
yield* checkValueBound("append canonical batch", request.batchJson);
|
|
293
|
+
yield* checkValueBound("append canonical batch", request.batchDigest);
|
|
294
|
+
yield* checkValueBound("append canonical batch", request.tailDigest);
|
|
295
|
+
yield* Effect.forEach(request.records, (record) => checkValueBound("append canonical record", record.recordJson), { discard: true });
|
|
296
|
+
return yield* withWriteTransaction("append transaction")(Effect.gen(function* () {
|
|
297
|
+
const recordIds = request.records.map((record) => record.recordId);
|
|
298
|
+
if (new Set(recordIds).size !== recordIds.length) return yield* DoAppendConflict.make({
|
|
299
|
+
message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
|
|
300
|
+
reason: "record-identity"
|
|
301
|
+
});
|
|
302
|
+
const threadRows = yield* sql`
|
|
303
|
+
SELECT
|
|
304
|
+
thread_id,
|
|
305
|
+
created_at,
|
|
306
|
+
tail_sequence,
|
|
307
|
+
tail_digest,
|
|
308
|
+
producer_epoch
|
|
309
|
+
FROM effect_agent_threads
|
|
310
|
+
WHERE thread_id = ${request.threadId}
|
|
311
|
+
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
312
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", request.threadId, threadRows);
|
|
313
|
+
if (request.producerEpoch !== thread.producer_epoch) return yield* DoFenceRejected.make({
|
|
314
|
+
producerEpoch: request.producerEpoch,
|
|
315
|
+
actualEpoch: thread.producer_epoch,
|
|
316
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`
|
|
317
|
+
});
|
|
318
|
+
const batchRows = yield* sql`
|
|
319
|
+
SELECT
|
|
320
|
+
thread_id,
|
|
321
|
+
batch_id,
|
|
322
|
+
first_sequence,
|
|
323
|
+
last_sequence,
|
|
324
|
+
batch_digest,
|
|
325
|
+
tail_digest,
|
|
326
|
+
batch_json
|
|
327
|
+
FROM effect_agent_canonical_batches
|
|
328
|
+
WHERE thread_id = ${request.threadId}
|
|
329
|
+
AND batch_id = ${request.batchId}
|
|
330
|
+
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
331
|
+
const batches = yield* decodeRows(Schema.Array(BatchRow), "effect_agent_canonical_batches", `${request.threadId}/${request.batchId}`, batchRows);
|
|
332
|
+
if (batches.length > 1) return yield* DoStorageCorruptionError.make({
|
|
333
|
+
table: "effect_agent_canonical_batches",
|
|
334
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
335
|
+
message: "A canonical batch primary key returned more than one row."
|
|
336
|
+
});
|
|
337
|
+
if (batches.length === 1) {
|
|
338
|
+
const existing = batches[0];
|
|
339
|
+
if (existing.batch_digest !== request.batchDigest) return yield* DoAppendConflict.make({
|
|
340
|
+
message: `Batch ${request.batchId} already exists with different canonical content.`,
|
|
341
|
+
reason: "batch-digest"
|
|
342
|
+
});
|
|
343
|
+
return RawAppendResult.make({
|
|
344
|
+
firstSequence: existing.first_sequence,
|
|
345
|
+
lastSequence: existing.last_sequence,
|
|
346
|
+
replayed: true,
|
|
347
|
+
tailDigest: existing.tail_digest
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
if (request.expectedTailSequence !== thread.tail_sequence || request.expectedTailDigest !== thread.tail_digest) return yield* DoAppendConflict.make({
|
|
351
|
+
message: `Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
352
|
+
reason: "tail",
|
|
353
|
+
actualTailSequence: thread.tail_sequence,
|
|
354
|
+
actualTailDigest: thread.tail_digest
|
|
355
|
+
});
|
|
356
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) return yield* DoStorageError.make({
|
|
357
|
+
operation: "append canonical batch",
|
|
358
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`
|
|
359
|
+
});
|
|
360
|
+
const existingRecords = [];
|
|
361
|
+
for (const chunk of chunked(recordIds, 90)) {
|
|
362
|
+
const existingRecordRows = yield* sql`
|
|
363
|
+
SELECT
|
|
364
|
+
thread_id,
|
|
365
|
+
sequence,
|
|
366
|
+
record_id,
|
|
367
|
+
batch_id,
|
|
368
|
+
record_json
|
|
369
|
+
FROM effect_agent_canonical_records
|
|
370
|
+
WHERE thread_id = ${request.threadId}
|
|
371
|
+
AND record_id IN ${sql.in([...chunk])}
|
|
372
|
+
ORDER BY sequence
|
|
373
|
+
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
374
|
+
existingRecords.push(...yield* decodeRows(Schema.Array(RecordRow), "effect_agent_canonical_records", `${request.threadId}/record_ids`, existingRecordRows));
|
|
375
|
+
}
|
|
376
|
+
if (existingRecords.length > 0) return yield* DoAppendConflict.make({
|
|
377
|
+
message: `Canonical record ID ${existingRecords[0].record_id} already exists.`,
|
|
378
|
+
reason: "record-identity"
|
|
379
|
+
});
|
|
380
|
+
const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(thread.tail_sequence + 1).pipe(Effect.mapError((error) => DoStorageError.make({
|
|
381
|
+
cause: error,
|
|
382
|
+
operation: "append canonical batch",
|
|
383
|
+
message: error.message
|
|
384
|
+
})));
|
|
385
|
+
const lastSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(firstSequence + request.records.length - 1).pipe(Effect.mapError((error) => DoStorageError.make({
|
|
386
|
+
cause: error,
|
|
387
|
+
operation: "append canonical batch",
|
|
388
|
+
message: error.message
|
|
389
|
+
})));
|
|
390
|
+
yield* sql`
|
|
391
|
+
INSERT INTO effect_agent_canonical_batches (
|
|
392
|
+
thread_id,
|
|
393
|
+
batch_id,
|
|
394
|
+
first_sequence,
|
|
395
|
+
last_sequence,
|
|
396
|
+
batch_digest,
|
|
397
|
+
tail_digest,
|
|
398
|
+
batch_json
|
|
399
|
+
) VALUES (
|
|
400
|
+
${request.threadId},
|
|
401
|
+
${request.batchId},
|
|
402
|
+
${firstSequence},
|
|
403
|
+
${lastSequence},
|
|
404
|
+
${request.batchDigest},
|
|
405
|
+
${request.tailDigest},
|
|
406
|
+
${request.batchJson}
|
|
407
|
+
)
|
|
408
|
+
`.pipe(Effect.mapError(storageError("insert canonical batch")));
|
|
409
|
+
yield* failpoint("append:after-batch-insert");
|
|
410
|
+
yield* Effect.forEach(request.records, (record, index) => Effect.gen(function* () {
|
|
411
|
+
yield* sql`
|
|
412
|
+
INSERT INTO effect_agent_canonical_records (
|
|
413
|
+
thread_id,
|
|
414
|
+
sequence,
|
|
415
|
+
record_id,
|
|
416
|
+
batch_id,
|
|
417
|
+
record_json
|
|
418
|
+
) VALUES (
|
|
419
|
+
${request.threadId},
|
|
420
|
+
${firstSequence + index},
|
|
421
|
+
${record.recordId},
|
|
422
|
+
${request.batchId},
|
|
423
|
+
${record.recordJson}
|
|
424
|
+
)
|
|
425
|
+
`.pipe(Effect.mapError(storageError("insert canonical record")));
|
|
426
|
+
yield* failpoint("append:after-record-insert");
|
|
427
|
+
}), { discard: true });
|
|
428
|
+
yield* sql`
|
|
429
|
+
UPDATE effect_agent_threads
|
|
430
|
+
SET
|
|
431
|
+
tail_sequence = ${lastSequence},
|
|
432
|
+
tail_digest = ${request.tailDigest},
|
|
433
|
+
producer_epoch = ${request.producerEpoch}
|
|
434
|
+
WHERE thread_id = ${request.threadId}
|
|
435
|
+
`.pipe(Effect.mapError(storageError("advance thread tail")));
|
|
436
|
+
yield* failpoint("append:after-tail-update");
|
|
437
|
+
return RawAppendResult.make({
|
|
438
|
+
firstSequence,
|
|
439
|
+
lastSequence,
|
|
440
|
+
replayed: false,
|
|
441
|
+
tailDigest: request.tailDigest
|
|
442
|
+
});
|
|
443
|
+
}));
|
|
444
|
+
});
|
|
445
|
+
const read = Effect.fn("DoJournal.read")(function* (request) {
|
|
446
|
+
const rows = yield* sql`
|
|
447
|
+
SELECT
|
|
448
|
+
thread_id,
|
|
449
|
+
sequence,
|
|
450
|
+
record_id,
|
|
451
|
+
batch_id,
|
|
452
|
+
record_json
|
|
453
|
+
FROM effect_agent_canonical_records
|
|
454
|
+
WHERE thread_id = ${request.threadId}
|
|
455
|
+
AND sequence > ${request.fromSequenceExclusive}
|
|
456
|
+
ORDER BY sequence
|
|
457
|
+
LIMIT ${request.limit}
|
|
458
|
+
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
459
|
+
return yield* decodeRows(Schema.Array(RecordRow), "effect_agent_canonical_records", `${request.threadId}>${request.fromSequenceExclusive}`, rows);
|
|
460
|
+
});
|
|
461
|
+
const exportThread = Effect.fn("DoJournal.exportThread")(function* (threadId) {
|
|
462
|
+
return yield* sql.withTransaction(Effect.gen(function* () {
|
|
463
|
+
const threadRows = yield* sql`
|
|
464
|
+
SELECT
|
|
465
|
+
thread_id,
|
|
466
|
+
created_at,
|
|
467
|
+
tail_sequence,
|
|
468
|
+
tail_digest,
|
|
469
|
+
producer_epoch
|
|
470
|
+
FROM effect_agent_threads
|
|
471
|
+
WHERE thread_id = ${threadId}
|
|
472
|
+
`.pipe(Effect.mapError(storageError("export thread")));
|
|
473
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", threadId, threadRows);
|
|
474
|
+
yield* failpoint("export:after-thread-read");
|
|
475
|
+
const batchRows = yield* sql`
|
|
476
|
+
SELECT
|
|
477
|
+
thread_id,
|
|
478
|
+
batch_id,
|
|
479
|
+
first_sequence,
|
|
480
|
+
last_sequence,
|
|
481
|
+
batch_digest,
|
|
482
|
+
tail_digest,
|
|
483
|
+
batch_json
|
|
484
|
+
FROM effect_agent_canonical_batches
|
|
485
|
+
WHERE thread_id = ${threadId}
|
|
486
|
+
ORDER BY first_sequence
|
|
487
|
+
`.pipe(Effect.mapError(storageError("export canonical batches")));
|
|
488
|
+
const recordRows = yield* sql`
|
|
489
|
+
SELECT
|
|
490
|
+
thread_id,
|
|
491
|
+
sequence,
|
|
492
|
+
record_id,
|
|
493
|
+
batch_id,
|
|
494
|
+
record_json
|
|
495
|
+
FROM effect_agent_canonical_records
|
|
496
|
+
WHERE thread_id = ${threadId}
|
|
497
|
+
ORDER BY sequence
|
|
498
|
+
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
499
|
+
const checkpointRows = yield* sql`
|
|
500
|
+
SELECT
|
|
501
|
+
thread_id,
|
|
502
|
+
through_sequence,
|
|
503
|
+
tail_digest,
|
|
504
|
+
checkpoint_json
|
|
505
|
+
FROM effect_agent_checkpoints
|
|
506
|
+
WHERE thread_id = ${threadId}
|
|
507
|
+
ORDER BY through_sequence
|
|
508
|
+
`.pipe(Effect.mapError(storageError("export checkpoints")));
|
|
509
|
+
return RawThreadExport.make({
|
|
510
|
+
thread,
|
|
511
|
+
batches: yield* decodeRows(Schema.Array(BatchRow), "effect_agent_canonical_batches", threadId, batchRows),
|
|
512
|
+
records: yield* decodeRows(Schema.Array(RecordRow), "effect_agent_canonical_records", threadId, recordRows),
|
|
513
|
+
checkpoints: yield* decodeRows(Schema.Array(CheckpointRow), "effect_agent_checkpoints", threadId, checkpointRows)
|
|
514
|
+
});
|
|
515
|
+
})).pipe(Effect.catchTag("SqlError", (error) => Effect.fail(storageError("export transaction")(error))));
|
|
516
|
+
});
|
|
517
|
+
const saveCheckpoint = Effect.fn("DoJournal.saveCheckpoint")(function* (checkpoint) {
|
|
518
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) return yield* DoStorageError.make({
|
|
519
|
+
operation: "save checkpoint",
|
|
520
|
+
message: "Checkpoint identity exceeds the Durable Object storage bounds."
|
|
521
|
+
});
|
|
522
|
+
yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
|
|
523
|
+
yield* withWriteTransaction("checkpoint transaction")(Effect.gen(function* () {
|
|
524
|
+
const threadRows = yield* sql`
|
|
525
|
+
SELECT
|
|
526
|
+
thread_id,
|
|
527
|
+
created_at,
|
|
528
|
+
tail_sequence,
|
|
529
|
+
tail_digest,
|
|
530
|
+
producer_epoch
|
|
531
|
+
FROM effect_agent_threads
|
|
532
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
533
|
+
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
534
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", checkpoint.threadId, threadRows);
|
|
535
|
+
if (checkpoint.throughSequence > thread.tail_sequence) return yield* DoCheckpointConflict.make({ message: `Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ${thread.tail_sequence}.` });
|
|
536
|
+
const checkpointRows = yield* sql`
|
|
537
|
+
SELECT
|
|
538
|
+
thread_id,
|
|
539
|
+
through_sequence,
|
|
540
|
+
tail_digest,
|
|
541
|
+
checkpoint_json
|
|
542
|
+
FROM effect_agent_checkpoints
|
|
543
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
544
|
+
AND through_sequence = ${checkpoint.throughSequence}
|
|
545
|
+
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
546
|
+
const existing = yield* decodeRows(Schema.Array(CheckpointRow), "effect_agent_checkpoints", `${checkpoint.threadId}/${checkpoint.throughSequence}`, checkpointRows);
|
|
547
|
+
if (existing.length > 1) return yield* DoStorageCorruptionError.make({
|
|
548
|
+
table: "effect_agent_checkpoints",
|
|
549
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
550
|
+
message: "A checkpoint primary key returned more than one row."
|
|
551
|
+
});
|
|
552
|
+
if (existing.length === 1) {
|
|
553
|
+
if (existing[0].tail_digest !== checkpoint.tailDigest || existing[0].checkpoint_json !== checkpoint.checkpointJson) return yield* DoCheckpointConflict.make({ message: "A different checkpoint already exists at this canonical sequence." });
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
yield* sql`
|
|
557
|
+
INSERT INTO effect_agent_checkpoints (
|
|
558
|
+
thread_id,
|
|
559
|
+
through_sequence,
|
|
560
|
+
tail_digest,
|
|
561
|
+
checkpoint_json
|
|
562
|
+
) VALUES (
|
|
563
|
+
${checkpoint.threadId},
|
|
564
|
+
${checkpoint.throughSequence},
|
|
565
|
+
${checkpoint.tailDigest},
|
|
566
|
+
${checkpoint.checkpointJson}
|
|
567
|
+
)
|
|
568
|
+
`.pipe(Effect.mapError(storageError("insert checkpoint")));
|
|
569
|
+
}));
|
|
570
|
+
});
|
|
571
|
+
const loadCheckpoint = Effect.fn("DoJournal.loadCheckpoint")(function* (threadId, atOrBeforeSequence) {
|
|
572
|
+
const rows = yield* sql`
|
|
573
|
+
SELECT
|
|
574
|
+
thread_id,
|
|
575
|
+
through_sequence,
|
|
576
|
+
tail_digest,
|
|
577
|
+
checkpoint_json
|
|
578
|
+
FROM effect_agent_checkpoints
|
|
579
|
+
WHERE thread_id = ${threadId}
|
|
580
|
+
AND through_sequence <= ${atOrBeforeSequence}
|
|
581
|
+
ORDER BY through_sequence DESC
|
|
582
|
+
LIMIT 1
|
|
583
|
+
`.pipe(Effect.mapError(storageError("load checkpoint")));
|
|
584
|
+
return yield* decodeRows(Schema.Array(CheckpointRow), "effect_agent_checkpoints", `${threadId}<=${atOrBeforeSequence}`, rows);
|
|
585
|
+
});
|
|
586
|
+
return {
|
|
587
|
+
append,
|
|
588
|
+
checkValueBound,
|
|
589
|
+
exportThread,
|
|
590
|
+
getThread,
|
|
591
|
+
getTailDigestAt: Effect.fn("DoJournal.getTailDigestAt")(function* (threadId, sequence) {
|
|
592
|
+
if (sequence === 0) {
|
|
593
|
+
const threads = yield* getThread(threadId);
|
|
594
|
+
return threads.length === 0 ? [] : [threads[0].tail_sequence === 0 ? threads[0].tail_digest : void 0].filter((value) => value !== void 0);
|
|
595
|
+
}
|
|
596
|
+
const rows = yield* sql`
|
|
597
|
+
SELECT
|
|
598
|
+
thread_id,
|
|
599
|
+
batch_id,
|
|
600
|
+
first_sequence,
|
|
601
|
+
last_sequence,
|
|
602
|
+
batch_digest,
|
|
603
|
+
tail_digest,
|
|
604
|
+
batch_json
|
|
605
|
+
FROM effect_agent_canonical_batches
|
|
606
|
+
WHERE thread_id = ${threadId}
|
|
607
|
+
AND last_sequence = ${sequence}
|
|
608
|
+
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
609
|
+
return (yield* decodeRows(Schema.Array(BatchRow), "effect_agent_canonical_batches", `${threadId}/${sequence}`, rows)).map((batch) => batch.tail_digest);
|
|
610
|
+
}),
|
|
611
|
+
loadCheckpoint,
|
|
612
|
+
materialize,
|
|
613
|
+
read,
|
|
614
|
+
saveCheckpoint,
|
|
615
|
+
scanStoredPayloads: Effect.fn("DoJournal.scanStoredPayloads")(function* () {
|
|
616
|
+
return yield* sql.withTransaction(Effect.gen(function* () {
|
|
617
|
+
const threads = yield* sql`
|
|
618
|
+
SELECT
|
|
619
|
+
thread_id,
|
|
620
|
+
created_at,
|
|
621
|
+
tail_sequence,
|
|
622
|
+
tail_digest,
|
|
623
|
+
producer_epoch
|
|
624
|
+
FROM effect_agent_threads
|
|
625
|
+
ORDER BY thread_id
|
|
626
|
+
`.pipe(Effect.mapError(storageError("scan threads")));
|
|
627
|
+
const batches = yield* sql`
|
|
628
|
+
SELECT
|
|
629
|
+
thread_id,
|
|
630
|
+
batch_id,
|
|
631
|
+
first_sequence,
|
|
632
|
+
last_sequence,
|
|
633
|
+
batch_digest,
|
|
634
|
+
tail_digest,
|
|
635
|
+
batch_json
|
|
636
|
+
FROM effect_agent_canonical_batches
|
|
637
|
+
ORDER BY thread_id, first_sequence
|
|
638
|
+
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
639
|
+
const records = yield* sql`
|
|
640
|
+
SELECT
|
|
641
|
+
thread_id,
|
|
642
|
+
sequence,
|
|
643
|
+
record_id,
|
|
644
|
+
batch_id,
|
|
645
|
+
record_json
|
|
646
|
+
FROM effect_agent_canonical_records
|
|
647
|
+
ORDER BY thread_id, sequence
|
|
648
|
+
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
649
|
+
const checkpoints = yield* sql`
|
|
650
|
+
SELECT
|
|
651
|
+
thread_id,
|
|
652
|
+
through_sequence,
|
|
653
|
+
tail_digest,
|
|
654
|
+
checkpoint_json
|
|
655
|
+
FROM effect_agent_checkpoints
|
|
656
|
+
ORDER BY thread_id, through_sequence
|
|
657
|
+
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
658
|
+
return {
|
|
659
|
+
threads: yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", "startup_scan", threads),
|
|
660
|
+
batches: yield* decodeRows(Schema.Array(BatchRow), "effect_agent_canonical_batches", "startup_scan", batches),
|
|
661
|
+
records: yield* decodeRows(Schema.Array(RecordRow), "effect_agent_canonical_records", "startup_scan", records),
|
|
662
|
+
checkpoints: yield* decodeRows(Schema.Array(CheckpointRow), "effect_agent_checkpoints", "startup_scan", checkpoints)
|
|
663
|
+
};
|
|
664
|
+
})).pipe(Effect.catchTag("SqlError", (error) => Effect.fail(storageError("startup scan transaction")(error))));
|
|
665
|
+
}),
|
|
666
|
+
withWriteTransaction
|
|
667
|
+
};
|
|
668
|
+
};
|
|
669
|
+
const initializeDoJournal = ensureCurrentStorage;
|
|
670
|
+
//#endregion
|
|
671
|
+
//#region src/DoThreadStore.ts
|
|
672
|
+
var DoThreadStore_exports = /* @__PURE__ */ __exportAll({
|
|
673
|
+
layer: () => layer,
|
|
674
|
+
storageConfigLayer: () => storageConfigLayer,
|
|
675
|
+
storageFailpointLayer: () => storageFailpointLayer,
|
|
676
|
+
threadStoreLayer: () => threadStoreLayer
|
|
677
|
+
});
|
|
678
|
+
const OffsetText = Schema.String.check(Schema.isMaxLength(4096));
|
|
679
|
+
const DO_OFFSET_PREFIX = "effect-agent-do@1:";
|
|
680
|
+
const ZERO_CANONICAL_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
681
|
+
const isDigest = Schema.is(Digest);
|
|
682
|
+
const isDoFenceRejected = Schema.is(DoFenceRejected);
|
|
683
|
+
const isDoAppendConflict = Schema.is(DoAppendConflict);
|
|
684
|
+
const isDoCheckpointConflict = Schema.is(DoCheckpointConflict);
|
|
685
|
+
const storeError = (operation, error) => ThreadStoreError.make({
|
|
686
|
+
cause: error,
|
|
687
|
+
operation,
|
|
688
|
+
message: error.message
|
|
689
|
+
});
|
|
690
|
+
const schemaStoreError = (operation, error) => ThreadStoreError.make({
|
|
691
|
+
cause: error,
|
|
692
|
+
operation,
|
|
693
|
+
message: error.message
|
|
694
|
+
});
|
|
695
|
+
const makeOffset = Effect.fn(function* (threadId, sequence) {
|
|
696
|
+
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(sequence).pipe(Effect.flatMap((validatedSequence) => Schema.decodeUnknownEffect(ObservationOffset)(`${DO_OFFSET_PREFIX}${encodeURIComponent(threadId)}:${validatedSequence}`)), Effect.mapError((error) => schemaStoreError("encode observation offset", error)));
|
|
697
|
+
});
|
|
698
|
+
const parseOffset = Effect.fn(function* (threadId, offset) {
|
|
699
|
+
if (offset === void 0) return ZERO_CANONICAL_SEQUENCE;
|
|
700
|
+
const text = yield* Schema.decodeUnknownEffect(OffsetText)(offset).pipe(Effect.mapError((error) => schemaStoreError("decode observation offset", error)));
|
|
701
|
+
const threadPrefix = `${DO_OFFSET_PREFIX}${encodeURIComponent(threadId)}:`;
|
|
702
|
+
if (!text.startsWith(threadPrefix)) return yield* ThreadStoreError.make({
|
|
703
|
+
operation: "decode observation offset",
|
|
704
|
+
message: "The observation offset belongs to a different adapter, storage version, or Thread."
|
|
705
|
+
});
|
|
706
|
+
const sequenceText = text.slice(threadPrefix.length);
|
|
707
|
+
if (!/^(0|[1-9][0-9]*)$/.test(sequenceText)) return yield* ThreadStoreError.make({
|
|
708
|
+
operation: "decode observation offset",
|
|
709
|
+
message: "The observation offset is malformed."
|
|
710
|
+
});
|
|
711
|
+
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(Number(sequenceText)).pipe(Effect.mapError((error) => schemaStoreError("decode observation offset", error)));
|
|
712
|
+
});
|
|
713
|
+
const mapFence = (threadId, error) => FenceRejected.make({
|
|
714
|
+
threadId,
|
|
715
|
+
actualEpoch: error.actualEpoch,
|
|
716
|
+
attemptedEpoch: error.producerEpoch
|
|
717
|
+
});
|
|
718
|
+
const encodeCanonicalRecord = Effect.fn(function* (record) {
|
|
719
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(record).pipe(Effect.mapError((error) => schemaStoreError("encode canonical record", error)));
|
|
720
|
+
});
|
|
721
|
+
const encodeCanonicalBatch = Effect.fn(function* (batch) {
|
|
722
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalBatch))(batch).pipe(Effect.mapError((error) => schemaStoreError("encode canonical batch", error)));
|
|
723
|
+
});
|
|
724
|
+
const encodeCheckpoint = Effect.fn(function* (checkpoint) {
|
|
725
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint).pipe(Effect.mapError((error) => schemaStoreError("encode checkpoint", error)));
|
|
726
|
+
});
|
|
727
|
+
const decodeEnvelope = Effect.fn(function* (row) {
|
|
728
|
+
const record = yield* Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(row.record_json).pipe(Effect.mapError((error) => ThreadStoreError.make({
|
|
729
|
+
operation: "decode canonical record",
|
|
730
|
+
message: error.message
|
|
731
|
+
})));
|
|
732
|
+
const threadId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.threadId)(row.thread_id).pipe(Effect.mapError((error) => schemaStoreError("decode thread identity", error)));
|
|
733
|
+
const offset = yield* makeOffset(threadId, row.sequence);
|
|
734
|
+
const batchId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.batchId)(row.batch_id).pipe(Effect.mapError((error) => schemaStoreError("decode batch identity", error)));
|
|
735
|
+
return CanonicalRecordEnvelope.make({
|
|
736
|
+
threadId,
|
|
737
|
+
batchId,
|
|
738
|
+
sequence: row.sequence,
|
|
739
|
+
offset,
|
|
740
|
+
record
|
|
741
|
+
});
|
|
742
|
+
});
|
|
743
|
+
const decodeCheckpoint = Effect.fn(function* (checkpointJson) {
|
|
744
|
+
return yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpointJson).pipe(Effect.mapError((error) => schemaStoreError("decode checkpoint", error)));
|
|
745
|
+
});
|
|
746
|
+
const requireThread = Effect.fn("DoThreadStore.requireThread")(function* (journal, threadId) {
|
|
747
|
+
const rows = yield* journal.getThread(threadId).pipe(Effect.mapError((error) => storeError("read thread", error)));
|
|
748
|
+
if (rows.length === 0) return yield* ThreadNotMaterialized.make({ threadId });
|
|
749
|
+
return rows[0];
|
|
750
|
+
});
|
|
751
|
+
const tailDigestAt = Effect.fn("DoThreadStore.tailDigestAt")(function* (journal, threadId, sequence) {
|
|
752
|
+
if (sequence === 0) return EMPTY_TAIL_DIGEST;
|
|
753
|
+
const digests = yield* journal.getTailDigestAt(threadId, sequence).pipe(Effect.mapError((error) => storeError("read checkpoint digest", error)));
|
|
754
|
+
if (digests.length !== 1) return yield* CheckpointRejected.make({
|
|
755
|
+
threadId,
|
|
756
|
+
reason: "digest-mismatch"
|
|
757
|
+
});
|
|
758
|
+
return yield* Schema.decodeUnknownEffect(Digest)(digests[0]).pipe(Effect.mapError((error) => schemaStoreError("decode checkpoint digest", error)));
|
|
759
|
+
});
|
|
760
|
+
const groupByKey = (rows, key) => {
|
|
761
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
762
|
+
for (const row of rows) {
|
|
763
|
+
const existing = grouped.get(key(row));
|
|
764
|
+
if (existing === void 0) grouped.set(key(row), [row]);
|
|
765
|
+
else existing.push(row);
|
|
766
|
+
}
|
|
767
|
+
return grouped;
|
|
768
|
+
};
|
|
769
|
+
/**
|
|
770
|
+
* Opt-in full integrity audit (`verifyOnOpen`). Every stored payload is decoded, re-encoded,
|
|
771
|
+
* and re-digested against the canonical chain. Routine opens skip this scan: per-operation
|
|
772
|
+
* Schema decoding plus the digest chain already fail clearly on corrupt rows.
|
|
773
|
+
*/
|
|
774
|
+
const decodeStartupPayloads = Effect.fn("DoThreadStore.decodeStartupPayloads")(function* (journal, crypto) {
|
|
775
|
+
const stored = yield* journal.scanStoredPayloads();
|
|
776
|
+
const batches = yield* Effect.forEach(stored.batches, (batch) => Schema.decodeEffect(Schema.fromJsonString(CanonicalBatch))(batch.batch_json).pipe(Effect.map((decoded) => ({
|
|
777
|
+
decoded,
|
|
778
|
+
row: batch
|
|
779
|
+
})), Effect.mapError((error) => DoStorageCorruptionError.make({
|
|
780
|
+
table: "effect_agent_canonical_batches",
|
|
781
|
+
rowKey: `${batch.thread_id}/${batch.batch_id}`,
|
|
782
|
+
message: error.message
|
|
783
|
+
}))));
|
|
784
|
+
const records = yield* Effect.forEach(stored.records, (record) => Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(record.record_json).pipe(Effect.map((decoded) => ({
|
|
785
|
+
decoded,
|
|
786
|
+
row: record
|
|
787
|
+
})), Effect.mapError((error) => DoStorageCorruptionError.make({
|
|
788
|
+
table: "effect_agent_canonical_records",
|
|
789
|
+
rowKey: `${record.thread_id}/${record.sequence}`,
|
|
790
|
+
message: error.message
|
|
791
|
+
}))));
|
|
792
|
+
const checkpoints = yield* Effect.forEach(stored.checkpoints, (checkpoint) => Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint.checkpoint_json).pipe(Effect.map((decoded) => ({
|
|
793
|
+
decoded,
|
|
794
|
+
row: checkpoint
|
|
795
|
+
})), Effect.mapError((error) => DoStorageCorruptionError.make({
|
|
796
|
+
table: "effect_agent_checkpoints",
|
|
797
|
+
rowKey: `${checkpoint.thread_id}/${checkpoint.through_sequence}`,
|
|
798
|
+
message: error.message
|
|
799
|
+
}))));
|
|
800
|
+
const batchesByThread = groupByKey(batches, ({ row }) => row.thread_id);
|
|
801
|
+
const recordsByThread = groupByKey(records, ({ row }) => row.thread_id);
|
|
802
|
+
const checkpointsByThread = groupByKey(checkpoints, ({ row }) => row.thread_id);
|
|
803
|
+
const materializedIds = new Set(stored.threads.map((thread) => thread.thread_id));
|
|
804
|
+
for (const thread of stored.threads) {
|
|
805
|
+
const threadBatches = batchesByThread.get(thread.thread_id) ?? [];
|
|
806
|
+
const threadRecords = recordsByThread.get(thread.thread_id) ?? [];
|
|
807
|
+
const threadCheckpoints = checkpointsByThread.get(thread.thread_id) ?? [];
|
|
808
|
+
const recordsByBatch = groupByKey(threadRecords, ({ row }) => row.batch_id);
|
|
809
|
+
let previousDigest = EMPTY_TAIL_DIGEST;
|
|
810
|
+
let expectedSequence = 1;
|
|
811
|
+
const tailDigests = /* @__PURE__ */ new Map([[0, EMPTY_TAIL_DIGEST]]);
|
|
812
|
+
for (const { decoded: canonicalBatch, row: batchRow } of threadBatches) {
|
|
813
|
+
const key = `${batchRow.thread_id}/${batchRow.batch_id}`;
|
|
814
|
+
if (canonicalBatch.batchId !== batchRow.batch_id || batchRow.first_sequence !== expectedSequence || batchRow.last_sequence !== batchRow.first_sequence + canonicalBatch.records.length - 1) return yield* DoStorageCorruptionError.make({
|
|
815
|
+
table: "effect_agent_canonical_batches",
|
|
816
|
+
rowKey: key,
|
|
817
|
+
message: "Canonical batch identity, sequence, or record count is inconsistent."
|
|
818
|
+
});
|
|
819
|
+
const digest = yield* digestCanonicalBatch(previousDigest, canonicalBatch).pipe(Effect.provideService(Crypto.Crypto, crypto), Effect.mapError((error) => DoStorageCorruptionError.make({
|
|
820
|
+
table: "effect_agent_canonical_batches",
|
|
821
|
+
rowKey: key,
|
|
822
|
+
message: error.message
|
|
823
|
+
})));
|
|
824
|
+
if (batchRow.batch_digest !== digest || batchRow.tail_digest !== digest) return yield* DoStorageCorruptionError.make({
|
|
825
|
+
table: "effect_agent_canonical_batches",
|
|
826
|
+
rowKey: key,
|
|
827
|
+
message: "Canonical batch digest does not match its decoded content and prior tail."
|
|
828
|
+
});
|
|
829
|
+
const batchRecords = recordsByBatch.get(batchRow.batch_id) ?? [];
|
|
830
|
+
if (batchRecords.length !== canonicalBatch.records.length) return yield* DoStorageCorruptionError.make({
|
|
831
|
+
table: "effect_agent_canonical_records",
|
|
832
|
+
rowKey: key,
|
|
833
|
+
message: "Canonical batch and record-table counts differ."
|
|
834
|
+
});
|
|
835
|
+
for (let index = 0; index < canonicalBatch.records.length; index++) {
|
|
836
|
+
const expectedRecord = canonicalBatch.records[index];
|
|
837
|
+
const storedRecord = batchRecords[index];
|
|
838
|
+
const expectedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(expectedRecord).pipe(Effect.mapError((error) => DoStorageCorruptionError.make({
|
|
839
|
+
table: "effect_agent_canonical_batches",
|
|
840
|
+
rowKey: key,
|
|
841
|
+
message: error.message
|
|
842
|
+
})));
|
|
843
|
+
const storedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(storedRecord.decoded).pipe(Effect.mapError((error) => DoStorageCorruptionError.make({
|
|
844
|
+
table: "effect_agent_canonical_records",
|
|
845
|
+
rowKey: `${key}/${storedRecord.row.sequence}`,
|
|
846
|
+
message: error.message
|
|
847
|
+
})));
|
|
848
|
+
if (storedRecord.row.sequence !== batchRow.first_sequence + index || storedRecord.row.record_id !== expectedRecord.recordId || expectedJson !== storedJson) return yield* DoStorageCorruptionError.make({
|
|
849
|
+
table: "effect_agent_canonical_records",
|
|
850
|
+
rowKey: `${key}/${storedRecord.row.sequence}`,
|
|
851
|
+
message: "Canonical record identity, sequence, or payload differs from its batch."
|
|
852
|
+
});
|
|
853
|
+
}
|
|
854
|
+
previousDigest = digest;
|
|
855
|
+
expectedSequence = batchRow.last_sequence + 1;
|
|
856
|
+
tailDigests.set(batchRow.last_sequence, digest);
|
|
857
|
+
}
|
|
858
|
+
if (threadRecords.length !== thread.tail_sequence || thread.tail_sequence !== expectedSequence - 1 || thread.tail_digest !== previousDigest) return yield* DoStorageCorruptionError.make({
|
|
859
|
+
table: "effect_agent_threads",
|
|
860
|
+
rowKey: thread.thread_id,
|
|
861
|
+
message: "Thread tail does not match its canonical batch chain."
|
|
862
|
+
});
|
|
863
|
+
for (const checkpoint of threadCheckpoints) if (checkpoint.decoded.threadId !== thread.thread_id || checkpoint.decoded.throughSequence !== checkpoint.row.through_sequence || checkpoint.decoded.tailDigest !== checkpoint.row.tail_digest || tailDigests.get(checkpoint.row.through_sequence) !== checkpoint.row.tail_digest) return yield* DoStorageCorruptionError.make({
|
|
864
|
+
table: "effect_agent_checkpoints",
|
|
865
|
+
rowKey: `${thread.thread_id}/${checkpoint.row.through_sequence}`,
|
|
866
|
+
message: "Checkpoint identity or digest is not bound to a canonical batch tail."
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
if (batches.some(({ row }) => !materializedIds.has(row.thread_id)) || records.some(({ row }) => !materializedIds.has(row.thread_id)) || checkpoints.some(({ row }) => !materializedIds.has(row.thread_id))) return yield* DoStorageCorruptionError.make({
|
|
870
|
+
table: "effect_agent_threads",
|
|
871
|
+
rowKey: "startup_scan",
|
|
872
|
+
message: "Canonical rows exist without a materialized Thread."
|
|
873
|
+
});
|
|
874
|
+
});
|
|
875
|
+
const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
|
|
876
|
+
const config = yield* DoStorageConfig;
|
|
877
|
+
const failpoint = yield* DoStorageFailpoint;
|
|
878
|
+
const sql = yield* SqlClient.SqlClient;
|
|
879
|
+
const crypto = yield* Crypto.Crypto;
|
|
880
|
+
const journal = yield* initializeDoJournal(sql, failpoint.hit, config.maxStoredValueBytes);
|
|
881
|
+
if (config.verifyOnOpen) yield* decodeStartupPayloads(journal, crypto);
|
|
882
|
+
const provideCrypto = (effect) => Effect.provideService(effect, Crypto.Crypto, crypto);
|
|
883
|
+
const hitFailpoint = Effect.fn((location) => failpoint.hit(location).pipe(Effect.mapError((error) => storeError(`storage failpoint ${location}`, error))));
|
|
884
|
+
const materialize = Effect.fn("DoThreadStore.materialize")(function* (request) {
|
|
885
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadMaterialization))(request).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
|
|
886
|
+
const now = yield* Clock.currentTimeMillis;
|
|
887
|
+
yield* hitFailpoint("materialize:before");
|
|
888
|
+
yield* journal.materialize(validated.threadId, new Date(now).toISOString(), EMPTY_TAIL_DIGEST, validated.producerEpoch).pipe(Effect.mapError((error) => error._tag === "DoFenceRejected" ? mapFence(validated.threadId, error) : storeError("materialize thread", error)));
|
|
889
|
+
yield* hitFailpoint("materialize:after");
|
|
890
|
+
});
|
|
891
|
+
const append = Effect.fn("DoThreadStore.append")(function* (request) {
|
|
892
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(FencedAppendRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate canonical append", error)));
|
|
893
|
+
yield* requireThread(journal, validated.threadId);
|
|
894
|
+
const tailDigest = yield* provideCrypto(digestCanonicalBatch(validated.expectedTailDigest, validated.batch)).pipe(Effect.mapError((error) => storeError("digest canonical append", error)));
|
|
895
|
+
const batchJson = yield* encodeCanonicalBatch(validated.batch);
|
|
896
|
+
const rawRecords = yield* Effect.forEach(validated.batch.records, (record) => encodeCanonicalRecord(record).pipe(Effect.map((recordJson) => ({
|
|
897
|
+
recordId: record.recordId,
|
|
898
|
+
recordJson
|
|
899
|
+
}))));
|
|
900
|
+
const rawRequest = yield* Schema.decodeUnknownEffect(RawAppendRequest)({
|
|
901
|
+
threadId: validated.threadId,
|
|
902
|
+
batchId: validated.batch.batchId,
|
|
903
|
+
batchDigest: tailDigest,
|
|
904
|
+
batchJson,
|
|
905
|
+
expectedTailSequence: validated.expectedTailSequence,
|
|
906
|
+
expectedTailDigest: validated.expectedTailDigest,
|
|
907
|
+
producerEpoch: validated.producerEpoch,
|
|
908
|
+
records: rawRecords,
|
|
909
|
+
tailDigest
|
|
910
|
+
}).pipe(Effect.mapError((error) => schemaStoreError("encode canonical append", error)));
|
|
911
|
+
yield* hitFailpoint("append:before");
|
|
912
|
+
const result = yield* journal.append(rawRequest).pipe(Effect.mapError((error) => {
|
|
913
|
+
if (isDoFenceRejected(error)) return mapFence(validated.threadId, error);
|
|
914
|
+
if (isDoAppendConflict(error)) return error.actualTailSequence !== void 0 && isDigest(error.actualTailDigest) ? AppendConflict.make({
|
|
915
|
+
threadId: validated.threadId,
|
|
916
|
+
batchId: validated.batch.batchId,
|
|
917
|
+
reason: error.reason,
|
|
918
|
+
actualTailSequence: error.actualTailSequence,
|
|
919
|
+
actualTailDigest: error.actualTailDigest
|
|
920
|
+
}) : AppendConflict.make({
|
|
921
|
+
threadId: validated.threadId,
|
|
922
|
+
batchId: validated.batch.batchId,
|
|
923
|
+
reason: error.reason
|
|
924
|
+
});
|
|
925
|
+
return storeError("append canonical batch", error);
|
|
926
|
+
}), Effect.flatMap((result) => Schema.decodeUnknownEffect(AppendResult)(result).pipe(Effect.mapError((error) => schemaStoreError("decode append result", error)))));
|
|
927
|
+
yield* hitFailpoint("append:after");
|
|
928
|
+
return result;
|
|
929
|
+
});
|
|
930
|
+
const loadRecords = Effect.fn("DoThreadStore.loadRecords")(function* (request) {
|
|
931
|
+
const rows = yield* journal.read(request).pipe(Effect.mapError((error) => storeError("read canonical records", error)));
|
|
932
|
+
return yield* Effect.forEach(rows, decodeEnvelope);
|
|
933
|
+
});
|
|
934
|
+
const readEffect = Effect.fn("DoThreadStore.read")(function* (request) {
|
|
935
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadRead))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread read", error)));
|
|
936
|
+
yield* requireThread(journal, validated.threadId);
|
|
937
|
+
const records = yield* loadRecords(RawReadRequest.make({
|
|
938
|
+
threadId: validated.threadId,
|
|
939
|
+
fromSequenceExclusive: validated.afterSequence ?? ZERO_CANONICAL_SEQUENCE,
|
|
940
|
+
limit: validated.limit
|
|
941
|
+
}));
|
|
942
|
+
return Stream.fromIterable(records);
|
|
943
|
+
});
|
|
944
|
+
const read = (request) => Stream.unwrap(readEffect(request));
|
|
945
|
+
const observeEffect = Effect.fn("DoThreadStore.observe")(function* (request) {
|
|
946
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadObservation))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread observation", error)));
|
|
947
|
+
yield* requireThread(journal, validated.threadId);
|
|
948
|
+
const initialSequence = yield* parseOffset(validated.threadId, validated.afterOffset);
|
|
949
|
+
const cursor = yield* Ref.make(initialSequence);
|
|
950
|
+
const poll = Effect.fn("DoThreadStore.observePoll")(function* () {
|
|
951
|
+
const fromSequenceExclusive = yield* Ref.get(cursor);
|
|
952
|
+
const records = yield* loadRecords(RawReadRequest.make({
|
|
953
|
+
threadId: validated.threadId,
|
|
954
|
+
fromSequenceExclusive,
|
|
955
|
+
limit: 1024
|
|
956
|
+
}));
|
|
957
|
+
if (records.length === 0) {
|
|
958
|
+
yield* Effect.sleep(config.observationPollInterval);
|
|
959
|
+
return [];
|
|
960
|
+
}
|
|
961
|
+
yield* Ref.set(cursor, records[records.length - 1].sequence);
|
|
962
|
+
return records;
|
|
963
|
+
});
|
|
964
|
+
return Stream.fromIterableEffectRepeat(poll());
|
|
965
|
+
});
|
|
966
|
+
const observe = (request) => Stream.unwrap(observeEffect(request));
|
|
967
|
+
const exportThread = Effect.fn("DoThreadStore.export")(function* (request) {
|
|
968
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadExportRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread export", error)));
|
|
969
|
+
yield* requireThread(journal, validated.threadId);
|
|
970
|
+
const exported = yield* journal.exportThread(validated.threadId).pipe(Effect.mapError((error) => storeError("export thread", error)));
|
|
971
|
+
const records = yield* Effect.forEach(exported.records, decodeEnvelope);
|
|
972
|
+
if (records.length > 65536) return yield* ThreadStoreError.make({
|
|
973
|
+
operation: "decode thread export",
|
|
974
|
+
message: "The thread exceeds the current export record limit."
|
|
975
|
+
});
|
|
976
|
+
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(exported.thread.tail_digest).pipe(Effect.mapError((error) => schemaStoreError("decode export tail digest", error)));
|
|
977
|
+
return ThreadExport.make({
|
|
978
|
+
format: "effect-agent/thread@1",
|
|
979
|
+
threadId: validated.threadId,
|
|
980
|
+
tailSequence: exported.thread.tail_sequence,
|
|
981
|
+
tailDigest,
|
|
982
|
+
records
|
|
983
|
+
});
|
|
984
|
+
});
|
|
985
|
+
const inspectTail = Effect.fn("DoThreadStore.inspectTail")(function* (request) {
|
|
986
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadTailRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate tail inspection", error)));
|
|
987
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
988
|
+
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(thread.tail_digest).pipe(Effect.mapError((error) => schemaStoreError("decode tail digest", error)));
|
|
989
|
+
return ThreadTail.make({
|
|
990
|
+
threadId: validated.threadId,
|
|
991
|
+
tailSequence: thread.tail_sequence,
|
|
992
|
+
tailDigest,
|
|
993
|
+
producerEpoch: thread.producer_epoch
|
|
994
|
+
});
|
|
995
|
+
});
|
|
996
|
+
const saveCheckpoint = Effect.fn("DoThreadStore.saveCheckpoint")(function* (request) {
|
|
997
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(SaveCheckpointRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
|
|
998
|
+
const thread = yield* requireThread(journal, validated.checkpoint.threadId);
|
|
999
|
+
if (validated.checkpoint.throughSequence > thread.tail_sequence) return yield* CheckpointRejected.make({
|
|
1000
|
+
threadId: validated.checkpoint.threadId,
|
|
1001
|
+
reason: "ahead-of-tail"
|
|
1002
|
+
});
|
|
1003
|
+
if ((yield* tailDigestAt(journal, validated.checkpoint.threadId, validated.checkpoint.throughSequence)) !== validated.checkpoint.tailDigest) return yield* CheckpointRejected.make({
|
|
1004
|
+
threadId: validated.checkpoint.threadId,
|
|
1005
|
+
reason: "digest-mismatch"
|
|
1006
|
+
});
|
|
1007
|
+
const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
|
|
1008
|
+
const raw = RawCheckpoint.make({
|
|
1009
|
+
threadId: validated.checkpoint.threadId,
|
|
1010
|
+
throughSequence: validated.checkpoint.throughSequence,
|
|
1011
|
+
tailDigest: validated.checkpoint.tailDigest,
|
|
1012
|
+
checkpointJson
|
|
1013
|
+
});
|
|
1014
|
+
yield* hitFailpoint("save-checkpoint:before");
|
|
1015
|
+
yield* journal.saveCheckpoint(raw).pipe(Effect.mapError((error) => isDoCheckpointConflict(error) ? CheckpointRejected.make({
|
|
1016
|
+
threadId: validated.checkpoint.threadId,
|
|
1017
|
+
reason: "digest-mismatch"
|
|
1018
|
+
}) : storeError("save checkpoint", error)));
|
|
1019
|
+
yield* hitFailpoint("save-checkpoint:after");
|
|
1020
|
+
});
|
|
1021
|
+
const loadCheckpoint = Effect.fn("DoThreadStore.loadCheckpoint")(function* (request) {
|
|
1022
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(LoadCheckpointRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint lookup", error)));
|
|
1023
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
1024
|
+
const rows = yield* journal.loadCheckpoint(validated.threadId, validated.atOrBeforeSequence ?? thread.tail_sequence).pipe(Effect.mapError((error) => storeError("load checkpoint", error)));
|
|
1025
|
+
if (rows.length === 0) return Option.none();
|
|
1026
|
+
if (rows.length !== 1) return yield* ThreadStoreError.make({
|
|
1027
|
+
operation: "load checkpoint",
|
|
1028
|
+
message: `Expected at most one checkpoint row but found ${rows.length}.`
|
|
1029
|
+
});
|
|
1030
|
+
const checkpoint = yield* decodeCheckpoint(rows[0].checkpoint_json);
|
|
1031
|
+
if ((yield* tailDigestAt(journal, checkpoint.threadId, checkpoint.throughSequence)) !== checkpoint.tailDigest) return yield* CheckpointRejected.make({
|
|
1032
|
+
threadId: checkpoint.threadId,
|
|
1033
|
+
reason: "digest-mismatch"
|
|
1034
|
+
});
|
|
1035
|
+
return Option.some(checkpoint);
|
|
1036
|
+
});
|
|
1037
|
+
const threadStore = ThreadStore.of({
|
|
1038
|
+
append,
|
|
1039
|
+
export: exportThread,
|
|
1040
|
+
inspectTail,
|
|
1041
|
+
materialize,
|
|
1042
|
+
observe,
|
|
1043
|
+
read,
|
|
1044
|
+
checkpoints: {
|
|
1045
|
+
save: saveCheckpoint,
|
|
1046
|
+
load: loadCheckpoint
|
|
1047
|
+
}
|
|
1048
|
+
});
|
|
1049
|
+
return Context.make(ThreadStore, threadStore);
|
|
1050
|
+
});
|
|
1051
|
+
/**
|
|
1052
|
+
* Durable Object Thread Store implementation with configuration, failpoint, SQL, and
|
|
1053
|
+
* Crypto authority kept visible in its input channel.
|
|
1054
|
+
*/
|
|
1055
|
+
const threadStoreLayer = Layer.effectContext(makeServices());
|
|
1056
|
+
/**
|
|
1057
|
+
* Validated Durable Object storage configuration Layer with the documented defaults applied.
|
|
1058
|
+
* Shared by the ThreadStore and SubmissionLedger convenience layers so their defaults
|
|
1059
|
+
* cannot drift.
|
|
1060
|
+
*/
|
|
1061
|
+
const storageConfigLayer = (options) => Layer.effect(DoStorageConfig)(Schema.decodeUnknownEffect(DoStorageConfigValue)({
|
|
1062
|
+
observationPollInterval: options.observationPollInterval ?? 25,
|
|
1063
|
+
ownershipLeaseDuration: options.ownershipLeaseDuration ?? Duration.toMillis(DEFAULT_OWNERSHIP_LEASE_DURATION),
|
|
1064
|
+
maxStoredValueBytes: options.maxStoredValueBytes ?? 19e5,
|
|
1065
|
+
verifyOnOpen: options.verifyOnOpen ?? false
|
|
1066
|
+
}).pipe(Effect.mapError((error) => DoStorageError.make({
|
|
1067
|
+
cause: error,
|
|
1068
|
+
operation: "configure Durable Object storage",
|
|
1069
|
+
message: error.message
|
|
1070
|
+
}))));
|
|
1071
|
+
/** The failpoint Layer selected by convenience options: explicit handler or the no-op default. */
|
|
1072
|
+
const storageFailpointLayer = (options) => options.failpoint === void 0 ? DoStorageFailpoint.layer : Layer.succeed(DoStorageFailpoint)({ hit: options.failpoint });
|
|
1073
|
+
/**
|
|
1074
|
+
* A composition-root convenience Layer for canonical Threads inside one Durable Object,
|
|
1075
|
+
* built over `ctx.storage`. Durable accepted work is served by the separate SubmissionLedger
|
|
1076
|
+
* port; point both at the SAME `ctx.storage` so claims fence the same producer epochs
|
|
1077
|
+
* (ADR-0011 D7's "same file" rule, transposed to one object's private database).
|
|
1078
|
+
*/
|
|
1079
|
+
const layer = (options) => Layer.unwrap(Effect.map(DoStorageConfig, (config) => threadStoreLayer.pipe(Layer.provide(Layer.mergeAll(Layer.succeed(DoStorageConfig)(config), storageFailpointLayer(options), SqliteClient.layer({ storage: options.storage }), BrowserCrypto.layer))))).pipe(Layer.provide(storageConfigLayer(options)));
|
|
1080
|
+
//#endregion
|
|
1081
|
+
export { threadStoreLayer as a, storageFailpointLayer as i, layer as n, decodeRows as o, storageConfigLayer as r, initializeDoJournal as s, DoThreadStore_exports as t };
|
|
1082
|
+
|
|
1083
|
+
//# sourceMappingURL=DoThreadStore-D9ebIYPv.mjs.map
|