@effect-agent/storage-cloudflare 0.1.0-beta.8 → 0.1.0-beta.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DoMemoryStore.d.mts +29 -0
- package/dist/DoMemoryStore.mjs +55 -0
- package/dist/DoMemoryStore.mjs.map +1 -0
- package/dist/DoMessageDeliveryStore.d.mts +17 -0
- package/dist/DoMessageDeliveryStore.mjs +32 -0
- package/dist/DoMessageDeliveryStore.mjs.map +1 -0
- package/dist/DoScheduleStore.d.mts +36 -0
- package/dist/DoScheduleStore.mjs +388 -0
- package/dist/DoScheduleStore.mjs.map +1 -0
- package/dist/DoStorageConfig.d.mts +45 -0
- package/dist/DoStorageConfig.mjs +52 -0
- package/dist/DoStorageConfig.mjs.map +1 -0
- package/dist/DoStorageError-Cmhvl4TQ.d.mts +98 -0
- package/dist/DoStorageError.d.mts +2 -0
- package/dist/DoStorageError.mjs +164 -0
- package/dist/DoStorageError.mjs.map +1 -0
- package/dist/DoStorageFailpoint.d.mts +17 -0
- package/dist/DoStorageFailpoint.mjs +14 -0
- package/dist/DoStorageFailpoint.mjs.map +1 -0
- package/dist/DoStorageFailpointTesting.d.mts +34 -0
- package/dist/DoStorageFailpointTesting.mjs +35 -0
- package/dist/DoStorageFailpointTesting.mjs.map +1 -0
- package/dist/DoStorageVersion-x5OOurvZ.d.mts +12 -0
- package/dist/DoStorageVersion.d.mts +2 -0
- package/dist/DoStorageVersion.mjs +8 -0
- package/dist/DoStorageVersion.mjs.map +1 -0
- package/dist/DoSubmissionLedger.d.mts +23 -0
- package/dist/DoSubmissionLedger.mjs +1849 -0
- package/dist/DoSubmissionLedger.mjs.map +1 -0
- package/dist/DoSubscriptionStore.d.mts +25 -0
- package/dist/DoSubscriptionStore.mjs +183 -0
- package/dist/DoSubscriptionStore.mjs.map +1 -0
- package/dist/DoThreadStore.d.mts +59 -0
- package/dist/DoThreadStore.mjs +465 -0
- package/dist/DoThreadStore.mjs.map +1 -0
- package/dist/MemoryProtocol.d.mts +38160 -0
- package/dist/MemoryProtocol.mjs +219 -0
- package/dist/MemoryProtocol.mjs.map +1 -0
- package/dist/PortProtocol.d.mts +3207 -0
- package/dist/PortProtocol.mjs +176 -0
- package/dist/PortProtocol.mjs.map +1 -0
- package/dist/PortRouting.d.mts +83 -0
- package/dist/PortRouting.mjs +410 -0
- package/dist/PortRouting.mjs.map +1 -0
- package/dist/do-journal-DHqCQeBU.mjs +991 -0
- package/dist/do-journal-DHqCQeBU.mjs.map +1 -0
- package/dist/index.d.mts +14 -1521
- package/dist/index.mjs +14 -3695
- package/dist/migrations-Ba_kOrSx.mjs +287 -0
- package/dist/migrations-Ba_kOrSx.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -45
- package/src/DoMemoryStore.ts +59 -0
- package/src/DoMessageDeliveryStore.ts +53 -0
- package/src/DoScheduleStore.ts +622 -0
- package/src/{errors.ts → DoStorageError.ts} +10 -3
- package/src/DoStorageFailpoint.ts +20 -0
- package/src/{do-storage-failpoint.ts → DoStorageFailpointTesting.ts} +5 -18
- package/src/DoStorageVersion.ts +2 -0
- package/src/{do-ledger.ts → DoSubmissionLedger.ts} +862 -163
- package/src/DoSubscriptionStore.ts +329 -0
- package/src/DoThreadStore.ts +1014 -0
- package/src/MemoryProtocol.ts +355 -0
- package/src/{port-protocol.ts → PortProtocol.ts} +44 -37
- package/src/{routing.ts → PortRouting.ts} +292 -264
- package/src/index.ts +13 -32
- package/src/{do-journal.ts → internal/do-journal.ts} +716 -232
- package/src/internal/message-delivery-schema.ts +24 -0
- package/src/{migrations.ts → internal/migrations.ts} +51 -35
- package/src/internal/recovery-checkpoint-schema.ts +17 -0
- package/dist/index.mjs.map +0 -1
- package/src/do-conversation-store.ts +0 -850
- /package/src/{do-storage-config.ts → DoStorageConfig.ts} +0 -0
|
@@ -1,21 +1,32 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { EMPTY_TAIL_DIGEST } from "@effect-agent/thread/Digest";
|
|
2
|
+
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread/Records";
|
|
3
|
+
import { checkV2ThreadLayout } from "@effect-agent/thread/SqlStorageV2Upgrade";
|
|
4
|
+
import {
|
|
5
|
+
MAX_THREAD_EXPORT_RECORDS,
|
|
6
|
+
CheckpointRejected,
|
|
7
|
+
FenceRejected,
|
|
8
|
+
ThreadNotMaterialized,
|
|
9
|
+
type SaveRecoveryCheckpointRequest,
|
|
10
|
+
} from "@effect-agent/thread/ThreadStore";
|
|
2
11
|
import { SqliteMigrator } from "@effect/sql-sqlite-do";
|
|
3
|
-
import { Effect, Schema } from "effect";
|
|
12
|
+
import { Effect, Schema, Stream } from "effect";
|
|
4
13
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
5
14
|
import { SqlError } from "effect/unstable/sql/SqlError";
|
|
6
15
|
|
|
7
16
|
import {
|
|
17
|
+
type DoStorageFailpointError,
|
|
8
18
|
DoAppendConflict,
|
|
9
19
|
DoCheckpointConflict,
|
|
10
20
|
DoFenceRejected,
|
|
11
21
|
DoStorageCompatibilityError,
|
|
12
22
|
DoStorageCorruptionError,
|
|
13
23
|
DoStorageError,
|
|
14
|
-
DoStorageFailpointError,
|
|
15
24
|
DoValueBoundExceeded,
|
|
16
25
|
type DoStorageFailpointLocation,
|
|
17
|
-
} from "
|
|
18
|
-
import {
|
|
26
|
+
} from "../DoStorageError.ts";
|
|
27
|
+
import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
|
|
28
|
+
import { CurrentDoStorageVersion, createNonterminalIndex, doMigrations } from "./migrations.ts";
|
|
29
|
+
import { createRecoveryCheckpointTable } from "./recovery-checkpoint-schema.ts";
|
|
19
30
|
|
|
20
31
|
/**
|
|
21
32
|
* Static schema ceiling for stored text columns. Writes are bounded in BYTES by the
|
|
@@ -25,18 +36,23 @@ import { CurrentDoStorageVersion, doMigrations } from "./migrations.ts";
|
|
|
25
36
|
*/
|
|
26
37
|
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(2_000_000));
|
|
27
38
|
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
28
|
-
const
|
|
39
|
+
const MAX_RECORDS_PER_THREAD = MAX_THREAD_EXPORT_RECORDS;
|
|
40
|
+
const ZERO_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
29
41
|
const MAX_IDENTIFIER_LENGTH = 1_024;
|
|
42
|
+
const MAX_READ_PAGE_JSON_BYTES = 4 * 1024 * 1024;
|
|
30
43
|
/** Durable Object SQL storage allows at most 100 bound parameters per statement. */
|
|
31
44
|
const MAX_BOUND_PARAMETERS = 100;
|
|
45
|
+
const isSqlError = Schema.is(SqlError);
|
|
32
46
|
|
|
33
47
|
const storedTextBytes = (value: string): number => new TextEncoder().encode(value).byteLength;
|
|
34
48
|
|
|
35
49
|
const chunked = <A>(values: ReadonlyArray<A>, size: number): Array<ReadonlyArray<A>> => {
|
|
36
50
|
const chunks: Array<ReadonlyArray<A>> = [];
|
|
51
|
+
|
|
37
52
|
for (let index = 0; index < values.length; index += size) {
|
|
38
53
|
chunks.push(values.slice(index, index + size));
|
|
39
54
|
}
|
|
55
|
+
|
|
40
56
|
return chunks;
|
|
41
57
|
};
|
|
42
58
|
|
|
@@ -48,8 +64,8 @@ class DoNameRow extends Schema.Class<DoNameRow>("DoNameRow")({
|
|
|
48
64
|
name: BoundedIdentifier,
|
|
49
65
|
}) {}
|
|
50
66
|
|
|
51
|
-
class
|
|
52
|
-
|
|
67
|
+
class ThreadRow extends Schema.Class<ThreadRow>("ThreadRow")({
|
|
68
|
+
thread_id: BoundedIdentifier,
|
|
53
69
|
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
54
70
|
producer_epoch: ProducerEpoch,
|
|
55
71
|
tail_digest: BoundedStoredText,
|
|
@@ -60,7 +76,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
60
76
|
batch_digest: BoundedStoredText,
|
|
61
77
|
batch_id: BoundedIdentifier,
|
|
62
78
|
batch_json: BoundedStoredText,
|
|
63
|
-
|
|
79
|
+
thread_id: BoundedIdentifier,
|
|
64
80
|
first_sequence: CanonicalSequence,
|
|
65
81
|
last_sequence: CanonicalSequence,
|
|
66
82
|
tail_digest: BoundedStoredText,
|
|
@@ -68,15 +84,22 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
68
84
|
|
|
69
85
|
class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
70
86
|
batch_id: BoundedIdentifier,
|
|
71
|
-
|
|
87
|
+
thread_id: BoundedIdentifier,
|
|
72
88
|
record_id: BoundedIdentifier,
|
|
73
89
|
record_json: BoundedStoredText,
|
|
74
90
|
sequence: CanonicalSequence,
|
|
75
91
|
}) {}
|
|
76
92
|
|
|
93
|
+
const ReadPlanRow = Schema.Struct({
|
|
94
|
+
sequence: CanonicalSequence,
|
|
95
|
+
record_json_bytes: Schema.Natural.check(Schema.isLessThanOrEqualTo(MAX_READ_PAGE_JSON_BYTES)),
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
type ReadPage = [typeof ReadPlanRow.Type, ...Array<typeof ReadPlanRow.Type>];
|
|
99
|
+
|
|
77
100
|
class CheckpointRow extends Schema.Class<CheckpointRow>("CheckpointRow")({
|
|
78
101
|
checkpoint_json: BoundedStoredText,
|
|
79
|
-
|
|
102
|
+
thread_id: BoundedIdentifier,
|
|
80
103
|
tail_digest: BoundedStoredText,
|
|
81
104
|
through_sequence: CanonicalSequence,
|
|
82
105
|
}) {}
|
|
@@ -94,7 +117,7 @@ export class RawAppendRequest extends Schema.Class<RawAppendRequest>(
|
|
|
94
117
|
batchDigest: BoundedStoredText,
|
|
95
118
|
batchId: BoundedIdentifier,
|
|
96
119
|
batchJson: BoundedStoredText,
|
|
97
|
-
|
|
120
|
+
threadId: BoundedIdentifier,
|
|
98
121
|
expectedTailDigest: BoundedStoredText,
|
|
99
122
|
expectedTailSequence: CanonicalSequence,
|
|
100
123
|
producerEpoch: ProducerEpoch,
|
|
@@ -114,7 +137,7 @@ export class RawAppendResult extends Schema.Class<RawAppendResult>(
|
|
|
114
137
|
export class RawReadRequest extends Schema.Class<RawReadRequest>(
|
|
115
138
|
"@effect-agent/storage-cloudflare/RawReadRequest",
|
|
116
139
|
)({
|
|
117
|
-
|
|
140
|
+
threadId: BoundedIdentifier,
|
|
118
141
|
fromSequenceExclusive: CanonicalSequence,
|
|
119
142
|
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
|
|
120
143
|
}) {}
|
|
@@ -123,17 +146,15 @@ export class RawCheckpoint extends Schema.Class<RawCheckpoint>(
|
|
|
123
146
|
"@effect-agent/storage-cloudflare/RawCheckpoint",
|
|
124
147
|
)({
|
|
125
148
|
checkpointJson: BoundedStoredText,
|
|
126
|
-
|
|
149
|
+
threadId: BoundedIdentifier,
|
|
127
150
|
tailDigest: BoundedStoredText,
|
|
128
151
|
throughSequence: CanonicalSequence,
|
|
129
152
|
}) {}
|
|
130
153
|
|
|
131
|
-
export class
|
|
132
|
-
"@effect-agent/storage-cloudflare/
|
|
154
|
+
export class RawThreadExport extends Schema.Class<RawThreadExport>(
|
|
155
|
+
"@effect-agent/storage-cloudflare/RawThreadExport",
|
|
133
156
|
)({
|
|
134
|
-
|
|
135
|
-
checkpoints: Schema.Array(CheckpointRow),
|
|
136
|
-
conversation: ConversationRow,
|
|
157
|
+
thread: ThreadRow,
|
|
137
158
|
records: Schema.Array(RecordRow),
|
|
138
159
|
}) {}
|
|
139
160
|
|
|
@@ -167,7 +188,7 @@ const storageError =
|
|
|
167
188
|
});
|
|
168
189
|
|
|
169
190
|
/** Decode raw Durable Object SQLite rows against a Schema, reporting failures as typed corruption. */
|
|
170
|
-
export const decodeRows = Effect.fn(
|
|
191
|
+
export const decodeRows = Effect.fn(
|
|
171
192
|
<A, I>(
|
|
172
193
|
schema: Schema.Codec<ReadonlyArray<A>, ReadonlyArray<I>>,
|
|
173
194
|
table: string,
|
|
@@ -186,7 +207,7 @@ export const decodeRows = Effect.fn("DoJournal.decodeRows")(
|
|
|
186
207
|
);
|
|
187
208
|
|
|
188
209
|
/** Decode exactly one raw row against a Schema, reporting failures as typed corruption. */
|
|
189
|
-
export const decodeSingleRow = Effect.fn(
|
|
210
|
+
export const decodeSingleRow = Effect.fn(
|
|
190
211
|
<A, I>(
|
|
191
212
|
schema: Schema.Codec<ReadonlyArray<A>, ReadonlyArray<I>>,
|
|
192
213
|
table: string,
|
|
@@ -208,6 +229,183 @@ export const decodeSingleRow = Effect.fn("DoJournal.decodeSingleRow")(
|
|
|
208
229
|
),
|
|
209
230
|
);
|
|
210
231
|
|
|
232
|
+
/** Column inventory of the supported v3 predecessor, independent of physical column order. */
|
|
233
|
+
const predecessorColumns = {
|
|
234
|
+
effect_agent_threads: [
|
|
235
|
+
"thread_id",
|
|
236
|
+
"created_at",
|
|
237
|
+
"tail_sequence",
|
|
238
|
+
"tail_digest",
|
|
239
|
+
"producer_epoch",
|
|
240
|
+
],
|
|
241
|
+
effect_agent_canonical_batches: [
|
|
242
|
+
"thread_id",
|
|
243
|
+
"batch_id",
|
|
244
|
+
"first_sequence",
|
|
245
|
+
"last_sequence",
|
|
246
|
+
"batch_digest",
|
|
247
|
+
"tail_digest",
|
|
248
|
+
"batch_json",
|
|
249
|
+
],
|
|
250
|
+
effect_agent_canonical_records: ["thread_id", "sequence", "record_id", "batch_id", "record_json"],
|
|
251
|
+
effect_agent_checkpoints: ["thread_id", "through_sequence", "tail_digest", "checkpoint_json"],
|
|
252
|
+
effect_agent_submissions: [
|
|
253
|
+
"submission_id",
|
|
254
|
+
"thread_id",
|
|
255
|
+
"queue_sequence",
|
|
256
|
+
"principal",
|
|
257
|
+
"idempotency_key",
|
|
258
|
+
"agent_id",
|
|
259
|
+
"agent_digests_json",
|
|
260
|
+
"deployment_id",
|
|
261
|
+
"input_json",
|
|
262
|
+
"input_digest",
|
|
263
|
+
"receipt_id",
|
|
264
|
+
"state",
|
|
265
|
+
"settled_outcome",
|
|
266
|
+
"created_at",
|
|
267
|
+
"ready_at",
|
|
268
|
+
"input_applied_record_id",
|
|
269
|
+
"input_applied_sequence",
|
|
270
|
+
"joined_host_submission_id",
|
|
271
|
+
"suspended_reason_json",
|
|
272
|
+
"suspended_at",
|
|
273
|
+
"unknown_reason",
|
|
274
|
+
"unknown_tool_call_ids_json",
|
|
275
|
+
"parent_submission_id",
|
|
276
|
+
"parent_tool_call_id",
|
|
277
|
+
"admission_group",
|
|
278
|
+
"admission_fence_json",
|
|
279
|
+
],
|
|
280
|
+
effect_agent_submission_ownership: [
|
|
281
|
+
"submission_id",
|
|
282
|
+
"attempt_id",
|
|
283
|
+
"ownership_token",
|
|
284
|
+
"producer_epoch",
|
|
285
|
+
"owner_producer_id",
|
|
286
|
+
"lease_expires_at",
|
|
287
|
+
],
|
|
288
|
+
effect_agent_attempts: [
|
|
289
|
+
"attempt_id",
|
|
290
|
+
"submission_id",
|
|
291
|
+
"thread_id",
|
|
292
|
+
"owner_producer_id",
|
|
293
|
+
"producer_epoch",
|
|
294
|
+
"claimed_at",
|
|
295
|
+
],
|
|
296
|
+
effect_agent_settlement_reservations: [
|
|
297
|
+
"submission_id",
|
|
298
|
+
"settlement_id",
|
|
299
|
+
"outcome",
|
|
300
|
+
"record_id",
|
|
301
|
+
"record_json",
|
|
302
|
+
"record_digest",
|
|
303
|
+
"reserved_at",
|
|
304
|
+
"finalized_at",
|
|
305
|
+
],
|
|
306
|
+
effect_agent_abort_intents: [
|
|
307
|
+
"submission_id",
|
|
308
|
+
"author",
|
|
309
|
+
"reason",
|
|
310
|
+
"requested_at",
|
|
311
|
+
"canonical_record_id",
|
|
312
|
+
],
|
|
313
|
+
effect_agent_approval_decisions: [
|
|
314
|
+
"submission_id",
|
|
315
|
+
"tool_call_id",
|
|
316
|
+
"decision",
|
|
317
|
+
"resolver",
|
|
318
|
+
"reason",
|
|
319
|
+
"decided_at",
|
|
320
|
+
],
|
|
321
|
+
effect_agent_unknown_resolutions: [
|
|
322
|
+
"submission_id",
|
|
323
|
+
"tool_call_id",
|
|
324
|
+
"author",
|
|
325
|
+
"reason",
|
|
326
|
+
"resolution_json",
|
|
327
|
+
"resolved_at",
|
|
328
|
+
],
|
|
329
|
+
effect_agent_child_reservations: [
|
|
330
|
+
"reservation_id",
|
|
331
|
+
"parent_submission_id",
|
|
332
|
+
"parent_tool_call_id",
|
|
333
|
+
"child_submission_id",
|
|
334
|
+
"status",
|
|
335
|
+
"allocation_json",
|
|
336
|
+
"allocation_digest",
|
|
337
|
+
"accounting_json",
|
|
338
|
+
"reserved_at",
|
|
339
|
+
"release_began_at",
|
|
340
|
+
"released_at",
|
|
341
|
+
],
|
|
342
|
+
effect_agent_meta: ["key", "value"],
|
|
343
|
+
effect_agent_child_settlements: [
|
|
344
|
+
"parent_submission_id",
|
|
345
|
+
"child_submission_id",
|
|
346
|
+
"child_outcome",
|
|
347
|
+
"recorded_at",
|
|
348
|
+
],
|
|
349
|
+
} as const;
|
|
350
|
+
|
|
351
|
+
const checkPredecessorLayout = Effect.fn("DoJournal.checkPredecessorLayout")(function* (
|
|
352
|
+
version: 3 | 4 | 5,
|
|
353
|
+
) {
|
|
354
|
+
const sql = yield* SqlClient.SqlClient;
|
|
355
|
+
|
|
356
|
+
const messageColumns =
|
|
357
|
+
version === 3
|
|
358
|
+
? predecessorColumns
|
|
359
|
+
: {
|
|
360
|
+
...predecessorColumns,
|
|
361
|
+
effect_agent_submissions: [
|
|
362
|
+
...predecessorColumns.effect_agent_submissions,
|
|
363
|
+
"worker_admission_json",
|
|
364
|
+
"message_admission_json",
|
|
365
|
+
],
|
|
366
|
+
effect_agent_message_deliveries: [
|
|
367
|
+
"owner_thread_id",
|
|
368
|
+
"message_id",
|
|
369
|
+
"version",
|
|
370
|
+
"state",
|
|
371
|
+
"deadline_at_millis",
|
|
372
|
+
"record_json",
|
|
373
|
+
],
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
const expectedColumns = {
|
|
377
|
+
...messageColumns,
|
|
378
|
+
...(version === 5
|
|
379
|
+
? {
|
|
380
|
+
effect_agent_recovery_checkpoints: [
|
|
381
|
+
"thread_id",
|
|
382
|
+
"through_sequence",
|
|
383
|
+
"tail_digest",
|
|
384
|
+
"checkpoint_json",
|
|
385
|
+
],
|
|
386
|
+
}
|
|
387
|
+
: {}),
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
for (const [table, expected] of Object.entries(expectedColumns)) {
|
|
391
|
+
const columns = yield* decodeRows(
|
|
392
|
+
Schema.Array(Schema.Struct({ name: BoundedIdentifier })),
|
|
393
|
+
table,
|
|
394
|
+
"schema",
|
|
395
|
+
yield* sql.unsafe(`PRAGMA table_info(${table})`),
|
|
396
|
+
);
|
|
397
|
+
|
|
398
|
+
const names = new Set<string>(expected);
|
|
399
|
+
|
|
400
|
+
if (columns.length !== names.size || columns.some((column) => !names.has(column.name)))
|
|
401
|
+
return yield* DoStorageCompatibilityError.make({
|
|
402
|
+
actualVersion: version,
|
|
403
|
+
supportedVersion: CurrentDoStorageVersion,
|
|
404
|
+
message: `The v${version} ${table} columns do not match the supported predecessor; no upgrade was committed.`,
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
|
|
211
409
|
const REQUIRED_TABLES = [
|
|
212
410
|
"effect_agent_abort_intents",
|
|
213
411
|
"effect_agent_approval_decisions",
|
|
@@ -217,7 +415,7 @@ const REQUIRED_TABLES = [
|
|
|
217
415
|
"effect_agent_checkpoints",
|
|
218
416
|
"effect_agent_child_reservations",
|
|
219
417
|
"effect_agent_child_settlements",
|
|
220
|
-
"
|
|
418
|
+
"effect_agent_threads",
|
|
221
419
|
"effect_agent_meta",
|
|
222
420
|
"effect_agent_settlement_reservations",
|
|
223
421
|
"effect_agent_submission_ownership",
|
|
@@ -226,7 +424,7 @@ const REQUIRED_TABLES = [
|
|
|
226
424
|
] as const;
|
|
227
425
|
|
|
228
426
|
/**
|
|
229
|
-
*
|
|
427
|
+
* Supported-predecessor or fresh storage gate (DEPLOY-008) over `effect_agent_meta` instead of
|
|
230
428
|
* `PRAGMA user_version` (unverified on Durable Object SQL storage; a meta table is portable
|
|
231
429
|
* regardless). No WAL check (Durable Object storage owns durability and confirms writes
|
|
232
430
|
* through output gates) and no busy timeout (a Durable Object has exactly one writer): the
|
|
@@ -243,6 +441,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
243
441
|
WHERE type = 'table'
|
|
244
442
|
AND name = 'effect_agent_meta'
|
|
245
443
|
`.pipe(Effect.mapError(storageError("read storage version table")));
|
|
444
|
+
|
|
246
445
|
const metaTables = yield* decodeRows(
|
|
247
446
|
Schema.Array(DoNameRow),
|
|
248
447
|
"sqlite_master",
|
|
@@ -258,6 +457,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
258
457
|
AND name LIKE 'effect_agent_%'
|
|
259
458
|
ORDER BY name
|
|
260
459
|
`.pipe(Effect.mapError(storageError("inspect unversioned storage")));
|
|
460
|
+
|
|
261
461
|
const existing = yield* decodeRows(
|
|
262
462
|
Schema.Array(DoNameRow),
|
|
263
463
|
"sqlite_master",
|
|
@@ -270,11 +470,17 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
270
470
|
actualVersion: 0,
|
|
271
471
|
supportedVersion: CurrentDoStorageVersion,
|
|
272
472
|
message:
|
|
273
|
-
"The Durable Object contains unversioned Effect Agent tables.
|
|
473
|
+
"The Durable Object contains unversioned Effect Agent tables. Refusing to mutate ambiguous stored data; retain it for inspection with its original writer.",
|
|
274
474
|
});
|
|
275
475
|
}
|
|
276
476
|
|
|
277
|
-
yield* SqliteMigrator.run({
|
|
477
|
+
yield* SqliteMigrator.run({
|
|
478
|
+
loader: doMigrations,
|
|
479
|
+
// An application can share this SQL client and own its own migration history.
|
|
480
|
+
// Keep bookkeeping outside effect_agent_% so interrupted unversioned schemas
|
|
481
|
+
// still fail the ambiguity check above.
|
|
482
|
+
table: "effect_sql_migrations_agent_threads",
|
|
483
|
+
}).pipe(
|
|
278
484
|
// SqliteMigrator depends on the generic client supplied by this adapter. The concrete
|
|
279
485
|
// Durable Object client is kept at the outer Layer boundary.
|
|
280
486
|
Effect.provideService(SqlClient.SqlClient, sql),
|
|
@@ -292,6 +498,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
292
498
|
FROM effect_agent_meta
|
|
293
499
|
WHERE key = 'storage_version'
|
|
294
500
|
`.pipe(Effect.mapError(storageError("read storage version")));
|
|
501
|
+
|
|
295
502
|
const version = yield* decodeSingleRow(
|
|
296
503
|
Schema.Array(DoMetaRow),
|
|
297
504
|
"effect_agent_meta",
|
|
@@ -299,18 +506,136 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
299
506
|
versionRows,
|
|
300
507
|
);
|
|
301
508
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
509
|
+
if (
|
|
510
|
+
version.value === "2" ||
|
|
511
|
+
version.value === "3" ||
|
|
512
|
+
version.value === "4" ||
|
|
513
|
+
version.value === "5"
|
|
514
|
+
) {
|
|
515
|
+
yield* sql
|
|
516
|
+
.withTransaction(
|
|
517
|
+
Effect.gen(function* () {
|
|
518
|
+
const current = yield* sql<{
|
|
519
|
+
value: string;
|
|
520
|
+
}>`SELECT value FROM effect_agent_meta WHERE key='storage_version'`;
|
|
521
|
+
|
|
522
|
+
if (current.length === 1 && current[0].value === String(CurrentDoStorageVersion))
|
|
523
|
+
return;
|
|
524
|
+
if (
|
|
525
|
+
current.length !== 1 ||
|
|
526
|
+
(current[0].value !== "2" &&
|
|
527
|
+
current[0].value !== "3" &&
|
|
528
|
+
current[0].value !== "4" &&
|
|
529
|
+
current[0].value !== "5")
|
|
530
|
+
)
|
|
531
|
+
return yield* DoStorageCompatibilityError.make({
|
|
532
|
+
actualVersion: -1,
|
|
533
|
+
supportedVersion: CurrentDoStorageVersion,
|
|
534
|
+
message: "Storage version changed while acquiring the upgrade transaction.",
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
const required = yield* decodeRows(
|
|
538
|
+
Schema.Array(DoNameRow),
|
|
539
|
+
"sqlite_master",
|
|
540
|
+
"required_tables",
|
|
541
|
+
yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name IN ${sql.in([...REQUIRED_TABLES])}`,
|
|
542
|
+
);
|
|
543
|
+
|
|
544
|
+
if (required.length !== REQUIRED_TABLES.length)
|
|
545
|
+
return yield* DoStorageCompatibilityError.make({
|
|
546
|
+
actualVersion: Number(current[0].value),
|
|
547
|
+
supportedVersion: CurrentDoStorageVersion,
|
|
548
|
+
message:
|
|
549
|
+
"The predecessor store is missing required tables. Retain the original store for inspection; no upgrade was committed.",
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
const recoveryTables =
|
|
553
|
+
yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name='effect_agent_recovery_checkpoints'`;
|
|
554
|
+
|
|
555
|
+
if (recoveryTables.length !== (current[0].value === "5" ? 1 : 0))
|
|
556
|
+
return yield* DoStorageCompatibilityError.make({
|
|
557
|
+
actualVersion: Number(current[0].value),
|
|
558
|
+
supportedVersion: CurrentDoStorageVersion,
|
|
559
|
+
message:
|
|
560
|
+
"The predecessor recovery checkpoint storage does not match its version; refusing ambiguous data without mutation.",
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
const indexes =
|
|
564
|
+
yield* sql`SELECT name FROM sqlite_master WHERE name='effect_agent_submissions_nonterminal'`;
|
|
565
|
+
|
|
566
|
+
if (indexes.length !== 0)
|
|
567
|
+
return yield* DoStorageCompatibilityError.make({
|
|
568
|
+
actualVersion: Number(current[0].value),
|
|
569
|
+
supportedVersion: CurrentDoStorageVersion,
|
|
570
|
+
message:
|
|
571
|
+
"The predecessor already contains the nonterminal index; refusing ambiguous storage without mutation.",
|
|
572
|
+
});
|
|
573
|
+
if (current[0].value === "2") {
|
|
574
|
+
yield* checkV2ThreadLayout();
|
|
575
|
+
for (const statement of [
|
|
576
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
|
|
577
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
|
|
578
|
+
sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`,
|
|
579
|
+
]) {
|
|
580
|
+
yield* failpoint("upgrade:before-mutation");
|
|
581
|
+
yield* statement;
|
|
582
|
+
yield* failpoint("upgrade:after-mutation");
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
if (current[0].value === "3") yield* checkPredecessorLayout(3);
|
|
586
|
+
if (current[0].value === "4") yield* checkPredecessorLayout(4);
|
|
587
|
+
if (current[0].value === "5") yield* checkPredecessorLayout(5);
|
|
588
|
+
if (current[0].value === "2" || current[0].value === "3") {
|
|
589
|
+
yield* failpoint("upgrade:before-mutation");
|
|
590
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
|
|
591
|
+
yield* failpoint("upgrade:after-mutation");
|
|
592
|
+
yield* failpoint("upgrade:before-mutation");
|
|
593
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
|
|
594
|
+
yield* failpoint("upgrade:after-mutation");
|
|
595
|
+
yield* failpoint("upgrade:before-mutation");
|
|
596
|
+
yield* createMessageDeliveryTables;
|
|
597
|
+
yield* failpoint("upgrade:after-mutation");
|
|
598
|
+
}
|
|
599
|
+
if (current[0].value !== "5") {
|
|
600
|
+
yield* failpoint("upgrade:before-mutation");
|
|
601
|
+
yield* createRecoveryCheckpointTable;
|
|
602
|
+
yield* failpoint("upgrade:after-mutation");
|
|
603
|
+
}
|
|
604
|
+
yield* failpoint("upgrade:before-mutation");
|
|
605
|
+
yield* createNonterminalIndex;
|
|
606
|
+
yield* failpoint("upgrade:after-mutation");
|
|
607
|
+
yield* failpoint("upgrade:before-version");
|
|
608
|
+
yield* sql`UPDATE effect_agent_meta SET value='6' WHERE key='storage_version'`;
|
|
609
|
+
yield* failpoint("upgrade:after-version");
|
|
610
|
+
}),
|
|
611
|
+
)
|
|
612
|
+
.pipe(
|
|
613
|
+
Effect.catchTag("DoStorageFailpointError", (error) =>
|
|
614
|
+
DoStorageError.make({
|
|
615
|
+
cause: error,
|
|
616
|
+
operation: "upgrade storage",
|
|
617
|
+
message: error.message,
|
|
618
|
+
}),
|
|
619
|
+
),
|
|
620
|
+
Effect.catchTag("StorageUpgradeError", (error) =>
|
|
621
|
+
DoStorageCorruptionError.make({
|
|
622
|
+
table: error.table,
|
|
623
|
+
rowKey: error.rowKey,
|
|
624
|
+
message: error.message,
|
|
625
|
+
}),
|
|
626
|
+
),
|
|
627
|
+
Effect.catchTag("SqlError", storageError("upgrade supported thread storage")),
|
|
628
|
+
);
|
|
629
|
+
} else if (version.value !== String(CurrentDoStorageVersion)) {
|
|
306
630
|
const actualVersion = Number.parseInt(version.value, 10);
|
|
631
|
+
|
|
307
632
|
return yield* DoStorageCompatibilityError.make({
|
|
308
633
|
actualVersion: Number.isSafeInteger(actualVersion) ? actualVersion : -1,
|
|
309
634
|
supportedVersion: CurrentDoStorageVersion,
|
|
310
635
|
message:
|
|
311
|
-
`The Durable Object uses
|
|
636
|
+
`The Durable Object uses unsupported storage version ${version.value}; ` +
|
|
312
637
|
`this build supports exactly version ${CurrentDoStorageVersion}. ` +
|
|
313
|
-
"
|
|
638
|
+
"Only supported v2, v3, v4 and v5 can be upgraded automatically. Keep the original store and use a compatible library version.",
|
|
314
639
|
});
|
|
315
640
|
}
|
|
316
641
|
}
|
|
@@ -318,22 +643,25 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
|
|
|
318
643
|
const requiredRows = yield* sql<Record<string, unknown>>`
|
|
319
644
|
SELECT name
|
|
320
645
|
FROM sqlite_master
|
|
321
|
-
WHERE type = 'table'
|
|
322
|
-
AND name IN ${sql.in([...REQUIRED_TABLES])}
|
|
646
|
+
WHERE (type = 'table'
|
|
647
|
+
AND name IN ${sql.in([...REQUIRED_TABLES, "effect_agent_message_deliveries", "effect_agent_recovery_checkpoints"])}
|
|
648
|
+
) OR (type = 'index' AND name = 'effect_agent_submissions_nonterminal')
|
|
323
649
|
ORDER BY name
|
|
324
650
|
`.pipe(Effect.mapError(storageError("verify storage tables")));
|
|
651
|
+
|
|
325
652
|
const required = yield* decodeRows(
|
|
326
653
|
Schema.Array(DoNameRow),
|
|
327
654
|
"sqlite_master",
|
|
328
655
|
"required_tables",
|
|
329
656
|
requiredRows,
|
|
330
657
|
);
|
|
331
|
-
|
|
658
|
+
|
|
659
|
+
if (required.length !== REQUIRED_TABLES.length + 3) {
|
|
332
660
|
return yield* DoStorageCompatibilityError.make({
|
|
333
661
|
actualVersion: CurrentDoStorageVersion,
|
|
334
662
|
supportedVersion: CurrentDoStorageVersion,
|
|
335
663
|
message:
|
|
336
|
-
"The Durable Object claims the current format but is missing required tables.
|
|
664
|
+
"The Durable Object claims the current format but is missing required tables or its nonterminal index. Retain the original store for inspection.",
|
|
337
665
|
});
|
|
338
666
|
}
|
|
339
667
|
|
|
@@ -351,6 +679,7 @@ const makeJournal = (
|
|
|
351
679
|
value: string,
|
|
352
680
|
): Effect.Effect<void, DoValueBoundExceeded> => {
|
|
353
681
|
const actualBytes = storedTextBytes(value);
|
|
682
|
+
|
|
354
683
|
return actualBytes > maxStoredValueBytes
|
|
355
684
|
? Effect.fail(
|
|
356
685
|
DoValueBoundExceeded.make({
|
|
@@ -378,14 +707,12 @@ const makeJournal = (
|
|
|
378
707
|
(operation: string) =>
|
|
379
708
|
<A, E>(effect: Effect.Effect<A, E>): Effect.Effect<A, E | DoStorageError> =>
|
|
380
709
|
sql.withTransaction(effect).pipe(
|
|
381
|
-
Effect.mapError((error) =>
|
|
382
|
-
error instanceof SqlError ? storageError(operation)(error) : error,
|
|
383
|
-
),
|
|
710
|
+
Effect.mapError((error) => (isSqlError(error) ? storageError(operation)(error) : error)),
|
|
384
711
|
Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
|
|
385
712
|
);
|
|
386
713
|
|
|
387
714
|
const materialize = Effect.fn("DoJournal.materialize")(function* (
|
|
388
|
-
|
|
715
|
+
threadId: string,
|
|
389
716
|
createdAt: string,
|
|
390
717
|
emptyTailDigest: string,
|
|
391
718
|
producerEpoch: ProducerEpoch,
|
|
@@ -393,54 +720,57 @@ const makeJournal = (
|
|
|
393
720
|
void,
|
|
394
721
|
DoFenceRejected | DoStorageCorruptionError | DoStorageError | DoValueBoundExceeded
|
|
395
722
|
> {
|
|
396
|
-
if (
|
|
723
|
+
if (threadId.length > MAX_IDENTIFIER_LENGTH) {
|
|
397
724
|
return yield* DoStorageError.make({
|
|
398
|
-
operation: "materialize
|
|
399
|
-
message: "
|
|
725
|
+
operation: "materialize thread",
|
|
726
|
+
message: "Thread identity exceeds the Durable Object storage bounds.",
|
|
400
727
|
});
|
|
401
728
|
}
|
|
402
|
-
yield* checkValueBound("materialize
|
|
729
|
+
yield* checkValueBound("materialize thread", emptyTailDigest);
|
|
403
730
|
yield* withWriteTransaction("materialize transaction")(
|
|
404
731
|
Effect.gen(function* () {
|
|
405
732
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
406
733
|
SELECT
|
|
407
|
-
|
|
734
|
+
thread_id,
|
|
408
735
|
created_at,
|
|
409
736
|
tail_sequence,
|
|
410
737
|
tail_digest,
|
|
411
738
|
producer_epoch
|
|
412
|
-
FROM
|
|
413
|
-
WHERE
|
|
414
|
-
`.pipe(Effect.mapError(storageError("read materialized
|
|
739
|
+
FROM effect_agent_threads
|
|
740
|
+
WHERE thread_id = ${threadId}
|
|
741
|
+
`.pipe(Effect.mapError(storageError("read materialized thread")));
|
|
742
|
+
|
|
415
743
|
const existing = yield* decodeRows(
|
|
416
|
-
Schema.Array(
|
|
417
|
-
"
|
|
418
|
-
|
|
744
|
+
Schema.Array(ThreadRow),
|
|
745
|
+
"effect_agent_threads",
|
|
746
|
+
threadId,
|
|
419
747
|
existingRows,
|
|
420
748
|
);
|
|
749
|
+
|
|
421
750
|
if (existing.length > 1) {
|
|
422
751
|
return yield* DoStorageCorruptionError.make({
|
|
423
|
-
table: "
|
|
424
|
-
rowKey:
|
|
425
|
-
message: "A
|
|
752
|
+
table: "effect_agent_threads",
|
|
753
|
+
rowKey: threadId,
|
|
754
|
+
message: "A thread primary key returned more than one row.",
|
|
426
755
|
});
|
|
427
756
|
}
|
|
428
757
|
if (existing.length === 0) {
|
|
429
758
|
yield* sql`
|
|
430
|
-
INSERT INTO
|
|
431
|
-
|
|
759
|
+
INSERT INTO effect_agent_threads (
|
|
760
|
+
thread_id,
|
|
432
761
|
created_at,
|
|
433
762
|
tail_sequence,
|
|
434
763
|
tail_digest,
|
|
435
764
|
producer_epoch
|
|
436
765
|
) VALUES (
|
|
437
|
-
${
|
|
766
|
+
${threadId},
|
|
438
767
|
${createdAt},
|
|
439
768
|
0,
|
|
440
769
|
${emptyTailDigest},
|
|
441
770
|
${producerEpoch}
|
|
442
771
|
)
|
|
443
|
-
`.pipe(Effect.mapError(storageError("materialize
|
|
772
|
+
`.pipe(Effect.mapError(storageError("materialize thread")));
|
|
773
|
+
|
|
444
774
|
return;
|
|
445
775
|
}
|
|
446
776
|
if (producerEpoch < existing[0].producer_epoch) {
|
|
@@ -452,41 +782,35 @@ const makeJournal = (
|
|
|
452
782
|
}
|
|
453
783
|
if (producerEpoch > existing[0].producer_epoch) {
|
|
454
784
|
yield* sql`
|
|
455
|
-
UPDATE
|
|
785
|
+
UPDATE effect_agent_threads
|
|
456
786
|
SET producer_epoch = ${producerEpoch}
|
|
457
|
-
WHERE
|
|
787
|
+
WHERE thread_id = ${threadId}
|
|
458
788
|
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
459
789
|
}
|
|
460
790
|
}),
|
|
461
791
|
);
|
|
462
792
|
});
|
|
463
793
|
|
|
464
|
-
const
|
|
465
|
-
conversationId: string,
|
|
466
|
-
) {
|
|
794
|
+
const getThread = Effect.fn("DoJournal.getThread")(function* (threadId: string) {
|
|
467
795
|
const rows = yield* sql<Record<string, unknown>>`
|
|
468
796
|
SELECT
|
|
469
|
-
|
|
797
|
+
thread_id,
|
|
470
798
|
created_at,
|
|
471
799
|
tail_sequence,
|
|
472
800
|
tail_digest,
|
|
473
801
|
producer_epoch
|
|
474
|
-
FROM
|
|
475
|
-
WHERE
|
|
476
|
-
`.pipe(Effect.mapError(storageError("read
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
"effect_agent_conversations",
|
|
480
|
-
conversationId,
|
|
481
|
-
rows,
|
|
482
|
-
);
|
|
802
|
+
FROM effect_agent_threads
|
|
803
|
+
WHERE thread_id = ${threadId}
|
|
804
|
+
`.pipe(Effect.mapError(storageError("read thread")));
|
|
805
|
+
|
|
806
|
+
return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
483
807
|
});
|
|
484
808
|
|
|
485
809
|
const append = Effect.fn("DoJournal.append")(function* (
|
|
486
810
|
request: RawAppendRequest,
|
|
487
811
|
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
488
812
|
if (
|
|
489
|
-
request.
|
|
813
|
+
request.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
490
814
|
request.batchId.length > MAX_IDENTIFIER_LENGTH ||
|
|
491
815
|
request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)
|
|
492
816
|
) {
|
|
@@ -504,9 +828,11 @@ const makeJournal = (
|
|
|
504
828
|
(record) => checkValueBound("append canonical record", record.recordJson),
|
|
505
829
|
{ discard: true },
|
|
506
830
|
);
|
|
831
|
+
|
|
507
832
|
return yield* withWriteTransaction("append transaction")(
|
|
508
833
|
Effect.gen(function* () {
|
|
509
834
|
const recordIds = request.records.map((record) => record.recordId);
|
|
835
|
+
|
|
510
836
|
if (new Set(recordIds).size !== recordIds.length) {
|
|
511
837
|
return yield* DoAppendConflict.make({
|
|
512
838
|
message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
|
|
@@ -514,34 +840,35 @@ const makeJournal = (
|
|
|
514
840
|
});
|
|
515
841
|
}
|
|
516
842
|
|
|
517
|
-
const
|
|
843
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
518
844
|
SELECT
|
|
519
|
-
|
|
845
|
+
thread_id,
|
|
520
846
|
created_at,
|
|
521
847
|
tail_sequence,
|
|
522
848
|
tail_digest,
|
|
523
849
|
producer_epoch
|
|
524
|
-
FROM
|
|
525
|
-
WHERE
|
|
850
|
+
FROM effect_agent_threads
|
|
851
|
+
WHERE thread_id = ${request.threadId}
|
|
526
852
|
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
853
|
+
|
|
854
|
+
const thread = yield* decodeSingleRow(
|
|
855
|
+
Schema.Array(ThreadRow),
|
|
856
|
+
"effect_agent_threads",
|
|
857
|
+
request.threadId,
|
|
858
|
+
threadRows,
|
|
532
859
|
);
|
|
533
860
|
|
|
534
|
-
if (request.producerEpoch !==
|
|
861
|
+
if (request.producerEpoch !== thread.producer_epoch) {
|
|
535
862
|
return yield* DoFenceRejected.make({
|
|
536
863
|
producerEpoch: request.producerEpoch,
|
|
537
|
-
actualEpoch:
|
|
538
|
-
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${
|
|
864
|
+
actualEpoch: thread.producer_epoch,
|
|
865
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`,
|
|
539
866
|
});
|
|
540
867
|
}
|
|
541
868
|
|
|
542
869
|
const batchRows = yield* sql<Record<string, unknown>>`
|
|
543
870
|
SELECT
|
|
544
|
-
|
|
871
|
+
thread_id,
|
|
545
872
|
batch_id,
|
|
546
873
|
first_sequence,
|
|
547
874
|
last_sequence,
|
|
@@ -549,31 +876,34 @@ const makeJournal = (
|
|
|
549
876
|
tail_digest,
|
|
550
877
|
batch_json
|
|
551
878
|
FROM effect_agent_canonical_batches
|
|
552
|
-
WHERE
|
|
879
|
+
WHERE thread_id = ${request.threadId}
|
|
553
880
|
AND batch_id = ${request.batchId}
|
|
554
881
|
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
882
|
+
|
|
555
883
|
const batches = yield* decodeRows(
|
|
556
884
|
Schema.Array(BatchRow),
|
|
557
885
|
"effect_agent_canonical_batches",
|
|
558
|
-
`${request.
|
|
886
|
+
`${request.threadId}/${request.batchId}`,
|
|
559
887
|
batchRows,
|
|
560
888
|
);
|
|
561
889
|
|
|
562
890
|
if (batches.length > 1) {
|
|
563
891
|
return yield* DoStorageCorruptionError.make({
|
|
564
892
|
table: "effect_agent_canonical_batches",
|
|
565
|
-
rowKey: `${request.
|
|
893
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
566
894
|
message: "A canonical batch primary key returned more than one row.",
|
|
567
895
|
});
|
|
568
896
|
}
|
|
569
897
|
if (batches.length === 1) {
|
|
570
898
|
const existing = batches[0];
|
|
899
|
+
|
|
571
900
|
if (existing.batch_digest !== request.batchDigest) {
|
|
572
901
|
return yield* DoAppendConflict.make({
|
|
573
902
|
message: `Batch ${request.batchId} already exists with different canonical content.`,
|
|
574
903
|
reason: "batch-digest",
|
|
575
904
|
});
|
|
576
905
|
}
|
|
906
|
+
|
|
577
907
|
return RawAppendResult.make({
|
|
578
908
|
firstSequence: existing.first_sequence,
|
|
579
909
|
lastSequence: existing.last_sequence,
|
|
@@ -583,46 +913,48 @@ const makeJournal = (
|
|
|
583
913
|
}
|
|
584
914
|
|
|
585
915
|
if (
|
|
586
|
-
request.expectedTailSequence !==
|
|
587
|
-
request.expectedTailDigest !==
|
|
916
|
+
request.expectedTailSequence !== thread.tail_sequence ||
|
|
917
|
+
request.expectedTailDigest !== thread.tail_digest
|
|
588
918
|
) {
|
|
589
919
|
return yield* DoAppendConflict.make({
|
|
590
920
|
message:
|
|
591
921
|
`Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} ` +
|
|
592
|
-
`but found ${
|
|
922
|
+
`but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
593
923
|
reason: "tail",
|
|
594
|
-
actualTailSequence:
|
|
595
|
-
actualTailDigest:
|
|
924
|
+
actualTailSequence: thread.tail_sequence,
|
|
925
|
+
actualTailDigest: thread.tail_digest,
|
|
596
926
|
});
|
|
597
927
|
}
|
|
598
|
-
if (
|
|
928
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) {
|
|
599
929
|
return yield* DoStorageError.make({
|
|
600
930
|
operation: "append canonical batch",
|
|
601
|
-
message: `
|
|
931
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`,
|
|
602
932
|
});
|
|
603
933
|
}
|
|
604
934
|
|
|
605
935
|
// Chunked to respect the Durable Object platform's 100-bound-parameter statement
|
|
606
936
|
// limit: a batch may carry up to 256 records.
|
|
607
937
|
const existingRecords: Array<RecordRow> = [];
|
|
938
|
+
|
|
608
939
|
for (const chunk of chunked(recordIds, MAX_BOUND_PARAMETERS - 10)) {
|
|
609
940
|
const existingRecordRows = yield* sql<Record<string, unknown>>`
|
|
610
941
|
SELECT
|
|
611
|
-
|
|
942
|
+
thread_id,
|
|
612
943
|
sequence,
|
|
613
944
|
record_id,
|
|
614
945
|
batch_id,
|
|
615
946
|
record_json
|
|
616
947
|
FROM effect_agent_canonical_records
|
|
617
|
-
WHERE
|
|
948
|
+
WHERE thread_id = ${request.threadId}
|
|
618
949
|
AND record_id IN ${sql.in([...chunk])}
|
|
619
950
|
ORDER BY sequence
|
|
620
951
|
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
952
|
+
|
|
621
953
|
existingRecords.push(
|
|
622
954
|
...(yield* decodeRows(
|
|
623
955
|
Schema.Array(RecordRow),
|
|
624
956
|
"effect_agent_canonical_records",
|
|
625
|
-
`${request.
|
|
957
|
+
`${request.threadId}/record_ids`,
|
|
626
958
|
existingRecordRows,
|
|
627
959
|
)),
|
|
628
960
|
);
|
|
@@ -635,7 +967,7 @@ const makeJournal = (
|
|
|
635
967
|
}
|
|
636
968
|
|
|
637
969
|
const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
638
|
-
|
|
970
|
+
thread.tail_sequence + 1,
|
|
639
971
|
).pipe(
|
|
640
972
|
Effect.mapError((error) =>
|
|
641
973
|
DoStorageError.make({
|
|
@@ -645,6 +977,7 @@ const makeJournal = (
|
|
|
645
977
|
}),
|
|
646
978
|
),
|
|
647
979
|
);
|
|
980
|
+
|
|
648
981
|
const lastSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
649
982
|
firstSequence + request.records.length - 1,
|
|
650
983
|
).pipe(
|
|
@@ -659,7 +992,7 @@ const makeJournal = (
|
|
|
659
992
|
|
|
660
993
|
yield* sql`
|
|
661
994
|
INSERT INTO effect_agent_canonical_batches (
|
|
662
|
-
|
|
995
|
+
thread_id,
|
|
663
996
|
batch_id,
|
|
664
997
|
first_sequence,
|
|
665
998
|
last_sequence,
|
|
@@ -667,7 +1000,7 @@ const makeJournal = (
|
|
|
667
1000
|
tail_digest,
|
|
668
1001
|
batch_json
|
|
669
1002
|
) VALUES (
|
|
670
|
-
${request.
|
|
1003
|
+
${request.threadId},
|
|
671
1004
|
${request.batchId},
|
|
672
1005
|
${firstSequence},
|
|
673
1006
|
${lastSequence},
|
|
@@ -684,13 +1017,13 @@ const makeJournal = (
|
|
|
684
1017
|
Effect.gen(function* () {
|
|
685
1018
|
yield* sql`
|
|
686
1019
|
INSERT INTO effect_agent_canonical_records (
|
|
687
|
-
|
|
1020
|
+
thread_id,
|
|
688
1021
|
sequence,
|
|
689
1022
|
record_id,
|
|
690
1023
|
batch_id,
|
|
691
1024
|
record_json
|
|
692
1025
|
) VALUES (
|
|
693
|
-
${request.
|
|
1026
|
+
${request.threadId},
|
|
694
1027
|
${firstSequence + index},
|
|
695
1028
|
${record.recordId},
|
|
696
1029
|
${request.batchId},
|
|
@@ -703,13 +1036,13 @@ const makeJournal = (
|
|
|
703
1036
|
);
|
|
704
1037
|
|
|
705
1038
|
yield* sql`
|
|
706
|
-
UPDATE
|
|
1039
|
+
UPDATE effect_agent_threads
|
|
707
1040
|
SET
|
|
708
1041
|
tail_sequence = ${lastSequence},
|
|
709
1042
|
tail_digest = ${request.tailDigest},
|
|
710
1043
|
producer_epoch = ${request.producerEpoch}
|
|
711
|
-
WHERE
|
|
712
|
-
`.pipe(Effect.mapError(storageError("advance
|
|
1044
|
+
WHERE thread_id = ${request.threadId}
|
|
1045
|
+
`.pipe(Effect.mapError(storageError("advance thread tail")));
|
|
713
1046
|
yield* failpoint("append:after-tail-update");
|
|
714
1047
|
|
|
715
1048
|
return RawAppendResult.make({
|
|
@@ -723,106 +1056,165 @@ const makeJournal = (
|
|
|
723
1056
|
});
|
|
724
1057
|
|
|
725
1058
|
const read = Effect.fn("DoJournal.read")(function* (request: RawReadRequest) {
|
|
726
|
-
|
|
1059
|
+
// Capture membership without retaining payloads. Append-only sequences keep each later
|
|
1060
|
+
// payload query inside this snapshot, even when new records arrive during consumption.
|
|
1061
|
+
const planRows = yield* sql<Record<string, unknown>>`
|
|
727
1062
|
SELECT
|
|
728
|
-
conversation_id,
|
|
729
1063
|
sequence,
|
|
730
|
-
|
|
731
|
-
batch_id,
|
|
732
|
-
record_json
|
|
1064
|
+
length(CAST(record_json AS BLOB)) AS record_json_bytes
|
|
733
1065
|
FROM effect_agent_canonical_records
|
|
734
|
-
WHERE
|
|
1066
|
+
WHERE thread_id = ${request.threadId}
|
|
735
1067
|
AND sequence > ${request.fromSequenceExclusive}
|
|
736
1068
|
ORDER BY sequence
|
|
737
1069
|
LIMIT ${request.limit}
|
|
738
1070
|
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
739
|
-
|
|
740
|
-
|
|
1071
|
+
|
|
1072
|
+
const plan = yield* decodeRows(
|
|
1073
|
+
Schema.Array(ReadPlanRow),
|
|
741
1074
|
"effect_agent_canonical_records",
|
|
742
|
-
`${request.
|
|
743
|
-
|
|
1075
|
+
`${request.threadId}>${request.fromSequenceExclusive}`,
|
|
1076
|
+
planRows,
|
|
744
1077
|
);
|
|
1078
|
+
|
|
1079
|
+
const mismatch = () =>
|
|
1080
|
+
DoStorageCorruptionError.make({
|
|
1081
|
+
table: "effect_agent_canonical_records",
|
|
1082
|
+
rowKey: `${request.threadId}>${request.fromSequenceExclusive}`,
|
|
1083
|
+
message: "Canonical read membership, sequence, or payload size changed from its read plan.",
|
|
1084
|
+
});
|
|
1085
|
+
|
|
1086
|
+
if (plan.length > request.limit) return yield* mismatch();
|
|
1087
|
+
|
|
1088
|
+
const pages: Array<ReadPage> = [];
|
|
1089
|
+
let page: ReadPage | undefined;
|
|
1090
|
+
let pageBytes = 0;
|
|
1091
|
+
let previousSequence = request.fromSequenceExclusive;
|
|
1092
|
+
|
|
1093
|
+
for (const row of plan) {
|
|
1094
|
+
if (row.sequence !== previousSequence + 1) return yield* mismatch();
|
|
1095
|
+
previousSequence = row.sequence;
|
|
1096
|
+
if (page === undefined || pageBytes + row.record_json_bytes > MAX_READ_PAGE_JSON_BYTES) {
|
|
1097
|
+
page = [row];
|
|
1098
|
+
pages.push(page);
|
|
1099
|
+
pageBytes = row.record_json_bytes;
|
|
1100
|
+
} else {
|
|
1101
|
+
page.push(row);
|
|
1102
|
+
pageBytes += row.record_json_bytes;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
const readPage = Effect.fn("DoJournal.readPage")(function* (page: ReadPage) {
|
|
1107
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
1108
|
+
SELECT thread_id, sequence, record_id, batch_id, record_json
|
|
1109
|
+
FROM effect_agent_canonical_records
|
|
1110
|
+
WHERE thread_id = ${request.threadId}
|
|
1111
|
+
AND sequence >= ${page[0].sequence}
|
|
1112
|
+
AND sequence <= ${page[page.length - 1].sequence}
|
|
1113
|
+
ORDER BY sequence
|
|
1114
|
+
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
1115
|
+
|
|
1116
|
+
const decoded = yield* decodeRows(
|
|
1117
|
+
Schema.Array(RecordRow),
|
|
1118
|
+
"effect_agent_canonical_records",
|
|
1119
|
+
`${request.threadId}/${page[0].sequence}`,
|
|
1120
|
+
rows,
|
|
1121
|
+
);
|
|
1122
|
+
|
|
1123
|
+
if (
|
|
1124
|
+
decoded.length !== page.length ||
|
|
1125
|
+
decoded.some(
|
|
1126
|
+
(row, index) =>
|
|
1127
|
+
row.thread_id !== request.threadId ||
|
|
1128
|
+
row.sequence !== page[index].sequence ||
|
|
1129
|
+
storedTextBytes(row.record_json) !== page[index].record_json_bytes,
|
|
1130
|
+
)
|
|
1131
|
+
) {
|
|
1132
|
+
return yield* mismatch();
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
return decoded;
|
|
1136
|
+
});
|
|
1137
|
+
|
|
1138
|
+
return {
|
|
1139
|
+
count: plan.length,
|
|
1140
|
+
records: Stream.fromIterable(pages).pipe(
|
|
1141
|
+
Stream.flatMap((page) => Stream.fromIterableEffect(readPage(page))),
|
|
1142
|
+
),
|
|
1143
|
+
};
|
|
745
1144
|
});
|
|
746
1145
|
|
|
747
|
-
const
|
|
748
|
-
conversationId: string,
|
|
749
|
-
) {
|
|
1146
|
+
const exportThread = Effect.fn("DoJournal.exportThread")(function* (threadId: string) {
|
|
750
1147
|
return yield* sql
|
|
751
1148
|
.withTransaction(
|
|
752
1149
|
Effect.gen(function* () {
|
|
753
|
-
const
|
|
1150
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
754
1151
|
SELECT
|
|
755
|
-
|
|
1152
|
+
thread_id,
|
|
756
1153
|
created_at,
|
|
757
1154
|
tail_sequence,
|
|
758
1155
|
tail_digest,
|
|
759
1156
|
producer_epoch
|
|
760
|
-
FROM
|
|
761
|
-
WHERE
|
|
762
|
-
`.pipe(Effect.mapError(storageError("export
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
1157
|
+
FROM effect_agent_threads
|
|
1158
|
+
WHERE thread_id = ${threadId}
|
|
1159
|
+
`.pipe(Effect.mapError(storageError("export thread")));
|
|
1160
|
+
|
|
1161
|
+
const thread = yield* decodeSingleRow(
|
|
1162
|
+
Schema.Array(ThreadRow),
|
|
1163
|
+
"effect_agent_threads",
|
|
1164
|
+
threadId,
|
|
1165
|
+
threadRows,
|
|
768
1166
|
);
|
|
769
|
-
yield* failpoint("export:after-conversation-read");
|
|
770
|
-
const batchRows = yield* sql<Record<string, unknown>>`
|
|
771
|
-
SELECT
|
|
772
|
-
conversation_id,
|
|
773
|
-
batch_id,
|
|
774
|
-
first_sequence,
|
|
775
|
-
last_sequence,
|
|
776
|
-
batch_digest,
|
|
777
|
-
tail_digest,
|
|
778
|
-
batch_json
|
|
779
|
-
FROM effect_agent_canonical_batches
|
|
780
|
-
WHERE conversation_id = ${conversationId}
|
|
781
|
-
ORDER BY first_sequence
|
|
782
|
-
`.pipe(Effect.mapError(storageError("export canonical batches")));
|
|
783
|
-
const recordRows = yield* sql<Record<string, unknown>>`
|
|
784
|
-
SELECT
|
|
785
|
-
conversation_id,
|
|
786
|
-
sequence,
|
|
787
|
-
record_id,
|
|
788
|
-
batch_id,
|
|
789
|
-
record_json
|
|
790
|
-
FROM effect_agent_canonical_records
|
|
791
|
-
WHERE conversation_id = ${conversationId}
|
|
792
|
-
ORDER BY sequence
|
|
793
|
-
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
794
|
-
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
795
|
-
SELECT
|
|
796
|
-
conversation_id,
|
|
797
|
-
through_sequence,
|
|
798
|
-
tail_digest,
|
|
799
|
-
checkpoint_json
|
|
800
|
-
FROM effect_agent_checkpoints
|
|
801
|
-
WHERE conversation_id = ${conversationId}
|
|
802
|
-
ORDER BY through_sequence
|
|
803
|
-
`.pipe(Effect.mapError(storageError("export checkpoints")));
|
|
804
1167
|
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
"
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
)
|
|
825
|
-
|
|
1168
|
+
yield* failpoint("export:after-thread-read");
|
|
1169
|
+
|
|
1170
|
+
if (thread.tail_sequence > MAX_RECORDS_PER_THREAD)
|
|
1171
|
+
return yield* DoStorageError.make({
|
|
1172
|
+
operation: "export thread",
|
|
1173
|
+
message: "The thread exceeds the current export record limit.",
|
|
1174
|
+
});
|
|
1175
|
+
const records: Array<RecordRow> = [];
|
|
1176
|
+
let afterSequence = ZERO_SEQUENCE;
|
|
1177
|
+
|
|
1178
|
+
while (afterSequence < thread.tail_sequence) {
|
|
1179
|
+
const limit = Math.min(1_024, thread.tail_sequence - afterSequence);
|
|
1180
|
+
|
|
1181
|
+
const request = RawReadRequest.make({
|
|
1182
|
+
threadId,
|
|
1183
|
+
fromSequenceExclusive: afterSequence,
|
|
1184
|
+
limit,
|
|
1185
|
+
});
|
|
1186
|
+
|
|
1187
|
+
const plan = yield* read(request);
|
|
1188
|
+
const page = yield* Stream.runCollect(plan.records);
|
|
1189
|
+
|
|
1190
|
+
if (
|
|
1191
|
+
page.length !== limit ||
|
|
1192
|
+
page.some((record, index) => record.sequence !== afterSequence + index + 1)
|
|
1193
|
+
) {
|
|
1194
|
+
return yield* DoStorageCorruptionError.make({
|
|
1195
|
+
table: "effect_agent_canonical_records",
|
|
1196
|
+
rowKey: threadId,
|
|
1197
|
+
message:
|
|
1198
|
+
"The exported canonical prefix is not contiguous through its captured tail.",
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
records.push(...page);
|
|
1202
|
+
afterSequence = page[page.length - 1].sequence;
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
const beyondTail =
|
|
1206
|
+
yield* sql`SELECT sequence FROM effect_agent_canonical_records WHERE thread_id=${threadId} AND sequence > ${thread.tail_sequence} LIMIT 1`.pipe(
|
|
1207
|
+
Effect.mapError(storageError("verify export tail")),
|
|
1208
|
+
);
|
|
1209
|
+
|
|
1210
|
+
if (beyondTail.length !== 0)
|
|
1211
|
+
return yield* DoStorageCorruptionError.make({
|
|
1212
|
+
table: "effect_agent_canonical_records",
|
|
1213
|
+
rowKey: threadId,
|
|
1214
|
+
message: "Canonical records exist beyond the captured thread tail.",
|
|
1215
|
+
});
|
|
1216
|
+
|
|
1217
|
+
return RawThreadExport.make({ thread, records });
|
|
826
1218
|
}),
|
|
827
1219
|
)
|
|
828
1220
|
.pipe(
|
|
@@ -835,7 +1227,7 @@ const makeJournal = (
|
|
|
835
1227
|
const saveCheckpoint = Effect.fn("DoJournal.saveCheckpoint")(function* (
|
|
836
1228
|
checkpoint: RawCheckpoint,
|
|
837
1229
|
): Effect.fn.Return<void, CheckpointError> {
|
|
838
|
-
if (checkpoint.
|
|
1230
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
|
|
839
1231
|
return yield* DoStorageError.make({
|
|
840
1232
|
operation: "save checkpoint",
|
|
841
1233
|
message: "Checkpoint identity exceeds the Durable Object storage bounds.",
|
|
@@ -844,50 +1236,54 @@ const makeJournal = (
|
|
|
844
1236
|
yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
|
|
845
1237
|
yield* withWriteTransaction("checkpoint transaction")(
|
|
846
1238
|
Effect.gen(function* () {
|
|
847
|
-
const
|
|
1239
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
848
1240
|
SELECT
|
|
849
|
-
|
|
1241
|
+
thread_id,
|
|
850
1242
|
created_at,
|
|
851
1243
|
tail_sequence,
|
|
852
1244
|
tail_digest,
|
|
853
1245
|
producer_epoch
|
|
854
|
-
FROM
|
|
855
|
-
WHERE
|
|
1246
|
+
FROM effect_agent_threads
|
|
1247
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
856
1248
|
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
1249
|
+
|
|
1250
|
+
const thread = yield* decodeSingleRow(
|
|
1251
|
+
Schema.Array(ThreadRow),
|
|
1252
|
+
"effect_agent_threads",
|
|
1253
|
+
checkpoint.threadId,
|
|
1254
|
+
threadRows,
|
|
862
1255
|
);
|
|
863
|
-
|
|
1256
|
+
|
|
1257
|
+
if (checkpoint.throughSequence > thread.tail_sequence) {
|
|
864
1258
|
return yield* DoCheckpointConflict.make({
|
|
865
1259
|
message:
|
|
866
1260
|
`Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
|
|
867
|
-
`${
|
|
1261
|
+
`${thread.tail_sequence}.`,
|
|
868
1262
|
});
|
|
869
1263
|
}
|
|
870
1264
|
|
|
871
1265
|
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
872
1266
|
SELECT
|
|
873
|
-
|
|
1267
|
+
thread_id,
|
|
874
1268
|
through_sequence,
|
|
875
1269
|
tail_digest,
|
|
876
1270
|
checkpoint_json
|
|
877
1271
|
FROM effect_agent_checkpoints
|
|
878
|
-
WHERE
|
|
1272
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
879
1273
|
AND through_sequence = ${checkpoint.throughSequence}
|
|
880
1274
|
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
1275
|
+
|
|
881
1276
|
const existing = yield* decodeRows(
|
|
882
1277
|
Schema.Array(CheckpointRow),
|
|
883
1278
|
"effect_agent_checkpoints",
|
|
884
|
-
`${checkpoint.
|
|
1279
|
+
`${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
885
1280
|
checkpointRows,
|
|
886
1281
|
);
|
|
1282
|
+
|
|
887
1283
|
if (existing.length > 1) {
|
|
888
1284
|
return yield* DoStorageCorruptionError.make({
|
|
889
1285
|
table: "effect_agent_checkpoints",
|
|
890
|
-
rowKey: `${checkpoint.
|
|
1286
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
891
1287
|
message: "A checkpoint primary key returned more than one row.",
|
|
892
1288
|
});
|
|
893
1289
|
}
|
|
@@ -900,17 +1296,18 @@ const makeJournal = (
|
|
|
900
1296
|
message: "A different checkpoint already exists at this canonical sequence.",
|
|
901
1297
|
});
|
|
902
1298
|
}
|
|
1299
|
+
|
|
903
1300
|
return;
|
|
904
1301
|
}
|
|
905
1302
|
|
|
906
1303
|
yield* sql`
|
|
907
1304
|
INSERT INTO effect_agent_checkpoints (
|
|
908
|
-
|
|
1305
|
+
thread_id,
|
|
909
1306
|
through_sequence,
|
|
910
1307
|
tail_digest,
|
|
911
1308
|
checkpoint_json
|
|
912
1309
|
) VALUES (
|
|
913
|
-
${checkpoint.
|
|
1310
|
+
${checkpoint.threadId},
|
|
914
1311
|
${checkpoint.throughSequence},
|
|
915
1312
|
${checkpoint.tailDigest},
|
|
916
1313
|
${checkpoint.checkpointJson}
|
|
@@ -920,45 +1317,124 @@ const makeJournal = (
|
|
|
920
1317
|
);
|
|
921
1318
|
});
|
|
922
1319
|
|
|
1320
|
+
const saveRecoveryCheckpoint = Effect.fn("DoJournal.saveRecoveryCheckpoint")(function* (
|
|
1321
|
+
request: SaveRecoveryCheckpointRequest,
|
|
1322
|
+
checkpointJson: string,
|
|
1323
|
+
) {
|
|
1324
|
+
const { checkpoint } = request;
|
|
1325
|
+
|
|
1326
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
|
|
1327
|
+
return yield* DoStorageError.make({
|
|
1328
|
+
operation: "save recovery checkpoint",
|
|
1329
|
+
message: "Checkpoint identity exceeds the Durable Object storage bounds.",
|
|
1330
|
+
});
|
|
1331
|
+
}
|
|
1332
|
+
yield* checkValueBound("save recovery checkpoint", checkpointJson);
|
|
1333
|
+
// Keep injected waits outside the storage-backed transaction callback.
|
|
1334
|
+
yield* failpoint("save-recovery-checkpoint:before");
|
|
1335
|
+
yield* withWriteTransaction("recovery checkpoint transaction")(
|
|
1336
|
+
Effect.gen(function* () {
|
|
1337
|
+
const threads = yield* getThread(checkpoint.threadId);
|
|
1338
|
+
const thread = threads[0];
|
|
1339
|
+
|
|
1340
|
+
if (thread === undefined)
|
|
1341
|
+
return yield* ThreadNotMaterialized.make({ threadId: checkpoint.threadId });
|
|
1342
|
+
if (request.producerEpoch !== thread.producer_epoch)
|
|
1343
|
+
return yield* FenceRejected.make({
|
|
1344
|
+
threadId: checkpoint.threadId,
|
|
1345
|
+
actualEpoch: thread.producer_epoch,
|
|
1346
|
+
attemptedEpoch: request.producerEpoch,
|
|
1347
|
+
});
|
|
1348
|
+
if (checkpoint.throughSequence > thread.tail_sequence)
|
|
1349
|
+
return yield* CheckpointRejected.make({
|
|
1350
|
+
threadId: checkpoint.threadId,
|
|
1351
|
+
reason: "ahead-of-tail",
|
|
1352
|
+
});
|
|
1353
|
+
|
|
1354
|
+
const digests =
|
|
1355
|
+
checkpoint.throughSequence === 0
|
|
1356
|
+
? [EMPTY_TAIL_DIGEST]
|
|
1357
|
+
: yield* getTailDigestAt(checkpoint.threadId, checkpoint.throughSequence);
|
|
1358
|
+
|
|
1359
|
+
if (digests.length !== 1 || digests[0] !== checkpoint.tailDigest)
|
|
1360
|
+
return yield* CheckpointRejected.make({
|
|
1361
|
+
threadId: checkpoint.threadId,
|
|
1362
|
+
reason: "digest-mismatch",
|
|
1363
|
+
});
|
|
1364
|
+
|
|
1365
|
+
yield* sql`
|
|
1366
|
+
INSERT INTO effect_agent_recovery_checkpoints (thread_id, through_sequence, tail_digest, checkpoint_json)
|
|
1367
|
+
VALUES (${checkpoint.threadId}, ${checkpoint.throughSequence}, ${checkpoint.tailDigest}, ${checkpointJson})
|
|
1368
|
+
ON CONFLICT (thread_id) DO UPDATE SET
|
|
1369
|
+
through_sequence = excluded.through_sequence,
|
|
1370
|
+
tail_digest = excluded.tail_digest,
|
|
1371
|
+
checkpoint_json = excluded.checkpoint_json
|
|
1372
|
+
WHERE excluded.through_sequence >= effect_agent_recovery_checkpoints.through_sequence
|
|
1373
|
+
`.pipe(Effect.mapError(storageError("save recovery checkpoint")));
|
|
1374
|
+
}),
|
|
1375
|
+
);
|
|
1376
|
+
yield* failpoint("save-recovery-checkpoint:after");
|
|
1377
|
+
});
|
|
1378
|
+
|
|
1379
|
+
const loadRecoveryCheckpoint = Effect.fn("DoJournal.loadRecoveryCheckpoint")(function* (
|
|
1380
|
+
threadId: string,
|
|
1381
|
+
) {
|
|
1382
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
1383
|
+
SELECT thread_id, through_sequence, tail_digest, checkpoint_json
|
|
1384
|
+
FROM effect_agent_recovery_checkpoints
|
|
1385
|
+
WHERE thread_id = ${threadId}
|
|
1386
|
+
`.pipe(Effect.mapError(storageError("load recovery checkpoint")));
|
|
1387
|
+
|
|
1388
|
+
return yield* decodeRows(
|
|
1389
|
+
Schema.Array(CheckpointRow),
|
|
1390
|
+
"effect_agent_recovery_checkpoints",
|
|
1391
|
+
threadId,
|
|
1392
|
+
rows,
|
|
1393
|
+
);
|
|
1394
|
+
});
|
|
1395
|
+
|
|
923
1396
|
const loadCheckpoint = Effect.fn("DoJournal.loadCheckpoint")(function* (
|
|
924
|
-
|
|
1397
|
+
threadId: string,
|
|
925
1398
|
atOrBeforeSequence: CanonicalSequence,
|
|
926
1399
|
) {
|
|
927
1400
|
const rows = yield* sql<Record<string, unknown>>`
|
|
928
1401
|
SELECT
|
|
929
|
-
|
|
1402
|
+
thread_id,
|
|
930
1403
|
through_sequence,
|
|
931
1404
|
tail_digest,
|
|
932
1405
|
checkpoint_json
|
|
933
1406
|
FROM effect_agent_checkpoints
|
|
934
|
-
WHERE
|
|
1407
|
+
WHERE thread_id = ${threadId}
|
|
935
1408
|
AND through_sequence <= ${atOrBeforeSequence}
|
|
936
1409
|
ORDER BY through_sequence DESC
|
|
937
1410
|
LIMIT 1
|
|
938
1411
|
`.pipe(Effect.mapError(storageError("load checkpoint")));
|
|
1412
|
+
|
|
939
1413
|
return yield* decodeRows(
|
|
940
1414
|
Schema.Array(CheckpointRow),
|
|
941
1415
|
"effect_agent_checkpoints",
|
|
942
|
-
`${
|
|
1416
|
+
`${threadId}<=${atOrBeforeSequence}`,
|
|
943
1417
|
rows,
|
|
944
1418
|
);
|
|
945
1419
|
});
|
|
946
1420
|
|
|
947
1421
|
const getTailDigestAt = Effect.fn("DoJournal.getTailDigestAt")(function* (
|
|
948
|
-
|
|
1422
|
+
threadId: string,
|
|
949
1423
|
sequence: CanonicalSequence,
|
|
950
1424
|
) {
|
|
951
1425
|
if (sequence === 0) {
|
|
952
|
-
const
|
|
953
|
-
|
|
1426
|
+
const threads = yield* getThread(threadId);
|
|
1427
|
+
|
|
1428
|
+
return threads.length === 0
|
|
954
1429
|
? []
|
|
955
|
-
: [
|
|
1430
|
+
: [threads[0].tail_sequence === 0 ? threads[0].tail_digest : undefined].filter(
|
|
956
1431
|
(value): value is string => value !== undefined,
|
|
957
1432
|
);
|
|
958
1433
|
}
|
|
1434
|
+
|
|
959
1435
|
const rows = yield* sql<Record<string, unknown>>`
|
|
960
1436
|
SELECT
|
|
961
|
-
|
|
1437
|
+
thread_id,
|
|
962
1438
|
batch_id,
|
|
963
1439
|
first_sequence,
|
|
964
1440
|
last_sequence,
|
|
@@ -966,15 +1442,17 @@ const makeJournal = (
|
|
|
966
1442
|
tail_digest,
|
|
967
1443
|
batch_json
|
|
968
1444
|
FROM effect_agent_canonical_batches
|
|
969
|
-
WHERE
|
|
1445
|
+
WHERE thread_id = ${threadId}
|
|
970
1446
|
AND last_sequence = ${sequence}
|
|
971
1447
|
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
1448
|
+
|
|
972
1449
|
const batches = yield* decodeRows(
|
|
973
1450
|
Schema.Array(BatchRow),
|
|
974
1451
|
"effect_agent_canonical_batches",
|
|
975
|
-
`${
|
|
1452
|
+
`${threadId}/${sequence}`,
|
|
976
1453
|
rows,
|
|
977
1454
|
);
|
|
1455
|
+
|
|
978
1456
|
return batches.map((batch) => batch.tail_digest);
|
|
979
1457
|
});
|
|
980
1458
|
|
|
@@ -982,19 +1460,20 @@ const makeJournal = (
|
|
|
982
1460
|
return yield* sql
|
|
983
1461
|
.withTransaction(
|
|
984
1462
|
Effect.gen(function* () {
|
|
985
|
-
const
|
|
1463
|
+
const threads = yield* sql<Record<string, unknown>>`
|
|
986
1464
|
SELECT
|
|
987
|
-
|
|
1465
|
+
thread_id,
|
|
988
1466
|
created_at,
|
|
989
1467
|
tail_sequence,
|
|
990
1468
|
tail_digest,
|
|
991
1469
|
producer_epoch
|
|
992
|
-
FROM
|
|
993
|
-
ORDER BY
|
|
994
|
-
`.pipe(Effect.mapError(storageError("scan
|
|
1470
|
+
FROM effect_agent_threads
|
|
1471
|
+
ORDER BY thread_id
|
|
1472
|
+
`.pipe(Effect.mapError(storageError("scan threads")));
|
|
1473
|
+
|
|
995
1474
|
const batches = yield* sql<Record<string, unknown>>`
|
|
996
1475
|
SELECT
|
|
997
|
-
|
|
1476
|
+
thread_id,
|
|
998
1477
|
batch_id,
|
|
999
1478
|
first_sequence,
|
|
1000
1479
|
last_sequence,
|
|
@@ -1002,33 +1481,36 @@ const makeJournal = (
|
|
|
1002
1481
|
tail_digest,
|
|
1003
1482
|
batch_json
|
|
1004
1483
|
FROM effect_agent_canonical_batches
|
|
1005
|
-
ORDER BY
|
|
1484
|
+
ORDER BY thread_id, first_sequence
|
|
1006
1485
|
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
1486
|
+
|
|
1007
1487
|
const records = yield* sql<Record<string, unknown>>`
|
|
1008
1488
|
SELECT
|
|
1009
|
-
|
|
1489
|
+
thread_id,
|
|
1010
1490
|
sequence,
|
|
1011
1491
|
record_id,
|
|
1012
1492
|
batch_id,
|
|
1013
1493
|
record_json
|
|
1014
1494
|
FROM effect_agent_canonical_records
|
|
1015
|
-
ORDER BY
|
|
1495
|
+
ORDER BY thread_id, sequence
|
|
1016
1496
|
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
1497
|
+
|
|
1017
1498
|
const checkpoints = yield* sql<Record<string, unknown>>`
|
|
1018
1499
|
SELECT
|
|
1019
|
-
|
|
1500
|
+
thread_id,
|
|
1020
1501
|
through_sequence,
|
|
1021
1502
|
tail_digest,
|
|
1022
1503
|
checkpoint_json
|
|
1023
1504
|
FROM effect_agent_checkpoints
|
|
1024
|
-
ORDER BY
|
|
1505
|
+
ORDER BY thread_id, through_sequence
|
|
1025
1506
|
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
1507
|
+
|
|
1026
1508
|
return {
|
|
1027
|
-
|
|
1028
|
-
Schema.Array(
|
|
1029
|
-
"
|
|
1509
|
+
threads: yield* decodeRows(
|
|
1510
|
+
Schema.Array(ThreadRow),
|
|
1511
|
+
"effect_agent_threads",
|
|
1030
1512
|
"startup_scan",
|
|
1031
|
-
|
|
1513
|
+
threads,
|
|
1032
1514
|
),
|
|
1033
1515
|
batches: yield* decodeRows(
|
|
1034
1516
|
Schema.Array(BatchRow),
|
|
@@ -1061,10 +1543,12 @@ const makeJournal = (
|
|
|
1061
1543
|
return {
|
|
1062
1544
|
append,
|
|
1063
1545
|
checkValueBound,
|
|
1064
|
-
|
|
1065
|
-
|
|
1546
|
+
exportThread,
|
|
1547
|
+
getThread,
|
|
1066
1548
|
getTailDigestAt,
|
|
1067
1549
|
loadCheckpoint,
|
|
1550
|
+
loadRecoveryCheckpoint,
|
|
1551
|
+
saveRecoveryCheckpoint,
|
|
1068
1552
|
materialize,
|
|
1069
1553
|
read,
|
|
1070
1554
|
saveCheckpoint,
|