@effect-agent/storage-cloudflare 0.1.0-beta.55 → 0.1.0-beta.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/DoMemoryStore.d.mts +2 -2
  2. package/dist/DoMemoryStore.mjs +2 -2
  3. package/dist/DoMessageDeliveryStore.d.mts +17 -0
  4. package/dist/DoMessageDeliveryStore.mjs +32 -0
  5. package/dist/DoMessageDeliveryStore.mjs.map +1 -0
  6. package/dist/DoScheduleStore.d.mts +1 -1
  7. package/dist/DoScheduleStore.mjs +2 -2
  8. package/dist/{DoStorageVersion-DxqcVgDE.d.mts → DoStorageVersion-Pt3jEHdW.d.mts} +2 -2
  9. package/dist/DoStorageVersion.d.mts +1 -1
  10. package/dist/DoStorageVersion.mjs +2 -2
  11. package/dist/DoSubmissionLedger.mjs +45 -9
  12. package/dist/DoSubmissionLedger.mjs.map +1 -1
  13. package/dist/DoSubscriptionStore.mjs +1 -1
  14. package/dist/DoThreadStore.d.mts +1 -1
  15. package/dist/DoThreadStore.mjs +432 -5
  16. package/dist/DoThreadStore.mjs.map +1 -0
  17. package/dist/MemoryProtocol.d.mts +1 -1
  18. package/dist/MemoryProtocol.mjs +1 -1
  19. package/dist/PortProtocol.d.mts +1334 -7
  20. package/dist/{DoThreadStore-CCSgDto6.mjs → do-journal-Brv7xSH5.mjs} +186 -456
  21. package/dist/do-journal-Brv7xSH5.mjs.map +1 -0
  22. package/dist/index.d.mts +5 -4
  23. package/dist/index.mjs +6 -5
  24. package/dist/{migrations-B-jiT7vx.mjs → migrations-soC86CQA.mjs} +30 -5
  25. package/dist/migrations-soC86CQA.mjs.map +1 -0
  26. package/package.json +1 -1
  27. package/src/DoMessageDeliveryStore.ts +53 -0
  28. package/src/DoSubmissionLedger.ts +118 -3
  29. package/src/index.ts +1 -0
  30. package/src/internal/do-journal.ts +174 -19
  31. package/src/internal/message-delivery-schema.ts +24 -0
  32. package/src/internal/migrations.ts +6 -1
  33. package/dist/DoThreadStore-CCSgDto6.mjs.map +0 -1
  34. package/dist/migrations-B-jiT7vx.mjs.map +0 -1
@@ -1,17 +1,10 @@
1
- import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
2
- import { DoStorageConfig, DoStorageConfigValue } from "./DoStorageConfig.mjs";
3
1
  import { DoAppendConflict, DoCheckpointConflict, DoFenceRejected, DoStorageCompatibilityError, DoStorageCorruptionError, DoStorageError, DoValueBoundExceeded } from "./DoStorageError.mjs";
4
- import { DoStorageFailpoint } from "./DoStorageFailpoint.mjs";
5
- import { n as doMigrations } from "./migrations-B-jiT7vx.mjs";
6
- import { SqliteClient, SqliteMigrator } from "@effect/sql-sqlite-do";
7
- import { Clock, Context, Crypto, Duration, Effect, Layer, Option, Ref, Schema, Stream } from "effect";
8
- import { checkV2ThreadLayout } from "@effect-agent/thread/SqlStorageV2Upgrade";
2
+ import { n as doMigrations, r as createMessageDeliveryTables } from "./migrations-soC86CQA.mjs";
3
+ import { Effect, Schema, Stream } from "effect";
9
4
  import * as SqlClient from "effect/unstable/sql/SqlClient";
10
- import { CanonicalBatch, CanonicalRecord, CanonicalRecordEnvelope, CanonicalSequence, Digest, ObservationOffset, ProducerEpoch } from "@effect-agent/thread/Records";
11
- import { EMPTY_TAIL_DIGEST, digestCanonicalBatch } from "@effect-agent/thread/Digest";
12
- import { DEFAULT_OWNERSHIP_LEASE_DURATION } from "@effect-agent/thread/SubmissionLedger";
13
- import { BrowserCrypto } from "@effect/platform-browser";
14
- import { AppendConflict, AppendResult, CheckpointRejected, FenceRejected, FencedAppendRequest, LoadCheckpointRequest, SaveCheckpointRequest, ThreadCheckpoint, ThreadExport, ThreadExportRequest, ThreadMaterialization, ThreadNotMaterialized, ThreadObservation, ThreadRead, ThreadStore, ThreadStoreError, ThreadTail, ThreadTailRequest } from "@effect-agent/thread/ThreadStore";
5
+ import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread/Records";
6
+ import { checkV2ThreadLayout } from "@effect-agent/thread/SqlStorageV2Upgrade";
7
+ import { SqliteMigrator } from "@effect/sql-sqlite-do";
15
8
  import { SqlError } from "effect/unstable/sql/SqlError";
16
9
  //#region src/internal/do-journal.ts
17
10
  /**
@@ -121,6 +114,147 @@ const decodeSingleRow = Effect.fn((schema, table, rowKey, rows) => decodeRows(sc
121
114
  rowKey,
122
115
  message: `Expected exactly one row but found ${decoded.length}.`
123
116
  })))));
117
+ /** Column inventory of the supported v3 predecessor, independent of physical column order. */
118
+ const predecessorColumns = {
119
+ effect_agent_threads: [
120
+ "thread_id",
121
+ "created_at",
122
+ "tail_sequence",
123
+ "tail_digest",
124
+ "producer_epoch"
125
+ ],
126
+ effect_agent_canonical_batches: [
127
+ "thread_id",
128
+ "batch_id",
129
+ "first_sequence",
130
+ "last_sequence",
131
+ "batch_digest",
132
+ "tail_digest",
133
+ "batch_json"
134
+ ],
135
+ effect_agent_canonical_records: [
136
+ "thread_id",
137
+ "sequence",
138
+ "record_id",
139
+ "batch_id",
140
+ "record_json"
141
+ ],
142
+ effect_agent_checkpoints: [
143
+ "thread_id",
144
+ "through_sequence",
145
+ "tail_digest",
146
+ "checkpoint_json"
147
+ ],
148
+ effect_agent_submissions: [
149
+ "submission_id",
150
+ "thread_id",
151
+ "queue_sequence",
152
+ "principal",
153
+ "idempotency_key",
154
+ "agent_id",
155
+ "agent_digests_json",
156
+ "deployment_id",
157
+ "input_json",
158
+ "input_digest",
159
+ "receipt_id",
160
+ "state",
161
+ "settled_outcome",
162
+ "created_at",
163
+ "ready_at",
164
+ "input_applied_record_id",
165
+ "input_applied_sequence",
166
+ "joined_host_submission_id",
167
+ "suspended_reason_json",
168
+ "suspended_at",
169
+ "unknown_reason",
170
+ "unknown_tool_call_ids_json",
171
+ "parent_submission_id",
172
+ "parent_tool_call_id",
173
+ "admission_group",
174
+ "admission_fence_json"
175
+ ],
176
+ effect_agent_submission_ownership: [
177
+ "submission_id",
178
+ "attempt_id",
179
+ "ownership_token",
180
+ "producer_epoch",
181
+ "owner_producer_id",
182
+ "lease_expires_at"
183
+ ],
184
+ effect_agent_attempts: [
185
+ "attempt_id",
186
+ "submission_id",
187
+ "thread_id",
188
+ "owner_producer_id",
189
+ "producer_epoch",
190
+ "claimed_at"
191
+ ],
192
+ effect_agent_settlement_reservations: [
193
+ "submission_id",
194
+ "settlement_id",
195
+ "outcome",
196
+ "record_id",
197
+ "record_json",
198
+ "record_digest",
199
+ "reserved_at",
200
+ "finalized_at"
201
+ ],
202
+ effect_agent_abort_intents: [
203
+ "submission_id",
204
+ "author",
205
+ "reason",
206
+ "requested_at",
207
+ "canonical_record_id"
208
+ ],
209
+ effect_agent_approval_decisions: [
210
+ "submission_id",
211
+ "tool_call_id",
212
+ "decision",
213
+ "resolver",
214
+ "reason",
215
+ "decided_at"
216
+ ],
217
+ effect_agent_unknown_resolutions: [
218
+ "submission_id",
219
+ "tool_call_id",
220
+ "author",
221
+ "reason",
222
+ "resolution_json",
223
+ "resolved_at"
224
+ ],
225
+ effect_agent_child_reservations: [
226
+ "reservation_id",
227
+ "parent_submission_id",
228
+ "parent_tool_call_id",
229
+ "child_submission_id",
230
+ "status",
231
+ "allocation_json",
232
+ "allocation_digest",
233
+ "accounting_json",
234
+ "reserved_at",
235
+ "release_began_at",
236
+ "released_at"
237
+ ],
238
+ effect_agent_meta: ["key", "value"],
239
+ effect_agent_child_settlements: [
240
+ "parent_submission_id",
241
+ "child_submission_id",
242
+ "child_outcome",
243
+ "recorded_at"
244
+ ]
245
+ };
246
+ const checkPredecessorLayout = Effect.fn("DoJournal.checkPredecessorLayout")(function* () {
247
+ const sql = yield* SqlClient.SqlClient;
248
+ for (const [table, expected] of Object.entries(predecessorColumns)) {
249
+ const columns = yield* decodeRows(Schema.Array(Schema.Struct({ name: BoundedIdentifier })), table, "schema", yield* sql.unsafe(`PRAGMA table_info(${table})`));
250
+ const names = new Set(expected);
251
+ if (columns.length !== names.size || columns.some((column) => !names.has(column.name))) return yield* DoStorageCompatibilityError.make({
252
+ actualVersion: 3,
253
+ supportedVersion: 4,
254
+ message: `The v3 ${table} columns do not match the supported predecessor; no upgrade was committed.`
255
+ });
256
+ }
257
+ });
124
258
  const REQUIRED_TABLES = [
125
259
  "effect_agent_abort_intents",
126
260
  "effect_agent_approval_decisions",
@@ -161,7 +295,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
161
295
  `.pipe(Effect.mapError(storageError("inspect unversioned storage")));
162
296
  if ((yield* decodeRows(Schema.Array(DoNameRow), "sqlite_master", "effect_agent_%", existingRows)).length > 0) return yield* DoStorageCompatibilityError.make({
163
297
  actualVersion: 0,
164
- supportedVersion: 3,
298
+ supportedVersion: 4,
165
299
  message: "The Durable Object contains unversioned Effect Agent tables. Refusing to mutate ambiguous stored data; retain it for inspection with its original writer."
166
300
  });
167
301
  yield* SqliteMigrator.run({ loader: doMigrations }).pipe(Effect.provideService(SqlClient.SqlClient, sql), Effect.mapError((error) => DoStorageError.make({
@@ -176,31 +310,43 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
176
310
  WHERE key = 'storage_version'
177
311
  `.pipe(Effect.mapError(storageError("read storage version")));
178
312
  const version = yield* decodeSingleRow(Schema.Array(DoMetaRow), "effect_agent_meta", "storage_version", versionRows);
179
- if (version.value === "2") yield* sql.withTransaction(Effect.gen(function* () {
313
+ if (version.value === "2" || version.value === "3") yield* sql.withTransaction(Effect.gen(function* () {
180
314
  const current = yield* sql`SELECT value FROM effect_agent_meta WHERE key='storage_version'`;
181
- if (current.length === 1 && current[0].value === "3") return;
182
- if (current.length !== 1 || current[0].value !== "2") return yield* DoStorageCompatibilityError.make({
315
+ if (current.length === 1 && current[0].value === String(4)) return;
316
+ if (current.length !== 1 || current[0].value !== "2" && current[0].value !== "3") return yield* DoStorageCompatibilityError.make({
183
317
  actualVersion: -1,
184
- supportedVersion: 3,
318
+ supportedVersion: 4,
185
319
  message: "Storage version changed while acquiring the upgrade transaction."
186
320
  });
187
321
  if ((yield* decodeRows(Schema.Array(DoNameRow), "sqlite_master", "required_tables", yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name IN ${sql.in([...REQUIRED_TABLES])}`)).length !== REQUIRED_TABLES.length) return yield* DoStorageCompatibilityError.make({
188
- actualVersion: 2,
189
- supportedVersion: 3,
190
- message: "The v2 store is missing required tables. Retain the original store for inspection; no upgrade was committed."
322
+ actualVersion: Number(current[0].value),
323
+ supportedVersion: 4,
324
+ message: "The predecessor store is missing required tables. Retain the original store for inspection; no upgrade was committed."
191
325
  });
192
- yield* checkV2ThreadLayout();
193
- for (const statement of [
194
- sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
195
- sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
196
- sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`
197
- ]) {
198
- yield* failpoint("upgrade:before-mutation");
199
- yield* statement;
200
- yield* failpoint("upgrade:after-mutation");
326
+ if (current[0].value === "2") {
327
+ yield* checkV2ThreadLayout();
328
+ for (const statement of [
329
+ sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
330
+ sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
331
+ sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`
332
+ ]) {
333
+ yield* failpoint("upgrade:before-mutation");
334
+ yield* statement;
335
+ yield* failpoint("upgrade:after-mutation");
336
+ }
201
337
  }
338
+ if (current[0].value === "3") yield* checkPredecessorLayout();
339
+ yield* failpoint("upgrade:before-mutation");
340
+ yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
341
+ yield* failpoint("upgrade:after-mutation");
342
+ yield* failpoint("upgrade:before-mutation");
343
+ yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
344
+ yield* failpoint("upgrade:after-mutation");
345
+ yield* failpoint("upgrade:before-mutation");
346
+ yield* createMessageDeliveryTables;
347
+ yield* failpoint("upgrade:after-mutation");
202
348
  yield* failpoint("upgrade:before-version");
203
- yield* sql`UPDATE effect_agent_meta SET value='3' WHERE key='storage_version'`;
349
+ yield* sql`UPDATE effect_agent_meta SET value='4' WHERE key='storage_version'`;
204
350
  yield* failpoint("upgrade:after-version");
205
351
  })).pipe(Effect.catchTag("DoStorageFailpointError", (error) => DoStorageError.make({
206
352
  cause: error,
@@ -210,13 +356,13 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
210
356
  table: error.table,
211
357
  rowKey: error.rowKey,
212
358
  message: error.message
213
- })), Effect.catchTag("SqlError", storageError("upgrade v2 thread storage")));
214
- else if (version.value !== String(3)) {
359
+ })), Effect.catchTag("SqlError", storageError("upgrade supported thread storage")));
360
+ else if (version.value !== String(4)) {
215
361
  const actualVersion = Number.parseInt(version.value, 10);
216
362
  return yield* DoStorageCompatibilityError.make({
217
363
  actualVersion: Number.isSafeInteger(actualVersion) ? actualVersion : -1,
218
- supportedVersion: 3,
219
- message: `The Durable Object uses unsupported storage version ${version.value}; this build supports exactly version 3. Only unpatched v2 can be upgraded automatically. Keep the original store and use a compatible library version.`
364
+ supportedVersion: 4,
365
+ message: `The Durable Object uses unsupported storage version ${version.value}; this build supports exactly version 4. Only supported v2 and v3 can be upgraded automatically. Keep the original store and use a compatible library version.`
220
366
  });
221
367
  }
222
368
  }
@@ -224,12 +370,12 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
224
370
  SELECT name
225
371
  FROM sqlite_master
226
372
  WHERE type = 'table'
227
- AND name IN ${sql.in([...REQUIRED_TABLES])}
373
+ AND name IN ${sql.in([...REQUIRED_TABLES, "effect_agent_message_deliveries"])}
228
374
  ORDER BY name
229
375
  `.pipe(Effect.mapError(storageError("verify storage tables")));
230
- if ((yield* decodeRows(Schema.Array(DoNameRow), "sqlite_master", "required_tables", requiredRows)).length !== REQUIRED_TABLES.length) return yield* DoStorageCompatibilityError.make({
231
- actualVersion: 3,
232
- supportedVersion: 3,
376
+ if ((yield* decodeRows(Schema.Array(DoNameRow), "sqlite_master", "required_tables", requiredRows)).length !== REQUIRED_TABLES.length + 1) return yield* DoStorageCompatibilityError.make({
377
+ actualVersion: 4,
378
+ supportedVersion: 4,
233
379
  message: "The Durable Object claims the current format but is missing required tables. Retain the original store for inspection."
234
380
  });
235
381
  return makeJournal(sql, failpoint, maxStoredValueBytes);
@@ -718,422 +864,6 @@ const makeJournal = (sql, failpoint, maxStoredValueBytes) => {
718
864
  };
719
865
  const initializeDoJournal = ensureCurrentStorage;
720
866
  //#endregion
721
- //#region src/DoThreadStore.ts
722
- var DoThreadStore_exports = /* @__PURE__ */ __exportAll({
723
- layer: () => layer,
724
- storageConfigLayer: () => storageConfigLayer,
725
- storageFailpointLayer: () => storageFailpointLayer,
726
- threadStoreLayer: () => threadStoreLayer
727
- });
728
- const OffsetText = Schema.String.check(Schema.isMaxLength(4096));
729
- const DO_OFFSET_PREFIX = "effect-agent-do@1:";
730
- const ZERO_CANONICAL_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
731
- const isDigest = Schema.is(Digest);
732
- const isDoFenceRejected = Schema.is(DoFenceRejected);
733
- const isDoAppendConflict = Schema.is(DoAppendConflict);
734
- const isDoCheckpointConflict = Schema.is(DoCheckpointConflict);
735
- const storeError = (operation, error) => ThreadStoreError.make({
736
- cause: error,
737
- operation,
738
- message: error.message
739
- });
740
- const schemaStoreError = (operation, error) => ThreadStoreError.make({
741
- cause: error,
742
- operation,
743
- message: error.message
744
- });
745
- const makeOffset = Effect.fn(function* (threadId, sequence) {
746
- return yield* Schema.decodeUnknownEffect(CanonicalSequence)(sequence).pipe(Effect.flatMap((validatedSequence) => Schema.decodeUnknownEffect(ObservationOffset)(`${DO_OFFSET_PREFIX}${encodeURIComponent(threadId)}:${validatedSequence}`)), Effect.mapError((error) => schemaStoreError("encode observation offset", error)));
747
- });
748
- const parseOffset = Effect.fn(function* (threadId, offset) {
749
- if (offset === void 0) return ZERO_CANONICAL_SEQUENCE;
750
- const text = yield* Schema.decodeUnknownEffect(OffsetText)(offset).pipe(Effect.mapError((error) => schemaStoreError("decode observation offset", error)));
751
- const threadPrefix = `${DO_OFFSET_PREFIX}${encodeURIComponent(threadId)}:`;
752
- if (!text.startsWith(threadPrefix)) return yield* ThreadStoreError.make({
753
- operation: "decode observation offset",
754
- message: "The observation offset belongs to a different adapter, storage version, or Thread."
755
- });
756
- const sequenceText = text.slice(threadPrefix.length);
757
- if (!/^(0|[1-9][0-9]*)$/.test(sequenceText)) return yield* ThreadStoreError.make({
758
- operation: "decode observation offset",
759
- message: "The observation offset is malformed."
760
- });
761
- return yield* Schema.decodeUnknownEffect(CanonicalSequence)(Number(sequenceText)).pipe(Effect.mapError((error) => schemaStoreError("decode observation offset", error)));
762
- });
763
- const mapFence = (threadId, error) => FenceRejected.make({
764
- threadId,
765
- actualEpoch: error.actualEpoch,
766
- attemptedEpoch: error.producerEpoch
767
- });
768
- const encodeCanonicalRecord = Effect.fn(function* (record) {
769
- return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(record).pipe(Effect.mapError((error) => schemaStoreError("encode canonical record", error)));
770
- });
771
- const encodeCanonicalBatch = Effect.fn(function* (batch) {
772
- return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalBatch))(batch).pipe(Effect.mapError((error) => schemaStoreError("encode canonical batch", error)));
773
- });
774
- const encodeCheckpoint = Effect.fn(function* (checkpoint) {
775
- return yield* Schema.encodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint).pipe(Effect.mapError((error) => schemaStoreError("encode checkpoint", error)));
776
- });
777
- const decodeEnvelope = Effect.fn(function* (row) {
778
- const record = yield* Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(row.record_json).pipe(Effect.mapError((error) => ThreadStoreError.make({
779
- operation: "decode canonical record",
780
- message: error.message
781
- })));
782
- const threadId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.threadId)(row.thread_id).pipe(Effect.mapError((error) => schemaStoreError("decode thread identity", error)));
783
- const offset = yield* makeOffset(threadId, row.sequence);
784
- const batchId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.batchId)(row.batch_id).pipe(Effect.mapError((error) => schemaStoreError("decode batch identity", error)));
785
- return CanonicalRecordEnvelope.make({
786
- threadId,
787
- batchId,
788
- sequence: row.sequence,
789
- offset,
790
- record
791
- });
792
- });
793
- const decodeCheckpoint = Effect.fn(function* (checkpointJson) {
794
- return yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpointJson).pipe(Effect.mapError((error) => schemaStoreError("decode checkpoint", error)));
795
- });
796
- const requireThread = Effect.fn("DoThreadStore.requireThread")(function* (journal, threadId) {
797
- const rows = yield* journal.getThread(threadId).pipe(Effect.mapError((error) => storeError("read thread", error)));
798
- if (rows.length === 0) return yield* ThreadNotMaterialized.make({ threadId });
799
- return rows[0];
800
- });
801
- const tailDigestAt = Effect.fn("DoThreadStore.tailDigestAt")(function* (journal, threadId, sequence) {
802
- if (sequence === 0) return EMPTY_TAIL_DIGEST;
803
- const digests = yield* journal.getTailDigestAt(threadId, sequence).pipe(Effect.mapError((error) => storeError("read checkpoint digest", error)));
804
- if (digests.length !== 1) return yield* CheckpointRejected.make({
805
- threadId,
806
- reason: "digest-mismatch"
807
- });
808
- return yield* Schema.decodeUnknownEffect(Digest)(digests[0]).pipe(Effect.mapError((error) => schemaStoreError("decode checkpoint digest", error)));
809
- });
810
- const groupByKey = (rows, key) => {
811
- const grouped = /* @__PURE__ */ new Map();
812
- for (const row of rows) {
813
- const existing = grouped.get(key(row));
814
- if (existing === void 0) grouped.set(key(row), [row]);
815
- else existing.push(row);
816
- }
817
- return grouped;
818
- };
819
- /**
820
- * Opt-in full integrity audit (`verifyOnOpen`). Every stored payload is decoded, re-encoded,
821
- * and re-digested against the canonical chain. Routine opens skip this scan: per-operation
822
- * Schema decoding plus the digest chain already fail clearly on corrupt rows.
823
- */
824
- const decodeStartupPayloads = Effect.fn("DoThreadStore.decodeStartupPayloads")(function* (journal, crypto) {
825
- const stored = yield* journal.scanStoredPayloads();
826
- const batches = yield* Effect.forEach(stored.batches, (batch) => Schema.decodeEffect(Schema.fromJsonString(CanonicalBatch))(batch.batch_json).pipe(Effect.map((decoded) => ({
827
- decoded,
828
- row: batch
829
- })), Effect.mapError((error) => DoStorageCorruptionError.make({
830
- table: "effect_agent_canonical_batches",
831
- rowKey: `${batch.thread_id}/${batch.batch_id}`,
832
- message: error.message
833
- }))));
834
- const records = yield* Effect.forEach(stored.records, (record) => Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(record.record_json).pipe(Effect.map((decoded) => ({
835
- decoded,
836
- row: record
837
- })), Effect.mapError((error) => DoStorageCorruptionError.make({
838
- table: "effect_agent_canonical_records",
839
- rowKey: `${record.thread_id}/${record.sequence}`,
840
- message: error.message
841
- }))));
842
- const checkpoints = yield* Effect.forEach(stored.checkpoints, (checkpoint) => Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint.checkpoint_json).pipe(Effect.map((decoded) => ({
843
- decoded,
844
- row: checkpoint
845
- })), Effect.mapError((error) => DoStorageCorruptionError.make({
846
- table: "effect_agent_checkpoints",
847
- rowKey: `${checkpoint.thread_id}/${checkpoint.through_sequence}`,
848
- message: error.message
849
- }))));
850
- const batchesByThread = groupByKey(batches, ({ row }) => row.thread_id);
851
- const recordsByThread = groupByKey(records, ({ row }) => row.thread_id);
852
- const checkpointsByThread = groupByKey(checkpoints, ({ row }) => row.thread_id);
853
- const materializedIds = new Set(stored.threads.map((thread) => thread.thread_id));
854
- for (const thread of stored.threads) {
855
- const threadBatches = batchesByThread.get(thread.thread_id) ?? [];
856
- const threadRecords = recordsByThread.get(thread.thread_id) ?? [];
857
- const threadCheckpoints = checkpointsByThread.get(thread.thread_id) ?? [];
858
- const recordsByBatch = groupByKey(threadRecords, ({ row }) => row.batch_id);
859
- let previousDigest = EMPTY_TAIL_DIGEST;
860
- let expectedSequence = 1;
861
- const tailDigests = /* @__PURE__ */ new Map([[0, EMPTY_TAIL_DIGEST]]);
862
- for (const { decoded: canonicalBatch, row: batchRow } of threadBatches) {
863
- const key = `${batchRow.thread_id}/${batchRow.batch_id}`;
864
- if (canonicalBatch.batchId !== batchRow.batch_id || batchRow.first_sequence !== expectedSequence || batchRow.last_sequence !== batchRow.first_sequence + canonicalBatch.records.length - 1) return yield* DoStorageCorruptionError.make({
865
- table: "effect_agent_canonical_batches",
866
- rowKey: key,
867
- message: "Canonical batch identity, sequence, or record count is inconsistent."
868
- });
869
- const digest = yield* digestCanonicalBatch(previousDigest, canonicalBatch).pipe(Effect.provideService(Crypto.Crypto, crypto), Effect.mapError((error) => DoStorageCorruptionError.make({
870
- table: "effect_agent_canonical_batches",
871
- rowKey: key,
872
- message: error.message
873
- })));
874
- if (batchRow.batch_digest !== digest || batchRow.tail_digest !== digest) return yield* DoStorageCorruptionError.make({
875
- table: "effect_agent_canonical_batches",
876
- rowKey: key,
877
- message: "Canonical batch digest does not match its decoded content and prior tail."
878
- });
879
- const batchRecords = recordsByBatch.get(batchRow.batch_id) ?? [];
880
- if (batchRecords.length !== canonicalBatch.records.length) return yield* DoStorageCorruptionError.make({
881
- table: "effect_agent_canonical_records",
882
- rowKey: key,
883
- message: "Canonical batch and record-table counts differ."
884
- });
885
- for (let index = 0; index < canonicalBatch.records.length; index++) {
886
- const expectedRecord = canonicalBatch.records[index];
887
- const storedRecord = batchRecords[index];
888
- const expectedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(expectedRecord).pipe(Effect.mapError((error) => DoStorageCorruptionError.make({
889
- table: "effect_agent_canonical_batches",
890
- rowKey: key,
891
- message: error.message
892
- })));
893
- const storedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(storedRecord.decoded).pipe(Effect.mapError((error) => DoStorageCorruptionError.make({
894
- table: "effect_agent_canonical_records",
895
- rowKey: `${key}/${storedRecord.row.sequence}`,
896
- message: error.message
897
- })));
898
- if (storedRecord.row.sequence !== batchRow.first_sequence + index || storedRecord.row.record_id !== expectedRecord.recordId || expectedJson !== storedJson) return yield* DoStorageCorruptionError.make({
899
- table: "effect_agent_canonical_records",
900
- rowKey: `${key}/${storedRecord.row.sequence}`,
901
- message: "Canonical record identity, sequence, or payload differs from its batch."
902
- });
903
- }
904
- previousDigest = digest;
905
- expectedSequence = batchRow.last_sequence + 1;
906
- tailDigests.set(batchRow.last_sequence, digest);
907
- }
908
- if (threadRecords.length !== thread.tail_sequence || thread.tail_sequence !== expectedSequence - 1 || thread.tail_digest !== previousDigest) return yield* DoStorageCorruptionError.make({
909
- table: "effect_agent_threads",
910
- rowKey: thread.thread_id,
911
- message: "Thread tail does not match its canonical batch chain."
912
- });
913
- for (const checkpoint of threadCheckpoints) if (checkpoint.decoded.threadId !== thread.thread_id || checkpoint.decoded.throughSequence !== checkpoint.row.through_sequence || checkpoint.decoded.tailDigest !== checkpoint.row.tail_digest || tailDigests.get(checkpoint.row.through_sequence) !== checkpoint.row.tail_digest) return yield* DoStorageCorruptionError.make({
914
- table: "effect_agent_checkpoints",
915
- rowKey: `${thread.thread_id}/${checkpoint.row.through_sequence}`,
916
- message: "Checkpoint identity or digest is not bound to a canonical batch tail."
917
- });
918
- }
919
- if (batches.some(({ row }) => !materializedIds.has(row.thread_id)) || records.some(({ row }) => !materializedIds.has(row.thread_id)) || checkpoints.some(({ row }) => !materializedIds.has(row.thread_id))) return yield* DoStorageCorruptionError.make({
920
- table: "effect_agent_threads",
921
- rowKey: "startup_scan",
922
- message: "Canonical rows exist without a materialized Thread."
923
- });
924
- });
925
- const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
926
- const config = yield* DoStorageConfig;
927
- const failpoint = yield* DoStorageFailpoint;
928
- const sql = yield* SqlClient.SqlClient;
929
- const crypto = yield* Crypto.Crypto;
930
- const journal = yield* initializeDoJournal(sql, failpoint.hit, config.maxStoredValueBytes);
931
- if (config.verifyOnOpen) yield* decodeStartupPayloads(journal, crypto);
932
- const provideCrypto = (effect) => Effect.provideService(effect, Crypto.Crypto, crypto);
933
- const hitFailpoint = Effect.fn((location) => failpoint.hit(location).pipe(Effect.mapError((error) => storeError(`storage failpoint ${location}`, error))));
934
- const materialize = Effect.fn("DoThreadStore.materialize")(function* (request) {
935
- const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadMaterialization))(request).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
936
- const now = yield* Clock.currentTimeMillis;
937
- yield* hitFailpoint("materialize:before");
938
- yield* journal.materialize(validated.threadId, new Date(now).toISOString(), EMPTY_TAIL_DIGEST, validated.producerEpoch).pipe(Effect.mapError((error) => error._tag === "DoFenceRejected" ? mapFence(validated.threadId, error) : storeError("materialize thread", error)));
939
- yield* hitFailpoint("materialize:after");
940
- });
941
- const append = Effect.fn("DoThreadStore.append")(function* (request) {
942
- const validated = yield* Schema.decodeUnknownEffect(Schema.toType(FencedAppendRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate canonical append", error)));
943
- yield* requireThread(journal, validated.threadId);
944
- const tailDigest = yield* provideCrypto(digestCanonicalBatch(validated.expectedTailDigest, validated.batch)).pipe(Effect.mapError((error) => storeError("digest canonical append", error)));
945
- const batchJson = yield* encodeCanonicalBatch(validated.batch);
946
- const rawRecords = yield* Effect.forEach(validated.batch.records, (record) => encodeCanonicalRecord(record).pipe(Effect.map((recordJson) => ({
947
- recordId: record.recordId,
948
- recordJson
949
- }))));
950
- const rawRequest = yield* Schema.decodeUnknownEffect(RawAppendRequest)({
951
- threadId: validated.threadId,
952
- batchId: validated.batch.batchId,
953
- batchDigest: tailDigest,
954
- batchJson,
955
- expectedTailSequence: validated.expectedTailSequence,
956
- expectedTailDigest: validated.expectedTailDigest,
957
- producerEpoch: validated.producerEpoch,
958
- records: rawRecords,
959
- tailDigest
960
- }).pipe(Effect.mapError((error) => schemaStoreError("encode canonical append", error)));
961
- yield* hitFailpoint("append:before");
962
- const result = yield* journal.append(rawRequest).pipe(Effect.mapError((error) => {
963
- if (isDoFenceRejected(error)) return mapFence(validated.threadId, error);
964
- if (isDoAppendConflict(error)) return error.actualTailSequence !== void 0 && isDigest(error.actualTailDigest) ? AppendConflict.make({
965
- threadId: validated.threadId,
966
- batchId: validated.batch.batchId,
967
- reason: error.reason,
968
- actualTailSequence: error.actualTailSequence,
969
- actualTailDigest: error.actualTailDigest
970
- }) : AppendConflict.make({
971
- threadId: validated.threadId,
972
- batchId: validated.batch.batchId,
973
- reason: error.reason
974
- });
975
- return storeError("append canonical batch", error);
976
- }), Effect.flatMap((result) => Schema.decodeUnknownEffect(AppendResult)(result).pipe(Effect.mapError((error) => schemaStoreError("decode append result", error)))));
977
- yield* hitFailpoint("append:after");
978
- return result;
979
- });
980
- const loadRecords = Effect.fn("DoThreadStore.loadRecords")(function* (request) {
981
- const result = yield* journal.read(request).pipe(Effect.mapError((error) => storeError("read canonical records", error)));
982
- return {
983
- count: result.count,
984
- records: result.records.pipe(Stream.mapError((error) => storeError("read canonical records", error)), Stream.mapEffect(decodeEnvelope))
985
- };
986
- });
987
- const readEffect = Effect.fn("DoThreadStore.read")(function* (request) {
988
- const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadRead))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread read", error)));
989
- yield* requireThread(journal, validated.threadId);
990
- return (yield* loadRecords(RawReadRequest.make({
991
- threadId: validated.threadId,
992
- fromSequenceExclusive: validated.afterSequence ?? ZERO_CANONICAL_SEQUENCE,
993
- limit: validated.limit
994
- }))).records;
995
- });
996
- const read = (request) => Stream.unwrap(readEffect(request));
997
- const observeEffect = Effect.fn("DoThreadStore.observe")(function* (request) {
998
- const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadObservation))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread observation", error)));
999
- yield* requireThread(journal, validated.threadId);
1000
- const initialSequence = yield* parseOffset(validated.threadId, validated.afterOffset);
1001
- const cursor = yield* Ref.make(initialSequence);
1002
- const poll = Effect.fn("DoThreadStore.observePoll")(function* () {
1003
- const fromSequenceExclusive = yield* Ref.get(cursor);
1004
- const result = yield* loadRecords(RawReadRequest.make({
1005
- threadId: validated.threadId,
1006
- fromSequenceExclusive,
1007
- limit: 1024
1008
- }));
1009
- if (result.count === 0) {
1010
- yield* Effect.sleep(config.observationPollInterval);
1011
- return Stream.empty;
1012
- }
1013
- return result.records.pipe(Stream.tap((record) => Ref.set(cursor, record.sequence)));
1014
- });
1015
- return Stream.fromEffectRepeat(poll()).pipe(Stream.flatten);
1016
- });
1017
- const observe = (request) => Stream.unwrap(observeEffect(request));
1018
- const exportThread = Effect.fn("DoThreadStore.export")(function* (request) {
1019
- const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadExportRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread export", error)));
1020
- yield* requireThread(journal, validated.threadId);
1021
- const exported = yield* journal.exportThread(validated.threadId).pipe(Effect.mapError((error) => storeError("export thread", error)));
1022
- const records = yield* Effect.forEach(exported.records, decodeEnvelope);
1023
- if (records.length > 65536) return yield* ThreadStoreError.make({
1024
- operation: "decode thread export",
1025
- message: "The thread exceeds the current export record limit."
1026
- });
1027
- const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(exported.thread.tail_digest).pipe(Effect.mapError((error) => schemaStoreError("decode export tail digest", error)));
1028
- return ThreadExport.make({
1029
- format: "effect-agent/thread@1",
1030
- threadId: validated.threadId,
1031
- tailSequence: exported.thread.tail_sequence,
1032
- tailDigest,
1033
- records
1034
- });
1035
- });
1036
- const inspectTail = Effect.fn("DoThreadStore.inspectTail")(function* (request) {
1037
- const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadTailRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate tail inspection", error)));
1038
- const thread = yield* requireThread(journal, validated.threadId);
1039
- const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(thread.tail_digest).pipe(Effect.mapError((error) => schemaStoreError("decode tail digest", error)));
1040
- return ThreadTail.make({
1041
- threadId: validated.threadId,
1042
- tailSequence: thread.tail_sequence,
1043
- tailDigest,
1044
- producerEpoch: thread.producer_epoch
1045
- });
1046
- });
1047
- const saveCheckpoint = Effect.fn("DoThreadStore.saveCheckpoint")(function* (request) {
1048
- const validated = yield* Schema.decodeUnknownEffect(Schema.toType(SaveCheckpointRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
1049
- const thread = yield* requireThread(journal, validated.checkpoint.threadId);
1050
- if (validated.checkpoint.throughSequence > thread.tail_sequence) return yield* CheckpointRejected.make({
1051
- threadId: validated.checkpoint.threadId,
1052
- reason: "ahead-of-tail"
1053
- });
1054
- if ((yield* tailDigestAt(journal, validated.checkpoint.threadId, validated.checkpoint.throughSequence)) !== validated.checkpoint.tailDigest) return yield* CheckpointRejected.make({
1055
- threadId: validated.checkpoint.threadId,
1056
- reason: "digest-mismatch"
1057
- });
1058
- const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
1059
- const raw = RawCheckpoint.make({
1060
- threadId: validated.checkpoint.threadId,
1061
- throughSequence: validated.checkpoint.throughSequence,
1062
- tailDigest: validated.checkpoint.tailDigest,
1063
- checkpointJson
1064
- });
1065
- yield* hitFailpoint("save-checkpoint:before");
1066
- yield* journal.saveCheckpoint(raw).pipe(Effect.mapError((error) => isDoCheckpointConflict(error) ? CheckpointRejected.make({
1067
- threadId: validated.checkpoint.threadId,
1068
- reason: "digest-mismatch"
1069
- }) : storeError("save checkpoint", error)));
1070
- yield* hitFailpoint("save-checkpoint:after");
1071
- });
1072
- const loadCheckpoint = Effect.fn("DoThreadStore.loadCheckpoint")(function* (request) {
1073
- const validated = yield* Schema.decodeUnknownEffect(Schema.toType(LoadCheckpointRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint lookup", error)));
1074
- const thread = yield* requireThread(journal, validated.threadId);
1075
- const rows = yield* journal.loadCheckpoint(validated.threadId, validated.atOrBeforeSequence ?? thread.tail_sequence).pipe(Effect.mapError((error) => storeError("load checkpoint", error)));
1076
- if (rows.length === 0) return Option.none();
1077
- if (rows.length !== 1) return yield* ThreadStoreError.make({
1078
- operation: "load checkpoint",
1079
- message: `Expected at most one checkpoint row but found ${rows.length}.`
1080
- });
1081
- const row = rows[0];
1082
- const checkpoint = yield* decodeCheckpoint(row.checkpoint_json);
1083
- if (row.thread_id !== validated.threadId || checkpoint.threadId !== row.thread_id || checkpoint.throughSequence !== row.through_sequence || checkpoint.tailDigest !== row.tail_digest) return yield* ThreadStoreError.make({
1084
- operation: "load checkpoint",
1085
- message: "Stored checkpoint metadata does not match its canonical row."
1086
- });
1087
- if ((yield* tailDigestAt(journal, checkpoint.threadId, checkpoint.throughSequence)) !== checkpoint.tailDigest) return yield* CheckpointRejected.make({
1088
- threadId: checkpoint.threadId,
1089
- reason: "digest-mismatch"
1090
- });
1091
- return Option.some(checkpoint);
1092
- });
1093
- const threadStore = ThreadStore.of({
1094
- append,
1095
- export: exportThread,
1096
- inspectTail,
1097
- materialize,
1098
- observe,
1099
- read,
1100
- checkpoints: {
1101
- save: saveCheckpoint,
1102
- load: loadCheckpoint
1103
- }
1104
- });
1105
- return Context.make(ThreadStore, threadStore);
1106
- });
1107
- /**
1108
- * Durable Object Thread Store implementation with configuration, failpoint, SQL, and
1109
- * Crypto authority kept visible in its input channel.
1110
- */
1111
- const threadStoreLayer = Layer.effectContext(makeServices());
1112
- /**
1113
- * Validated Durable Object storage configuration Layer with the documented defaults applied.
1114
- * Shared by the ThreadStore and SubmissionLedger convenience layers so their defaults
1115
- * cannot drift.
1116
- */
1117
- const storageConfigLayer = (options) => Layer.effect(DoStorageConfig)(Schema.decodeUnknownEffect(DoStorageConfigValue)({
1118
- observationPollInterval: options.observationPollInterval ?? 25,
1119
- ownershipLeaseDuration: options.ownershipLeaseDuration ?? Duration.toMillis(DEFAULT_OWNERSHIP_LEASE_DURATION),
1120
- maxStoredValueBytes: options.maxStoredValueBytes ?? 19e5,
1121
- verifyOnOpen: options.verifyOnOpen ?? false
1122
- }).pipe(Effect.mapError((error) => DoStorageError.make({
1123
- cause: error,
1124
- operation: "configure Durable Object storage",
1125
- message: error.message
1126
- }))));
1127
- /** The failpoint Layer selected by convenience options: explicit handler or the no-op default. */
1128
- const storageFailpointLayer = (options) => options.failpoint === void 0 ? DoStorageFailpoint.layer : Layer.succeed(DoStorageFailpoint)({ hit: options.failpoint });
1129
- /**
1130
- * A composition-root convenience Layer for canonical Threads inside one Durable Object,
1131
- * built over `ctx.storage`. Durable accepted work is served by the separate SubmissionLedger
1132
- * port; point both at the SAME `ctx.storage` so claims fence the same producer epochs
1133
- * (ADR-0011 D7's "same file" rule, transposed to one object's private database).
1134
- */
1135
- const layer = (options) => Layer.unwrap(Effect.map(DoStorageConfig, (config) => threadStoreLayer.pipe(Layer.provide(Layer.mergeAll(Layer.succeed(DoStorageConfig)(config), storageFailpointLayer(options), SqliteClient.layer({ storage: options.storage }), BrowserCrypto.layer))))).pipe(Layer.provide(storageConfigLayer(options)));
1136
- //#endregion
1137
- export { threadStoreLayer as a, storageFailpointLayer as i, layer as n, decodeRows as o, storageConfigLayer as r, initializeDoJournal as s, DoThreadStore_exports as t };
867
+ export { initializeDoJournal as a, decodeRows as i, RawCheckpoint as n, RawReadRequest as r, RawAppendRequest as t };
1138
868
 
1139
- //# sourceMappingURL=DoThreadStore-CCSgDto6.mjs.map
869
+ //# sourceMappingURL=do-journal-Brv7xSH5.mjs.map