@effect-agent/storage-sqlite 0.1.0-beta.42 → 0.1.0-beta.44

package/src/sqlite-thread-store.ts

@@ -131,10 +131,13 @@ const parseOffset = Effect.fn("SqliteThreadStore.parseOffset")(function* (
   offset: ObservationOffset | undefined,
 ): Effect.fn.Return<CanonicalSequence, ThreadStoreError> {
   if (offset === undefined) return ZERO_CANONICAL_SEQUENCE;
+
   const text = yield* Schema.decodeUnknownEffect(OffsetText)(offset).pipe(
     Effect.mapError((error) => schemaStoreError("decode observation offset", error)),
   );
+
   const threadPrefix = `${SQLITE_OFFSET_PREFIX}${encodeURIComponent(threadId)}:`;
+
   if (!text.startsWith(threadPrefix)) {
     return yield* ThreadStoreError.make({
       operation: "decode observation offset",
@@ -142,12 +145,14 @@ const parseOffset = Effect.fn("SqliteThreadStore.parseOffset")(function* (
     });
   }
   const sequenceText = text.slice(threadPrefix.length);
+
   if (!/^(0|[1-9][0-9]*)$/.test(sequenceText)) {
     return yield* ThreadStoreError.make({
       operation: "decode observation offset",
       message: "The observation offset is malformed.",
     });
   }
+
   return yield* Schema.decodeUnknownEffect(CanonicalSequence)(Number(sequenceText)).pipe(
     Effect.mapError((error) => schemaStoreError("decode observation offset", error)),
   );
@@ -200,13 +205,17 @@ const decodeEnvelope = Effect.fn("SqliteThreadStore.decodeEnvelope")(function* (
       }),
     ),
   );
+
   const threadId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.threadId)(
     row.thread_id,
   ).pipe(Effect.mapError((error) => schemaStoreError("decode thread identity", error)));
+
   const offset = yield* makeOffset(threadId, row.sequence);
+
   const batchId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.batchId)(
     row.batch_id,
   ).pipe(Effect.mapError((error) => schemaStoreError("decode batch identity", error)));
+
   return CanonicalRecordEnvelope.make({
     threadId,
     batchId,
@@ -231,9 +240,11 @@ const requireThread = Effect.fn("SqliteThreadStore.requireThread")(function* (
   const rows = yield* journal
     .getThread(threadId)
     .pipe(Effect.mapError((error) => storeError("read thread", error)));
+
   if (rows.length === 0) {
     return yield* ThreadNotMaterialized.make({ threadId });
   }
+
   return rows[0];
 });
 
@@ -243,15 +254,18 @@ const tailDigestAt = Effect.fn("SqliteThreadStore.tailDigestAt")(function* (
   sequence: CanonicalSequence,
 ) {
   if (sequence === 0) return EMPTY_TAIL_DIGEST;
+
   const digests = yield* journal
     .getTailDigestAt(threadId, sequence)
     .pipe(Effect.mapError((error) => storeError("read checkpoint digest", error)));
+
   if (digests.length !== 1) {
     return yield* CheckpointRejected.make({
       threadId,
       reason: "digest-mismatch",
     });
   }
+
   return yield* Schema.decodeUnknownEffect(Digest)(digests[0]).pipe(
     Effect.mapError((error) => schemaStoreError("decode checkpoint digest", error)),
   );
@@ -262,14 +276,17 @@ const groupByKey = <A>(
   key: (row: A) => string,
 ): ReadonlyMap<string, ReadonlyArray<A>> => {
   const grouped = new Map<string, Array<A>>();
+
   for (const row of rows) {
     const existing = grouped.get(key(row));
+
     if (existing === undefined) {
       grouped.set(key(row), [row]);
     } else {
       existing.push(row);
     }
   }
+
   return grouped;
 };
 
@@ -283,6 +300,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
   crypto: Crypto.Crypto,
 ) {
   const stored = yield* journal.scanStoredPayloads();
+
   const batches = yield* Effect.forEach(stored.batches, (batch) =>
     Schema.decodeEffect(Schema.fromJsonString(CanonicalBatch))(batch.batch_json).pipe(
       Effect.map((decoded) => ({ decoded, row: batch })),
@@ -295,6 +313,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
       ),
     ),
   );
+
   const records = yield* Effect.forEach(stored.records, (record) =>
     Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(record.record_json).pipe(
       Effect.map((decoded) => ({ decoded, row: record })),
@@ -307,6 +326,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
       ),
     ),
   );
+
   const checkpoints = yield* Effect.forEach(stored.checkpoints, (checkpoint) =>
     Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint.checkpoint_json).pipe(
       Effect.map((decoded) => ({ decoded, row: checkpoint })),
@@ -336,6 +356,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
 
     for (const { decoded: canonicalBatch, row: batchRow } of threadBatches) {
       const key = `${batchRow.thread_id}/${batchRow.batch_id}`;
+
       if (
         canonicalBatch.batchId !== batchRow.batch_id ||
         batchRow.first_sequence !== expectedSequence ||
@@ -358,6 +379,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
           }),
         ),
       );
+
       if (batchRow.batch_digest !== digest || batchRow.tail_digest !== digest) {
         return yield* SqliteStorageCorruptionError.make({
           table: "effect_agent_canonical_batches",
@@ -367,6 +389,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
       }
 
       const batchRecords = recordsByBatch.get(batchRow.batch_id) ?? [];
+
       if (batchRecords.length !== canonicalBatch.records.length) {
         return yield* SqliteStorageCorruptionError.make({
           table: "effect_agent_canonical_records",
@@ -377,6 +400,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
       for (let index = 0; index < canonicalBatch.records.length; index++) {
         const expectedRecord = canonicalBatch.records[index];
         const storedRecord = batchRecords[index];
+
         const expectedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(
           expectedRecord,
         ).pipe(
@@ -388,6 +412,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
             }),
           ),
         );
+
         const storedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(
           storedRecord.decoded,
         ).pipe(
@@ -399,6 +424,7 @@ const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads
             }),
           ),
         );
+
         if (
           storedRecord.row.sequence !== batchRow.first_sequence + index ||
           storedRecord.row.record_id !== expectedRecord.recordId ||
@@ -463,12 +489,14 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
   const failpoint = yield* SqliteStorageFailpoint;
   const crypto = yield* Crypto.Crypto;
   const journal = yield* initializeSqliteJournal();
+
   if (config.verifyOnOpen) {
     yield* decodeStartupPayloads(journal, crypto);
   }
 
   const provideCrypto = <A, E>(effect: Effect.Effect<A, E, Crypto.Crypto>) =>
     Effect.provideService(effect, Crypto.Crypto, crypto);
+
   const hitFailpoint = Effect.fn("SqliteThreadStore.hitFailpoint")(
     (location: SqliteStorageFailpointLocation): Effect.Effect<void, ThreadStoreError> =>
       failpoint
@@ -482,7 +510,9 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
     const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadMaterialization))(
       request,
     ).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
+
     const now = yield* Clock.currentTimeMillis;
+
     yield* hitFailpoint("materialize:before");
     yield* journal
       .materialize(
@@ -507,11 +537,15 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
     const validated = yield* Schema.decodeUnknownEffect(Schema.toType(FencedAppendRequest))(
       request,
     ).pipe(Effect.mapError((error) => schemaStoreError("validate canonical append", error)));
+
     yield* requireThread(journal, validated.threadId);
+
     const tailDigest = yield* provideCrypto(
       digestCanonicalBatch(validated.expectedTailDigest, validated.batch),
     ).pipe(Effect.mapError((error) => storeError("digest canonical append", error)));
+
     const batchJson = yield* encodeCanonicalBatch(validated.batch);
+
     const rawRecords = yield* Effect.forEach(validated.batch.records, (record) =>
       encodeCanonicalRecord(record).pipe(
         Effect.map((recordJson) => ({
@@ -520,6 +554,7 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
         })),
       ),
     );
+
     const rawRequest = yield* Schema.decodeUnknownEffect(RawAppendRequest)({
       threadId: validated.threadId,
       batchId: validated.batch.batchId,
@@ -531,7 +566,9 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
       records: rawRecords,
       tailDigest,
     }).pipe(Effect.mapError((error) => schemaStoreError("encode canonical append", error)));
+
     yield* hitFailpoint("append:before");
+
     const result = yield* journal.append(rawRequest).pipe(
       Effect.mapError((error) => {
         if (isSqliteFenceRejected(error)) {
@@ -552,6 +589,7 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
                 reason: error.reason,
               });
         }
+
         return storeError("append canonical batch", error);
       }),
       Effect.flatMap((result) =>
@@ -560,7 +598,9 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
         ),
       ),
     );
+
     yield* hitFailpoint("append:after");
+
     return result;
   });
 
@@ -570,6 +610,7 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
     const rows = yield* journal
       .read(request)
       .pipe(Effect.mapError((error) => storeError("read canonical records", error)));
+
     return yield* Effect.forEach(rows, decodeEnvelope);
   });
 
@@ -577,7 +618,9 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
     const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadRead))(request).pipe(
       Effect.mapError((error) => schemaStoreError("validate thread read", error)),
     );
+
     yield* requireThread(journal, validated.threadId);
+
     const records = yield* loadRecords(
       RawReadRequest.make({
         threadId: validated.threadId,
@@ -585,8 +628,10 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
         limit: validated.limit,
       }),
     );
+
     return Stream.fromIterable(records);
   });
+
   const read: ThreadStore["Service"]["read"] = (request) => Stream.unwrap(readEffect(request));
 
   const observeEffect = Effect.fn("SqliteThreadStore.observe")(function* (
@@ -595,11 +640,14 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
     const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadObservation))(
       request,
     ).pipe(Effect.mapError((error) => schemaStoreError("validate thread observation", error)));
+
     yield* requireThread(journal, validated.threadId);
     const initialSequence = yield* parseOffset(validated.threadId, validated.afterOffset);
     const cursor = yield* Ref.make(initialSequence);
+
     const poll = Effect.fn("SqliteThreadStore.observePoll")(function* () {
       const fromSequenceExclusive = yield* Ref.get(cursor);
+
       const records = yield* loadRecords(
         RawReadRequest.make({
           threadId: validated.threadId,
@@ -607,15 +655,20 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
           limit: 1_024,
         }),
       );
+
       if (records.length === 0) {
         yield* Effect.sleep(config.observationPollInterval);
+
         return [];
       }
       yield* Ref.set(cursor, records[records.length - 1].sequence);
+
       return records;
     });
+
     return Stream.fromIterableEffectRepeat(poll());
   });
+
   const observe: ThreadStore["Service"]["observe"] = (request) =>
     Stream.unwrap(observeEffect(request));
 
@@ -624,20 +677,26 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
       const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadExportRequest))(
         request,
       ).pipe(Effect.mapError((error) => schemaStoreError("validate thread export", error)));
+
       yield* requireThread(journal, validated.threadId);
+
       const exported = yield* journal
         .exportThread(validated.threadId)
         .pipe(Effect.mapError((error) => storeError("export thread", error)));
+
       const records = yield* Effect.forEach(exported.records, decodeEnvelope);
+
       if (records.length > 65_536) {
         return yield* ThreadStoreError.make({
           operation: "decode thread export",
           message: "The thread exceeds the current export record limit.",
         });
       }
+
       const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(
         exported.thread.tail_digest,
       ).pipe(Effect.mapError((error) => schemaStoreError("decode export tail digest", error)));
+
       return ThreadExport.make({
         format: "effect-agent/thread@1",
         threadId: validated.threadId,
@@ -654,10 +713,13 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
     const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadTailRequest))(
       request,
     ).pipe(Effect.mapError((error) => schemaStoreError("validate tail inspection", error)));
+
     const thread = yield* requireThread(journal, validated.threadId);
+
     const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(thread.tail_digest).pipe(
       Effect.mapError((error) => schemaStoreError("decode tail digest", error)),
     );
+
     return ThreadTail.make({
       threadId: validated.threadId,
       tailSequence: thread.tail_sequence,
@@ -671,18 +733,22 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
       const validated = yield* Schema.decodeUnknownEffect(Schema.toType(SaveCheckpointRequest))(
         request,
       ).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
+
       const thread = yield* requireThread(journal, validated.checkpoint.threadId);
+
       if (validated.checkpoint.throughSequence > thread.tail_sequence) {
         return yield* CheckpointRejected.make({
           threadId: validated.checkpoint.threadId,
           reason: "ahead-of-tail",
         });
       }
+
       const canonicalDigest = yield* tailDigestAt(
         journal,
         validated.checkpoint.threadId,
         validated.checkpoint.throughSequence,
       );
+
       if (canonicalDigest !== validated.checkpoint.tailDigest) {
         return yield* CheckpointRejected.make({
           threadId: validated.checkpoint.threadId,
@@ -690,12 +756,14 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
         });
       }
       const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
+
       const raw = RawCheckpoint.make({
         threadId: validated.checkpoint.threadId,
         throughSequence: validated.checkpoint.throughSequence,
         tailDigest: validated.checkpoint.tailDigest,
         checkpointJson,
       });
+
       yield* hitFailpoint("save-checkpoint:before");
       yield* journal.saveCheckpoint(raw).pipe(
         Effect.mapError((error) =>
@@ -716,10 +784,13 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
       const validated = yield* Schema.decodeUnknownEffect(Schema.toType(LoadCheckpointRequest))(
         request,
       ).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint lookup", error)));
+
       const thread = yield* requireThread(journal, validated.threadId);
+
       const rows = yield* journal
         .loadCheckpoint(validated.threadId, validated.atOrBeforeSequence ?? thread.tail_sequence)
         .pipe(Effect.mapError((error) => storeError("load checkpoint", error)));
+
       if (rows.length === 0) return Option.none();
       if (rows.length !== 1) {
         return yield* ThreadStoreError.make({
@@ -728,17 +799,20 @@ const makeServices = Effect.fn("SqliteThreadStore.makeServices")(function* () {
         });
       }
       const checkpoint = yield* decodeCheckpoint(rows[0].checkpoint_json);
+
       const canonicalDigest = yield* tailDigestAt(
         journal,
         checkpoint.threadId,
         checkpoint.throughSequence,
       );
+
       if (canonicalDigest !== checkpoint.tailDigest) {
         return yield* CheckpointRejected.make({
           threadId: checkpoint.threadId,
           reason: "digest-mismatch",
         });
       }
+
       return Option.some(checkpoint);
     },
   );