@structure-ai/eventsourcing-nisshi 0.2.0 → 0.3.0

package/README.md

@@ -80,3 +80,9 @@ The sidecar and broker storage may share one PostgreSQL server, but use separate
 ## Tests
 
 `bun test` skips broker suites unless `NISSHI_URL` is set; PostgreSQL-sidecar scenarios additionally require `DATABASE_URL`. CI installs a pinned Nisshi release, starts PostgreSQL, and runs both sidecar suites. The wire quirks this client encodes (trailing `throttle_time_ms` in Produce, metadata-triggered topic auto-creation, empty-topic high watermark of 1, whole-batch fetch granularity) are pinned by tests in `test/protocol.test.ts`.
+
+## Persistence failures
+
+The event store and sidecar adapters expose `PersistenceError` for SQL, broker and envelope failures, retaining the operation and diagnostic cause. Version races remain `ConcurrencyConflict`. See the [shared error contract](../eventsourcing/README.md#persistence-failures).
+
+`runPendingRelay` retries only connection failures and broker errors marked retriable. SQL and permanent broker failures stop the relay with a typed error; supervise the returned effect. Produce failures wrap `NisshiProduceError` in `PersistenceError`. Pending rows may still be published later, so a failed append is not proof that no events will appear. See [ADR-0020](../../docs/decisions/0020-typed-persistence-and-cause-boundaries.md).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@structure-ai/eventsourcing-nisshi",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Nisshi (Kafka-API broker) event store with a SQL sidecar for optimistic concurrency, snapshots, checkpoints, and inbox dedupe.",
   "license": "MIT",
   "type": "module",
@@ -28,8 +28,8 @@
     "@effect/sql": "^0.52.1",
     "@effect/sql-sqlite-bun": "^0.53.0",
     "@effect/sql-pg": "^0.53.0",
-    "@structure-ai/domain": "0.2.0",
-    "@structure-ai/eventsourcing": "0.2.0"
+    "@structure-ai/domain": "0.3.0",
+    "@structure-ai/eventsourcing": "0.3.0"
   },
   "devDependencies": {
     "typescript": "^5.9.2",

package/src/EventStore.ts

@@ -1,6 +1,6 @@
 import * as SqlClient from "@effect/sql/SqlClient";
-import type { SqlError } from "@effect/sql/SqlError";
-import { ConcurrencyConflict } from "@structure-ai/domain";
+import { SqlError } from "@effect/sql/SqlError";
+import { ConcurrencyConflict, PersistenceError } from "@structure-ai/domain";
 import {
   type AppendEvent,
   type AppendResult,
@@ -10,7 +10,7 @@ import {
   type StoredEvent,
   type StoredEventMetadata,
 } from "@structure-ai/eventsourcing";
-import { Effect, Layer, Stream } from "effect";
+import { Cause, Effect, Layer, Stream } from "effect";
 import { decodeWireEvent, encodeWireEvent, validateWireEvent } from "./envelope.js";
 import { NisshiClient } from "./protocol/client.js";
 import { NisshiProduceError } from "./protocol/errors.js";
@@ -109,7 +109,7 @@ const make = (
       streamName: string,
       expectedVersion: number,
       events: ReadonlyArray<AppendEvent>,
-    ): Effect.Effect<AppendResult, ConcurrencyConflict | SqlError> =>
+    ): Effect.Effect<AppendResult, ConcurrencyConflict | SqlError | PersistenceError> =>
       sql
         .withTransaction(
           Effect.gen(function* () {
@@ -141,22 +141,27 @@ const make = (
             }
             yield* Effect.forEach(
               events,
-              (event, index) => {
-                const wire = {
-                  type: event.type,
-                  schemaVersion: event.schemaVersion,
-                  version: expectedVersion + index + 1,
-                  payload: event.payload,
-                  metadata: event.metadata,
-                };
-                if (validate) {
-                  validateWireEvent(wire, streamName);
-                }
-                return sql`
+              (event, index) =>
+                Effect.gen(function* () {
+                  const wire = {
+                    type: event.type,
+                    schemaVersion: event.schemaVersion,
+                    version: expectedVersion + index + 1,
+                    payload: event.payload,
+                    metadata: event.metadata,
+                  };
+                  const encoded = yield* Effect.try({
+                    try: () => {
+                      if (validate) validateWireEvent(wire, streamName);
+                      return JSON.stringify(wire);
+                    },
+                    catch: (cause) => new PersistenceError({ operation: "nisshi.encode", cause }),
+                  });
+                  return yield* sql`
                 INSERT INTO ${sql(tables.pending)} (stream_name, version, topic, record_value)
-                VALUES (${streamName}, ${expectedVersion + index + 1}, ${topic}, ${JSON.stringify(wire)})
+                VALUES (${streamName}, ${expectedVersion + index + 1}, ${topic}, ${encoded})
               `;
-              },
+                }),
               { discard: true },
             );
             return {
@@ -166,66 +171,95 @@ const make = (
           }),
         )
         .pipe(
-          Effect.catchIf(
-            (error): error is SqlError => isUniqueViolation(error),
-            () =>
-              Effect.flatMap(Effect.orDie(currentVersion(streamName)), (actual) =>
-                Effect.fail(conflict(streamName, expectedVersion, actual)),
-              ),
-          ),
+          Effect.catchAllCause((cause) => {
+            if (
+              !Cause.isFailType(cause) ||
+              !(cause.error instanceof SqlError) ||
+              !isUniqueViolation(cause.error)
+            )
+              return Effect.failCause(cause);
+            return Effect.flatMap(currentVersion(streamName), (actual) =>
+              Effect.fail(conflict(streamName, expectedVersion, actual)),
+            );
+          }),
         );
 
-    /** Best-effort reservation rollback: only succeeds if nobody built on top. Never masks the original failure. */
+    /** Conditional rollback; its failure is retained alongside the produce failure. */
     const rollback = (
       streamName: string,
       expectedVersion: number,
       count: number,
-    ): Effect.Effect<void> =>
-      sql
-        .withTransaction(
-          Effect.gen(function* () {
-            yield* sql`
+    ): Effect.Effect<void, SqlError> =>
+      sql.withTransaction(
+        Effect.gen(function* () {
+          yield* sql`
               UPDATE ${sql(tables.streams)}
               SET last_version = ${expectedVersion}
               WHERE stream_name = ${streamName} AND last_version = ${expectedVersion + count}
             `;
-            yield* sql`
+          yield* sql`
               DELETE FROM ${sql(tables.pending)}
               WHERE stream_name = ${streamName}
                 AND version > ${expectedVersion} AND version <= ${expectedVersion + count}
             `;
-          }),
-        )
-        .pipe(
-          Effect.ignore,
-          Effect.catchAllDefect(() => Effect.void),
-        );
+        }),
+      );
 
     const append: EventStoreService["append"] = (streamName, expectedVersion, events) =>
       Effect.gen(function* () {
+        const records = yield* Effect.try({
+          try: () =>
+            events.map((event, index) => ({
+              key: new TextEncoder().encode(streamName),
+              value: encodeWireEvent({
+                type: event.type,
+                schemaVersion: event.schemaVersion,
+                version: expectedVersion + index + 1,
+                payload: event.payload,
+                metadata: event.metadata,
+              }),
+            })),
+          catch: (cause) => new PersistenceError({ operation: "nisshi.encode", cause }),
+        });
         const result = yield* reserve(streamName, expectedVersion, events).pipe(
-          Effect.catchTag("SqlError", (error) => Effect.die(error)),
+          Effect.mapError((cause) =>
+            cause instanceof SqlError
+              ? new PersistenceError({ operation: "nisshi.reserve", cause })
+              : cause,
+          ),
         );
         if (events.length === 0) {
           return result;
         }
-        const records = events.map((event, index) => ({
-          key: new TextEncoder().encode(streamName),
-          value: encodeWireEvent({
-            type: event.type,
-            schemaVersion: event.schemaVersion,
-            version: expectedVersion + index + 1,
-            payload: event.payload,
-            metadata: event.metadata,
-          }),
-        }));
         yield* client.produce(topic, records).pipe(
-          Effect.catchAllCause((cause) =>
-            Effect.gen(function* () {
-              yield* rollback(streamName, expectedVersion, events.length);
-              return yield* Effect.die(new NisshiProduceError({ topic, cause }));
-            }),
-          ),
+          Effect.catchAllCause((cause) => {
+            const original = Cause.map(
+              cause,
+              (error) =>
+                new PersistenceError({
+                  operation: "nisshi.produce",
+                  cause: new NisshiProduceError({ topic, cause: error }),
+                }),
+            );
+            if (!Cause.isFailType(cause)) return Effect.failCause(original);
+            return rollback(streamName, expectedVersion, events.length).pipe(
+              Effect.uninterruptible,
+              Effect.matchCauseEffect({
+                onSuccess: () => Effect.failCause(original),
+                onFailure: (cleanup) =>
+                  Effect.failCause(
+                    Cause.sequential(
+                      original,
+                      Cause.map(
+                        cleanup,
+                        (error) =>
+                          new PersistenceError({ operation: "nisshi.rollback", cause: error }),
+                      ),
+                    ),
+                  ),
+              }),
+            );
+          }),
         );
         // Confirm: events are durable in the topic. A failure here leaves the
         // pending rows for the relay; a re-produce duplicates are tolerated
@@ -235,8 +269,13 @@ const make = (
           WHERE stream_name = ${streamName}
             AND version > ${expectedVersion} AND version <= ${expectedVersion + events.length}
         `.pipe(
-          Effect.catchTag("SqlError", (error) =>
-            Effect.logWarning(`pending confirm failed: ${error.message}`),
+          Effect.mapError((cause) => new PersistenceError({ operation: "nisshi.confirm", cause })),
+          Effect.catchAllCause((cause) =>
+            Cause.isFailType(cause)
+              ? Effect.logWarning(
+                  "Nisshi pending confirmation failed; relay will recover pending rows",
+                )
+              : Effect.failCause(cause),
           ),
         );
         return result;
@@ -250,12 +289,18 @@ const make = (
         readonly value: Uint8Array;
       }>;
     }
-    const readTopic = (fromOffset: bigint): Effect.Effect<TopicRecords> =>
+    const readTopic = (fromOffset: bigint): Effect.Effect<TopicRecords, PersistenceError> =>
       Effect.gen(function* () {
         const out: { offset: bigint; streamName: string; value: Uint8Array }[] = [];
         let offset = fromOffset;
         for (;;) {
-          const page = yield* client.fetch(topic, offset, maxBytes).pipe(Effect.orDie);
+          const page = yield* client
+            .fetch(topic, offset, maxBytes)
+            .pipe(
+              Effect.mapError(
+                (cause) => new PersistenceError({ operation: "nisshi.fetch", cause }),
+              ),
+            );
           if (page.records.length === 0) {
             return { records: out };
           }
@@ -278,24 +323,29 @@ const make = (
       append,
       read: (streamName, readOptions) =>
         Stream.unwrap(
-          Effect.map(readTopic(0n), ({ records }) => {
-            const fromVersion = readOptions?.fromVersion ?? 1;
-            const seen = new Set<number>();
-            const events: StoredEvent[] = [];
-            for (const record of records) {
-              if (record.streamName !== streamName) {
-                continue;
-              }
-              const envelope = decodeWireEvent(record.value, streamName);
-              if (envelope.version < fromVersion || seen.has(envelope.version)) {
-                continue;
+          Effect.flatMap(readTopic(0n), ({ records }) =>
+            Effect.gen(function* () {
+              const fromVersion = readOptions?.fromVersion ?? 1;
+              const seen = new Set<number>();
+              const events: StoredEvent[] = [];
+              for (const record of records) {
+                if (record.streamName !== streamName) {
+                  continue;
+                }
+                const envelope = yield* Effect.try({
+                  try: () => decodeWireEvent(record.value, streamName),
+                  catch: (cause) => new PersistenceError({ operation: "nisshi.decode", cause }),
+                });
+                if (envelope.version < fromVersion || seen.has(envelope.version)) {
+                  continue;
+                }
+                seen.add(envelope.version);
+                events.push(storedEvent(record.offset, streamName, envelope));
               }
-              seen.add(envelope.version);
-              events.push(storedEvent(record.offset, streamName, envelope));
-            }
-            events.sort((a, b) => a.version - b.version);
-            return Stream.fromIterable(events);
-          }),
+              events.sort((a, b) => a.version - b.version);
+              return Stream.fromIterable(events);
+            }),
+          ),
         ),
       readAll: (readOptions) =>
         Stream.unwrap(
@@ -315,7 +365,13 @@ const make = (
               if (limit !== undefined && events.length >= limit) {
                 break;
               }
-              const page = yield* client.fetch(topic, offset, maxBytes).pipe(Effect.orDie);
+              const page = yield* client
+                .fetch(topic, offset, maxBytes)
+                .pipe(
+                  Effect.mapError(
+                    (cause) => new PersistenceError({ operation: "nisshi.fetch", cause }),
+                  ),
+                );
               if (page.records.length === 0) {
                 break;
               }
@@ -324,7 +380,10 @@ const make = (
                 const event = storedEvent(
                   record.offset,
                   streamName,
-                  decodeWireEvent(record.value, streamName),
+                  yield* Effect.try({
+                    try: () => decodeWireEvent(record.value, streamName),
+                    catch: (cause) => new PersistenceError({ operation: "nisshi.decode", cause }),
+                  }),
                 );
                 if (keep(event)) {
                   events.push(event);

package/src/Stores.ts

@@ -1,4 +1,5 @@
 import * as SqlClient from "@effect/sql/SqlClient";
+import { PersistenceError } from "@structure-ai/domain";
 import {
   CheckpointStore,
   type CheckpointStoreService,
@@ -29,26 +30,44 @@ export const snapshotStoreLayer = (
       const sql = yield* SqlClient.SqlClient;
       const service: SnapshotStoreService = {
         load: (streamName) =>
-          Effect.map(
+          Effect.flatMap(
             sql<SnapshotRow>`
               SELECT state, version FROM ${sql(tables.snapshots)} WHERE stream_name = ${streamName}
-            `.pipe(Effect.orDie),
-            (rows): Option.Option<{ state: unknown; version: number }> => {
-              const row = rows[0];
-              return row === undefined
-                ? Option.none()
-                : Option.some({
-                    state: JSON.parse(row.state) as unknown,
-                    version: Number(row.version),
-                  });
-            },
+            `.pipe(
+              Effect.mapError(
+                (cause) => new PersistenceError({ operation: "nisshi-sidecar", cause }),
+              ),
+            ),
+            (rows) =>
+              Effect.try({
+                try: (): Option.Option<{ state: unknown; version: number }> => {
+                  const row = rows[0];
+                  return row === undefined
+                    ? Option.none()
+                    : Option.some({
+                        state: JSON.parse(row.state) as unknown,
+                        version: Number(row.version),
+                      });
+                },
+                catch: (cause) => new PersistenceError({ operation: "snapshot.decode", cause }),
+              }),
           ),
         save: (streamName, snapshot) =>
-          Effect.asVoid(sql`
+          Effect.gen(function* () {
+            const encoded = yield* Effect.try({
+              try: () => JSON.stringify(snapshot.state ?? null),
+              catch: (cause) => new PersistenceError({ operation: "snapshot.encode", cause }),
+            });
+            return yield* Effect.asVoid(sql`
             INSERT INTO ${sql(tables.snapshots)} (stream_name, state, version)
-            VALUES (${streamName}, ${JSON.stringify(snapshot.state ?? null)}, ${snapshot.version})
+            VALUES (${streamName}, ${encoded}, ${snapshot.version})
             ON CONFLICT (stream_name) DO UPDATE SET state = excluded.state, version = excluded.version
-          `).pipe(Effect.orDie),
+          `).pipe(
+              Effect.mapError(
+                (cause) => new PersistenceError({ operation: "nisshi-sidecar", cause }),
+              ),
+            );
+          }),
       };
       return SnapshotStore.of(service);
     }),
@@ -69,7 +88,11 @@ export const checkpointStoreLayer = (
           Effect.map(
             sql<{ readonly position: number | bigint | string | null }>`
               SELECT position FROM ${sql(tables.checkpoints)} WHERE name = ${name}
-            `.pipe(Effect.orDie),
+            `.pipe(
+              Effect.mapError(
+                (cause) => new PersistenceError({ operation: "nisshi-sidecar", cause }),
+              ),
+            ),
             (rows) => toBigInt(rows[0]?.position),
           ),
         save: (name, position) =>
@@ -77,7 +100,11 @@ export const checkpointStoreLayer = (
             INSERT INTO ${sql(tables.checkpoints)} (name, position)
             VALUES (${name}, ${position})
             ON CONFLICT (name) DO UPDATE SET position = excluded.position
-          `).pipe(Effect.orDie),
+          `).pipe(
+            Effect.mapError(
+              (cause) => new PersistenceError({ operation: "nisshi-sidecar", cause }),
+            ),
+          ),
       };
       return CheckpointStore.of(service);
     }),
@@ -99,7 +126,11 @@ export const inboxLayer = (
             sql<{ readonly one: number }>`
               SELECT 1 AS one FROM ${sql(tables.inbox)}
               WHERE consumer_id = ${consumerId} AND message_id = ${messageId}
-            `.pipe(Effect.orDie),
+            `.pipe(
+              Effect.mapError(
+                (cause) => new PersistenceError({ operation: "nisshi-sidecar", cause }),
+              ),
+            ),
             (rows) => rows.length > 0,
           ),
         markProcessed: (consumerId, messageId) =>
@@ -107,7 +138,11 @@ export const inboxLayer = (
             INSERT INTO ${sql(tables.inbox)} (consumer_id, message_id)
             VALUES (${consumerId}, ${messageId})
             ON CONFLICT DO NOTHING
-          `).pipe(Effect.orDie),
+          `).pipe(
+            Effect.mapError(
+              (cause) => new PersistenceError({ operation: "nisshi-sidecar", cause }),
+            ),
+          ),
       };
       return Inbox.of(service);
     }),

package/src/layer.ts

@@ -2,6 +2,7 @@ import type * as SqlClient from "@effect/sql/SqlClient";
 import type { SqlError } from "@effect/sql/SqlError";
 import { PgClient } from "@effect/sql-pg";
 import { SqliteClient } from "@effect/sql-sqlite-bun";
+import { PersistenceError } from "@structure-ai/domain";
 import type {
   CheckpointStore,
   EventStore,
@@ -82,7 +83,7 @@ export const layerPg = (
   options: NisshiPgConfig,
 ): Layer.Layer<
   StoreServices | PgClient.PgClient | SqlClient.SqlClient | NisshiClient,
-  SqlError | NisshiConnectionError
+  SqlError | NisshiConnectionError | PersistenceError
 > => {
   const client = nisshiClientLayer({
     brokerUrl: options.brokerUrl,
@@ -101,7 +102,13 @@ export const layerPg = (
       ? client
       : Layer.effectDiscard(
           Effect.flatMap(NisshiClient, (c) =>
-            c.ensureTopic(options.topic ?? "events").pipe(Effect.orDie),
+            c
+              .ensureTopic(options.topic ?? "events")
+              .pipe(
+                Effect.mapError(
+                  (cause) => new PersistenceError({ operation: "nisshi.ensure-topic", cause }),
+                ),
+              ),
           ),
         ).pipe(Layer.provideMerge(client));
   return storesLayer(options).pipe(Layer.provideMerge(migrated), Layer.provideMerge(ensured));
@@ -116,7 +123,7 @@ export const layer = (
   options: NisshiAdaptersConfig,
 ): Layer.Layer<
   StoreServices | SqliteClient.SqliteClient | SqlClient.SqlClient | NisshiClient,
-  ConfigError | SqlError | NisshiConnectionError
+  ConfigError | SqlError | NisshiConnectionError | PersistenceError
 > => {
   const client = nisshiClientLayer({
     brokerUrl: options.brokerUrl,
@@ -130,7 +137,13 @@ export const layer = (
       ? client
       : Layer.effectDiscard(
           Effect.flatMap(NisshiClient, (c) =>
-            c.ensureTopic(options.topic ?? "events").pipe(Effect.orDie),
+            c
+              .ensureTopic(options.topic ?? "events")
+              .pipe(
+                Effect.mapError(
+                  (cause) => new PersistenceError({ operation: "nisshi.ensure-topic", cause }),
+                ),
+              ),
           ),
         ).pipe(Layer.provideMerge(client));
   return storesLayer(options).pipe(Layer.provideMerge(migrated), Layer.provideMerge(ensured));

package/src/relay.ts

@@ -1,6 +1,6 @@
 import * as SqlClient from "@effect/sql/SqlClient";
 import type { SqlError } from "@effect/sql/SqlError";
-import { Effect } from "effect";
+import { Cause, Effect } from "effect";
 import { NisshiClient } from "./protocol/client.js";
 import type { NisshiApiError, NisshiConnectionError } from "./protocol/errors.js";
 import { type SidecarOptions, sidecarTables } from "./sidecar.js";
@@ -64,13 +64,24 @@ export const drainPending = (
 /** Runs `drainPending` forever, sleeping `pollInterval` between passes. */
 export const runPendingRelay = (
   options?: RelayOptions & { readonly pollInterval?: number },
-): Effect.Effect<never, never, SqlClient.SqlClient | NisshiClient> =>
+): Effect.Effect<
+  never,
+  NisshiApiError | NisshiConnectionError | SqlError,
+  SqlClient.SqlClient | NisshiClient
+> =>
   Effect.gen(function* () {
     const interval = options?.pollInterval ?? 500;
     for (;;) {
       yield* drainPending(options).pipe(
-        Effect.ignore,
-        Effect.catchAllDefect(() => Effect.void),
+        Effect.catchAllCause((cause) => {
+          if (
+            Cause.isFailType(cause) &&
+            (cause.error._tag === "NisshiConnectionError" ||
+              (cause.error._tag === "NisshiApiError" && cause.error.retriable))
+          )
+            return Effect.logWarning("Nisshi relay will retry a transient broker failure");
+          return Effect.failCause(cause);
+        }),
       );
       yield* Effect.sleep(interval);
     }