@structure-ai/eventsourcing-nisshi 0.0.10

package/src/index.ts

@@ -0,0 +1,44 @@
+export { type EventStoreOptions, eventStoreLayer } from "./EventStore.js";
+export {
+  decodeWireEvent,
+  encodeWireEvent,
+  validateWireEvent,
+  type WireEvent,
+} from "./envelope.js";
+export {
+  layer,
+  layerPg,
+  type NisshiAdaptersConfig,
+  type NisshiPgConfig,
+  type StoreServices,
+  storesLayer,
+} from "./layer.js";
+export {
+  decodeRecordBatch,
+  encodeRecordBatch,
+  type FetchedRecord,
+  type RecordToProduce,
+} from "./protocol/batch.js";
+export {
+  type FetchPage,
+  NisshiClient,
+  type NisshiClientService,
+  nisshiClientLayer,
+} from "./protocol/client.js";
+export {
+  ApiKey,
+  type NisshiConnection,
+  openConnection,
+  PinnedVersion,
+} from "./protocol/connection.js";
+export {
+  NisshiApiError,
+  NisshiConnectionError,
+  NisshiProduceError,
+  NisshiProtocolError,
+  NisshiTopicConfigurationError,
+} from "./protocol/errors.js";
+export { drainPending, type RelayOptions, runPendingRelay } from "./relay.js";
+export { checkpointStoreLayer, inboxLayer, snapshotStoreLayer } from "./Stores.js";
+export { envelopeJsonSchema, writeSchemaFiles } from "./schemas.js";
+export { migrate, type SidecarOptions, type SidecarTables, sidecarTables } from "./sidecar.js";

package/src/layer.ts

@@ -0,0 +1,137 @@
+import type * as SqlClient from "@effect/sql/SqlClient";
+import type { SqlError } from "@effect/sql/SqlError";
+import { PgClient } from "@effect/sql-pg";
+import { SqliteClient } from "@effect/sql-sqlite-bun";
+import type {
+  CheckpointStore,
+  EventStore,
+  Inbox,
+  SnapshotStore,
+} from "@structure-ai/eventsourcing";
+import { Effect, Layer, Redacted } from "effect";
+import type { ConfigError } from "effect/ConfigError";
+import { type EventStoreOptions, eventStoreLayer } from "./EventStore.js";
+import { NisshiClient, nisshiClientLayer } from "./protocol/client.js";
+import type { NisshiConnectionError } from "./protocol/errors.js";
+import { checkpointStoreLayer, inboxLayer, snapshotStoreLayer } from "./Stores.js";
+import { migrate } from "./sidecar.js";
+
+/**
+ * The four ports this package implements. There is no `Outbox` by design:
+ * the event topic itself is the publication (ADR-0015); apps needing
+ * arbitrary notifications run consumers that produce derived topics.
+ */
+export type StoreServices = EventStore | SnapshotStore | CheckpointStore | Inbox;
+
+/**
+ * All adapters over an existing `SqlClient` (the sidecar) and an existing
+ * `NisshiClient`. Run `migrate` first, or use `layer` or `layerPg`, which do.
+ */
+export const storesLayer = (
+  options?: EventStoreOptions,
+): Layer.Layer<StoreServices, never, SqlClient.SqlClient | NisshiClient> =>
+  Layer.mergeAll(
+    eventStoreLayer(options),
+    snapshotStoreLayer(options),
+    checkpointStoreLayer(options),
+    inboxLayer(options),
+  );
+
+/** Configuration for the all-in-one `layer`: broker plus sqlite sidecar. */
+export interface NisshiAdaptersConfig extends EventStoreOptions {
+  /** Broker listener, e.g. `tcp://127.0.0.1:9092`. Must equal the broker's advertised listener. */
+  readonly brokerUrl: string;
+  /** Client id sent with every request (default `structure-nisshi`). */
+  readonly clientId?: string;
+  /** Per-request timeout in millis (default 10 000). */
+  readonly timeoutMillis?: number;
+  /** Sidecar database file, or `":memory:"` for an in-memory sidecar. */
+  readonly filename: string;
+  /** Create the topic (single partition) at layer start when missing (default true). */
+  readonly createTopic?: boolean;
+}
+
+/** Configuration for the all-in-one `layerPg`: broker plus PostgreSQL sidecar. */
+export interface NisshiPgConfig extends EventStoreOptions {
+  /** Broker listener, e.g. `tcp://127.0.0.1:9092`. Must equal the broker's advertised listener. */
+  readonly brokerUrl: string;
+  /** Client id sent with every request (default `structure-nisshi`). */
+  readonly clientId?: string;
+  /** Per-request timeout in millis (default 10 000). */
+  readonly timeoutMillis?: number;
+  /**
+   * Postgres connection URL. Defaults to the `DATABASE_URL` environment
+   * variable; when neither is set, the client falls back to libpq-style
+   * defaults (localhost:5432, OS user).
+   */
+  readonly url?: string;
+  /** Maximum pool connections (driver default when omitted). */
+  readonly maxConnections?: number;
+  /** `application_name` reported to the server. */
+  readonly applicationName?: string;
+  /** Create the topic (single partition) at layer start when missing (default true). */
+  readonly createTopic?: boolean;
+}
+
+/**
+ * Everything in one layer over a PostgreSQL sidecar: broker connection, pg
+ * `SqlClient` (schema migration runs at layer build), topic verification,
+ * and the four port adapters. The sidecar client is exposed too.
+ */
+export const layerPg = (
+  options: NisshiPgConfig,
+): Layer.Layer<
+  StoreServices | PgClient.PgClient | SqlClient.SqlClient | NisshiClient,
+  SqlError | NisshiConnectionError
+> => {
+  const client = nisshiClientLayer({
+    brokerUrl: options.brokerUrl,
+    clientId: options.clientId,
+    timeoutMillis: options.timeoutMillis,
+  });
+  const url = options.url ?? process.env.DATABASE_URL;
+  const sidecar = PgClient.layer({
+    ...(url !== undefined ? { url: Redacted.make(url) } : {}),
+    ...(options.maxConnections !== undefined ? { maxConnections: options.maxConnections } : {}),
+    ...(options.applicationName !== undefined ? { applicationName: options.applicationName } : {}),
+  });
+  const migrated = Layer.effectDiscard(migrate(options)).pipe(Layer.provideMerge(sidecar));
+  const ensured =
+    options.createTopic === false
+      ? client
+      : Layer.effectDiscard(
+          Effect.flatMap(NisshiClient, (c) =>
+            c.ensureTopic(options.topic ?? "events").pipe(Effect.orDie),
+          ),
+        ).pipe(Layer.provideMerge(client));
+  return storesLayer(options).pipe(Layer.provideMerge(migrated), Layer.provideMerge(ensured));
+};
+/**
+ * Everything in one layer over a sqlite sidecar: broker connection, sqlite
+ * `SqlClient` (schema migration runs at layer build), topic verification,
+ * and the four port adapters. The sidecar `SqlClient` is exposed for
+ * callers' own queries. For a PostgreSQL sidecar use `layerPg`.
+ */
+export const layer = (
+  options: NisshiAdaptersConfig,
+): Layer.Layer<
+  StoreServices | SqliteClient.SqliteClient | SqlClient.SqlClient | NisshiClient,
+  ConfigError | SqlError | NisshiConnectionError
+> => {
+  const client = nisshiClientLayer({
+    brokerUrl: options.brokerUrl,
+    clientId: options.clientId,
+    timeoutMillis: options.timeoutMillis,
+  });
+  const sidecar = SqliteClient.layer({ filename: options.filename });
+  const migrated = Layer.effectDiscard(migrate(options)).pipe(Layer.provideMerge(sidecar));
+  const ensured =
+    options.createTopic === false
+      ? client
+      : Layer.effectDiscard(
+          Effect.flatMap(NisshiClient, (c) =>
+            c.ensureTopic(options.topic ?? "events").pipe(Effect.orDie),
+          ),
+        ).pipe(Layer.provideMerge(client));
+  return storesLayer(options).pipe(Layer.provideMerge(migrated), Layer.provideMerge(ensured));
+};

package/src/protocol/batch.ts

@@ -0,0 +1,126 @@
+import { crc32c, Reader, Writer } from "./primitives.js";
+
+/** One record to produce: byte key (may be null) and byte value. */
+export interface RecordToProduce {
+  readonly key: Uint8Array | null;
+  readonly value: Uint8Array;
+}
+
+/** One record read back: its absolute offset plus key and value bytes. */
+export interface FetchedRecord {
+  readonly offset: bigint;
+  readonly key: Uint8Array | null;
+  readonly value: Uint8Array;
+}
+
+/**
+ * Encodes one uncompressed message-format-v2 record batch. Offsets inside the
+ * batch are `0..n-1` deltas; the broker assigns the base offset. The CRC
+ * covers everything from attributes to the end of the batch.
+ */
+export const encodeRecordBatch = (records: ReadonlyArray<RecordToProduce>): Uint8Array => {
+  if (records.length === 0) {
+    throw new Error("kafka protocol: empty record batch");
+  }
+  const now = BigInt(Date.now());
+  const encoded = new Writer();
+  for (let i = 0; i < records.length; i++) {
+    const record = records[i];
+    if (record === undefined) {
+      continue;
+    }
+    const body = new Writer()
+      .i8(0) // record attributes (unused, must be 0)
+      .varint(0) // timestamp delta
+      .varint(i); // offset delta
+    if (record.key === null) {
+      body.varint(-1);
+    } else {
+      body.varint(record.key.length).raw(record.key);
+    }
+    body.varint(record.value.length).raw(record.value).varint(0); // headers
+    const bodyBytes = body.out();
+    encoded.varint(bodyBytes.length).raw(bodyBytes);
+  }
+
+  const batchBody = new Writer()
+    .i16(0) // attributes: no compression, no flags
+    .i32(records.length - 1) // lastOffsetDelta
+    .i64(now) // firstTimestamp
+    .i64(now) // maxTimestamp
+    .i64(-1) // producerId
+    .i16(-1) // producerEpoch
+    .i32(-1) // baseSequence
+    .i32(records.length)
+    .raw(encoded.out());
+  const bodyBytes = batchBody.out();
+
+  return new Writer()
+    .i64(0) // baseOffset (assigned by the broker)
+    .i32(4 + 1 + 4 + bodyBytes.length) // batchLength: from partitionLeaderEpoch on
+    .i32(-1) // partitionLeaderEpoch
+    .i8(2) // magic
+    .i32(crc32c(bodyBytes))
+    .raw(bodyBytes)
+    .out();
+};
+
+/**
+ * Decodes every record of one message-format-v2 batch starting at `buf[0]`,
+ * projecting each record's absolute offset from the batch base offset.
+ * Batches from Nisshi are always uncompressed; a compressed batch is an
+ * error rather than a silent skip.
+ */
+export const decodeRecordBatch = (buf: Uint8Array): ReadonlyArray<FetchedRecord> => {
+  if (buf.length === 0) {
+    return []; // Nisshi encodes "no records" as a zero-length field
+  }
+  const r = new Reader(buf);
+  const baseOffset = r.i64();
+  r.i32(); // batchLength
+  r.i32(); // partitionLeaderEpoch
+  const magic = r.i8();
+  if (magic !== 2) {
+    throw new Error(`kafka protocol: unsupported record batch magic ${magic}`);
+  }
+  r.u32(); // crc
+  const attributes = r.i16();
+  if ((attributes & 0x07) !== 0) {
+    throw new Error("kafka protocol: compressed record batches are not supported");
+  }
+  r.i32(); // lastOffsetDelta
+  r.i64(); // firstTimestamp
+  r.i64(); // maxTimestamp
+  r.i64(); // producerId
+  r.i16(); // producerEpoch
+  r.i32(); // baseSequence
+  const count = r.i32();
+  const out: FetchedRecord[] = [];
+  for (let i = 0; i < count; i++) {
+    const bodyLength = r.varint();
+    const end = r.pos + bodyLength;
+    r.i8(); // attributes
+    r.varint(); // timestamp delta
+    const offsetDelta = r.varint();
+    const keyLength = r.varint();
+    const key = keyLength < 0 ? null : buf.subarray(r.pos, r.pos + keyLength);
+    if (keyLength >= 0) {
+      r.pos += keyLength;
+    }
+    const valueLength = r.varint();
+    const value = buf.subarray(r.pos, r.pos + valueLength);
+    r.pos += valueLength;
+    const headers = r.varint();
+    for (let h = 0; h < headers; h++) {
+      const headerKeyLength = r.varint();
+      r.pos += headerKeyLength;
+      const headerValueLength = r.varint();
+      if (headerValueLength >= 0) {
+        r.pos += headerValueLength;
+      }
+    }
+    out.push({ offset: baseOffset + BigInt(offsetDelta), key, value });
+    r.pos = end;
+  }
+  return out;
+};

package/src/protocol/client.ts

@@ -0,0 +1,340 @@
+import { Context, Effect, Layer } from "effect";
+import {
+  decodeRecordBatch,
+  encodeRecordBatch,
+  type FetchedRecord,
+  type RecordToProduce,
+} from "./batch.js";
+import { ApiKey, type NisshiConnection, openConnection, PinnedVersion } from "./connection.js";
+import {
+  apiError,
+  NisshiApiError,
+  type NisshiConnectionError,
+  NisshiTopicConfigurationError,
+} from "./errors.js";
+import { Reader } from "./primitives.js";
+
+/** Kafka error codes this client interprets specially. */
+const ErrorCode = {
+  none: 0,
+  unknownTopicOrPartition: 3,
+  topicAlreadyExists: 36,
+} as const;
+
+/** One page of fetch results plus the partition high watermark (log end offset). */
+export interface FetchPage {
+  readonly records: ReadonlyArray<FetchedRecord>;
+  /** Offset of the next record to be written — read up to `highWatermark - 1`. */
+  readonly highWatermark: bigint;
+}
+
+/** The wire-level client: single broker, single partition per topic. */
+export interface NisshiClientService {
+  /** Creates `topic` with one partition; an existing topic is fine if it has exactly one. */
+  readonly ensureTopic: (
+    topic: string,
+  ) => Effect.Effect<void, NisshiTopicConfigurationError | NisshiApiError | NisshiConnectionError>;
+  /** Lists topic names known to the broker. */
+  readonly listTopics: () => Effect.Effect<
+    ReadonlyArray<string>,
+    NisshiApiError | NisshiConnectionError
+  >;
+  /**
+   * Produces `records` in one batch with acks=all; returns the base offset the
+   * broker assigned. Records are ordered within the batch.
+   */
+  readonly produce: (
+    topic: string,
+    records: ReadonlyArray<RecordToProduce>,
+  ) => Effect.Effect<bigint, NisshiApiError | NisshiConnectionError>;
+  /**
+   * Reads one page of records starting at `offset` (inclusive), up to
+   * `maxBytes` of batch payload. Returns an empty page when caught up.
+   */
+  readonly fetch: (
+    topic: string,
+    offset: bigint,
+    maxBytes: number,
+  ) => Effect.Effect<FetchPage, NisshiApiError | NisshiConnectionError>;
+  /**
+   * The log end offset: the offset the *next* record will get. Note Nisshi's
+   * ListOffsets(LATEST) reports the last written record's offset — this
+   * method normalizes the difference (and is covered by tests).
+   */
+  readonly endOffset: (
+    topic: string,
+  ) => Effect.Effect<bigint, NisshiApiError | NisshiConnectionError>;
+}
+
+/** Service tag for the Nisshi wire client. */
+export class NisshiClient extends Context.Tag("@structure-ai/eventsourcing-nisshi/NisshiClient")<
+  NisshiClient,
+  NisshiClientService
+>() {}
+
+const make = (brokerUrl: string, clientId: string, timeoutMillis: number) =>
+  Effect.gen(function* () {
+    const connection: NisshiConnection = yield* openConnection(brokerUrl, clientId);
+    const { request } = connection;
+
+    const partitionCount = (topic: string) =>
+      Effect.map(
+        request(
+          ApiKey.metadata,
+          PinnedVersion.metadata,
+          (w) => {
+            w.i32(1); // one requested topic
+            w.str(topic);
+          },
+          timeoutMillis,
+        ),
+        (body): number => {
+          const r = new Reader(body);
+          const brokers = r.i32();
+          for (let i = 0; i < brokers; i++) {
+            r.i32();
+            r.str();
+            r.i32();
+          }
+          const topics = r.i32();
+          for (let i = 0; i < topics; i++) {
+            const errorCode = r.i16();
+            const name = r.str();
+            const partitions = r.i32();
+            const count = partitions;
+            for (let p = 0; p < partitions; p++) {
+              r.i16(); // partition_error_code
+              r.i32(); // partition_id
+              r.i32(); // leader_id
+              const replicas = r.i32();
+              r.pos += replicas * 4;
+              const isr = r.i32();
+              r.pos += isr * 4;
+            }
+            if (errorCode === ErrorCode.none && name === topic) {
+              return count;
+            }
+          }
+          return -1;
+        },
+      );
+
+    const ensureTopic: NisshiClientService["ensureTopic"] = (topic) =>
+      Effect.gen(function* () {
+        // Create FIRST, verify second: requesting Metadata for an unknown
+        // topic makes Nisshi auto-create it — with FOUR partitions. Never
+        // probe for a topic that may not exist.
+        const body = yield* request(
+          ApiKey.createTopics,
+          PinnedVersion.createTopics,
+          (w) => {
+            w.i32(1); // one topic
+            w.str(topic);
+            w.i32(1); // num_partitions
+            w.i16(1); // replication_factor
+            w.i32(0); // assignments
+            w.i32(0); // configs
+            w.i32(timeoutMillis); // timeout
+            w.i8(0); // validate_only = false
+          },
+          timeoutMillis,
+        );
+        const r = new Reader(body);
+        // Nisshi's CreateTopics v4 response leads with throttle_time_ms.
+        r.i32(); // throttle_time_ms
+        const count = r.i32();
+        for (let i = 0; i < count; i++) {
+          const name = r.str();
+          const code = r.i16();
+          r.str(); // message
+          if (name === topic && code !== ErrorCode.none && code !== ErrorCode.topicAlreadyExists) {
+            return yield* Effect.fail(
+              apiError(code, `CreateTopics(${topic}) failed with code ${code}`),
+            );
+          }
+        }
+        const created = yield* partitionCount(topic);
+        if (created !== 1) {
+          return yield* Effect.fail(
+            new NisshiTopicConfigurationError({
+              topic,
+              reason: `topic reports ${created} partitions; the single-partition contract requires exactly 1 (see ADR-0015)`,
+            }),
+          );
+        }
+      });
+
+    const listTopics: NisshiClientService["listTopics"] = () =>
+      Effect.map(
+        request(
+          ApiKey.metadata,
+          PinnedVersion.metadata,
+          (w) => {
+            w.i32(-1); // all topics
+          },
+          timeoutMillis,
+        ),
+        (body): ReadonlyArray<string> => {
+          const r = new Reader(body);
+          const brokers = r.i32();
+          for (let i = 0; i < brokers; i++) {
+            r.i32();
+            r.str();
+            r.i32();
+          }
+          const topics = r.i32();
+          const names: string[] = [];
+          for (let i = 0; i < topics; i++) {
+            const errorCode = r.i16();
+            const name = r.str() ?? "?";
+            const partitions = r.i32();
+            for (let p = 0; p < partitions; p++) {
+              r.i16(); // partition_error_code
+              r.i32(); // partition_id
+              r.i32(); // leader_id
+              const replicas = r.i32();
+              r.pos += replicas * 4;
+              const isr = r.i32();
+              r.pos += isr * 4;
+            }
+            if (errorCode === ErrorCode.none) {
+              names.push(name);
+            }
+          }
+          return names;
+        },
+      );
+
+    const produce: NisshiClientService["produce"] = (topic, records) =>
+      Effect.gen(function* () {
+        const batch = encodeRecordBatch(records);
+        const body = yield* request(
+          ApiKey.produce,
+          PinnedVersion.produce,
+          (w) => {
+            w.str(null); // transactional id
+            w.i16(-1); // acks = all
+            w.i32(timeoutMillis);
+            w.i32(1); // one topic
+            w.str(topic);
+            w.i32(1); // one partition
+            w.i32(0); // partition 0 — single-partition contract
+            w.bytes(batch);
+          },
+          timeoutMillis,
+        );
+        // Produce v3 response: [topics [partitions]] then trailing throttle.
+        const r = new Reader(body);
+        const topics = r.i32();
+        for (let i = 0; i < topics; i++) {
+          const name = r.str();
+          const partitions = r.i32();
+          for (let p = 0; p < partitions; p++) {
+            r.i32(); // partition index
+            const code = r.i16();
+            const baseOffset = r.i64();
+            r.i64(); // log_append_time
+            if (name === topic) {
+              if (code !== ErrorCode.none) {
+                return yield* Effect.fail(
+                  apiError(code, `Produce(${topic}) failed with code ${code}`),
+                );
+              }
+              return baseOffset;
+            }
+          }
+        }
+        return yield* Effect.fail(
+          new NisshiApiError({
+            code: -1,
+            message: `Produce(${topic}): no partition in response`,
+            retriable: true,
+          }),
+        );
+      });
+
+    const fetch: NisshiClientService["fetch"] = (topic, offset, maxBytes) =>
+      Effect.gen(function* () {
+        const body = yield* request(
+          ApiKey.fetch,
+          PinnedVersion.fetch,
+          (w) => {
+            w.i32(-1); // replica id
+            w.i32(250); // max wait (ms)
+            w.i32(1); // min bytes
+            w.i32(maxBytes); // max bytes
+            w.i8(0); // isolation level
+            w.i32(1); // one topic
+            w.str(topic);
+            w.i32(1); // one partition
+            w.i32(0); // partition 0
+            w.i64(offset);
+            w.i32(maxBytes);
+          },
+          timeoutMillis,
+        );
+        // Fetch v4 response: throttle, [topics [partitions]], records last.
+        const r = new Reader(body);
+        r.i32(); // throttle_time_ms
+        const topics = r.i32();
+        for (let i = 0; i < topics; i++) {
+          const name = r.str();
+          const partitions = r.i32();
+          for (let p = 0; p < partitions; p++) {
+            r.i32(); // partition index
+            const code = r.i16();
+            const highWatermark = r.i64();
+            r.i64(); // last stable offset
+            const aborted = r.i32();
+            if (aborted > 0) {
+              for (let a = 0; a < aborted; a++) {
+                r.i64();
+                r.i32();
+              }
+            }
+            const records = r.bytes();
+            if (name !== topic) {
+              continue;
+            }
+            if (code === ErrorCode.unknownTopicOrPartition) {
+              return { records: [], highWatermark: 0n };
+            }
+            if (code !== ErrorCode.none) {
+              return yield* Effect.fail(apiError(code, `Fetch(${topic}) failed with code ${code}`));
+            }
+            // Kafka returns the WHOLE batch containing the requested offset;
+            // drop the leading records below it.
+            const decoded = records === null ? [] : decodeRecordBatch(records);
+            return {
+              highWatermark,
+              records: decoded.filter((record) => record.offset >= offset),
+            };
+          }
+        }
+        return { records: [], highWatermark: 0n };
+      });
+
+    const endOffset: NisshiClientService["endOffset"] = (topic) =>
+      Effect.map(fetch(topic, 0n, 1), (page) =>
+        // An empty partition reports a high watermark of 1 on Nisshi — treat
+        // "no records at offset 0" as the true log end (0).
+        page.records.length === 0 ? 0n : page.highWatermark,
+      );
+
+    return { ensureTopic, listTopics, produce, fetch, endOffset } satisfies NisshiClientService;
+  });
+
+/** Layer over one broker connection; the socket closes with the layer scope. */
+export const nisshiClientLayer = (options: {
+  readonly brokerUrl: string;
+  readonly clientId?: string | undefined;
+  readonly timeoutMillis?: number | undefined;
+}): Layer.Layer<NisshiClient, NisshiConnectionError> =>
+  Layer.scoped(
+    NisshiClient,
+    make(
+      options.brokerUrl,
+      options.clientId ?? "structure-nisshi",
+      options.timeoutMillis ?? 10_000,
+    ),
+  );