@structure-ai/eventsourcing-nisshi 0.0.10

package/src/protocol/connection.ts

@@ -0,0 +1,225 @@
+import { Socket } from "node:net";
+import { Deferred, Effect, Ref } from "effect";
+import type { Scope } from "effect/Scope";
+import { NisshiConnectionError } from "./errors.js";
+import { Reader, Writer } from "./primitives.js";
+
+/** Kafka API keys this client speaks. */
+export const ApiKey = {
+  produce: 0,
+  fetch: 1,
+  listOffsets: 2,
+  metadata: 3,
+  createTopics: 19,
+  apiVersions: 18,
+} as const;
+
+/**
+ * Protocol versions we pin — non-flexible framing, the simplest shape per
+ * API. Verified against Nisshi v0.7.0-pre.2. Notably CreateTopics v0 wedges
+ * the broker (missing assignments array); v4 is the safe choice.
+ */
+export const PinnedVersion = {
+  produce: 3,
+  fetch: 4,
+  listOffsets: 1,
+  metadata: 0,
+  createTopics: 4,
+  apiVersions: 0,
+} as const;
+
+/** A live, handshaked connection. Requests are serialized; responses match by correlation id. */
+export interface NisshiConnection {
+  /** API key → [minVersion, maxVersion] as reported by the broker. */
+  readonly versions: ReadonlyMap<number, readonly [number, number]>;
+  /** Sends one request, awaits its response body (correlation id stripped). */
+  readonly request: (
+    apiKey: number,
+    apiVersion: number,
+    build: (writer: Writer) => void,
+    timeoutMillis: number,
+  ) => Effect.Effect<Uint8Array, NisshiConnectionError>;
+}
+
+interface HostPort {
+  readonly host: string;
+  readonly port: number;
+}
+
+const parseUrl = (url: string): HostPort => {
+  const parsed = new URL(url);
+  const port = Number(parsed.port);
+  if (parsed.protocol !== "tcp:" || !parsed.hostname || Number.isNaN(port)) {
+    throw new Error(`nisshi: broker url must be tcp://host:port, got ${url}`);
+  }
+  return { host: parsed.hostname, port };
+};
+
+const assertVersion = (
+  versions: ReadonlyMap<number, readonly [number, number]>,
+  name: string,
+  key: number,
+  pinned: number,
+): void => {
+  const range = versions.get(key);
+  if (range === undefined) {
+    throw new NisshiConnectionError({ reason: `broker does not expose ${name} (api key ${key})` });
+  }
+  if (pinned < range[0] || pinned > range[1]) {
+    throw new NisshiConnectionError({
+      reason: `broker supports ${name} v${range[0]}-${range[1]}, client pins v${pinned}`,
+    });
+  }
+};
+
+/**
+ * Connects to `brokerUrl`, handshakes with ApiVersions, verifies every
+ * pinned version is offered, and returns the connection. The socket lives
+ * for the surrounding scope.
+ */
+export const openConnection = (
+  brokerUrl: string,
+  clientId: string,
+): Effect.Effect<NisshiConnection, NisshiConnectionError, Scope> =>
+  Effect.gen(function* () {
+    const { host, port } = yield* Effect.sync(() => parseUrl(brokerUrl)).pipe(
+      Effect.mapError(
+        (cause) => new NisshiConnectionError({ reason: `invalid broker url ${brokerUrl}`, cause }),
+      ),
+    );
+
+    const socket: Socket = yield* Effect.acquireRelease(
+      Effect.async<Socket, NisshiConnectionError>((resume, signal) => {
+        const s = new Socket();
+        const onAbort = () => s.destroy();
+        signal.addEventListener("abort", onAbort, { once: true });
+        s.once("connect", () => {
+          signal.removeEventListener("abort", onAbort);
+          resume(Effect.succeed(s));
+        });
+        s.once("error", (err) => {
+          signal.removeEventListener("abort", onAbort);
+          resume(Effect.fail(new NisshiConnectionError({ reason: "connect failed", cause: err })));
+        });
+        s.connect(port, host);
+      }),
+      (s) => Effect.sync(() => s.destroy()),
+    );
+
+    const pending = yield* Ref.make<
+      Map<number, Deferred.Deferred<Uint8Array, NisshiConnectionError>>
+    >(new Map());
+    let correlation = 0;
+
+    // Responses may split across TCP chunks; buffer until whole frames.
+    let receive = Buffer.alloc(0);
+    socket.on("data", (chunk: Buffer) => {
+      receive = Buffer.concat([receive, chunk]);
+      for (;;) {
+        if (receive.length < 4) {
+          return;
+        }
+        const size = receive.readInt32BE(0);
+        if (size < 4 || receive.length < 4 + size) {
+          return;
+        }
+        const body = new Uint8Array(receive.subarray(4, 4 + size));
+        receive = receive.subarray(4 + size);
+        const corr =
+          body.length >= 4 ? new DataView(body.buffer, body.byteOffset, 4).getInt32(0) : -1;
+        if (corr >= 0) {
+          takePending(pending, corr, (deferred) => {
+            Effect.runFork(Deferred.succeed(deferred, body.subarray(4)));
+          });
+        }
+      }
+    });
+    socket.on("close", () => failAll(pending, "connection closed"));
+    socket.on("error", (err) => failAll(pending, String(err)));
+
+    const request = (
+      apiKey: number,
+      apiVersion: number,
+      build: (writer: Writer) => void,
+      timeoutMillis: number,
+    ): Effect.Effect<Uint8Array, NisshiConnectionError> =>
+      Effect.gen(function* () {
+        const deferred = yield* Deferred.make<Uint8Array, NisshiConnectionError>();
+        correlation += 1;
+        const corr = correlation;
+        yield* Ref.update(pending, (map) => new Map(map).set(corr, deferred));
+
+        const writer = new Writer().i16(apiKey).i16(apiVersion).i32(corr).str(clientId);
+        build(writer);
+        const payload = writer.out();
+        // `false` only signals backpressure (data is still queued); real
+        // failures surface through the close/error handlers.
+        connection_socket_write(socket, payload);
+
+        return yield* Deferred.await(deferred).pipe(
+          Effect.timeoutFail({
+            duration: timeoutMillis,
+            onTimeout: () =>
+              new NisshiConnectionError({
+                reason: `request timed out (api ${apiKey} v${apiVersion})`,
+              }),
+          }),
+          Effect.ensuring(Effect.sync(() => takePending(pending, corr, () => {}))),
+        );
+      });
+
+    // Handshake: ApiVersions v0, then verify every pinned version is offered.
+    const body = yield* request(ApiKey.apiVersions, PinnedVersion.apiVersions, () => {}, 5000);
+    const reader = new Reader(body);
+    const errorCode = reader.i16();
+    if (errorCode !== 0) {
+      return yield* Effect.fail(
+        new NisshiConnectionError({ reason: `ApiVersions rejected with code ${errorCode}` }),
+      );
+    }
+    const count = reader.i32();
+    const versions = new Map<number, readonly [number, number]>();
+    for (let i = 0; i < count; i++) {
+      versions.set(reader.i16(), [reader.i16(), reader.i16()]);
+    }
+    assertVersion(versions, "Produce", ApiKey.produce, PinnedVersion.produce);
+    assertVersion(versions, "Fetch", ApiKey.fetch, PinnedVersion.fetch);
+    assertVersion(versions, "Metadata", ApiKey.metadata, PinnedVersion.metadata);
+    assertVersion(versions, "CreateTopics", ApiKey.createTopics, PinnedVersion.createTopics);
+    assertVersion(versions, "ListOffsets", ApiKey.listOffsets, PinnedVersion.listOffsets);
+
+    return { versions, request };
+  });
+
+// --- helpers running on the socket thread (outside Effect) ---
+type PendingMap = Ref.Ref<Map<number, Deferred.Deferred<Uint8Array, NisshiConnectionError>>>;
+
+function connection_socket_write(socket: Socket, payload: Uint8Array): void {
+  socket.write(new Writer().i32(payload.length).raw(payload).out());
+}
+
+const takePending = (
+  pending: PendingMap,
+  corr: number,
+  use: (deferred: Deferred.Deferred<Uint8Array, NisshiConnectionError>) => void,
+): void => {
+  let found: Deferred.Deferred<Uint8Array, NisshiConnectionError> | undefined;
+  Effect.runSync(
+    Ref.modify(pending, (map) => {
+      found = map.get(corr);
+      const next = new Map(map);
+      next.delete(corr);
+      return [found, next];
+    }),
+  );
+  if (found !== undefined) {
+    use(found);
+  }
+};
+
+const failAll = (pending: PendingMap, reason: string): void => {
+  const all = Effect.runSync(Ref.getAndSet(pending, new Map()));
+  for (const deferred of all.values()) {
+    Effect.runFork(Deferred.fail(deferred, new NisshiConnectionError({ reason })));
+  }
+};

package/src/protocol/errors.ts

@@ -0,0 +1,63 @@
+import { Data } from "effect";
+
+/**
+ * Connection-level failure: socket closed, handshake rejected, or request
+ * timed out. Always transient — a stateless Nisshi broker comes back and a
+ * retry re-establishes everything.
+ */
+export class NisshiConnectionError extends Data.TaggedError("NisshiConnectionError")<{
+  readonly reason: string;
+  readonly cause?: unknown;
+}> {
+  readonly classification = "transient" as const;
+}
+
+/** Broker answered a request with a nonzero Kafka error code. */
+export class NisshiApiError extends Data.TaggedError("NisshiApiError")<{
+  readonly code: number;
+  readonly message: string;
+  readonly retriable: boolean;
+}> {
+  readonly classification: "transient" | "permanent" = this.retriable ? "transient" : "permanent";
+}
+
+/** The topic exists but does not match the single-partition contract. */
+export class NisshiTopicConfigurationError extends Data.TaggedError(
+  "NisshiTopicConfigurationError",
+)<{
+  readonly topic: string;
+  readonly reason: string;
+}> {
+  readonly classification = "permanent" as const;
+}
+
+/** A produce attempt failed after versions were reserved — see the ADR. */
+export class NisshiProduceError extends Data.TaggedError("NisshiProduceError")<{
+  readonly topic: string;
+  readonly cause: unknown;
+}> {
+  readonly classification = "transient" as const;
+}
+
+/** Response bytes violate the wire protocol or the envelope contract. */
+export class NisshiProtocolError extends Data.TaggedError("NisshiProtocolError")<{
+  readonly reason: string;
+}> {
+  readonly classification = "permanent" as const;
+}
+
+// Kafka error codes we care about. Everything unlisted maps to retriable:
+// on a stateless broker most odd failures clear by the next attempt.
+const NON_RETRIABLE_CODES = new Set([
+  1, // OFFSET_OUT_OF_RANGE
+  3, // UNKNOWN_TOPIC_OR_PARTITION
+  17, // INVALID_TOPIC_EXCEPTION
+]);
+
+/** Maps a Kafka error code to a `NisshiApiError`. */
+export const apiError = (code: number, message: string): NisshiApiError =>
+  new NisshiApiError({
+    code,
+    message,
+    retriable: !NON_RETRIABLE_CODES.has(code),
+  });

package/src/protocol/primitives.ts

@@ -0,0 +1,251 @@
+/**
+ * Kafka wire-protocol primitives: big-endian writer/reader, zigzag varints,
+ * and CRC32-Castagnoli. The record-batch format and request/response framing
+ * are layered on top (see `batch.ts` and `connection.ts`).
+ *
+ * Length fields inside record batches are *signed* zigzag varints per the
+ * Kafka message-format-v2 spec (`null` is -1); arrays on the request/response
+ * level are int32-counted. Verified against Nisshi v0.7.0-pre.2.
+ */
+
+/** Appends big-endian and varint fields; call `out()` once at the end. */
+export class Writer {
+  private readonly parts: Uint8Array[] = [];
+  private length = 0;
+
+  private push(bytes: Uint8Array): void {
+    this.parts.push(bytes);
+    this.length += bytes.length;
+  }
+
+  /** One byte. */
+  i8(value: number): this {
+    this.push(new Uint8Array([value & 0xff]));
+    return this;
+  }
+
+  i16(value: number): this {
+    const b = new Uint8Array(2);
+    b[0] = (value >> 8) & 0xff;
+    b[1] = value & 0xff;
+    this.push(b);
+    return this;
+  }
+
+  i32(value: number): this {
+    const b = new Uint8Array(4);
+    const v = value >>> 0;
+    b[0] = (v >>> 24) & 0xff;
+    b[1] = (v >>> 16) & 0xff;
+    b[2] = (v >>> 8) & 0xff;
+    b[3] = v & 0xff;
+    this.push(b);
+    return this;
+  }
+
+  i64(value: bigint | number): this {
+    let v = BigInt(value);
+    const b = new Uint8Array(8);
+    for (let i = 7; i >= 0; i--) {
+      b[i] = Number(v & 0xffn);
+      v >>= 8n;
+    }
+    this.push(b);
+    return this;
+  }
+
+  /** int32-length-prefixed raw bytes (request/response level). */
+  bytes(value: Uint8Array): this {
+    this.i32(value.length);
+    this.push(value);
+    return this;
+  }
+
+  /** int16-length-prefixed string; `null` becomes -1. */
+  str(value: string | null): this {
+    if (value === null) {
+      return this.i16(-1);
+    }
+    const b = new TextEncoder().encode(value);
+    this.i16(b.length);
+    this.push(b);
+    return this;
+  }
+
+  raw(value: Uint8Array): this {
+    this.push(value);
+    return this;
+  }
+
+  /** Unsigned protobuf varint (used only inside zigzag encoding). */
+  uvarint(value: number): this {
+    let v = value;
+    for (;;) {
+      let byte = v & 0x7f;
+      v = Math.floor(v / 128);
+      if (v !== 0) {
+        byte |= 0x80;
+      }
+      this.push(new Uint8Array([byte]));
+      if (v === 0) {
+        return this;
+      }
+    }
+  }
+
+  /** Signed zigzag varint (record-batch lengths, deltas; `null` is -1). */
+  varint(value: number): this {
+    return this.uvarint((value << 1) ^ (value >> 31));
+  }
+
+  out(): Uint8Array {
+    const out = new Uint8Array(this.length);
+    let at = 0;
+    for (const part of this.parts) {
+      out.set(part, at);
+      at += part.length;
+    }
+    return out;
+  }
+}
+
+/** Reads big-endian and varint fields sequentially over one response body. */
+export class Reader {
+  /** Byte cursor — public for `batch.ts` record parsing. */
+  public pos = 0;
+
+  constructor(private readonly buf: Uint8Array) {}
+
+  private need(count: number): void {
+    if (this.pos + count > this.buf.length) {
+      throw new Error(
+        `kafka protocol: truncated response (need ${count} bytes at ${this.pos}, have ${this.buf.length - this.pos})`,
+      );
+    }
+  }
+
+  /** Indexed byte access that refuses to silently yield `undefined`. */
+  private at(index: number): number {
+    const byte = this.buf[index];
+    if (byte === undefined) {
+      throw new Error(`kafka protocol: byte read at ${index} out of bounds`);
+    }
+    return byte;
+  }
+
+  i8(): number {
+    this.need(1);
+    const v = this.at(this.pos);
+    this.pos += 1;
+    return v >= 128 ? v - 256 : v;
+  }
+
+  i16(): number {
+    this.need(2);
+    const v = (this.at(this.pos) << 8) | this.at(this.pos + 1);
+    this.pos += 2;
+    return v >= 0x8000 ? v - 0x10000 : v;
+  }
+
+  u16(): number {
+    this.need(2);
+    const v = (this.at(this.pos) << 8) | this.at(this.pos + 1);
+    this.pos += 2;
+    return v;
+  }
+
+  i32(): number {
+    this.need(4);
+    const v =
+      ((this.at(this.pos) << 24) |
+        (this.at(this.pos + 1) << 16) |
+        (this.at(this.pos + 2) << 8) |
+        this.at(this.pos + 3)) >>>
+      0;
+    this.pos += 4;
+    return v | 0;
+  }
+
+  i64(): bigint {
+    this.need(8);
+    let v = 0n;
+    for (let i = 0; i < 8; i++) {
+      v = (v << 8n) | BigInt(this.at(this.pos + i));
+    }
+    this.pos += 8;
+    return BigInt.asIntN(64, v);
+  }
+
+  u32(): number {
+    this.need(4);
+    const v = this.i32();
+    return v >>> 0;
+  }
+
+  str(): string | null {
+    const len = this.i16();
+    if (len < 0) {
+      return null;
+    }
+    this.need(len);
+    const s = new TextDecoder().decode(this.buf.subarray(this.pos, this.pos + len));
+    this.pos += len;
+    return s;
+  }
+
+  /** int32-length-prefixed bytes; `null` length yields null. */
+  bytes(): Uint8Array | null {
+    const len = this.i32();
+    if (len < 0) {
+      return null;
+    }
+    this.need(len);
+    const b = this.buf.slice(this.pos, this.pos + len);
+    this.pos += len;
+    return b;
+  }
+
+  uvarint(): number {
+    let v = 0;
+    let shift = 0;
+    for (;;) {
+      this.need(1);
+      const byte = this.at(this.pos);
+      this.pos += 1;
+      v |= (byte & 0x7f) << shift;
+      if ((byte & 0x80) === 0) {
+        return v >>> 0;
+      }
+      shift += 7;
+      if (shift > 31) {
+        throw new Error("kafka protocol: varint overflow");
+      }
+    }
+  }
+
+  varint(): number {
+    const v = this.uvarint();
+    return (v >>> 1) ^ -(v & 1);
+  }
+}
+
+const CRC32C_TABLE = (() => {
+  const table = new Uint32Array(256);
+  for (let i = 0; i < 256; i++) {
+    let c = i;
+    for (let k = 0; k < 8; k++) {
+      c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1;
+    }
+    table[i] = c >>> 0;
+  }
+  return table;
+})();
+
+/** CRC32-Castagnoli over `value` (record-batch v2 integrity field). */
+export const crc32c = (value: Uint8Array): number => {
+  let c = 0xffffffff;
+  for (const byte of value) {
+    c = (CRC32C_TABLE[(c ^ byte) & 0xff] ?? 0) ^ (c >>> 8);
+  }
+  return (c ^ 0xffffffff) >>> 0;
+};

package/src/relay.ts

@@ -0,0 +1,77 @@
+import * as SqlClient from "@effect/sql/SqlClient";
+import type { SqlError } from "@effect/sql/SqlError";
+import { Effect } from "effect";
+import { NisshiClient } from "./protocol/client.js";
+import type { NisshiApiError, NisshiConnectionError } from "./protocol/errors.js";
+import { type SidecarOptions, sidecarTables } from "./sidecar.js";
+
+export interface RelayOptions extends SidecarOptions {
+  /** Entries fetched per poll (default 32). */
+  readonly batchSize?: number;
+}
+
+interface PendingRow {
+  readonly stream_name: string;
+  readonly version: number | bigint | string;
+  readonly topic: string;
+  readonly record_value: string;
+}
+
+/**
+ * Produces every pending row whose append crashed between reservation and
+ * confirmation, then removes it. There are no dead letters here: pending
+ * rows are domain facts that already passed the ledger — the relay retries
+ * until the topic accepts them. A crash after produce but before delete
+ * re-produces a duplicate; readers dedupe by `(stream, version)`.
+ */
+export const drainPending = (
+  options?: RelayOptions,
+): Effect.Effect<
+  void,
+  NisshiApiError | NisshiConnectionError | SqlError,
+  SqlClient.SqlClient | NisshiClient
+> =>
+  Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const client = yield* NisshiClient;
+    const tables = sidecarTables(options);
+
+    for (;;) {
+      const rows = yield* sql<PendingRow>`
+        SELECT stream_name, version, topic, record_value
+        FROM ${sql(tables.pending)}
+        ORDER BY stream_name, version
+        LIMIT ${options?.batchSize ?? 32}
+      `;
+      if (rows.length === 0) {
+        return;
+      }
+      for (const row of rows) {
+        yield* client.produce(row.topic, [
+          {
+            key: new TextEncoder().encode(row.stream_name),
+            value: new TextEncoder().encode(row.record_value),
+          },
+        ]);
+        yield* Effect.asVoid(sql`
+          DELETE FROM ${sql(tables.pending)}
+          WHERE stream_name = ${row.stream_name} AND version = ${row.version}
+        `);
+      }
+    }
+  });
+
+/** Runs `drainPending` forever, sleeping `pollInterval` between passes. */
+export const runPendingRelay = (
+  options?: RelayOptions & { readonly pollInterval?: number },
+): Effect.Effect<never, never, SqlClient.SqlClient | NisshiClient> =>
+  Effect.gen(function* () {
+    const interval = options?.pollInterval ?? 500;
+    for (;;) {
+      yield* drainPending(options).pipe(
+        Effect.ignore,
+        Effect.catchAllDefect(() => Effect.void),
+      );
+      yield* Effect.sleep(interval);
+    }
+  });

package/src/schemas.ts

@@ -0,0 +1,47 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { Effect } from "effect";
+
+/**
+ * JSON Schema for the event envelope, matching `WireEvent`. Mount the
+ * generated file in the broker's schema registry (`--schema-registry
+ * file://<dir>` with one `<topic>.json` per topic) to get broker-side
+ * rejection of malformed records; see the package README for wiring.
+ */
+export const envelopeJsonSchema = {
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: "https://structure.dev/nisshi/event-envelope.json",
+  title: "Event envelope",
+  type: "object",
+  required: ["type", "schemaVersion", "version", "payload", "metadata"],
+  additionalProperties: false,
+  properties: {
+    type: { type: "string", minLength: 1 },
+    schemaVersion: { type: "integer", minimum: 1 },
+    version: { type: "integer", minimum: 1 },
+    payload: {},
+    metadata: {
+      type: "object",
+      required: ["occurredAt"],
+      properties: {
+        occurredAt: { type: "string", minLength: 1 },
+      },
+    },
+  },
+} as const;
+
+/**
+ * Writes one `<topic>.json` per topic into `dir` (created if missing).
+ * Point the Nisshi broker at the directory to enforce the envelope
+ * server-side.
+ */
+export const writeSchemaFiles = (dir: string, topics: ReadonlyArray<string>): Effect.Effect<void> =>
+  Effect.forEach(topics, (topic) =>
+    Effect.gen(function* () {
+      const file = `${dir.replace(/\/$/, "")}/${topic}.json`;
+      yield* Effect.promise(() => mkdir(dirname(file), { recursive: true }));
+      yield* Effect.promise(() =>
+        writeFile(file, `${JSON.stringify(envelopeJsonSchema, null, 2)}\n`, "utf8"),
+      );
+    }),
+  ).pipe(Effect.asVoid);