@streamotter/gateway 0.1.0-rc.1

package/src/sources/kafkajs-patch.ts

@@ -0,0 +1,41 @@
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+
+/** Mirrors KafkaJS's CHECK_PENDING_REQUESTS_INTERVAL. */
+const CHECK_PENDING_REQUESTS_INTERVAL_MS = 10;
+
+interface RequestQueueInternals {
+  pending: unknown[];
+  throttledUntil: number;
+  throttleCheckTimeoutId: NodeJS.Timeout | null;
+  checkPendingRequests(): void;
+}
+
+let applied = false;
+
+/**
+ * KafkaJS 2.2.4's RequestQueue.scheduleCheckPendingRequests schedules
+ * setTimeout(throttledUntil - Date.now()) even when nothing is pending. When not
+ * throttled that delay is hugely negative, Node clamps it to 1 ms (and on Node 24+
+ * prints TimeoutNegativeWarning), and the check reschedules itself: every open
+ * connection spins a 1 ms timer until it is destroyed. This replacement schedules a
+ * check only while requests are queued, never with a negative delay. It applies
+ * only to the exact pinned version and stays behind the source adapter boundary.
+ */
+export function patchKafkaJsRequestQueue(): boolean {
+  if (applied) return true;
+  const version = (require("kafkajs/package.json") as { version: string }).version;
+  if (version !== "2.2.4") return false;
+  const RequestQueue = require("kafkajs/src/network/requestQueue/index.js") as { prototype: RequestQueueInternals & { scheduleCheckPendingRequests(): void } };
+  RequestQueue.prototype.scheduleCheckPendingRequests = function (this: RequestQueueInternals): void {
+    if (this.throttleCheckTimeoutId !== null || this.pending.length === 0) return;
+    const throttledFor = this.throttledUntil - Date.now();
+    this.throttleCheckTimeoutId = setTimeout(() => {
+      this.throttleCheckTimeoutId = null;
+      this.checkPendingRequests();
+    }, throttledFor > 0 ? throttledFor : CHECK_PENDING_REQUESTS_INTERVAL_MS);
+  };
+  applied = true;
+  return true;
+}

package/src/sources/types.ts

@@ -0,0 +1,42 @@
+import type { DiagnosticStep, ErrorCode, GatewayLogger, Json, SourceRecord, SourceStatus } from "@streamotter/contracts";
+
+/** A record as read by an adapter, before gateway decoding and validation. */
+export interface SourceInput {
+  key: string | null;
+  /** Encoded broker value; null for a tombstone. Absent for fixture records. */
+  bytes?: Uint8Array | null;
+  /** Pre-decoded fixture value. */
+  value?: Json;
+  position: SourceRecord["position"];
+}
+
+/**
+ * commit: processing completed; the adapter may commit past this record.
+ * pause: the record is poison or unprocessable; do not commit it or anything after it.
+ * abandon: the gateway is stopping; do not commit.
+ */
+export type ProcessOutcome = { kind: "commit" } | { kind: "pause"; code: ErrorCode } | { kind: "abandon" };
+
+export interface SourceSink {
+  process(input: SourceInput): Promise<ProcessOutcome>;
+  /** Adapter-observed readiness changes (connection loss, rebalance, recovery). */
+  setStatus(status: SourceStatus["status"], reason?: ErrorCode): void;
+  readonly logger: GatewayLogger;
+  readonly stopSignal: AbortSignal;
+}
+
+/**
+ * Internal broker boundary. Broker-specific behavior stays behind this interface;
+ * it is not a public third-party adapter API.
+ */
+export interface SourceAdapter {
+  readonly kind: "kafka" | "fixture";
+  /** Opens resources and resolves once the source can deliver records. */
+  start(): Promise<void>;
+  /** Stops consumption; commits only completed processing. */
+  stop(deadline: number): Promise<void>;
+  /** Retries a paused source at its uncommitted position. */
+  resume(): Promise<void>;
+  /** Staged connectivity diagnostics with redacted messages. */
+  check(deadline: number): Promise<DiagnosticStep[]>;
+}

package/src/transport/socketio.ts

@@ -0,0 +1,125 @@
+import type { Server as HttpServer } from "node:http";
+import { Server, type Socket } from "socket.io";
+import {
+  EVENTS, HELLO_TIMEOUT_MS,
+  type ClientToServerEvents, type DataFrame, type ErrorFrame, type Hello, type Principal, type ServerToClientEvents,
+  type StreamError, type SubscriptionFrame
+} from "@streamotter/contracts";
+import type { ConnectionTransport } from "./types.ts";
+
+export interface HandshakeResult {
+  principal: Principal;
+  previewSessionId: string | null;
+}
+
+export interface SessionHandlers {
+  handleSubscribe(payload: unknown, reply: unknown): void;
+  handleUnsubscribe(payload: unknown, reply: unknown): void;
+  handleResync(payload: unknown, reply: unknown): void;
+  handleReceipt(payload: unknown): void;
+  handleUnknown(event: string, args: readonly unknown[]): void;
+  handleTransportClosed(): void;
+  open(): void;
+}
+
+export interface TransportCallbacks {
+  /** Validates origin and credentials; resolves to a principal or a public error. */
+  authenticate(input: { auth: unknown; origin: string | undefined }): Promise<{ ok: true; value: HandshakeResult } | { ok: false; error: StreamError }>;
+  /** Creates the session for an authenticated connection. */
+  openSession(result: HandshakeResult, transport: ConnectionTransport): SessionHandlers;
+}
+
+type SocketData = { handshake?: HandshakeResult };
+type StreamSocket = Socket<ClientToServerEvents, ServerToClientEvents, Record<string, never>, SocketData>;
+
+const KNOWN_EVENTS: ReadonlySet<string> = new Set([EVENTS.subscribe, EVENTS.unsubscribe, EVENTS.resync, EVENTS.receipt]);
+
+class SocketIoConnection implements ConnectionTransport {
+  readonly #socket: StreamSocket;
+
+  constructor(socket: StreamSocket) {
+    this.#socket = socket;
+  }
+
+  sendHello(hello: Hello): void {
+    this.#socket.emit("so:hello", hello);
+  }
+
+  sendState(frame: SubscriptionFrame): void {
+    this.#socket.emit("so:state", frame);
+  }
+
+  sendData(frame: DataFrame): void {
+    this.#socket.emit("so:data", frame);
+  }
+
+  sendError(frame: ErrorFrame): void {
+    this.#socket.emit("so:error", frame);
+  }
+
+  close(): void {
+    this.#socket.disconnect(true);
+  }
+}
+
+/**
+ * Socket.IO adapter: namespace "/", WebSocket-only transport, connection-state
+ * recovery disabled (V1 owns resynchronization), control frames bounded by
+ * maxControlFrameBytes. Authentication runs in handshake middleware so failures
+ * reach the SDK as structured connect errors.
+ */
+export function attachSocketIo(httpServer: HttpServer, options: {
+  path: string;
+  maxControlFrameBytes: number;
+  callbacks: TransportCallbacks;
+}): Server {
+  const io = new Server<ClientToServerEvents, ServerToClientEvents, Record<string, never>, SocketData>(httpServer, {
+    path: options.path,
+    serveClient: false,
+    transports: ["websocket"],
+    allowUpgrades: false,
+    maxHttpBufferSize: options.maxControlFrameBytes,
+    connectTimeout: HELLO_TIMEOUT_MS,
+    pingInterval: 25_000,
+    pingTimeout: 20_000
+  });
+
+  io.use((socket, next) => {
+    const origin = socket.handshake.headers.origin;
+    options.callbacks.authenticate({ auth: socket.handshake.auth, origin }).then(result => {
+      if (!result.ok) {
+        const error = new Error(result.error.message) as Error & { data?: StreamError };
+        error.data = result.error;
+        next(error);
+        return;
+      }
+      socket.data.handshake = result.value;
+      next();
+    }, () => {
+      const error = new Error("An unexpected error occurred.") as Error & { data?: StreamError };
+      error.data = { code: "INTERNAL", message: "An unexpected error occurred.", retryable: true, requestId: "" };
+      next(error);
+    });
+  });
+
+  io.on("connection", socket => {
+    const handshake = socket.data.handshake;
+    if (handshake === undefined) {
+      socket.disconnect(true);
+      return;
+    }
+    const session = options.callbacks.openSession(handshake, new SocketIoConnection(socket));
+    const untyped = socket as unknown as Socket;
+    untyped.on(EVENTS.subscribe, (payload: unknown, reply: unknown) => session.handleSubscribe(payload, reply));
+    untyped.on(EVENTS.unsubscribe, (payload: unknown, reply: unknown) => session.handleUnsubscribe(payload, reply));
+    untyped.on(EVENTS.resync, (payload: unknown, reply: unknown) => session.handleResync(payload, reply));
+    untyped.on(EVENTS.receipt, (payload: unknown) => session.handleReceipt(payload));
+    untyped.onAny((event: unknown, ...args: unknown[]) => {
+      if (typeof event !== "string" || !KNOWN_EVENTS.has(event)) session.handleUnknown(String(event), args);
+    });
+    socket.on("disconnect", () => session.handleTransportClosed());
+    session.open();
+  });
+
+  return io;
+}

package/src/transport/types.ts

@@ -0,0 +1,10 @@
+import type { DataFrame, ErrorFrame, Hello, SubscriptionFrame } from "@streamotter/contracts";
+
+/** Server-side view of one client transport connection. Transport specifics stay in the adapter. */
+export interface ConnectionTransport {
+  sendHello(hello: Hello): void;
+  sendState(frame: SubscriptionFrame): void;
+  sendData(frame: DataFrame): void;
+  sendError(frame: ErrorFrame): void;
+  close(): void;
+}