@nestarc/inbox 0.0.0-stage → 0.2.0

package/examples/src/common.ts

@@ -0,0 +1,112 @@
+import "reflect-metadata";
+import { Module } from "@nestjs/common";
+import { NestFactory } from "@nestjs/core";
+import { PrismaPg } from "@prisma/adapter-pg";
+import {
+  InboxModule,
+  InboxPermanentError,
+  type InboxHandler,
+} from "@nestarc/inbox";
+import { PrismaClient, type Prisma } from "./generated/prisma/client";
+import { setTimeout } from "node:timers/promises";
+
+export const tenantId = "example-tenant";
+export type OrderPayload = { orderId: string };
+
+export function exampleDatabaseUrl(): string {
+  const value = process.env.INBOX_EXAMPLE_DATABASE_URL;
+  if (!value)
+    throw new Error(
+      "Set INBOX_EXAMPLE_DATABASE_URL to the dedicated local fixture.",
+    );
+  const url = new URL(value);
+  const isolatedTest =
+    process.env.INBOX_TEST_DATABASE === "1" &&
+    process.env.INBOX_TEST_DATABASE_URL === value &&
+    process.env.INBOX_TEST_DATABASE_NAME === url.pathname.slice(1) &&
+    /^\/nestarc_inbox_test_[a-f0-9]{16}$/.test(url.pathname) &&
+    url.username === "inbox_test";
+  if (
+    !["localhost", "127.0.0.1", "[::1]"].includes(url.hostname) ||
+    (!/^\/inbox_examples(?:_[a-zA-Z0-9_]+)?$/.test(url.pathname) &&
+      !isolatedTest)
+  ) {
+    throw new Error("Examples require a local inbox_examples database.");
+  }
+  return value;
+}
+
+export function createPrisma(): PrismaClient {
+  return new PrismaClient({
+    adapter: new PrismaPg({ connectionString: exampleDatabaseUrl(), max: 12 }),
+  });
+}
+
+export function projectionHandler(
+  consumer: string,
+  eventType: string,
+): InboxHandler<OrderPayload, Prisma.TransactionClient> {
+  return {
+    consumer,
+    eventType,
+    schemaVersion: 1,
+    validate: (payload) =>
+      !!payload &&
+      typeof payload === "object" &&
+      !Array.isArray(payload) &&
+      typeof payload.orderId === "string" &&
+      payload.orderId.length > 0,
+    async handle(payload, ctx) {
+      if (ctx.scope.kind !== "tenant") throw new InboxPermanentError();
+      await ctx.tx.exampleProjection.create({
+        data: { tenantId: ctx.scope.id, orderId: payload.orderId },
+      });
+    },
+  };
+}
+
+export async function startInbox(consumer: string, eventType: string) {
+  const prisma = createPrisma();
+  @Module({
+    imports: [
+      InboxModule.forRoot({
+        prisma,
+        handlers: [projectionHandler(consumer, eventType)],
+        worker: { enabled: true, pollIntervalMs: 25, concurrency: 2 },
+      }),
+    ],
+  })
+  class AppModule {}
+  try {
+    const app = await NestFactory.createApplicationContext(AppModule, {
+      logger: ["error", "warn"],
+    });
+    return {
+      app,
+      prisma,
+      async close() {
+        try {
+          await app.close();
+        } finally {
+          await prisma.$disconnect();
+        }
+      },
+    };
+  } catch (error) {
+    await prisma.$disconnect();
+    throw error;
+  }
+}
+
+export async function waitUntil(
+  check: () => Promise<boolean>,
+  label: string,
+  timeoutMs = 15_000,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await check()) return;
+    await setTimeout(25);
+  }
+  throw new Error(`Timed out waiting for ${label}`);
+}

package/examples/src/idempotency-ingress/main.ts

@@ -0,0 +1,211 @@
+import "reflect-metadata";
+import assert from "node:assert/strict";
+import { createHash, randomUUID } from "node:crypto";
+import { createServer } from "node:http";
+import type { AddressInfo } from "node:net";
+import { PostgresStorage } from "@nestarc/idempotency/postgres";
+import {
+  InboxConflictError,
+  InboxService,
+  InboxValidationError,
+} from "@nestarc/inbox";
+import { Pool } from "pg";
+import { exampleDatabaseUrl, startInbox, tenantId, waitUntil } from "../common";
+
+/** An HTTP response cache may expire; the producer event ID must survive that expiry. */
+async function main() {
+  const fixture = await startInbox("example.idempotency.v1", "order.created");
+  const pool = new Pool({ connectionString: exampleDatabaseUrl(), max: 4 });
+  const tableName = `inbox_example_http_${randomUUID().replaceAll("-", "")}`;
+  const storage = new PostgresStorage({ pool, tableName });
+  const inbox = fixture.app.get(InboxService);
+  let acceptCalls = 0;
+
+  const server = createServer((request, response) => {
+    const send = (status: number, body: string, replayed = false) => {
+      response.writeHead(status, {
+        "content-type": "application/json",
+        ...(replayed ? { "idempotency-replayed": "true" } : {}),
+      });
+      response.end(body);
+    };
+    void (async () => {
+      if (request.method !== "POST" || request.url !== "/events")
+        return send(404, '{"error":"not found"}');
+      const header = request.headers["idempotency-key"];
+      if (typeof header !== "string" || !/^[a-zA-Z0-9-]{1,128}$/.test(header))
+        return send(400, '{"error":"invalid idempotency key"}');
+      const chunks: Buffer[] = [];
+      let size = 0;
+      for await (const chunk of request) {
+        const bytes = Buffer.from(chunk);
+        size += bytes.length;
+        if (size > 4096) return send(413, '{"error":"body too large"}');
+        chunks.push(bytes);
+      }
+      let event: { id: string; orderId: string };
+      try {
+        const input: unknown = JSON.parse(
+          Buffer.concat(chunks).toString("utf8"),
+        );
+        if (!input || typeof input !== "object" || Array.isArray(input))
+          throw new Error("invalid body");
+        const value = input as Record<string, unknown>;
+        if (
+          Object.keys(value).some((key) => !["id", "orderId"].includes(key)) ||
+          typeof value.id !== "string" ||
+          typeof value.orderId !== "string" ||
+          !/^[a-f0-9-]{36}$/.test(value.id) ||
+          !/^[a-f0-9-]{36}$/.test(value.orderId)
+        )
+          throw new Error("invalid event");
+        event = { id: value.id, orderId: value.orderId };
+      } catch {
+        return send(400, '{"error":"invalid event"}');
+      }
+
+      // This local fixture's authenticated account is fixed. A real route must authenticate
+      // before deriving tenantId and must scope its HTTP key by tenant and endpoint.
+      const key = `${tenantId}:POST:/events:${header}`;
+      const fingerprint = createHash("sha256")
+        .update(JSON.stringify(event))
+        .digest("hex");
+      const claim = await storage.create(key, fingerprint, 5);
+      if (!claim.acquired) {
+        const current = await storage.get(key);
+        if (!current) return send(503, '{"error":"retry acquisition"}');
+        if (current.fingerprint !== fingerprint)
+          return send(422, '{"error":"idempotency content conflict"}');
+        if (current.status !== "COMPLETED")
+          return send(409, '{"error":"request is processing"}');
+        return send(current.statusCode!, current.responseBody!, true);
+      }
+      const token = claim.token;
+      if (!token) return send(503, '{"error":"missing request ownership"}');
+      try {
+        acceptCalls++;
+        const receipt = await inbox.accept({
+          scope: { kind: "tenant", id: tenantId },
+          source: "example-http-client",
+          consumer: "example.idempotency.v1",
+          messageId: event.id,
+          eventType: "order.created",
+          schemaVersion: 1,
+          payload: { orderId: event.orderId },
+        });
+        const body = JSON.stringify({
+          inboxId: receipt.id,
+          kind: receipt.kind,
+        });
+        const completion = await storage.complete(
+          key,
+          token,
+          { statusCode: 202, body },
+          1,
+        );
+        if (completion !== "ok")
+          return send(503, '{"error":"request ownership lost"}');
+        return send(202, body);
+      } catch (error) {
+        // A retry uses the same producer event ID, even if the acceptance result was lost.
+        await storage.delete(key, token).catch(() => undefined);
+        if (error instanceof InboxConflictError)
+          return send(422, '{"error":"event identity content conflict"}');
+        if (error instanceof InboxValidationError)
+          return send(400, '{"error":"invalid inbox envelope"}');
+        return send(503, '{"error":"acceptance unavailable"}');
+      }
+    })().catch(() => send(503, '{"error":"storage unavailable"}'));
+  });
+
+  try {
+    // A dedicated fixture only. Deploy the package's SQL migration explicitly in production.
+    await PostgresStorage.createSchema(pool, tableName);
+    await new Promise<void>((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(0, "127.0.0.1", () => {
+        server.removeListener("error", reject);
+        resolve();
+      });
+    });
+    const endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}/events`;
+    const idempotencyKey = randomUUID();
+    const event = { id: randomUUID(), orderId: randomUUID() };
+    const post = (body = event) =>
+      fetch(endpoint, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          "idempotency-key": idempotencyKey,
+        },
+        body: JSON.stringify(body),
+        signal: AbortSignal.timeout(5000),
+      });
+
+    const first = await post();
+    assert.equal(first.status, 202);
+    const firstBody = await first.text();
+    assert.equal(JSON.parse(firstBody).kind, "accepted");
+    const replay = await post();
+    assert.equal(replay.status, 202);
+    assert.equal(replay.headers.get("idempotency-replayed"), "true");
+    assert.equal(await replay.text(), firstBody);
+    assert.equal(acceptCalls, 1);
+    const conflict = await post({ ...event, orderId: randomUUID() });
+    assert.equal(conflict.status, 422);
+    await conflict.text();
+    assert.equal(acceptCalls, 1);
+
+    await waitUntil(
+      async () =>
+        (await storage.get(`${tenantId}:POST:/events:${idempotencyKey}`)) ===
+        null,
+      "actual HTTP response retention expiry",
+      5000,
+    );
+    // The HTTP record has expired, but Inbox still rejects event identity reuse.
+    const durableConflict = await post({ ...event, orderId: randomUUID() });
+    assert.equal(durableConflict.status, 422);
+    await durableConflict.text();
+    const afterExpiry = await post();
+    assert.equal(afterExpiry.status, 202);
+    const afterBody = (await afterExpiry.json()) as {
+      inboxId: string;
+      kind: string;
+    };
+    assert.equal(afterBody.kind, "duplicate");
+    assert.equal(afterBody.inboxId, JSON.parse(firstBody).inboxId);
+    assert.equal(acceptCalls, 3);
+    await waitUntil(
+      async () =>
+        (await fixture.prisma.exampleProjection.count({
+          where: { tenantId, orderId: event.orderId },
+        })) === 1,
+      "one durable business projection despite HTTP cache expiry",
+    );
+    console.log(
+      JSON.stringify({
+        responseReplay: true,
+        afterExpiry: "duplicate",
+        projections: 1,
+      }),
+    );
+  } finally {
+    await new Promise<void>((resolve) => {
+      server.close(() => resolve());
+      server.closeAllConnections();
+    });
+    try {
+      await pool.query(`DROP TABLE IF EXISTS "${tableName}"`);
+    } finally {
+      await storage.close();
+      await pool.end();
+      await fixture.close();
+    }
+  }
+}
+
+void main().catch((error: unknown) => {
+  console.error(error);
+  process.exitCode = 1;
+});

package/examples/src/kafka-ingress/main.ts

@@ -0,0 +1,209 @@
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { setTimeout } from "node:timers/promises";
+import { Kafka, logLevel } from "kafkajs";
+import {
+  InboxConflictError,
+  InboxService,
+  InboxValidationError,
+} from "@nestarc/inbox";
+import { tenantId, waitUntil } from "../common";
+import { parseBrokerEvent, startBrokerInbox } from "../broker-common";
+
+async function main() {
+  const broker = process.env.INBOX_EXAMPLE_KAFKA_BROKER;
+  if (!broker || !/^127\.0\.0\.1:\d+$/.test(broker))
+    throw new Error(
+      "Set INBOX_EXAMPLE_KAFKA_BROKER to the local fixture host:port.",
+    );
+  const name = `inbox-example-${randomUUID()}`;
+  const topic = `${name}-orders`;
+  const deadTopic = `${name}-dead`;
+  const kafka = new Kafka({
+    clientId: name,
+    brokers: [broker],
+    logLevel: logLevel.ERROR,
+    connectionTimeout: 5_000,
+    requestTimeout: 10_000,
+    retry: { retries: 3, initialRetryTime: 100, maxRetryTime: 1_000 },
+  });
+  const admin = kafka.admin();
+  const producer = kafka.producer();
+  const consumer = kafka.consumer({ groupId: name, sessionTimeout: 30_000 });
+  const fixture = await startBrokerInbox("example.kafka.v1", "order.created");
+  let callbackError: unknown;
+  let accepted = 0;
+  let quarantined = 0;
+  let transportRetries = 0;
+  let injectAcceptanceFailure = true;
+  const createdTopics: string[] = [];
+  try {
+    await admin.connect();
+    for (const candidate of [topic, deadTopic]) {
+      await admin.createTopics({
+        topics: [{ topic: candidate, numPartitions: 1, replicationFactor: 1 }],
+        waitForLeaders: true,
+      });
+      createdTopics.push(candidate);
+    }
+    await producer.connect();
+    await consumer.connect();
+    await consumer.subscribe({ topics: [topic], fromBeginning: true });
+    const inbox = fixture.app.get(InboxService);
+    await consumer.run({
+      autoCommit: false,
+      // Sequential processing inside each partition prevents committing across a gap.
+      partitionsConsumedConcurrently: 1,
+      eachMessage: async ({
+        topic: currentTopic,
+        partition,
+        message,
+        heartbeat,
+      }) => {
+        try {
+          let received = false;
+          for (let attempt = 1; attempt <= 3; attempt++) {
+            try {
+              if (!message.value || message.value.byteLength > 256 * 1024)
+                throw new InboxValidationError("Missing or oversized event.");
+              let parsed: unknown;
+              try {
+                parsed = JSON.parse(message.value.toString("utf8"));
+              } catch {
+                throw new InboxValidationError("Invalid JSON.");
+              }
+              const event = parseBrokerEvent(parsed);
+              // Fixture-only failure: demonstrate that this offset is not committed yet.
+              if (injectAcceptanceFailure) {
+                injectAcceptanceFailure = false;
+                throw new Error("Injected acceptance failure");
+              }
+              await inbox.accept({
+                scope: { kind: "tenant", id: tenantId },
+                source: "example-kafka-orders",
+                consumer: "example.kafka.v1",
+                messageId: event.eventId,
+                eventType: "order.created",
+                schemaVersion: 1,
+                payload: { orderId: event.orderId },
+              });
+              received = true;
+              accepted++;
+              break;
+            } catch (error) {
+              const permanent =
+                error instanceof InboxValidationError ||
+                error instanceof InboxConflictError;
+              if (permanent || attempt === 3) {
+                // A durable DLQ publish must succeed before advancing the source offset.
+                // A crash between publish and commit can duplicate DLQ entries; preserve
+                // source coordinates so the DLQ consumer can deduplicate them.
+                await producer.send({
+                  topic: deadTopic,
+                  acks: -1,
+                  messages: [
+                    {
+                      key: `${currentTopic}/${partition}/${message.offset}`,
+                      value: message.value,
+                      headers: {
+                        sourceTopic: currentTopic,
+                        sourcePartition: String(partition),
+                        sourceOffset: message.offset,
+                      },
+                    },
+                  ],
+                });
+                quarantined++;
+                break;
+              }
+              transportRetries++;
+              await heartbeat();
+              await setTimeout(100 * attempt);
+            }
+          }
+          // Both durable acceptance and durable quarantine allow transport progress.
+          // Handler execution is asynchronous; its retry schedule belongs to Inbox.
+          await consumer.commitOffsets([
+            {
+              topic: currentTopic,
+              partition,
+              offset: String(BigInt(message.offset) + 1n),
+            },
+          ]);
+          if (received) await heartbeat();
+        } catch (error) {
+          callbackError = error;
+          throw error;
+        }
+      },
+    });
+    const orderId = randomUUID();
+    const eventId = randomUUID();
+    await producer.send({
+      topic,
+      acks: -1,
+      messages: [
+        { value: JSON.stringify({ eventId, orderId }) },
+        { value: JSON.stringify({ eventId, orderId }) },
+        { value: "invalid-json" },
+      ],
+    });
+    await waitUntil(
+      async () => {
+        if (callbackError) throw callbackError;
+        const offsets = await admin.fetchOffsets({
+          groupId: name,
+          topics: [topic],
+        });
+        return (
+          accepted === 2 &&
+          quarantined === 1 &&
+          offsets[0]?.partitions[0]?.offset === "3" &&
+          (await fixture.prisma.exampleProjection.count({
+            where: { orderId, tenantId },
+          })) === 1
+        );
+      },
+      "Kafka durable acceptance, contiguous offset commit, and quarantine",
+      30_000,
+    );
+    assert.equal(transportRetries, 1);
+    assert.equal(fixture.retries(), 1);
+    const deadOffsets = await admin.fetchTopicOffsets(deadTopic);
+    assert.equal(deadOffsets[0]?.high, "1");
+    console.log(
+      JSON.stringify({
+        broker: "Kafka protocol / Redpanda fixture",
+        accepted,
+        quarantined,
+        transportRetries,
+        handlerRetries: fixture.retries(),
+        projections: 1,
+      }),
+    );
+  } finally {
+    try {
+      await consumer.disconnect();
+    } finally {
+      try {
+        await producer.disconnect();
+      } finally {
+        try {
+          if (createdTopics.length)
+            await admin.deleteTopics({ topics: createdTopics });
+        } finally {
+          try {
+            await admin.disconnect();
+          } finally {
+            await fixture.close();
+          }
+        }
+      }
+    }
+  }
+}
+
+void main().catch((error: unknown) => {
+  console.error(error);
+  process.exitCode = 1;
+});