@nestarc/inbox 0.0.0-stage → 0.2.0

package/examples/scripts/with-brokers.cjs

@@ -0,0 +1,249 @@
+const { spawn } = require("node:child_process");
+const { randomBytes } = require("node:crypto");
+const { createServer } = require("node:net");
+const { resolve } = require("node:path");
+const { setTimeout } = require("node:timers/promises");
+
+const root = resolve(__dirname, "../..");
+const images = {
+  kafka: "docker.redpanda.com/redpandadata/redpanda:v26.2.6",
+  redis: "redis:7.4.2-alpine",
+  sqs: "softwaremill/elasticmq-native:1.7.1",
+};
+
+function capture(command, args) {
+  return new Promise((done, reject) => {
+    const process = spawn(command, args, { cwd: root });
+    let output = "";
+    let error = "";
+    process.stdout.on("data", (chunk) => {
+      output += chunk;
+    });
+    process.stderr.on("data", (chunk) => {
+      error += chunk;
+    });
+    process.on("error", reject);
+    process.on("close", (code) =>
+      code === 0
+        ? done(output.trim())
+        : reject(new Error(`${command} failed: ${error || output}`)),
+    );
+  });
+}
+
+async function unusedLoopbackPort() {
+  const server = createServer();
+  return new Promise((done, reject) => {
+    server.on("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+      const port = server.address().port;
+      server.close((error) => (error ? reject(error) : done(port)));
+    });
+  });
+}
+
+/** Each call owns and removes only the container IDs it creates. */
+async function withBrokers(run) {
+  const containers = [];
+  let interrupted;
+  const onSignal = (signal) => {
+    interrupted = signal;
+  };
+  const onInt = () => onSignal("SIGINT");
+  const onTerm = () => onSignal("SIGTERM");
+  process.on("SIGINT", onInt);
+  process.on("SIGTERM", onTerm);
+  const checkInterrupted = () => {
+    if (interrupted) throw new Error(`Interrupted by ${interrupted}`);
+  };
+  async function create(kind, args) {
+    checkInterrupted();
+    const id = await capture("docker", [
+      "create",
+      "--name",
+      `nestarc-inbox-${kind}-${randomBytes(8).toString("hex")}`,
+      "--label",
+      "dev.nestarc.inbox.fixture=true",
+      ...args,
+    ]);
+    if (!/^[a-f0-9]{64}$/.test(id))
+      throw new Error("Unexpected fixture container identity");
+    containers.push(id);
+    checkInterrupted();
+    await capture("docker", ["start", id]);
+    return id;
+  }
+  async function binding(id, port) {
+    const result = await capture("docker", ["port", id, `${port}/tcp`]);
+    if (!/^127\.0\.0\.1:\d+$/.test(result))
+      throw new Error("Broker must have one loopback port binding");
+    return result;
+  }
+  async function ready(label, probe) {
+    const deadline = Date.now() + 60_000;
+    let lastError;
+    while (Date.now() < deadline) {
+      checkInterrupted();
+      try {
+        await probe();
+        return;
+      } catch (error) {
+        lastError = error;
+        await setTimeout(300);
+      }
+    }
+    throw new Error(
+      `${label} fixture did not become ready: ${String(lastError)}`,
+    );
+  }
+  try {
+    await capture("docker", ["info", "--format", "{{.ServerVersion}}"]);
+    // Kafka metadata advertises the actual random host port. If another process
+    // acquires this port before Docker binds it, creation fails and cleans up.
+    const kafkaPort = await unusedLoopbackPort();
+    const kafkaId = await create("kafka", [
+      "--publish",
+      `127.0.0.1:${kafkaPort}:9092`,
+      images.kafka,
+      "redpanda",
+      "start",
+      "--mode",
+      "dev-container",
+      "--smp",
+      "1",
+      "--memory",
+      "1G",
+      "--reserve-memory",
+      "0M",
+      "--kafka-addr",
+      "0.0.0.0:9092",
+      "--advertise-kafka-addr",
+      `127.0.0.1:${kafkaPort}`,
+    ]);
+    const redisId = await create("redis", [
+      "--publish",
+      "127.0.0.1::6379",
+      "--tmpfs",
+      "/data",
+      images.redis,
+    ]);
+    const sqsId = await create("sqs", [
+      "--publish",
+      "127.0.0.1::9324",
+      images.sqs,
+    ]);
+    const kafkaBroker = await binding(kafkaId, 9092);
+    const redisBroker = await binding(redisId, 6379);
+    const sqsEndpoint = `http://${await binding(sqsId, 9324)}`;
+    const { Kafka, logLevel } = require("kafkajs");
+    await ready("Kafka", async () => {
+      const admin = new Kafka({
+        brokers: [kafkaBroker],
+        logLevel: logLevel.NOTHING,
+        connectionTimeout: 1_000,
+        requestTimeout: 2_000,
+        retry: { retries: 0 },
+      }).admin();
+      try {
+        await admin.connect();
+        await admin.listTopics();
+      } finally {
+        await admin.disconnect();
+      }
+    });
+    await ready("Redis", async () => {
+      if (
+        (await capture("docker", ["exec", redisId, "redis-cli", "ping"])) !==
+        "PONG"
+      )
+        throw new Error("Redis did not respond PONG");
+    });
+    const { SQSClient, ListQueuesCommand } = require("@aws-sdk/client-sqs");
+    const sqs = new SQSClient({
+      endpoint: sqsEndpoint,
+      region: "us-east-1",
+      credentials: { accessKeyId: "local", secretAccessKey: "local" },
+      maxAttempts: 1,
+      requestHandler: { connectionTimeout: 1_000, requestTimeout: 2_000 },
+    });
+    try {
+      await ready("ElasticMQ", () => sqs.send(new ListQueuesCommand({})));
+    } finally {
+      sqs.destroy();
+    }
+    checkInterrupted();
+    console.log(
+      "Disposable Kafka (Redpanda), Redis, and SQS-compatible ElasticMQ fixtures ready.",
+    );
+    return await run({
+      INBOX_EXAMPLE_KAFKA_BROKER: kafkaBroker,
+      INBOX_EXAMPLE_REDIS_URL: `redis://${redisBroker}`,
+      INBOX_EXAMPLE_SQS_ENDPOINT: sqsEndpoint,
+    });
+  } catch (error) {
+    for (const id of containers) {
+      try {
+        console.error(await capture("docker", ["logs", "--tail", "20", id]));
+      } catch {
+        /* preserve the original test failure */
+      }
+    }
+    throw error;
+  } finally {
+    process.removeListener("SIGINT", onInt);
+    process.removeListener("SIGTERM", onTerm);
+    const results = await Promise.allSettled(
+      containers.map((id) =>
+        capture("docker", ["rm", "--force", "--volumes", id]),
+      ),
+    );
+    for (const result of results) {
+      if (result.status === "rejected") {
+        console.error(`Fixture cleanup failed: ${result.reason}`);
+        process.exitCode = 1;
+      }
+    }
+  }
+}
+
+async function main() {
+  let child;
+  const onInt = () => child?.kill("SIGINT");
+  const onTerm = () => child?.kill("SIGTERM");
+  process.on("SIGINT", onInt);
+  process.on("SIGTERM", onTerm);
+  try {
+    await withBrokers(
+      (env) =>
+        new Promise((done, reject) => {
+          child = spawn(
+            process.execPath,
+            [
+              "scripts/with-test-db.mjs",
+              process.execPath,
+              "examples/scripts/smoke.cjs",
+              "--brokers",
+            ],
+            { cwd: root, env: { ...process.env, ...env }, stdio: "inherit" },
+          );
+          child.on("error", reject);
+          child.on("close", (code) => {
+            child = undefined;
+            code === 0
+              ? done()
+              : reject(new Error(`Broker examples exited ${code}`));
+          });
+        }),
+    );
+  } finally {
+    process.removeListener("SIGINT", onInt);
+    process.removeListener("SIGTERM", onTerm);
+  }
+}
+
+module.exports = { withBrokers };
+if (require.main === module)
+  main().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+  });

package/examples/scripts/with-rabbitmq.cjs

@@ -0,0 +1,131 @@
+const { spawn } = require("node:child_process");
+const { randomBytes } = require("node:crypto");
+const { resolve } = require("node:path");
+const { setTimeout } = require("node:timers/promises");
+
+const root = resolve(__dirname, "../..");
+let containerId;
+let child;
+let interrupted;
+
+function capture(command, args) {
+  return new Promise((done, reject) => {
+    const process = spawn(command, args, { cwd: root });
+    let output = "";
+    let error = "";
+    process.stdout.on("data", (chunk) => {
+      output += chunk;
+    });
+    process.stderr.on("data", (chunk) => {
+      error += chunk;
+    });
+    process.on("error", reject);
+    process.on("close", (code) =>
+      code === 0
+        ? done(output.trim())
+        : reject(new Error(`${command} failed: ${error || output}`)),
+    );
+  });
+}
+
+for (const signal of ["SIGINT", "SIGTERM"]) {
+  process.on(signal, () => {
+    interrupted = signal;
+    child?.kill(signal);
+  });
+}
+
+async function withRabbitmq(run) {
+  try {
+    containerId = await capture("docker", [
+      "create",
+      "--name",
+      `nestarc-inbox-rabbitmq-${randomBytes(8).toString("hex")}`,
+      "--label",
+      "dev.nestarc.inbox.fixture=true",
+      "--publish",
+      "127.0.0.1::5672",
+      "--tmpfs",
+      "/var/lib/rabbitmq",
+      "rabbitmq:4-alpine",
+    ]);
+    if (!/^[a-f0-9]{64}$/.test(containerId))
+      throw new Error("Unexpected fixture container identity");
+    await capture("docker", ["start", containerId]);
+    const deadline = Date.now() + 60_000;
+    let ready = false;
+    let lastReadinessError;
+    while (Date.now() < deadline && !interrupted) {
+      try {
+        await capture("docker", [
+          "exec",
+          "--user",
+          "rabbitmq",
+          containerId,
+          "rabbitmq-diagnostics",
+          "-q",
+          "check_port_connectivity",
+        ]);
+        ready = true;
+        break;
+      } catch (error) {
+        lastReadinessError = error;
+        await setTimeout(500);
+      }
+    }
+    if (!ready) {
+      console.error(
+        await capture("docker", ["logs", "--tail", "150", containerId]),
+      );
+      throw new Error(
+        `RabbitMQ fixture did not become ready: ${String(lastReadinessError)}`,
+      );
+    }
+    const binding = await capture("docker", ["port", containerId, "5672/tcp"]);
+    const match = /^127\.0\.0\.1:(\d+)$/.exec(binding);
+    if (!match) throw new Error("RabbitMQ must be bound to one loopback port");
+    console.log(`Disposable RabbitMQ fixture ready on 127.0.0.1:${match[1]}.`);
+    return await run(`amqp://guest:guest@127.0.0.1:${match[1]}`);
+  } finally {
+    if (containerId && /^[a-f0-9]{64}$/.test(containerId)) {
+      await capture("docker", ["rm", "--force", "--volumes", containerId]);
+    }
+  }
+}
+
+async function main() {
+  process.exitCode = await withRabbitmq(
+    (url) =>
+      new Promise((done, reject) => {
+        child = spawn(
+          process.execPath,
+          [
+            "scripts/with-test-db.mjs",
+            process.execPath,
+            "examples/scripts/smoke.cjs",
+            "--rabbitmq",
+          ],
+          {
+            cwd: root,
+            env: {
+              ...process.env,
+              INBOX_EXAMPLE_AMQP_URL: url,
+            },
+            stdio: "inherit",
+          },
+        );
+        child.on("error", reject);
+        child.on("close", (code) => {
+          child = undefined;
+          done(code ?? 1);
+        });
+      }),
+  );
+}
+module.exports = { withRabbitmq };
+if (require.main === module) {
+  main().catch((error) => {
+    console.error(error.message);
+    process.exitCode = 1;
+  });
+}

package/examples/sql/application.sql

@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS inbox_example_orders (
+  id text PRIMARY KEY,
+  tenant_id text NOT NULL
+);
+CREATE TABLE IF NOT EXISTS inbox_example_projections (
+  tenant_id text NOT NULL,
+  order_id text NOT NULL,
+  PRIMARY KEY (tenant_id, order_id)
+);
+CREATE TABLE IF NOT EXISTS inbox_example_receipts (
+  tenant_id text NOT NULL,
+  order_id text NOT NULL,
+  PRIMARY KEY (tenant_id, order_id)
+);

package/examples/sql/tenancy.sql

@@ -0,0 +1,26 @@
+-- Apply only to the dedicated local example database. The runtime role below
+-- is provisioned separately by the runnable example and never owns these tables.
+CREATE TABLE IF NOT EXISTS public.example_tenancy_records (
+  id uuid PRIMARY KEY,
+  tenant_id text NOT NULL,
+  inbox_id uuid NOT NULL,
+  note text NOT NULL
+);
+CREATE INDEX IF NOT EXISTS example_tenancy_records_tenant_idx
+  ON public.example_tenancy_records (tenant_id);
+ALTER TABLE public.example_tenancy_records ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.example_tenancy_records FORCE ROW LEVEL SECURITY;
+DROP POLICY IF EXISTS tenant_isolation ON public.example_tenancy_records;
+CREATE POLICY tenant_isolation ON public.example_tenancy_records
+  USING (tenant_id = NULLIF(current_setting('app.current_tenant', true), ''))
+  WITH CHECK (tenant_id = NULLIF(current_setting('app.current_tenant', true), ''));
+DROP POLICY IF EXISTS tenant_required ON public.example_tenancy_records;
+CREATE POLICY tenant_required ON public.example_tenancy_records AS RESTRICTIVE
+  USING (NULLIF(current_setting('app.current_tenant', true), '') IS NOT NULL)
+  WITH CHECK (NULLIF(current_setting('app.current_tenant', true), '') IS NOT NULL);
+
+-- Intentionally shared records have their own table and permissions.
+CREATE TABLE IF NOT EXISTS public.example_tenancy_shared (
+  id uuid PRIMARY KEY,
+  inbox_id uuid NOT NULL
+);

package/examples/src/broker-common.ts

@@ -0,0 +1,79 @@
+import "reflect-metadata";
+import { Module } from "@nestjs/common";
+import { NestFactory } from "@nestjs/core";
+import { InboxModule, InboxValidationError } from "@nestarc/inbox";
+import { createPrisma, projectionHandler } from "./common";
+
+export type BrokerEvent = { eventId: string; orderId: string };
+
+/** The producer's event ID survives transport redelivery and republishing. */
+export function parseBrokerEvent(value: unknown): BrokerEvent {
+  if (
+    !value ||
+    typeof value !== "object" ||
+    Array.isArray(value) ||
+    typeof (value as BrokerEvent).eventId !== "string" ||
+    !(value as BrokerEvent).eventId.trim() ||
+    Buffer.byteLength((value as BrokerEvent).eventId) > 256 ||
+    typeof (value as BrokerEvent).orderId !== "string" ||
+    !(value as BrokerEvent).orderId.trim() ||
+    Buffer.byteLength((value as BrokerEvent).orderId) > 256
+  )
+    throw new InboxValidationError("Expected an eventId and orderId.");
+  return value as BrokerEvent;
+}
+
+/** Local smoke fixture: fail the first handler attempt, then let Inbox retry it. */
+export async function startBrokerInbox(consumer: string, eventType: string) {
+  const prisma = createPrisma();
+  const projection = projectionHandler(consumer, eventType);
+  let retries = 0;
+  @Module({
+    imports: [
+      InboxModule.forRoot({
+        prisma,
+        handlers: [
+          {
+            ...projection,
+            async handle(payload, context) {
+              if (context.attempt === 1)
+                throw new Error("Injected first handler failure");
+              await projection.handle(payload, context);
+            },
+          },
+        ],
+        worker: {
+          enabled: true,
+          pollIntervalMs: 25,
+          concurrency: 2,
+          retryBaseMs: 10,
+          retryMaxMs: 20,
+        },
+        observer: (event) => {
+          if (event.type === "retry") retries++;
+        },
+      }),
+    ],
+  })
+  class AppModule {}
+  try {
+    const app = await NestFactory.createApplicationContext(AppModule, {
+      logger: ["error", "warn"],
+    });
+    return {
+      app,
+      prisma,
+      retries: () => retries,
+      async close() {
+        try {
+          await app.close();
+        } finally {
+          await prisma.$disconnect();
+        }
+      },
+    };
+  } catch (error) {
+    await prisma.$disconnect();
+    throw error;
+  }
+}

package/examples/src/bullmq-ingress/main.ts

@@ -0,0 +1,165 @@
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { Queue, Worker, UnrecoverableError } from "bullmq";
+import {
+  InboxConflictError,
+  InboxService,
+  InboxValidationError,
+} from "@nestarc/inbox";
+import { tenantId, waitUntil } from "../common";
+import {
+  parseBrokerEvent,
+  startBrokerInbox,
+  type BrokerEvent,
+} from "../broker-common";
+
+async function main() {
+  const value = process.env.INBOX_EXAMPLE_REDIS_URL;
+  if (!value)
+    throw new Error("Set INBOX_EXAMPLE_REDIS_URL to the local Redis fixture.");
+  const url = new URL(value);
+  if (
+    url.protocol !== "redis:" ||
+    url.hostname !== "127.0.0.1" ||
+    url.username ||
+    url.password
+  )
+    throw new Error(
+      "This example uses only the unauthenticated local Redis fixture.",
+    );
+  const connection = {
+    host: url.hostname,
+    port: Number(url.port || 6379),
+    maxRetriesPerRequest: null,
+    connectTimeout: 5_000,
+  };
+  const name = `inbox-example-${randomUUID()}`;
+  let callbackError: unknown;
+  const queue = new Queue<BrokerEvent>(name, {
+    connection: { ...connection, maxRetriesPerRequest: 1 },
+    defaultJobOptions: {
+      attempts: 3,
+      backoff: { type: "exponential", delay: 100 },
+      removeOnComplete: false,
+      removeOnFail: false,
+    },
+  });
+  queue.on("error", (error) => {
+    callbackError = error;
+  });
+  const fixture = await startBrokerInbox(
+    "example.bullmq.v1",
+    "order.created",
+  ).catch(async (error: unknown) => {
+    await queue.close();
+    throw error;
+  });
+  const inbox = fixture.app.get(InboxService);
+  let transportRetries = 0;
+  let injectAcceptanceFailure = true;
+  const worker = new Worker<BrokerEvent>(
+    name,
+    async (job) => {
+      try {
+        const event = parseBrokerEvent(job.data);
+        // Fixture-only failure: BullMQ retries acceptance, not the business handler.
+        if (injectAcceptanceFailure) {
+          injectAcceptanceFailure = false;
+          transportRetries++;
+          throw new Error("Injected acceptance failure");
+        }
+        const receipt = await inbox.accept({
+          scope: { kind: "tenant", id: tenantId },
+          source: "example-bullmq-orders",
+          consumer: "example.bullmq.v1",
+          messageId: event.eventId,
+          eventType: "order.created",
+          schemaVersion: 1,
+          payload: { orderId: event.orderId },
+        });
+        // Return only after the Inbox transaction commits. This completes the queue job.
+        return receipt.kind;
+      } catch (error) {
+        if (
+          error instanceof InboxValidationError ||
+          error instanceof InboxConflictError
+        )
+          throw new UnrecoverableError(
+            "Permanent ingress rejection; inspect the retained failed job.",
+          );
+        throw error;
+      }
+    },
+    { connection, concurrency: 1, maxStalledCount: 1 },
+  );
+  worker.on("error", (error) => {
+    callbackError = error;
+  });
+  try {
+    await worker.waitUntilReady();
+    const orderId = randomUUID();
+    const eventId = randomUUID();
+    // Different job IDs deliberately prove deduplication belongs to Inbox.
+    const jobs = await queue.addBulk([
+      {
+        name: "order",
+        data: { eventId, orderId },
+        opts: { jobId: randomUUID() },
+      },
+      {
+        name: "order",
+        data: { eventId, orderId },
+        opts: { jobId: randomUUID() },
+      },
+      {
+        name: "invalid",
+        data: { eventId: "", orderId: "" },
+        opts: { jobId: randomUUID() },
+      },
+    ]);
+    await waitUntil(async () => {
+      if (callbackError) throw callbackError;
+      return (
+        (await jobs[0]!.getState()) === "completed" &&
+        (await jobs[1]!.getState()) === "completed" &&
+        (await jobs[2]!.getState()) === "failed" &&
+        (await fixture.prisma.exampleProjection.count({
+          where: { orderId, tenantId },
+        })) === 1
+      );
+    }, "BullMQ acceptance retries, permanent failed jobs, and one projection");
+    const failed = await queue.getJob(jobs[2]!.id!);
+    assert.equal(failed?.attemptsMade, 1);
+    assert.equal(transportRetries, 1);
+    assert.equal(fixture.retries(), 1);
+    console.log(
+      JSON.stringify({
+        broker: "BullMQ / Redis",
+        completedJobs: 2,
+        quarantinedJobs: 1,
+        transportRetries,
+        handlerRetries: fixture.retries(),
+        projections: 1,
+      }),
+    );
+  } finally {
+    try {
+      await worker.close();
+    } finally {
+      try {
+        await queue.obliterate({ force: true });
+      } finally {
+        try {
+          await queue.close();
+        } finally {
+          await fixture.close();
+        }
+      }
+    }
+  }
+}
+
+void main().catch((error: unknown) => {
+  console.error(error);
+  process.exitCode = 1;
+});