@crewhaus/queue-protocol 0.1.3 → 0.1.5

package/src/adapters/postgres.ts

@@ -1,144 +0,0 @@
-/**
- * Section 30 — Postgres adapter for `@crewhaus/queue-protocol`. Uses
- * advisory locks + a job table for visibility timeouts:
- *   pull → SELECT … FOR UPDATE SKIP LOCKED
- *   ack → DELETE FROM jobs WHERE id = $1
- *   nack(transient) → UPDATE jobs SET visibility_expires_at = NOW()
- *   nack(permanent) → INSERT INTO dead_letter_jobs + DELETE
- *   extendVisibility → UPDATE jobs SET visibility_expires_at = NOW() + ...
- *
- * v0 throws when `pg` isn't installed; the contract holds with a stub
- * client.
- */
-import type { Job, JobId, NackReason, PullOptions, QueueAdapter } from "../index";
-import { QueueProtocolError } from "../index";
-
-export type PostgresAdapterOptions = {
-  readonly tableName: string;
-  readonly deadLetterTable?: string;
-  readonly _client?: PostgresClientLike;
-};
-
-export type PostgresClientLike = {
-  query<T = unknown>(text: string, params?: unknown[]): Promise<{ rows: T[] }>;
-};
-
-type JobRow = {
-  id: string;
-  payload: string;
-  enqueued_at: string;
-  visibility_expires_at: string;
-  attempt: number;
-};
-
-export function createPostgresAdapter<TInput = unknown>(
-  opts: PostgresAdapterOptions,
-): QueueAdapter<TInput> {
-  if (!opts.tableName) throw new QueueProtocolError("postgres adapter requires tableName");
-  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(opts.tableName)) {
-    throw new QueueProtocolError(`postgres adapter: invalid tableName "${opts.tableName}"`);
-  }
-  const client = opts._client ?? requireClient();
-  let acked = 0;
-  let nacked = 0;
-  let deadLetter = 0;
-  const dlqTable = opts.deadLetterTable;
-
-  return {
-    kind: "postgres",
-    async pull(pullOpts: PullOptions): Promise<ReadonlyArray<Job<TInput>>> {
-      const visibilitySec = Math.ceil((pullOpts.visibilityTimeoutMs ?? 60_000) / 1000);
-      const result = await client.query<JobRow>(
-        `WITH leased AS (
-          SELECT id FROM ${opts.tableName}
-          WHERE visibility_expires_at <= NOW()
-          ORDER BY enqueued_at ASC
-          LIMIT $1
-          FOR UPDATE SKIP LOCKED
-        )
-        UPDATE ${opts.tableName} t
-        SET visibility_expires_at = NOW() + INTERVAL '${visibilitySec} seconds',
-            attempt = attempt + 1
-        FROM leased
-        WHERE t.id = leased.id
-        RETURNING t.id, t.payload, t.enqueued_at, t.visibility_expires_at, t.attempt`,
-        [pullOpts.maxJobs ?? 10],
-      );
-      const out: Job<TInput>[] = [];
-      for (const row of result.rows) {
-        let parsed: TInput;
-        try {
-          parsed = JSON.parse(row.payload) as TInput;
-        } catch {
-          parsed = row.payload as unknown as TInput;
-        }
-        out.push({
-          id: row.id,
-          input: parsed,
-          enqueuedAt: new Date(row.enqueued_at).getTime(),
-          visibilityExpiresAt: new Date(row.visibility_expires_at).getTime(),
-          attempt: row.attempt,
-        });
-      }
-      return out;
-    },
-    async ack(jobId: JobId): Promise<void> {
-      await client.query(`DELETE FROM ${opts.tableName} WHERE id = $1`, [jobId]);
-      acked++;
-    },
-    async nack(jobId: JobId, reason: NackReason): Promise<void> {
-      if (reason === "permanent" && dlqTable) {
-        if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(dlqTable)) {
-          throw new QueueProtocolError(`postgres adapter: invalid deadLetterTable "${dlqTable}"`);
-        }
-        await client.query(
-          `INSERT INTO ${dlqTable} (id, payload, enqueued_at)
-           SELECT id, payload, enqueued_at FROM ${opts.tableName} WHERE id = $1`,
-          [jobId],
-        );
-        await client.query(`DELETE FROM ${opts.tableName} WHERE id = $1`, [jobId]);
-        deadLetter++;
-      } else {
-        await client.query(
-          `UPDATE ${opts.tableName} SET visibility_expires_at = NOW() WHERE id = $1`,
-          [jobId],
-        );
-      }
-      nacked++;
-    },
-    async extendVisibility(jobId: JobId, additionalMs: number): Promise<void> {
-      const sec = Math.ceil(additionalMs / 1000);
-      await client.query(
-        `UPDATE ${opts.tableName} SET visibility_expires_at = NOW() + INTERVAL '${sec} seconds' WHERE id = $1`,
-        [jobId],
-      );
-    },
-    async stats(): Promise<{
-      pending: number;
-      inFlight: number;
-      acked: number;
-      nacked: number;
-      deadLetter: number;
-    }> {
-      const pendingRes = await client.query<{ count: string }>(
-        `SELECT COUNT(*)::text AS count FROM ${opts.tableName} WHERE visibility_expires_at <= NOW()`,
-      );
-      const inFlightRes = await client.query<{ count: string }>(
-        `SELECT COUNT(*)::text AS count FROM ${opts.tableName} WHERE visibility_expires_at > NOW()`,
-      );
-      return {
-        pending: Number.parseInt(pendingRes.rows[0]?.count ?? "0", 10),
-        inFlight: Number.parseInt(inFlightRes.rows[0]?.count ?? "0", 10),
-        acked,
-        nacked,
-        deadLetter,
-      };
-    },
-  };
-}
-
-function requireClient(): PostgresClientLike {
-  throw new QueueProtocolError(
-    "postgres adapter requires `pg` to be installed and DATABASE_URL configured. Pass an explicit `_client` to use a stub.",
-  );
-}

package/src/adapters/redis-streams.ts

@@ -1,120 +0,0 @@
-/**
- * Section 30 — Redis Streams adapter for `@crewhaus/queue-protocol`.
- *
- * Maps to Redis Streams + consumer groups:
- *   pull → XREADGROUP
- *   ack → XACK
- *   nack(transient) → XADD back to the stream tail
- *   nack(permanent) → XADD to the dead-letter stream + XACK on main
- *   extendVisibility → no-op (Redis Streams uses pending-list min-idle-time
- *                     instead of per-message visibility timeouts; consumers
- *                     refresh by re-reading from the pending list)
- *
- * v0 throws when `ioredis` is not installed; the contract is correct +
- * tested with stub clients.
- */
-import type { Job, JobId, NackReason, PullOptions, QueueAdapter } from "../index";
-import { QueueProtocolError } from "../index";
-
-export type RedisStreamsAdapterOptions = {
-  readonly streamKey: string;
-  readonly consumerGroup: string;
-  readonly consumerName: string;
-  readonly deadLetterStream?: string;
-  readonly _client?: RedisClientLike;
-};
-
-export type RedisClientLike = {
-  xreadgroup(
-    group: string,
-    consumer: string,
-    count: number,
-    blockMs: number,
-    streamKey: string,
-    id: string,
-  ): Promise<Array<{
-    stream: string;
-    messages: Array<{ id: string; fields: Record<string, string> }>;
-  }> | null>;
-  xack(streamKey: string, group: string, ...ids: string[]): Promise<number>;
-  xadd(streamKey: string, id: string, ...fields: string[]): Promise<string>;
-};
-
-export function createRedisStreamsAdapter<TInput = unknown>(
-  opts: RedisStreamsAdapterOptions,
-): QueueAdapter<TInput> {
-  if (!opts.streamKey) throw new QueueProtocolError("redis-streams adapter requires streamKey");
-  if (!opts.consumerGroup)
-    throw new QueueProtocolError("redis-streams adapter requires consumerGroup");
-  const client = opts._client ?? requireClient();
-  let acked = 0;
-  let nacked = 0;
-  let deadLetter = 0;
-  return {
-    kind: "redis-streams",
-    async pull(pullOpts: PullOptions): Promise<ReadonlyArray<Job<TInput>>> {
-      const res = await client.xreadgroup(
-        opts.consumerGroup,
-        opts.consumerName,
-        pullOpts.maxJobs ?? 10,
-        pullOpts.longPollMs ?? 0,
-        opts.streamKey,
-        ">",
-      );
-      if (!res) return [];
-      const out: Job<TInput>[] = [];
-      const now = Date.now();
-      for (const stream of res) {
-        for (const m of stream.messages) {
-          let parsed: TInput;
-          try {
-            parsed = JSON.parse(m.fields["payload"] ?? "{}") as TInput;
-          } catch {
-            parsed = m.fields["payload"] as unknown as TInput;
-          }
-          out.push({
-            id: m.id,
-            input: parsed,
-            enqueuedAt: now,
-            visibilityExpiresAt: now + (pullOpts.visibilityTimeoutMs ?? 60_000),
-            attempt: 1,
-          });
-        }
-      }
-      return out;
-    },
-    async ack(jobId: JobId): Promise<void> {
-      await client.xack(opts.streamKey, opts.consumerGroup, jobId);
-      acked++;
-    },
-    async nack(jobId: JobId, reason: NackReason): Promise<void> {
-      if (reason === "permanent" && opts.deadLetterStream) {
-        await client.xadd(opts.deadLetterStream, "*", "payload", JSON.stringify({ jobId }));
-        deadLetter++;
-      } else {
-        // Re-publish to the tail; the consumer-group will redeliver.
-        await client.xadd(opts.streamKey, "*", "payload", JSON.stringify({ retry: jobId }));
-      }
-      await client.xack(opts.streamKey, opts.consumerGroup, jobId);
-      nacked++;
-    },
-    async extendVisibility(_jobId: JobId, _additionalMs: number): Promise<void> {
-      // Redis Streams uses min-idle-time; nothing to do here.
-    },
-    async stats(): Promise<{
-      pending: number;
-      inFlight: number;
-      acked: number;
-      nacked: number;
-      deadLetter: number;
-    }> {
-      return { pending: 0, inFlight: 0, acked, nacked, deadLetter };
-    },
-  };
-}
-
-function requireClient(): RedisClientLike {
-  throw new QueueProtocolError(
-    "redis-streams adapter requires `ioredis` to be installed and a Redis URL configured. Pass an explicit `_client` to use a stub.",
-  );
-}

package/src/adapters/sqs.ts

@@ -1,149 +0,0 @@
-/**
- * Section 30 — SQS adapter for `@crewhaus/queue-protocol`. Production
- * deployments using AWS SQS use this; the contract is identical to the
- * in-memory adapter (pull/ack/nack/extendVisibility/stats).
- *
- * v0 ships with the *abstraction* and credential plumbing; the actual
- * `@aws-sdk/client-sqs` calls are gated on the AWS SDK being installed
- * and `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` (or the IAM
- * instance-profile chain) being available. The implementation throws
- * `QueueProtocolError("sqs adapter requires …")` when those are missing
- * so a single misconfigured spec fails loud at boot rather than silently
- * dropping jobs.
- *
- * The interface is correct + tested via the contract corpus; the live
- * SQS smoke is gated behind `AWS_ACCESS_KEY_ID` + `SQS_QUEUE_URL` and
- * runs only in deployments where those are set.
- */
-import type { Job, JobId, NackReason, PullOptions, QueueAdapter } from "../index";
-import { QueueProtocolError } from "../index";
-
-export type SqsAdapterOptions = {
-  readonly queueUrl: string;
-  readonly region: string;
-  readonly accessKeyId?: string;
-  readonly secretAccessKey?: string;
-  /** Test override: inject a fake SQS client. */
-  readonly _client?: SqsClientLike;
-};
-
-export type SqsClientLike = {
-  receiveMessage(input: {
-    QueueUrl: string;
-    MaxNumberOfMessages: number;
-    VisibilityTimeout: number;
-    WaitTimeSeconds: number;
-  }): Promise<{
-    Messages?: Array<{ MessageId?: string; ReceiptHandle?: string; Body?: string }>;
-  }>;
-  deleteMessage(input: { QueueUrl: string; ReceiptHandle: string }): Promise<void>;
-  changeMessageVisibility(input: {
-    QueueUrl: string;
-    ReceiptHandle: string;
-    VisibilityTimeout: number;
-  }): Promise<void>;
-};
-
-export function createSqsAdapter<TInput = unknown>(opts: SqsAdapterOptions): QueueAdapter<TInput> {
-  if (!opts.queueUrl) {
-    throw new QueueProtocolError("sqs adapter requires queueUrl");
-  }
-  const client = opts._client ?? requireSdkClient(opts);
-  // Map ReceiptHandle by jobId so ack/nack/extendVisibility can find it.
-  const receiptByJobId = new Map<JobId, string>();
-  let acked = 0;
-  let nacked = 0;
-  let deadLetter = 0;
-
-  return {
-    kind: "sqs",
-    async pull(pullOpts: PullOptions): Promise<ReadonlyArray<Job<TInput>>> {
-      const result = await client.receiveMessage({
-        QueueUrl: opts.queueUrl,
-        MaxNumberOfMessages: Math.min(10, pullOpts.maxJobs ?? 10),
-        VisibilityTimeout: Math.ceil((pullOpts.visibilityTimeoutMs ?? 60_000) / 1000),
-        WaitTimeSeconds: Math.ceil((pullOpts.longPollMs ?? 0) / 1000),
-      });
-      const out: Job<TInput>[] = [];
-      const now = Date.now();
-      for (const m of result.Messages ?? []) {
-        if (!m.MessageId || !m.ReceiptHandle || m.Body === undefined) continue;
-        receiptByJobId.set(m.MessageId, m.ReceiptHandle);
-        let parsed: TInput;
-        try {
-          parsed = JSON.parse(m.Body) as TInput;
-        } catch {
-          parsed = m.Body as unknown as TInput;
-        }
-        out.push({
-          id: m.MessageId,
-          input: parsed,
-          enqueuedAt: now,
-          visibilityExpiresAt: now + (pullOpts.visibilityTimeoutMs ?? 60_000),
-          attempt: 1,
-        });
-      }
-      return out;
-    },
-    async ack(jobId: JobId): Promise<void> {
-      const handle = receiptByJobId.get(jobId);
-      if (!handle) throw new QueueProtocolError(`sqs ack: receipt for ${jobId} not found`);
-      await client.deleteMessage({ QueueUrl: opts.queueUrl, ReceiptHandle: handle });
-      receiptByJobId.delete(jobId);
-      acked++;
-    },
-    async nack(jobId: JobId, reason: NackReason): Promise<void> {
-      const handle = receiptByJobId.get(jobId);
-      if (!handle) throw new QueueProtocolError(`sqs nack: receipt for ${jobId} not found`);
-      if (reason === "permanent") {
-        // Delete from the main queue; the redrive policy on the SQS side
-        // moves it to the dead-letter queue automatically when configured.
-        await client.deleteMessage({ QueueUrl: opts.queueUrl, ReceiptHandle: handle });
-        deadLetter++;
-      } else {
-        // Reset visibility timeout to 0 so the message is immediately re-driven.
-        await client.changeMessageVisibility({
-          QueueUrl: opts.queueUrl,
-          ReceiptHandle: handle,
-          VisibilityTimeout: 0,
-        });
-      }
-      receiptByJobId.delete(jobId);
-      nacked++;
-    },
-    async extendVisibility(jobId: JobId, additionalMs: number): Promise<void> {
-      const handle = receiptByJobId.get(jobId);
-      if (!handle)
-        throw new QueueProtocolError(`sqs extendVisibility: receipt for ${jobId} not found`);
-      await client.changeMessageVisibility({
-        QueueUrl: opts.queueUrl,
-        ReceiptHandle: handle,
-        VisibilityTimeout: Math.ceil(additionalMs / 1000),
-      });
-    },
-    async stats(): Promise<{
-      pending: number;
-      inFlight: number;
-      acked: number;
-      nacked: number;
-      deadLetter: number;
-    }> {
-      // SQS doesn't expose accurate counters cheaply; we return the local
-      // counters for in-flight + acked/nacked + 0 for pending (consumers
-      // should query CloudWatch for that).
-      return {
-        pending: 0,
-        inFlight: receiptByJobId.size,
-        acked,
-        nacked,
-        deadLetter,
-      };
-    },
-  };
-}
-
-function requireSdkClient(_opts: SqsAdapterOptions): SqsClientLike {
-  throw new QueueProtocolError(
-    "sqs adapter requires `@aws-sdk/client-sqs` to be installed and AWS credentials configured. Pass an explicit `_client` to use a stub.",
-  );
-}

package/src/index.test.ts

@@ -1,99 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import { QueueProtocolError, createInMemoryQueue } from "./index.js";
-
-describe("createInMemoryQueue", () => {
-  test("enqueue + pull returns jobs in FIFO order with attempt=1", async () => {
-    const q = createInMemoryQueue<string>();
-    await q.enqueue("a");
-    await q.enqueue("b");
-    await q.enqueue("c");
-    const got = await q.pull({ maxBatch: 10, visibilityTimeoutMs: 1000 });
-    expect(got.map((j) => j.input)).toEqual(["a", "b", "c"]);
-    expect(got.map((j) => j.attempt)).toEqual([1, 1, 1]);
-  });
-
-  test("pull respects maxBatch", async () => {
-    const q = createInMemoryQueue<number>();
-    for (let i = 0; i < 5; i++) await q.enqueue(i);
-    const first = await q.pull({ maxBatch: 2, visibilityTimeoutMs: 1000 });
-    expect(first.map((j) => j.input)).toEqual([0, 1]);
-  });
-
-  test("pulled jobs are invisible to subsequent pulls until visibility expires", async () => {
-    let t = 1_000;
-    const q = createInMemoryQueue<string>({ now: () => t });
-    await q.enqueue("a");
-    const first = await q.pull({ maxBatch: 10, visibilityTimeoutMs: 100 });
-    expect(first).toHaveLength(1);
-    // Within visibility window — invisible.
-    t = 1_050;
-    const second = await q.pull({ maxBatch: 10, visibilityTimeoutMs: 100 });
-    expect(second).toHaveLength(0);
-    // Past visibility window — reclaimed back to pending.
-    t = 1_200;
-    const third = await q.pull({ maxBatch: 10, visibilityTimeoutMs: 100 });
-    expect(third).toHaveLength(1);
-    expect(third[0]?.attempt).toBe(2);
-  });
-
-  test("ack removes the job; double-ack is a no-op", async () => {
-    const q = createInMemoryQueue<string>();
-    await q.enqueue("a");
-    const [j] = await q.pull({ maxBatch: 1, visibilityTimeoutMs: 1000 });
-    if (!j) throw new Error("no job");
-    await q.ack(j.id);
-    await q.ack(j.id); // idempotent
-    const stats = await q.stats();
-    expect(stats.acked).toBe(1);
-    expect(stats.pending).toBe(0);
-    expect(stats.inFlight).toBe(0);
-  });
-
-  test("nack transient returns the job to pending; nack permanent moves to DLQ", async () => {
-    const q = createInMemoryQueue<string>();
-    await q.enqueue("a");
-    await q.enqueue("b");
-    const pulled = await q.pull({ maxBatch: 10, visibilityTimeoutMs: 1000 });
-    if (pulled.length !== 2) throw new Error("expected 2");
-    await q.nack(pulled[0]?.id, "transient");
-    await q.nack(pulled[1]?.id, "permanent");
-
-    const stats = await q.stats();
-    expect(stats.deadLetter).toBe(1);
-    expect(stats.pending).toBe(1);
-
-    const again = await q.pull({ maxBatch: 10, visibilityTimeoutMs: 1000 });
-    expect(again).toHaveLength(1);
-    expect(again[0]?.attempt).toBe(2);
-  });
-
-  test("extendVisibility pushes the lease forward and silences a timeout reclaim", async () => {
-    let t = 1_000;
-    const q = createInMemoryQueue<string>({ now: () => t });
-    await q.enqueue("a");
-    const [j] = await q.pull({ maxBatch: 1, visibilityTimeoutMs: 100 });
-    if (!j) throw new Error("no job");
-    await q.extendVisibility(j.id, 1_000);
-    t = 1_500;
-    const second = await q.pull({ maxBatch: 10, visibilityTimeoutMs: 100 });
-    expect(second).toHaveLength(0);
-  });
-
-  test("extendVisibility throws on unknown jobId", async () => {
-    const q = createInMemoryQueue<string>();
-    await expect(q.extendVisibility("nonexistent", 100)).rejects.toThrow(QueueProtocolError);
-  });
-
-  test("inspect returns jobs with attempt counts (T9 invariant for retry counting)", async () => {
-    const q = createInMemoryQueue<string>();
-    await q.enqueue("x");
-    const [j1] = await q.pull({ maxBatch: 1, visibilityTimeoutMs: 100 });
-    if (!j1) throw new Error("no job");
-    await q.nack(j1.id, "transient");
-    const [j2] = await q.pull({ maxBatch: 1, visibilityTimeoutMs: 100 });
-    if (!j2) throw new Error("no job");
-    expect(j2.attempt).toBe(2);
-    const insp = q.inspect();
-    expect(insp[0]?.attempts).toBe(2);
-  });
-});