@open-mercato/queue 0.4.2-canary-c02407ff85

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@open-mercato/queue",
+  "version": "0.4.2-canary-c02407ff85",
+  "description": "Multi-strategy job queue with local and BullMQ support",
+  "type": "module",
+  "main": "./dist/index.js",
+  "scripts": {
+    "build": "node build.mjs",
+    "watch": "node watch.mjs",
+    "test": "jest --config jest.config.cjs",
+    "typecheck": "tsc --noEmit"
+  },
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./dist/index.js"
+    },
+    "./worker": {
+      "types": "./src/worker/runner.ts",
+      "default": "./dist/worker/runner.js"
+    },
+    "./*": {
+      "types": [
+        "./src/*.ts"
+      ],
+      "default": "./dist/*.js"
+    },
+    "./*/*": {
+      "types": [
+        "./src/*/*.ts"
+      ],
+      "default": "./dist/*/*.js"
+    }
+  },
+  "peerDependencies": {
+    "bullmq": "^5.0.0"
+  },
+  "peerDependenciesMeta": {
+    "bullmq": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/jest": "^30.0.0",
+    "@types/node": "^20.0.0",
+    "jest": "^30.2.0",
+    "ts-jest": "^29.4.6"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "stableVersion": "0.4.1"
+}

package/src/__tests__/local.strategy.test.ts

@@ -0,0 +1,192 @@
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+import { createQueue } from '../factory'
+import type { QueuedJob } from '../types'
+
+function readJson(p: string) { return JSON.parse(fs.readFileSync(p, 'utf8')) }
+
+describe('Queue - local strategy', () => {
+  const origCwd = process.cwd()
+  let tmp: string
+
+  beforeEach(() => {
+    tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'queue-test-'))
+    process.chdir(tmp)
+  })
+
+  afterEach(() => {
+    process.chdir(origCwd)
+    try { fs.rmSync(tmp, { recursive: true, force: true }) } catch {}
+  })
+
+  test('enqueue adds job to queue file', async () => {
+    const queue = createQueue<{ value: number }>('test-queue', 'local')
+    const queuePath = path.join('.queue', 'test-queue', 'queue.json')
+
+    const jobId = await queue.enqueue({ value: 42 })
+
+    expect(typeof jobId).toBe('string')
+    expect(jobId.length).toBeGreaterThan(0)
+
+    const jobs = readJson(queuePath)
+    expect(jobs.length).toBe(1)
+    expect(jobs[0].payload).toEqual({ value: 42 })
+    expect(jobs[0].id).toBe(jobId)
+
+    await queue.close()
+  })
+
+  test('process executes handler for each job', async () => {
+    const queue = createQueue<{ value: number }>('test-queue', 'local')
+    const processed: QueuedJob<{ value: number }>[] = []
+
+    await queue.enqueue({ value: 1 })
+    await queue.enqueue({ value: 2 })
+    await queue.enqueue({ value: 3 })
+
+    // Use limit to trigger batch mode (without limit, enters continuous polling mode)
+    const result = await queue.process((job) => {
+      processed.push(job)
+    }, { limit: 10 })
+
+    expect(result).toBeDefined()
+    expect(result!.processed).toBe(3)
+    expect(result!.failed).toBe(0)
+    expect(processed.length).toBe(3)
+    expect(processed.map(j => j.payload.value)).toEqual([1, 2, 3])
+
+    await queue.close()
+  })
+
+  test('process with limit only processes specified number of jobs', async () => {
+    const queue = createQueue<{ value: number }>('test-queue', 'local')
+    const processed: number[] = []
+
+    await queue.enqueue({ value: 1 })
+    await queue.enqueue({ value: 2 })
+    await queue.enqueue({ value: 3 })
+
+    const result = await queue.process(
+      (job) => { processed.push(job.payload.value) },
+      { limit: 2 }
+    )
+
+    expect(result!.processed).toBe(2)
+    expect(processed).toEqual([1, 2])
+
+    // Process remaining (use limit to stay in batch mode)
+    const result2 = await queue.process(
+      (job) => { processed.push(job.payload.value) },
+      { limit: 10 }
+    )
+
+    expect(result2!.processed).toBe(1)
+    expect(processed).toEqual([1, 2, 3])
+
+    await queue.close()
+  })
+
+  test('clear removes all jobs from queue', async () => {
+    const queue = createQueue<{ value: number }>('test-queue', 'local')
+    const queuePath = path.join('.queue', 'test-queue', 'queue.json')
+
+    await queue.enqueue({ value: 1 })
+    await queue.enqueue({ value: 2 })
+
+    const before = readJson(queuePath)
+    expect(before.length).toBe(2)
+
+    const result = await queue.clear()
+    expect(result.removed).toBe(2)
+
+    const after = readJson(queuePath)
+    expect(after.length).toBe(0)
+
+    await queue.close()
+  })
+
+  test('getJobCounts returns correct counts', async () => {
+    const queue = createQueue<{ value: number }>('test-queue', 'local')
+
+    await queue.enqueue({ value: 1 })
+    await queue.enqueue({ value: 2 })
+    await queue.enqueue({ value: 3 })
+
+    const counts = await queue.getJobCounts()
+    expect(counts.waiting).toBe(3)
+    expect(counts.completed).toBe(0)
+
+    await queue.process(() => {}, { limit: 1 })
+
+    const counts2 = await queue.getJobCounts()
+    expect(counts2.waiting).toBe(2)
+    expect(counts2.completed).toBe(1)
+
+    await queue.close()
+  })
+
+  test('queue name is used for directory', async () => {
+    const queue = createQueue('my-custom-queue', 'local')
+    const queueDir = path.join('.queue', 'my-custom-queue')
+
+    await queue.enqueue({ data: 'test' })
+
+    expect(fs.existsSync(queueDir)).toBe(true)
+    expect(fs.existsSync(path.join(queueDir, 'queue.json'))).toBe(true)
+    expect(fs.existsSync(path.join(queueDir, 'state.json'))).toBe(true)
+
+    await queue.close()
+  })
+
+  test('custom baseDir option is respected', async () => {
+    const customDir = path.join(tmp, 'custom-queue-dir')
+    const queue = createQueue('test', 'local', { baseDir: customDir })
+
+    await queue.enqueue({ data: 'test' })
+
+    expect(fs.existsSync(path.join(customDir, 'test', 'queue.json'))).toBe(true)
+
+    await queue.close()
+  })
+
+  test('handler errors are caught and counted as failures', async () => {
+    const queue = createQueue<{ shouldFail: boolean }>('test-queue', 'local')
+
+    await queue.enqueue({ shouldFail: false })
+    await queue.enqueue({ shouldFail: true })
+    await queue.enqueue({ shouldFail: false })
+
+    // Use limit to trigger batch mode (without limit, enters continuous polling mode)
+    const result = await queue.process((job) => {
+      if (job.payload.shouldFail) {
+        throw new Error('Intentional test error')
+      }
+    }, { limit: 10 })
+
+    expect(result!.processed).toBe(2)
+    expect(result!.failed).toBe(1)
+
+    await queue.close()
+  })
+
+  test('job context contains correct information', async () => {
+    const queue = createQueue<{ value: number }>('context-test', 'local')
+    let capturedContext: any = null
+
+    const jobId = await queue.enqueue({ value: 42 })
+
+    // Use limit to trigger batch mode
+    await queue.process((job, ctx) => {
+      capturedContext = ctx
+    }, { limit: 10 })
+
+    expect(capturedContext).not.toBeNull()
+    expect(capturedContext.jobId).toBe(jobId)
+    expect(capturedContext.attemptNumber).toBe(1)
+    expect(capturedContext.queueName).toBe('context-test')
+
+    await queue.close()
+  })
+})

package/src/factory.ts

@@ -0,0 +1,55 @@
+import type { Queue, LocalQueueOptions, AsyncQueueOptions } from './types'
+import { createLocalQueue } from './strategies/local'
+import { createAsyncQueue } from './strategies/async'
+
+/**
+ * Creates a queue instance with the specified strategy.
+ *
+ * @template T - The payload type for jobs in this queue
+ * @param name - Unique name for the queue
+ * @param strategy - Queue strategy: 'local' for file-based, 'async' for BullMQ
+ * @param options - Strategy-specific options
+ * @returns A Queue instance
+ *
+ * @example
+ * ```typescript
+ * // Local file-based queue
+ * const localQueue = createQueue<MyJobData>('my-queue', 'local')
+ *
+ * // BullMQ-based queue
+ * const asyncQueue = createQueue<MyJobData>('my-queue', 'async', {
+ *   connection: { url: 'redis://localhost:6379' },
+ *   concurrency: 5
+ * })
+ * ```
+ */
+export function createQueue<T = unknown>(
+  name: string,
+  strategy: 'local',
+  options?: LocalQueueOptions
+): Queue<T>
+
+export function createQueue<T = unknown>(
+  name: string,
+  strategy: 'async',
+  options?: AsyncQueueOptions
+): Queue<T>
+
+// General overload for dynamic strategy (union type)
+export function createQueue<T = unknown>(
+  name: string,
+  strategy: 'local' | 'async',
+  options?: LocalQueueOptions | AsyncQueueOptions
+): Queue<T>
+
+export function createQueue<T = unknown>(
+  name: string,
+  strategy: 'local' | 'async',
+  options?: LocalQueueOptions | AsyncQueueOptions
+): Queue<T> {
+  if (strategy === 'async') {
+    return createAsyncQueue<T>(name, options as AsyncQueueOptions)
+  }
+
+  return createLocalQueue<T>(name, options as LocalQueueOptions)
+}

package/src/index.ts

@@ -0,0 +1,28 @@
+/**
+ * @open-mercato/queue
+ *
+ * Multi-strategy job queue package supporting local (file-based) and async (BullMQ) strategies.
+ *
+ * @example
+ * ```typescript
+ * import { createQueue } from '@open-mercato/queue'
+ *
+ * // Create a local queue
+ * const queue = createQueue<{ userId: string }>('my-queue', 'local')
+ *
+ * // Enqueue a job
+ * await queue.enqueue({ userId: '123' })
+ *
+ * // Process jobs
+ * await queue.process(async (job, ctx) => {
+ *   console.log(`Processing job ${ctx.jobId}:`, job.payload)
+ * })
+ * ```
+ */
+
+export * from './types'
+export { createQueue } from './factory'
+
+// Worker utilities
+export * from './worker/registry'
+export { runWorker, createRoutedHandler } from './worker/runner'

package/src/strategies/async.ts

@@ -0,0 +1,209 @@
+import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult } from '../types'
+
+// BullMQ interface types - we define the shape we use to maintain type safety
+// while keeping bullmq as an optional peer dependency
+type ConnectionOptions = { host?: string; port?: number; password?: string } | string
+
+interface BullQueueInterface<T> {
+  add: (name: string, data: T, opts?: { removeOnComplete?: boolean; removeOnFail?: number }) => Promise<{ id?: string }>
+  obliterate: (opts?: { force?: boolean }) => Promise<void>
+  close: () => Promise<void>
+  getJobCounts: (...states: string[]) => Promise<Record<string, number>>
+}
+
+interface BullWorkerInterface {
+  on: (event: string, handler: (...args: unknown[]) => void) => void
+  close: () => Promise<void>
+}
+
+interface BullMQModule {
+  Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>
+  Worker: new <T>(
+    name: string,
+    processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,
+    opts: { connection: ConnectionOptions; concurrency: number }
+  ) => BullWorkerInterface
+}
+
+/**
+ * Resolves Redis connection options from various sources.
+ */
+function resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {
+  // Priority: explicit options > environment variables
+  const url = options?.url ?? process.env.REDIS_URL ?? process.env.QUEUE_REDIS_URL
+
+  if (url) {
+    return url
+  }
+
+  if (options?.host) {
+    return {
+      host: options.host,
+      port: options.port ?? 6379,
+      password: options.password,
+    }
+  }
+
+  // Default to localhost
+  return { host: 'localhost', port: 6379 }
+}
+
+/**
+ * Creates a BullMQ-based async queue.
+ *
+ * This strategy provides:
+ * - Persistent job storage in Redis
+ * - Automatic retries with exponential backoff
+ * - Concurrent job processing
+ * - Job prioritization and scheduling
+ *
+ * @template T - The payload type for jobs
+ * @param name - Queue name
+ * @param options - Async queue options
+ */
+export function createAsyncQueue<T = unknown>(
+  name: string,
+  options?: AsyncQueueOptions
+): Queue<T> {
+  const connection = resolveConnection(options?.connection)
+  const concurrency = options?.concurrency ?? 1
+
+  let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null
+  let bullWorker: BullWorkerInterface | null = null
+  let bullmqModule: BullMQModule | null = null
+
+  // -------------------------------------------------------------------------
+  // Lazy BullMQ initialization
+  // -------------------------------------------------------------------------
+
+  async function getBullMQ(): Promise<BullMQModule> {
+    if (!bullmqModule) {
+      try {
+        bullmqModule = await import('bullmq') as unknown as BullMQModule
+      } catch {
+        throw new Error(
+          'BullMQ is required for async queue strategy. Install it with: npm install bullmq'
+        )
+      }
+    }
+    return bullmqModule
+  }
+
+  async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {
+    if (!bullQueue) {
+      const { Queue: BullQueueClass } = await getBullMQ()
+      bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection })
+    }
+    return bullQueue
+  }
+
+  // -------------------------------------------------------------------------
+  // Queue Implementation
+  // -------------------------------------------------------------------------
+
+  async function enqueue(data: T): Promise<string> {
+    const queue = await getQueue()
+    const jobData: QueuedJob<T> = {
+      id: crypto.randomUUID(),
+      payload: data,
+      createdAt: new Date().toISOString(),
+    }
+
+    const job = await queue.add(jobData.id, jobData, {
+      removeOnComplete: true,
+      removeOnFail: 1000, // Keep last 1000 failed jobs
+    })
+
+    return job.id ?? jobData.id
+  }
+
+  async function process(handler: JobHandler<T>): Promise<ProcessResult> {
+    const { Worker } = await getBullMQ()
+
+    // Create worker that processes jobs
+    bullWorker = new Worker<QueuedJob<T>>(
+      name,
+      async (job) => {
+        const jobData = job.data
+        await handler(jobData, {
+          jobId: job.id ?? jobData.id,
+          attemptNumber: job.attemptsMade + 1,
+          queueName: name,
+        })
+      },
+      {
+        connection,
+        concurrency,
+      }
+    )
+
+    // Set up event handlers
+    bullWorker.on('completed', (job) => {
+      const jobWithId = job as { id?: string }
+      console.log(`[queue:${name}] Job ${jobWithId.id} completed`)
+    })
+
+    bullWorker.on('failed', (job, err) => {
+      const jobWithId = job as { id?: string } | undefined
+      const error = err as Error
+      console.error(`[queue:${name}] Job ${jobWithId?.id} failed:`, error.message)
+    })
+
+    bullWorker.on('error', (err) => {
+      const error = err as Error
+      console.error(`[queue:${name}] Worker error:`, error.message)
+    })
+
+    console.log(`[queue:${name}] Worker started with concurrency ${concurrency}`)
+
+    // For async strategy, return a sentinel result indicating worker mode
+    // processed=-1 signals that this is a continuous worker, not a batch process
+    return { processed: -1, failed: -1, lastJobId: undefined }
+  }
+
+  async function clear(): Promise<{ removed: number }> {
+    const queue = await getQueue()
+
+    // Obliterate removes all jobs from the queue
+    await queue.obliterate({ force: true })
+
+    return { removed: -1 } // BullMQ obliterate doesn't return count
+  }
+
+  async function close(): Promise<void> {
+    if (bullWorker) {
+      await bullWorker.close()
+      bullWorker = null
+    }
+    if (bullQueue) {
+      await bullQueue.close()
+      bullQueue = null
+    }
+  }
+
+  async function getJobCounts(): Promise<{
+    waiting: number
+    active: number
+    completed: number
+    failed: number
+  }> {
+    const queue = await getQueue()
+    const counts = await queue.getJobCounts('waiting', 'active', 'completed', 'failed')
+    return {
+      waiting: counts.waiting ?? 0,
+      active: counts.active ?? 0,
+      completed: counts.completed ?? 0,
+      failed: counts.failed ?? 0,
+    }
+  }
+
+  return {
+    name,
+    strategy: 'async',
+    enqueue,
+    process,
+    clear,
+    close,
+    getJobCounts,
+  }
+}