@open-mercato/queue 0.4.2-canary-c02407ff85

package/src/strategies/local.ts

@@ -0,0 +1,303 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import crypto from 'node:crypto'
+import type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult } from '../types'
+
+type LocalState = {
+  lastProcessedId?: string
+  completedCount?: number
+  failedCount?: number
+}
+
+type StoredJob<T> = QueuedJob<T>
+
+/** Default polling interval in milliseconds */
+const DEFAULT_POLL_INTERVAL = 1000
+
+/**
+ * Creates a file-based local queue.
+ *
+ * Jobs are stored in JSON files within a directory structure:
+ * - `.queue/<name>/queue.json` - Array of queued jobs
+ * - `.queue/<name>/state.json` - Processing state (last processed ID)
+ *
+ * **Limitations:**
+ * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)
+ * - Not suitable for production or multi-process environments
+ * - No retry mechanism for failed jobs
+ *
+ * @template T - The payload type for jobs
+ * @param name - Queue name (used for directory naming)
+ * @param options - Local queue options
+ */
+export function createLocalQueue<T = unknown>(
+  name: string,
+  options?: LocalQueueOptions
+): Queue<T> {
+  const baseDir = options?.baseDir ?? path.resolve('.queue')
+  const queueDir = path.join(baseDir, name)
+  const queueFile = path.join(queueDir, 'queue.json')
+  const stateFile = path.join(queueDir, 'state.json')
+  // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially
+  const concurrency = options?.concurrency ?? 1
+  const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL
+
+  // Worker state for continuous polling
+  let pollingTimer: ReturnType<typeof setInterval> | null = null
+  let isProcessing = false
+  let activeHandler: JobHandler<T> | null = null
+
+  // -------------------------------------------------------------------------
+  // File Operations
+  // -------------------------------------------------------------------------
+
+  function ensureDir(): void {
+    // Use atomic operations to handle race conditions
+    try {
+      fs.mkdirSync(queueDir, { recursive: true })
+    } catch (e: unknown) {
+      const error = e as NodeJS.ErrnoException
+      if (error.code !== 'EEXIST') throw error
+    }
+
+    // Initialize queue file with exclusive create flag
+    try {
+      fs.writeFileSync(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })
+    } catch (e: unknown) {
+      const error = e as NodeJS.ErrnoException
+      if (error.code !== 'EEXIST') throw error
+    }
+
+    // Initialize state file with exclusive create flag
+    try {
+      fs.writeFileSync(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })
+    } catch (e: unknown) {
+      const error = e as NodeJS.ErrnoException
+      if (error.code !== 'EEXIST') throw error
+    }
+  }
+
+  function readQueue(): StoredJob<T>[] {
+    ensureDir()
+    try {
+      const content = fs.readFileSync(queueFile, 'utf8')
+      return JSON.parse(content) as StoredJob<T>[]
+    } catch (error: unknown) {
+      const e = error as NodeJS.ErrnoException
+      if (e.code === 'ENOENT') {
+        return []
+      }
+      console.error(`[queue:${name}] Failed to read queue file:`, e.message)
+      throw new Error(`Queue file corrupted or unreadable: ${e.message}`)
+    }
+  }
+
+  function writeQueue(jobs: StoredJob<T>[]): void {
+    ensureDir()
+    fs.writeFileSync(queueFile, JSON.stringify(jobs, null, 2), 'utf8')
+  }
+
+  function readState(): LocalState {
+    ensureDir()
+    try {
+      const content = fs.readFileSync(stateFile, 'utf8')
+      return JSON.parse(content) as LocalState
+    } catch {
+      return {}
+    }
+  }
+
+  function writeState(state: LocalState): void {
+    ensureDir()
+    fs.writeFileSync(stateFile, JSON.stringify(state, null, 2), 'utf8')
+  }
+
+  function generateId(): string {
+    return crypto.randomUUID()
+  }
+
+  // -------------------------------------------------------------------------
+  // Queue Implementation
+  // -------------------------------------------------------------------------
+
+  async function enqueue(data: T): Promise<string> {
+    const jobs = readQueue()
+    const job: StoredJob<T> = {
+      id: generateId(),
+      payload: data,
+      createdAt: new Date().toISOString(),
+    }
+    jobs.push(job)
+    writeQueue(jobs)
+    return job.id
+  }
+
+  /**
+   * Process pending jobs in a single batch (internal helper).
+   */
+  async function processBatch(
+    handler: JobHandler<T>,
+    options?: ProcessOptions
+  ): Promise<ProcessResult> {
+    const state = readState()
+    const jobs = readQueue()
+
+    // Find jobs that haven't been processed yet
+    const lastProcessedIndex = state.lastProcessedId
+      ? jobs.findIndex((j) => j.id === state.lastProcessedId)
+      : -1
+
+    const pendingJobs = jobs.slice(lastProcessedIndex + 1)
+    const jobsToProcess = options?.limit
+      ? pendingJobs.slice(0, options.limit)
+      : pendingJobs
+
+    let processed = 0
+    let failed = 0
+    let lastJobId: string | undefined
+    const jobIdsToRemove = new Set<string>()
+
+    for (const job of jobsToProcess) {
+      try {
+        await Promise.resolve(
+          handler(job, {
+            jobId: job.id,
+            attemptNumber: 1,
+            queueName: name,
+          })
+        )
+        processed++
+        lastJobId = job.id
+        jobIdsToRemove.add(job.id)
+        console.log(`[queue:${name}] Job ${job.id} completed`)
+      } catch (error) {
+        console.error(`[queue:${name}] Job ${job.id} failed:`, error)
+        failed++
+        lastJobId = job.id
+        jobIdsToRemove.add(job.id) // Remove failed jobs too (matching async strategy)
+      }
+    }
+
+    // Remove processed jobs from queue (matching async removeOnComplete behavior)
+    if (jobIdsToRemove.size > 0) {
+      const updatedJobs = jobs.filter((j) => !jobIdsToRemove.has(j.id))
+      writeQueue(updatedJobs)
+
+      // Update state with running counts
+      const newState: LocalState = {
+        lastProcessedId: lastJobId,
+        completedCount: (state.completedCount ?? 0) + processed,
+        failedCount: (state.failedCount ?? 0) + failed,
+      }
+      writeState(newState)
+    }
+
+    return { processed, failed, lastJobId }
+  }
+
+  /**
+   * Poll for and process new jobs.
+   */
+  async function pollAndProcess(): Promise<void> {
+    // Skip if already processing to avoid concurrent file access
+    if (isProcessing || !activeHandler) return
+
+    isProcessing = true
+    try {
+      await processBatch(activeHandler)
+    } catch (error) {
+      console.error(`[queue:${name}] Polling error:`, error)
+    } finally {
+      isProcessing = false
+    }
+  }
+
+  async function process(
+    handler: JobHandler<T>,
+    options?: ProcessOptions
+  ): Promise<ProcessResult> {
+    // If limit is specified, do a single batch (backward compatibility)
+    if (options?.limit) {
+      return processBatch(handler, options)
+    }
+
+    // Start continuous polling mode (like BullMQ Worker)
+    activeHandler = handler
+
+    // Process any pending jobs immediately
+    await processBatch(handler)
+
+    // Start polling interval for new jobs
+    pollingTimer = setInterval(() => {
+      pollAndProcess().catch((err) => {
+        console.error(`[queue:${name}] Poll cycle error:`, err)
+      })
+    }, pollInterval)
+
+    console.log(`[queue:${name}] Worker started with concurrency ${concurrency}`)
+
+    // Return sentinel value indicating continuous worker mode (like async strategy)
+    return { processed: -1, failed: -1, lastJobId: undefined }
+  }
+
+  async function clear(): Promise<{ removed: number }> {
+    const jobs = readQueue()
+    const removed = jobs.length
+    writeQueue([])
+    // Reset state but preserve counts for historical tracking
+    const state = readState()
+    writeState({
+      completedCount: state.completedCount,
+      failedCount: state.failedCount,
+    })
+    return { removed }
+  }
+
+  async function close(): Promise<void> {
+    // Stop polling timer
+    if (pollingTimer) {
+      clearInterval(pollingTimer)
+      pollingTimer = null
+    }
+    activeHandler = null
+
+    // Wait for any in-progress processing to complete (with timeout)
+    const SHUTDOWN_TIMEOUT = 5000
+    const startTime = Date.now()
+
+    while (isProcessing) {
+      if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {
+        console.warn(`[queue:${name}] Force closing after ${SHUTDOWN_TIMEOUT}ms timeout`)
+        break
+      }
+      await new Promise((resolve) => setTimeout(resolve, 50))
+    }
+  }
+
+  async function getJobCounts(): Promise<{
+    waiting: number
+    active: number
+    completed: number
+    failed: number
+  }> {
+    const state = readState()
+    const jobs = readQueue()
+
+    return {
+      waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)
+      active: 0, // Local strategy doesn't track active jobs
+      completed: state.completedCount ?? 0,
+      failed: state.failedCount ?? 0,
+    }
+  }
+
+  return {
+    name,
+    strategy: 'local',
+    enqueue,
+    process,
+    clear,
+    close,
+    getJobCounts,
+  }
+}

package/src/types.ts

@@ -0,0 +1,238 @@
+/**
+ * Queue Package Type Definitions
+ *
+ * Provides type-safe abstractions for multi-strategy job queues.
+ */
+
+// ============================================================================
+// Core Job Types
+// ============================================================================
+
+/**
+ * Represents a job stored in the queue.
+ * @template T - The payload type for this job
+ */
+export type QueuedJob<T = unknown> = {
+  /** Unique identifier for the job */
+  id: string
+  /** The job payload data */
+  payload: T
+  /** ISO timestamp when the job was created */
+  createdAt: string
+  /** Optional metadata for the job */
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * Context provided to job handlers during processing.
+ */
+export type JobContext = {
+  /** Unique identifier of the current job */
+  jobId: string
+  /** Current attempt number (1-based) */
+  attemptNumber: number
+  /** Name of the queue being processed */
+  queueName: string
+}
+
+/**
+ * Handler function that processes jobs from the queue.
+ * @template T - The payload type this handler expects
+ */
+export type JobHandler<T = unknown> = (
+  job: QueuedJob<T>,
+  ctx: JobContext
+) => Promise<void> | void
+
+// ============================================================================
+// Strategy Types
+// ============================================================================
+
+/** Available queue strategy types */
+export type QueueStrategyType = 'local' | 'async'
+
+/**
+ * Options for local (file-based) queue strategy.
+ */
+export type LocalQueueOptions = {
+  /** Base directory for queue files. Defaults to '.queue' */
+  baseDir?: string
+  /** Number of concurrent job processors. Defaults to 1 */
+  concurrency?: number
+  /** Polling interval in milliseconds for continuous processing. Defaults to 1000 */
+  pollInterval?: number
+}
+
+/**
+ * Redis connection options for async strategy.
+ */
+export type RedisConnectionOptions = {
+  /** Redis connection URL (e.g., redis://localhost:6379) */
+  url?: string
+  /** Redis host */
+  host?: string
+  /** Redis port */
+  port?: number
+  /** Redis password */
+  password?: string
+}
+
+/**
+ * Options for async (BullMQ) queue strategy.
+ */
+export type AsyncQueueOptions = {
+  /** Redis connection configuration */
+  connection?: RedisConnectionOptions
+  /** Number of concurrent job processors. Defaults to 1 */
+  concurrency?: number
+}
+
+/**
+ * Conditional options type based on strategy.
+ * Local strategy gets file options, async gets Redis options.
+ */
+export type QueueOptions<S extends QueueStrategyType> = S extends 'async'
+  ? AsyncQueueOptions
+  : LocalQueueOptions
+
+// ============================================================================
+// Process Types
+// ============================================================================
+
+/**
+ * Options for the process operation.
+ */
+export type ProcessOptions = {
+  /** Maximum number of jobs to process (local strategy only) */
+  limit?: number
+}
+
+/**
+ * Result returned after processing jobs.
+ */
+export type ProcessResult = {
+  /** Number of jobs successfully processed */
+  processed: number
+  /** Number of jobs that failed */
+  failed: number
+  /** ID of the last processed job */
+  lastJobId?: string
+}
+
+// ============================================================================
+// Queue Interface
+// ============================================================================
+
+/**
+ * Main queue interface that all strategies must implement.
+ * @template T - The payload type for jobs in this queue
+ */
+export interface Queue<T = unknown> {
+  /** Name of this queue */
+  readonly name: string
+  /** Strategy type used by this queue */
+  readonly strategy: QueueStrategyType
+
+  /**
+   * Add a job to the queue.
+   * @param data - The job payload
+   * @returns Promise resolving to the job ID
+   */
+  enqueue(data: T): Promise<string>
+
+  /**
+   * Process jobs from the queue.
+   *
+   * For local strategy: processes jobs synchronously and returns result with counts.
+   * For async strategy: starts a worker and returns sentinel result (processed=-1).
+   *
+   * @param handler - Function to handle each job
+   * @param options - Processing options
+   * @returns ProcessResult with counts (or sentinel for async worker mode)
+   */
+  process(handler: JobHandler<T>, options?: ProcessOptions): Promise<ProcessResult>
+
+  /**
+   * Remove all jobs from the queue.
+   * @returns Promise with count of removed jobs
+   */
+  clear(): Promise<{ removed: number }>
+
+  /**
+   * Close the queue and release resources.
+   */
+  close(): Promise<void>
+
+  /**
+   * Get current job counts by status.
+   * For async strategy: returns counts from BullMQ.
+   * For local strategy: waiting/completed based on last processed ID.
+   */
+  getJobCounts(): Promise<{
+    waiting: number
+    active: number
+    completed: number
+    failed: number
+  }>
+}
+
+// ============================================================================
+// Factory Types
+// ============================================================================
+
+/**
+ * Discriminated union for queue creation options.
+ */
+export type CreateQueueConfig<S extends QueueStrategyType = QueueStrategyType> =
+  S extends 'async'
+    ? { strategy: 'async' } & AsyncQueueOptions
+    : { strategy: 'local' } & LocalQueueOptions
+
+/**
+ * Factory function signature for creating queues.
+ */
+export type CreateQueueFn = <T = unknown>(
+  name: string,
+  strategy: QueueStrategyType,
+  options?: QueueOptions<QueueStrategyType>
+) => Queue<T>
+
+// ============================================================================
+// Worker Discovery Types
+// ============================================================================
+
+/**
+ * Metadata exported by worker files for auto-discovery.
+ *
+ * @example
+ * ```typescript
+ * // src/modules/example/workers/my-queue.ts
+ * export const metadata: WorkerMeta = {
+ *   queue: 'my-queue',
+ *   concurrency: 5,
+ * }
+ * ```
+ */
+export type WorkerMeta = {
+  /** Queue name this worker processes */
+  queue: string
+  /** Optional unique identifier (defaults to <module>:workers:<filename>) */
+  id?: string
+  /** Worker concurrency (default: 1) */
+  concurrency?: number
+}
+
+/**
+ * Descriptor for a discovered and registered worker.
+ * @template T - The job payload type this worker handles
+ */
+export type WorkerDescriptor<T = unknown> = {
+  /** Unique identifier for this worker */
+  id: string
+  /** Queue name to process */
+  queue: string
+  /** Handler function */
+  handler: JobHandler<T>
+  /** Concurrency level */
+  concurrency: number
+}

package/src/worker/__tests__/registry.test.ts

@@ -0,0 +1,176 @@
+import type { WorkerDescriptor } from '../../types'
+import {
+  registerWorker,
+  registerModuleWorkers,
+  getWorkers,
+  getWorkersByQueue,
+  getWorker,
+  getRegisteredQueues,
+  clearWorkers,
+} from '../registry'
+
+describe('Worker Registry', () => {
+  beforeEach(() => {
+    clearWorkers()
+  })
+
+  afterEach(() => {
+    clearWorkers()
+  })
+
+  describe('registerWorker', () => {
+    it('should register a single worker', () => {
+      const worker: WorkerDescriptor = {
+        id: 'test:worker',
+        queue: 'test-queue',
+        concurrency: 1,
+        handler: async () => {},
+      }
+
+      registerWorker(worker)
+
+      const result = getWorker('test:worker')
+      expect(result).toBeDefined()
+      expect(result?.id).toBe('test:worker')
+      expect(result?.queue).toBe('test-queue')
+      expect(result?.concurrency).toBe(1)
+    })
+
+    it('should overwrite existing worker with same id', () => {
+      const worker1: WorkerDescriptor = {
+        id: 'test:worker',
+        queue: 'queue-1',
+        concurrency: 1,
+        handler: async () => {},
+      }
+
+      const worker2: WorkerDescriptor = {
+        id: 'test:worker',
+        queue: 'queue-2',
+        concurrency: 5,
+        handler: async () => {},
+      }
+
+      registerWorker(worker1)
+      registerWorker(worker2)
+
+      const workers = getWorkers()
+      expect(workers.length).toBe(1)
+      expect(workers[0].queue).toBe('queue-2')
+      expect(workers[0].concurrency).toBe(5)
+    })
+  })
+
+  describe('registerModuleWorkers', () => {
+    it('should register multiple workers at once', () => {
+      const workers: WorkerDescriptor[] = [
+        { id: 'module:worker1', queue: 'queue-a', concurrency: 1, handler: async () => {} },
+        { id: 'module:worker2', queue: 'queue-b', concurrency: 2, handler: async () => {} },
+        { id: 'module:worker3', queue: 'queue-a', concurrency: 3, handler: async () => {} },
+      ]
+
+      registerModuleWorkers(workers)
+
+      expect(getWorkers().length).toBe(3)
+      expect(getWorker('module:worker1')).toBeDefined()
+      expect(getWorker('module:worker2')).toBeDefined()
+      expect(getWorker('module:worker3')).toBeDefined()
+    })
+
+    it('should handle empty array', () => {
+      registerModuleWorkers([])
+      expect(getWorkers().length).toBe(0)
+    })
+  })
+
+  describe('getWorkers', () => {
+    it('should return empty array when no workers registered', () => {
+      expect(getWorkers()).toEqual([])
+    })
+
+    it('should return all registered workers', () => {
+      registerWorker({ id: 'w1', queue: 'q1', concurrency: 1, handler: async () => {} })
+      registerWorker({ id: 'w2', queue: 'q2', concurrency: 2, handler: async () => {} })
+
+      const workers = getWorkers()
+      expect(workers.length).toBe(2)
+      expect(workers.map(w => w.id).sort()).toEqual(['w1', 'w2'])
+    })
+  })
+
+  describe('getWorkersByQueue', () => {
+    it('should return workers for a specific queue', () => {
+      registerModuleWorkers([
+        { id: 'events:w1', queue: 'events', concurrency: 1, handler: async () => {} },
+        { id: 'search:w1', queue: 'fulltext-indexing', concurrency: 2, handler: async () => {} },
+        { id: 'events:w2', queue: 'events', concurrency: 1, handler: async () => {} },
+      ])
+
+      const eventsWorkers = getWorkersByQueue('events')
+      expect(eventsWorkers.length).toBe(2)
+      expect(eventsWorkers.every(w => w.queue === 'events')).toBe(true)
+
+      const searchWorkers = getWorkersByQueue('fulltext-indexing')
+      expect(searchWorkers.length).toBe(1)
+      expect(searchWorkers[0].id).toBe('search:w1')
+    })
+
+    it('should return empty array for unknown queue', () => {
+      registerWorker({ id: 'w1', queue: 'known-queue', concurrency: 1, handler: async () => {} })
+
+      const workers = getWorkersByQueue('unknown-queue')
+      expect(workers).toEqual([])
+    })
+  })
+
+  describe('getWorker', () => {
+    it('should return undefined for unknown worker id', () => {
+      expect(getWorker('unknown:worker')).toBeUndefined()
+    })
+
+    it('should return the worker for known id', () => {
+      const handler = async () => {}
+      registerWorker({ id: 'known:worker', queue: 'my-queue', concurrency: 5, handler })
+
+      const worker = getWorker('known:worker')
+      expect(worker).toBeDefined()
+      expect(worker?.queue).toBe('my-queue')
+      expect(worker?.concurrency).toBe(5)
+      expect(worker?.handler).toBe(handler)
+    })
+  })
+
+  describe('getRegisteredQueues', () => {
+    it('should return empty array when no workers registered', () => {
+      expect(getRegisteredQueues()).toEqual([])
+    })
+
+    it('should return unique queue names', () => {
+      registerModuleWorkers([
+        { id: 'w1', queue: 'events', concurrency: 1, handler: async () => {} },
+        { id: 'w2', queue: 'fulltext-indexing', concurrency: 1, handler: async () => {} },
+        { id: 'w3', queue: 'events', concurrency: 1, handler: async () => {} },
+        { id: 'w4', queue: 'vector-indexing', concurrency: 1, handler: async () => {} },
+      ])
+
+      const queues = getRegisteredQueues().sort()
+      expect(queues).toEqual(['events', 'fulltext-indexing', 'vector-indexing'])
+    })
+  })
+
+  describe('clearWorkers', () => {
+    it('should remove all registered workers', () => {
+      registerModuleWorkers([
+        { id: 'w1', queue: 'q1', concurrency: 1, handler: async () => {} },
+        { id: 'w2', queue: 'q2', concurrency: 1, handler: async () => {} },
+      ])
+
+      expect(getWorkers().length).toBe(2)
+
+      clearWorkers()
+
+      expect(getWorkers()).toEqual([])
+      expect(getRegisteredQueues()).toEqual([])
+    })
+  })
+})