@mantiq/queue 0.0.1

package/src/QueueManager.ts

@@ -0,0 +1,76 @@
+import type { QueueDriver } from './contracts/QueueDriver.ts'
+import { QueueError } from './errors/QueueError.ts'
+import type { EventDispatcher } from '@mantiq/core'
+
+export interface QueueConnectionConfig {
+  driver: string
+  [key: string]: any
+}
+
+export interface QueueConfig {
+  default: string
+  connections: Record<string, QueueConnectionConfig>
+}
+
+type DriverFactory = (config: QueueConnectionConfig) => QueueDriver
+
+/**
+ * Manages queue driver instances.
+ * Follows the same Manager pattern as DatabaseManager / CacheManager.
+ */
+export class QueueManager {
+  private drivers = new Map<string, QueueDriver>()
+  private customDrivers = new Map<string, DriverFactory>()
+
+  /** Optional event dispatcher — set by QueueServiceProvider.boot() */
+  static _dispatcher: EventDispatcher | null = null
+
+  constructor(
+    private readonly config: QueueConfig,
+    private readonly builtInDrivers: Map<string, DriverFactory>,
+  ) {}
+
+  /** Get a queue driver by connection name (lazy-created and cached) */
+  driver(name?: string): QueueDriver {
+    const connName = name ?? this.config.default
+    if (this.drivers.has(connName)) return this.drivers.get(connName)!
+
+    const connConfig = this.config.connections[connName]
+    if (!connConfig) {
+      throw new QueueError(`Queue connection "${connName}" is not configured`, { connection: connName })
+    }
+
+    const driver = this.createDriver(connConfig)
+    this.drivers.set(connName, driver)
+    return driver
+  }
+
+  /** Register a custom driver factory */
+  extend(name: string, factory: DriverFactory): void {
+    this.customDrivers.set(name, factory)
+  }
+
+  /** Get the default connection name */
+  getDefaultDriver(): string {
+    return this.config.default
+  }
+
+  /** Get the full config */
+  getConfig(): QueueConfig {
+    return this.config
+  }
+
+  private createDriver(config: QueueConnectionConfig): QueueDriver {
+    const driverName = config.driver
+
+    // Check custom drivers first
+    const custom = this.customDrivers.get(driverName)
+    if (custom) return custom(config)
+
+    // Check built-in drivers
+    const builtIn = this.builtInDrivers.get(driverName)
+    if (builtIn) return builtIn(config)
+
+    throw new QueueError(`Unknown queue driver "${driverName}"`, { driver: driverName })
+  }
+}

package/src/QueueServiceProvider.ts

@@ -0,0 +1,88 @@
+import { ServiceProvider } from '@mantiq/core'
+import { QueueManager } from './QueueManager.ts'
+import type { QueueConfig, QueueConnectionConfig } from './QueueManager.ts'
+import { SyncDriver } from './drivers/SyncDriver.ts'
+import { SQLiteDriver } from './drivers/SQLiteDriver.ts'
+import { RedisDriver } from './drivers/RedisDriver.ts'
+import { SqsDriver } from './drivers/SqsDriver.ts'
+import { KafkaDriver } from './drivers/KafkaDriver.ts'
+import { setQueueManager, QUEUE_MANAGER } from './helpers/queue.ts'
+import { setPendingDispatchResolver } from './PendingDispatch.ts'
+import { setChainResolver } from './JobChain.ts'
+import { setBatchResolver } from './JobBatch.ts'
+
+/**
+ * Registers the QueueManager in the container and wires up helpers.
+ *
+ * @example — with @mantiq/core:
+ * ```ts
+ * // In your app's providers array:
+ * providers: [QueueServiceProvider]
+ * ```
+ */
+export class QueueServiceProvider extends ServiceProvider {
+  override register(): void {
+    this.app.singleton(QueueManager, () => {
+      const config: QueueConfig = (this.app as any).config?.().get?.('queue') ?? {
+        default: 'sync',
+        connections: {
+          sync: { driver: 'sync' },
+        },
+      }
+
+      const builtInDrivers = new Map<string, (cfg: QueueConnectionConfig) => any>([
+        ['sync', () => new SyncDriver()],
+        ['sqlite', (cfg) => new SQLiteDriver(cfg.database ?? 'queue.sqlite')],
+        ['redis', (cfg) => new RedisDriver(cfg as any)],
+        ['sqs', (cfg) => new SqsDriver(cfg as any)],
+        ['kafka', (cfg) => new KafkaDriver(cfg as any)],
+      ])
+
+      const manager = new QueueManager(config, builtInDrivers)
+
+      // Wire up helpers
+      setQueueManager(manager)
+      setPendingDispatchResolver(() => manager)
+      setChainResolver(() => manager)
+      setBatchResolver(() => manager)
+
+      return manager
+    })
+
+    // Also register under the symbol for container lookups
+    this.app.singleton(QUEUE_MANAGER as any, () => {
+      return this.app.make(QueueManager)
+    })
+  }
+
+  override boot(): void {
+    // Wire up the event dispatcher if available
+    try {
+      const dispatcher = (this.app as any).make?.('events') ?? null
+      if (dispatcher) {
+        QueueManager._dispatcher = dispatcher
+      }
+    } catch {
+      // Events package not installed — fine, events are optional
+    }
+  }
+}
+
+/**
+ * Standalone factory for using @mantiq/queue without @mantiq/core.
+ */
+export function createQueueManager(config: QueueConfig): QueueManager {
+  const builtInDrivers = new Map<string, (cfg: QueueConnectionConfig) => any>([
+    ['sync', () => new SyncDriver()],
+    ['sqlite', (cfg) => new SQLiteDriver(cfg.database ?? 'queue.sqlite')],
+  ])
+
+  const manager = new QueueManager(config, builtInDrivers)
+
+  setQueueManager(manager)
+  setPendingDispatchResolver(() => manager)
+  setChainResolver(() => manager)
+  setBatchResolver(() => manager)
+
+  return manager
+}

package/src/Worker.ts

@@ -0,0 +1,282 @@
+import { QueueManager } from './QueueManager.ts'
+import type { QueueDriver } from './contracts/QueueDriver.ts'
+import type { QueuedJob, SerializedPayload } from './contracts/JobContract.ts'
+import { Job } from './Job.ts'
+import { resolveJob } from './JobRegistry.ts'
+import { MaxAttemptsExceededError, JobTimeoutError } from './errors/QueueError.ts'
+import {
+  JobProcessing,
+  JobProcessed,
+  JobFailed,
+  JobExceptionOccurred,
+} from './events/QueueEvents.ts'
+
+export interface WorkerOptions {
+  /** Queue names to listen on (comma-separated). Default: 'default' */
+  queue?: string | undefined
+  /** Seconds to sleep when no jobs are available. Default: 3 */
+  sleep?: number | undefined
+  /** Default max attempts. Overridden by job's own tries setting. Default: 3 */
+  tries?: number | undefined
+  /** Default timeout in seconds. Overridden by job's own timeout. Default: 60 */
+  timeout?: number | undefined
+  /** Stop the worker when the queue is empty. Default: false */
+  stopWhenEmpty?: boolean | undefined
+  /** Maximum number of jobs to process before stopping. Default: 0 (unlimited) */
+  maxJobs?: number | undefined
+  /** Maximum time in seconds to run before stopping. Default: 0 (unlimited) */
+  maxTime?: number | undefined
+  /** Connection name to use. Default: manager's default */
+  connection?: string | undefined
+}
+
+/**
+ * Queue worker — poll loop that pops jobs, executes them, and handles
+ * retries, failures, chain continuation, and batch progress.
+ */
+export class Worker {
+  private running = false
+  private jobsProcessed = 0
+  private startedAt = 0
+
+  constructor(
+    private readonly manager: QueueManager,
+    private readonly options: WorkerOptions = {},
+  ) {}
+
+  /** Start the poll loop */
+  async run(): Promise<void> {
+    this.running = true
+    this.jobsProcessed = 0
+    this.startedAt = Math.floor(Date.now() / 1000)
+
+    const queues = (this.options.queue ?? 'default').split(',').map((q) => q.trim())
+    const sleep = (this.options.sleep ?? 3) * 1000
+    const connection = this.options.connection
+    const driver = this.manager.driver(connection)
+
+    while (this.running) {
+      let processed = false
+
+      for (const queue of queues) {
+        const job = await driver.pop(queue)
+        if (job) {
+          await this.processJob(job, driver)
+          processed = true
+          this.jobsProcessed++
+          break
+        }
+      }
+
+      // Check stop conditions
+      if (!processed && this.options.stopWhenEmpty) {
+        break
+      }
+
+      if (this.options.maxJobs && this.jobsProcessed >= this.options.maxJobs) {
+        break
+      }
+
+      if (this.options.maxTime) {
+        const elapsed = Math.floor(Date.now() / 1000) - this.startedAt
+        if (elapsed >= this.options.maxTime) break
+      }
+
+      if (!processed) {
+        await new Promise((resolve) => setTimeout(resolve, sleep))
+      }
+    }
+
+    this.running = false
+  }
+
+  /** Process a single popped job */
+  async processJob(queuedJob: QueuedJob, driver: QueueDriver): Promise<void> {
+    const { payload } = queuedJob
+
+    // Check if this is a batch job and the batch is cancelled
+    if (payload.batchId) {
+      const batch = await driver.findBatch(payload.batchId)
+      if (batch?.cancelledAt) {
+        await driver.delete(queuedJob)
+        return
+      }
+    }
+
+    // Resolve the job class
+    const JobClass = resolveJob(payload.jobName)
+    if (!JobClass) {
+      const error = new Error(`Job class "${payload.jobName}" not found in registry`)
+      await driver.fail(queuedJob, error)
+      await this.fireEvent(new JobFailed(payload, error))
+      return
+    }
+
+    // Reconstruct the job instance
+    const job = Object.assign(new (JobClass as any)(), payload.data) as Job
+    job.queue = payload.queue
+    job.connection = payload.connection
+    job.tries = payload.tries
+    job.backoff = payload.backoff
+    job.timeout = payload.timeout
+    job.attempts = queuedJob.attempts
+    job.jobId = queuedJob.id
+
+    const maxTries = payload.tries || this.options.tries || 3
+    const timeout = payload.timeout || this.options.timeout || 60
+
+    await this.fireEvent(new JobProcessing(payload))
+
+    try {
+      // Execute with timeout
+      await this.runWithTimeout(job, timeout)
+
+      // Success — delete the job
+      await driver.delete(queuedJob)
+      await this.fireEvent(new JobProcessed(payload))
+
+      // Handle chain continuation
+      await this.handleChainContinuation(payload, driver)
+
+      // Handle batch progress
+      if (payload.batchId) {
+        await this.handleBatchProgress(payload.batchId, driver, true)
+      }
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error))
+
+      await this.fireEvent(new JobExceptionOccurred(payload, err))
+
+      if (queuedJob.attempts >= maxTries) {
+        // Permanently failed — preserve the original error (e.g. JobTimeoutError)
+        const failError = err instanceof JobTimeoutError
+          ? err
+          : new MaxAttemptsExceededError(payload.jobName, maxTries)
+        await driver.fail(queuedJob, failError)
+        await this.fireEvent(new JobFailed(payload, failError))
+
+        // Call job's failed() hook
+        if (job.failed) {
+          try { await job.failed(failError) } catch { /* ignore */ }
+        }
+
+        // Handle chain failure
+        await this.handleChainFailure(payload, driver)
+
+        // Handle batch progress
+        if (payload.batchId) {
+          await this.handleBatchProgress(payload.batchId, driver, false)
+        }
+      } else {
+        // Retry
+        const delay = job.getBackoffDelay(queuedJob.attempts)
+        await driver.release(queuedJob, delay)
+      }
+    }
+  }
+
+  /** Stop the worker gracefully */
+  stop(): void {
+    this.running = false
+  }
+
+  /** Whether the worker is currently running */
+  isRunning(): boolean {
+    return this.running
+  }
+
+  /** Number of jobs processed so far */
+  getJobsProcessed(): number {
+    return this.jobsProcessed
+  }
+
+  // ── Private helpers ──────────────────────────────────────────────
+
+  private async runWithTimeout(job: Job, timeoutSeconds: number): Promise<void> {
+    const timeoutMs = timeoutSeconds * 1000
+    let timeoutId: ReturnType<typeof setTimeout> | null = null
+
+    const timeoutPromise = new Promise<never>((_, reject) => {
+      timeoutId = setTimeout(() => {
+        reject(new JobTimeoutError(job.constructor.name, timeoutSeconds))
+      }, timeoutMs)
+    })
+
+    try {
+      await Promise.race([job.handle(), timeoutPromise])
+    } finally {
+      if (timeoutId !== null) clearTimeout(timeoutId)
+    }
+  }
+
+  /** After a successful job, dispatch the next job in the chain */
+  private async handleChainContinuation(payload: SerializedPayload, driver: QueueDriver): Promise<void> {
+    if (!payload.chainedJobs || payload.chainedJobs.length === 0) return
+
+    const [next, ...remaining] = payload.chainedJobs
+    if (!next) return
+
+    // Pass remaining chain and catch job to the next payload
+    if (remaining.length > 0) {
+      next.chainedJobs = remaining
+    }
+    if (payload.chainCatchJob) {
+      next.chainCatchJob = payload.chainCatchJob
+    }
+
+    await driver.push(next, next.queue, next.delay)
+  }
+
+  /** When a chained job permanently fails, dispatch the catch handler */
+  private async handleChainFailure(payload: SerializedPayload, driver: QueueDriver): Promise<void> {
+    if (!payload.chainCatchJob) return
+    const catchPayload = payload.chainCatchJob
+    await driver.push(catchPayload, catchPayload.queue, catchPayload.delay)
+  }
+
+  /** Update batch progress and trigger lifecycle callbacks when complete */
+  private async handleBatchProgress(batchId: string, driver: QueueDriver, success: boolean): Promise<void> {
+    const updated = await driver.updateBatchProgress(
+      batchId,
+      success ? 1 : 0,
+      success ? 0 : 1,
+    )
+    if (!updated) return
+
+    const { totalJobs, processedJobs, failedJobs, options, finishedAt } = updated
+
+    // Check if batch is complete
+    if (processedJobs + failedJobs < totalJobs) return
+    if (finishedAt !== null) return // Already handled
+
+    await driver.markBatchFinished(batchId)
+
+    const hasFailures = failedJobs > 0
+    const allowFailures = options.allowFailures
+
+    // Dispatch then/catch/finally callbacks
+    if (!hasFailures || allowFailures) {
+      if (options.thenJob) {
+        await driver.push(options.thenJob, options.thenJob.queue, 0)
+      }
+    }
+
+    if (hasFailures && !allowFailures) {
+      if (options.catchJob) {
+        await driver.push(options.catchJob, options.catchJob.queue, 0)
+      }
+    }
+
+    // Finally always runs
+    if (options.finallyJob) {
+      await driver.push(options.finallyJob, options.finallyJob.queue, 0)
+    }
+  }
+
+  private async fireEvent(event: { timestamp: Date }): Promise<void> {
+    const dispatcher = QueueManager._dispatcher
+    if (dispatcher) {
+      await dispatcher.emit(event as any)
+    }
+  }
+}

package/src/commands/MakeJobCommand.ts

@@ -0,0 +1,29 @@
+import { GeneratorCommand } from '@mantiq/cli'
+import type { ParsedArgs } from '@mantiq/cli'
+
+export class MakeJobCommand extends GeneratorCommand {
+  override name = 'make:job'
+  override description = 'Create a new job class'
+  override usage = 'make:job <name>'
+
+  override directory() { return 'app/Jobs' }
+  override suffix() { return '' }
+
+  override stub(name: string, _args: ParsedArgs): string {
+    return `import { Job } from '@mantiq/queue'
+
+export class ${name} extends Job {
+  override queue = 'default'
+  override tries = 3
+
+  constructor() {
+    super()
+  }
+
+  override async handle(): Promise<void> {
+    //
+  }
+}
+`
+  }
+}

package/src/commands/QueueFailedCommand.ts

@@ -0,0 +1,50 @@
+import { Command } from '@mantiq/cli'
+import type { ParsedArgs } from '@mantiq/cli'
+import type { QueueManager } from '../QueueManager.ts'
+
+export class QueueFailedCommand extends Command {
+  override name = 'queue:failed'
+  override description = 'List all failed jobs'
+
+  constructor(private readonly manager: QueueManager) {
+    super()
+  }
+
+  override async handle(_args: ParsedArgs): Promise<number> {
+    const driver = this.manager.driver()
+    const failed = await driver.getFailedJobs()
+
+    if (failed.length === 0) {
+      this.io.info('No failed jobs.')
+      return 0
+    }
+
+    // Table header
+    const header = ['ID', 'Queue', 'Job', 'Failed At', 'Error']
+    const rows = failed.map((j) => [
+      String(j.id),
+      j.queue,
+      j.payload.jobName,
+      new Date(j.failedAt * 1000).toISOString(),
+      j.exception.split('\n')[0] ?? '',
+    ])
+
+    // Simple table output
+    const colWidths = header.map((h, i) => {
+      return Math.max(h.length, ...rows.map((r) => (r[i] ?? '').length))
+    })
+
+    const separator = colWidths.map((w) => '-'.repeat(w + 2)).join('+')
+    const formatRow = (row: string[]) =>
+      row.map((cell, i) => ` ${cell.padEnd(colWidths[i]!)} `).join('|')
+
+    console.log(formatRow(header))
+    console.log(separator)
+    for (const row of rows) {
+      console.log(formatRow(row))
+    }
+
+    console.log(`\n${failed.length} failed job(s) total.`)
+    return 0
+  }
+}

package/src/commands/QueueFlushCommand.ts

@@ -0,0 +1,19 @@
+import { Command } from '@mantiq/cli'
+import type { ParsedArgs } from '@mantiq/cli'
+import type { QueueManager } from '../QueueManager.ts'
+
+export class QueueFlushCommand extends Command {
+  override name = 'queue:flush'
+  override description = 'Delete all failed jobs'
+
+  constructor(private readonly manager: QueueManager) {
+    super()
+  }
+
+  override async handle(_args: ParsedArgs): Promise<number> {
+    const driver = this.manager.driver()
+    await driver.flushFailedJobs()
+    this.io.success('All failed jobs deleted.')
+    return 0
+  }
+}

package/src/commands/QueueRetryCommand.ts

@@ -0,0 +1,52 @@
+import { Command } from '@mantiq/cli'
+import type { ParsedArgs } from '@mantiq/cli'
+import type { QueueManager } from '../QueueManager.ts'
+
+export class QueueRetryCommand extends Command {
+  override name = 'queue:retry'
+  override description = 'Retry failed job(s)'
+  override usage = 'queue:retry <id|all>'
+
+  constructor(private readonly manager: QueueManager) {
+    super()
+  }
+
+  override async handle(args: ParsedArgs): Promise<number> {
+    const target = args.args[0]
+    if (!target) {
+      this.io.error('Please provide a failed job ID or "all".')
+      return 1
+    }
+
+    const driver = this.manager.driver()
+
+    if (target === 'all') {
+      const failed = await driver.getFailedJobs()
+      if (failed.length === 0) {
+        this.io.info('No failed jobs to retry.')
+        return 0
+      }
+
+      for (const job of failed) {
+        await driver.push(job.payload, job.queue)
+        await driver.forgetFailedJob(job.id)
+      }
+
+      this.io.success(`Retried ${failed.length} failed job(s).`)
+      return 0
+    }
+
+    const id = isNaN(Number(target)) ? target : Number(target)
+    const job = await driver.findFailedJob(id)
+    if (!job) {
+      this.io.error(`Failed job [${target}] not found.`)
+      return 1
+    }
+
+    await driver.push(job.payload, job.queue)
+    await driver.forgetFailedJob(job.id)
+
+    this.io.success(`Retried failed job [${target}].`)
+    return 0
+  }
+}

package/src/commands/QueueWorkCommand.ts

@@ -0,0 +1,49 @@
+import { Command } from '@mantiq/cli'
+import type { ParsedArgs } from '@mantiq/cli'
+import { Worker } from '../Worker.ts'
+import type { QueueManager } from '../QueueManager.ts'
+
+export class QueueWorkCommand extends Command {
+  override name = 'queue:work'
+  override description = 'Start processing jobs on the queue'
+  override usage = 'queue:work [--queue=default] [--sleep=3] [--tries=3] [--timeout=60] [--stop-when-empty] [--max-jobs=0] [--max-time=0] [--connection=]'
+
+  constructor(private readonly manager: QueueManager) {
+    super()
+  }
+
+  override async handle(args: ParsedArgs): Promise<number> {
+    const options = {
+      queue: String(args.flags['queue'] ?? 'default'),
+      sleep: Number(args.flags['sleep'] ?? 3),
+      tries: Number(args.flags['tries'] ?? 3),
+      timeout: Number(args.flags['timeout'] ?? 60),
+      stopWhenEmpty: Boolean(args.flags['stop-when-empty']),
+      maxJobs: Number(args.flags['max-jobs'] ?? 0) || undefined,
+      maxTime: Number(args.flags['max-time'] ?? 0) || undefined,
+      connection: args.flags['connection'] ? String(args.flags['connection']) : undefined,
+    }
+
+    this.io.info(`Processing jobs on queue [${options.queue}]...`)
+
+    const worker = new Worker(this.manager, options)
+
+    // Graceful shutdown on SIGINT/SIGTERM
+    const shutdown = () => {
+      this.io.info('Shutting down worker...')
+      worker.stop()
+    }
+    process.on('SIGINT', shutdown)
+    process.on('SIGTERM', shutdown)
+
+    try {
+      await worker.run()
+    } finally {
+      process.off('SIGINT', shutdown)
+      process.off('SIGTERM', shutdown)
+    }
+
+    this.io.success(`Worker stopped. Processed ${worker.getJobsProcessed()} job(s).`)
+    return 0
+  }
+}