@mantiq/queue 0.0.1

package/README.md

@@ -0,0 +1,19 @@
+# @mantiq/queue
+
+Job dispatching, chains, batches, and scheduling for MantiqJS — sync, SQLite, Redis, SQS, and Kafka drivers.
+
+Part of [MantiqJS](https://github.com/abdullahkhan/mantiq) — a batteries-included TypeScript web framework for Bun.
+
+## Installation
+
+```bash
+bun add @mantiq/queue
+```
+
+## Documentation
+
+See the [MantiqJS repository](https://github.com/abdullahkhan/mantiq) for full documentation.
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@mantiq/queue",
+  "version": "0.0.1",
+  "description": "Job dispatching, workers, retry logic",
+  "type": "module",
+  "license": "MIT",
+  "author": "Abdullah Khan",
+  "homepage": "https://github.com/abdullahkhan/mantiq/tree/main/packages/queue",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/abdullahkhan/mantiq.git",
+    "directory": "packages/queue"
+  },
+  "bugs": {
+    "url": "https://github.com/abdullahkhan/mantiq/issues"
+  },
+  "keywords": [
+    "mantiq",
+    "mantiqjs",
+    "bun",
+    "typescript",
+    "framework",
+    "queue"
+  ],
+  "engines": {
+    "bun": ">=1.1.0"
+  },
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "exports": {
+    ".": {
+      "bun": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  },
+  "files": [
+    "src/",
+    "package.json",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "bun build ./src/index.ts --outdir ./dist --target bun",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@mantiq/core": "workspace:*",
+    "@mantiq/cli": "workspace:*"
+  },
+  "devDependencies": {
+    "bun-types": "latest",
+    "typescript": "^5.7.0"
+  },
+  "peerDependencies": {
+    "ioredis": ">=5.0.0",
+    "@aws-sdk/client-sqs": ">=3.0.0",
+    "kafkajs": ">=2.0.0"
+  },
+  "peerDependenciesMeta": {
+    "ioredis": {
+      "optional": true
+    },
+    "@aws-sdk/client-sqs": {
+      "optional": true
+    },
+    "kafkajs": {
+      "optional": true
+    }
+  }
+}

package/src/Job.ts

@@ -0,0 +1,122 @@
+import type { SerializedPayload } from './contracts/JobContract.ts'
+
+/**
+ * Keys that belong to the Job base class config, NOT user data.
+ * Used by serialize() to separate job config from user-defined properties.
+ */
+const JOB_BASE_KEYS = new Set([
+  'queue', 'connection', 'tries', 'backoff', 'timeout',
+  'delay', 'attempts', 'jobId',
+])
+
+/**
+ * Abstract base class for all queueable jobs.
+ *
+ * Subclasses define their own properties (the "data") and implement handle().
+ * The framework serializes user properties automatically for queue storage.
+ *
+ * @example
+ * ```ts
+ * export class ProcessPayment extends Job {
+ *   override queue = 'payments'
+ *   override tries = 5
+ *   override backoff = 'exponential:30'
+ *
+ *   constructor(public orderId: number, public amount: number) { super() }
+ *
+ *   async handle(): Promise<void> {
+ *     // process the payment...
+ *   }
+ *
+ *   override async failed(error: Error): Promise<void> {
+ *     // notify admin of failure
+ *   }
+ * }
+ * ```
+ */
+export abstract class Job {
+  /** Queue name this job should be dispatched to */
+  queue = 'default'
+
+  /** Connection name (null = default connection) */
+  connection: string | null = null
+
+  /** Maximum number of attempts before permanent failure */
+  tries = 3
+
+  /**
+   * Backoff strategy between retries.
+   * - `'0'` — no delay
+   * - `'30'` — fixed 30s delay
+   * - `'30,60,120'` — custom delays per attempt
+   * - `'exponential:30'` — exponential backoff starting at 30s
+   */
+  backoff = '0'
+
+  /** Maximum execution time in seconds */
+  timeout = 60
+
+  /** Delay in seconds before the job becomes available (set by PendingDispatch) */
+  delay = 0
+
+  /** Current attempt number (set by Worker, starts at 0) */
+  attempts = 0
+
+  /** Queue driver's job ID (set after push) */
+  jobId: string | number | null = null
+
+  /** Execute the job logic */
+  abstract handle(): Promise<void>
+
+  /** Called when the job has permanently failed (optional) */
+  failed?(error: Error): Promise<void>
+
+  /**
+   * Serialize this job for queue storage.
+   * User-defined properties (anything not in JOB_BASE_KEYS) become the `data` object.
+   */
+  serialize(): SerializedPayload {
+    const data: Record<string, any> = {}
+    for (const key of Object.keys(this)) {
+      if (!JOB_BASE_KEYS.has(key)) {
+        data[key] = (this as any)[key]
+      }
+    }
+
+    return {
+      jobName: this.constructor.name,
+      data,
+      queue: this.queue,
+      connection: this.connection,
+      tries: this.tries,
+      backoff: this.backoff,
+      timeout: this.timeout,
+      delay: this.delay,
+    }
+  }
+
+  /**
+   * Calculate the backoff delay for a given attempt number.
+   * @returns delay in seconds
+   */
+  getBackoffDelay(attempt: number): number {
+    const raw = this.backoff.trim()
+
+    if (raw === '0' || raw === '') return 0
+
+    // Exponential: 'exponential:30' → 30, 60, 120, 240, ...
+    if (raw.startsWith('exponential:')) {
+      const base = parseInt(raw.slice('exponential:'.length), 10) || 1
+      return base * Math.pow(2, attempt - 1)
+    }
+
+    // Comma-separated: '30,60,120'
+    if (raw.includes(',')) {
+      const parts = raw.split(',').map((s) => parseInt(s.trim(), 10))
+      return parts[Math.min(attempt - 1, parts.length - 1)] ?? 0
+    }
+
+    // Fixed: '30'
+    return parseInt(raw, 10) || 0
+  }
+}

package/src/JobBatch.ts

@@ -0,0 +1,188 @@
+import type { Job } from './Job.ts'
+import type { QueueManager } from './QueueManager.ts'
+import type { BatchRecord, BatchOptions, SerializedPayload } from './contracts/JobContract.ts'
+
+/** Resolve the QueueManager lazily */
+let _resolveManager: (() => QueueManager) | null = null
+
+export function setBatchResolver(resolver: () => QueueManager): void {
+  _resolveManager = resolver
+}
+
+/**
+ * Represents a dispatched batch — used to check progress and status.
+ */
+export class Batch {
+  constructor(private record: BatchRecord, private manager: QueueManager) {}
+
+  get id(): string { return this.record.id }
+  get name(): string { return this.record.name }
+  get totalJobs(): number { return this.record.totalJobs }
+  get processedJobs(): number { return this.record.processedJobs }
+  get failedJobs(): number { return this.record.failedJobs }
+  get cancelled(): boolean { return this.record.cancelledAt !== null }
+  get createdAt(): number { return this.record.createdAt }
+  get finishedAt(): number | null { return this.record.finishedAt }
+
+  /** Progress as a percentage (0–100) */
+  progress(): number {
+    if (this.record.totalJobs === 0) return 100
+    return Math.round(
+      ((this.record.processedJobs + this.record.failedJobs) / this.record.totalJobs) * 100,
+    )
+  }
+
+  /** Whether all jobs have been processed (success or failure) */
+  finished(): boolean {
+    return this.record.finishedAt !== null
+  }
+
+  /** Whether any jobs in the batch have failed */
+  hasFailures(): boolean {
+    return this.record.failedJobs > 0
+  }
+
+  /** Cancel this batch — pending jobs with this batchId will be skipped by the Worker */
+  async cancel(): Promise<void> {
+    const driver = this.manager.driver(this.record.options.connection ?? undefined)
+    await driver.cancelBatch(this.record.id)
+    this.record.cancelledAt = Math.floor(Date.now() / 1000)
+  }
+
+  /** Refresh the batch status from the driver */
+  async fresh(): Promise<Batch> {
+    const driver = this.manager.driver(this.record.options.connection ?? undefined)
+    const updated = await driver.findBatch(this.record.id)
+    if (updated) this.record = updated
+    return this
+  }
+}
+
+/**
+ * Fluent builder for dispatching a batch of jobs in parallel.
+ *
+ * @example
+ * ```ts
+ * const batch = await Bus.batch([
+ *   new ImportCsvChunk(file, 0, 1000),
+ *   new ImportCsvChunk(file, 1000, 2000),
+ *   new ImportCsvChunk(file, 2000, 3000),
+ * ])
+ *   .then(new NotifyImportComplete(file))
+ *   .catch(new NotifyImportFailed(file))
+ *   .finally(new CleanupTempFiles(file))
+ *   .name('csv-import')
+ *   .onQueue('imports')
+ *   .dispatch()
+ * ```
+ */
+export class PendingBatch {
+  private jobs: Job[]
+  private thenJob: Job | null = null
+  private catchJob: Job | null = null
+  private finallyJob: Job | null = null
+  private _name = ''
+  private _queue = 'default'
+  private _connection: string | null = null
+  private _allowFailures = false
+
+  private constructor(jobs: Job[]) {
+    this.jobs = jobs
+  }
+
+  static of(jobs: Job[]): PendingBatch {
+    if (jobs.length === 0) {
+      throw new Error('PendingBatch.of() requires at least one job')
+    }
+    return new PendingBatch(jobs)
+  }
+
+  /** Job to dispatch when all jobs succeed (or if allowFailures is true) */
+  then(job: Job): this {
+    this.thenJob = job
+    return this
+  }
+
+  /** Job to dispatch when any job fails (and allowFailures is false) */
+  catch(job: Job): this {
+    this.catchJob = job
+    return this
+  }
+
+  /** Job to dispatch when the batch completes, regardless of success/failure */
+  finally(job: Job): this {
+    this.finallyJob = job
+    return this
+  }
+
+  /** Set a human-readable name for this batch */
+  name(name: string): this {
+    this._name = name
+    return this
+  }
+
+  /** Override the queue for all batch jobs */
+  onQueue(queue: string): this {
+    this._queue = queue
+    return this
+  }
+
+  /** Override the connection for all batch jobs */
+  onConnection(connection: string): this {
+    this._connection = connection
+    return this
+  }
+
+  /** Allow the batch to succeed even if some jobs fail */
+  allowFailures(): this {
+    this._allowFailures = true
+    return this
+  }
+
+  /** Dispatch all batch jobs and create the batch record */
+  async dispatch(): Promise<Batch> {
+    if (!_resolveManager) {
+      throw new Error('QueueManager not initialized. Call setBatchResolver() first.')
+    }
+
+    const manager = _resolveManager()
+    const driver = manager.driver(this._connection ?? undefined)
+
+    const batchId = crypto.randomUUID()
+
+    const options: BatchOptions = {
+      thenJob: this.thenJob?.serialize(),
+      catchJob: this.catchJob?.serialize(),
+      finallyJob: this.finallyJob?.serialize(),
+      allowFailures: this._allowFailures,
+      queue: this._queue,
+      connection: this._connection,
+    }
+
+    const record: BatchRecord = {
+      id: batchId,
+      name: this._name,
+      totalJobs: this.jobs.length,
+      processedJobs: 0,
+      failedJobs: 0,
+      failedJobIds: [],
+      options,
+      cancelledAt: null,
+      createdAt: Math.floor(Date.now() / 1000),
+      finishedAt: null,
+    }
+
+    await driver.createBatch(record)
+
+    // Push all jobs with the batchId attached
+    for (const job of this.jobs) {
+      const payload = job.serialize()
+      payload.batchId = batchId
+      payload.queue = this._queue
+      if (this._connection) payload.connection = this._connection
+      await driver.push(payload, this._queue, job.delay)
+    }
+
+    return new Batch(record, manager)
+  }
+}

package/src/JobChain.ts

@@ -0,0 +1,119 @@
+import type { Job } from './Job.ts'
+import type { QueueManager } from './QueueManager.ts'
+import type { SerializedPayload } from './contracts/JobContract.ts'
+
+/** Resolve the QueueManager lazily */
+let _resolveManager: (() => QueueManager) | null = null
+
+export function setChainResolver(resolver: () => QueueManager): void {
+  _resolveManager = resolver
+}
+
+/**
+ * Sequential job execution — each job runs only after the previous one succeeds.
+ * If any job fails permanently, the chain stops and the optional catch handler runs.
+ *
+ * Chain state is stored in the serialized payload: `chainedJobs` and `chainCatchJob`.
+ * The Worker checks these after successful completion and dispatches the next job.
+ *
+ * @example
+ * ```ts
+ * await Chain.of([
+ *   new ProcessPodcast(podcast),
+ *   new OptimizeAudio(podcast),
+ *   new PublishPodcast(podcast),
+ * ]).catch(new NotifyFailure(podcast)).dispatch()
+ * ```
+ */
+export class Chain {
+  private jobs: Job[]
+  private catchJob: Job | null = null
+  private _queue: string | null = null
+  private _connection: string | null = null
+
+  private constructor(jobs: Job[]) {
+    this.jobs = jobs
+  }
+
+  /** Create a chain from an ordered list of jobs */
+  static of(jobs: Job[]): Chain {
+    if (jobs.length === 0) {
+      throw new Error('Chain.of() requires at least one job')
+    }
+    return new Chain(jobs)
+  }
+
+  /** Set a job to run if any job in the chain fails permanently */
+  catch(job: Job): this {
+    this.catchJob = job
+    return this
+  }
+
+  /** Override the queue name for all jobs in the chain */
+  onQueue(queue: string): this {
+    this._queue = queue
+    return this
+  }
+
+  /** Override the connection for all jobs in the chain */
+  onConnection(connection: string): this {
+    this._connection = connection
+    return this
+  }
+
+  /**
+   * Dispatch the chain.
+   * The first job is pushed to the queue with the remaining jobs serialized
+   * in its payload as `chainedJobs`. The Worker handles continuation.
+   */
+  async dispatch(): Promise<void> {
+    if (!_resolveManager) {
+      throw new Error('QueueManager not initialized. Call setChainResolver() first.')
+    }
+
+    const manager = _resolveManager()
+    const [first, ...rest] = this.jobs
+
+    // Serialize the first job
+    const payload = first!.serialize()
+
+    // Attach remaining jobs as chained payloads
+    if (rest.length > 0) {
+      payload.chainedJobs = rest.map((j) => {
+        const p = j.serialize()
+        if (this._queue) p.queue = this._queue
+        if (this._connection) p.connection = this._connection
+        return p
+      })
+    }
+
+    // Attach catch handler
+    if (this.catchJob) {
+      const catchPayload = this.catchJob.serialize()
+      if (this._queue) catchPayload.queue = this._queue
+      if (this._connection) catchPayload.connection = this._connection
+      payload.chainCatchJob = catchPayload
+    }
+
+    // Apply queue/connection overrides to first job
+    if (this._queue) payload.queue = this._queue
+    if (this._connection) payload.connection = this._connection
+
+    const queue = payload.queue
+    const connection = payload.connection
+    const driver = manager.driver(connection ?? undefined)
+
+    await driver.push(payload, queue, first!.delay)
+  }
+
+  /**
+   * Makes Chain thenable so `await Chain.of([...]).dispatch()` and
+   * `await Chain.of([...])` both work.
+   */
+  then<TResult1 = void, TResult2 = never>(
+    onfulfilled?: ((value: void) => TResult1 | PromiseLike<TResult1>) | null,
+    onrejected?: ((reason: any) => TResult2 | PromiseLike<TResult2>) | null,
+  ): PromiseLike<TResult1 | TResult2> {
+    return this.dispatch().then(onfulfilled, onrejected)
+  }
+}

package/src/JobRegistry.ts

@@ -0,0 +1,37 @@
+import type { Job } from './Job.ts'
+import type { Constructor } from './contracts/JobContract.ts'
+
+/**
+ * Maps job class names to their constructors.
+ * Used by the Worker to reconstruct job instances from serialized payloads.
+ *
+ * Jobs must be registered before the worker starts processing.
+ */
+const registry = new Map<string, Constructor<Job>>()
+
+/** Register a job class so it can be deserialized by the worker */
+export function registerJob(jobClass: Constructor<Job>): void {
+  registry.set(jobClass.name, jobClass)
+}
+
+/** Register multiple job classes at once */
+export function registerJobs(jobClasses: Constructor<Job>[]): void {
+  for (const cls of jobClasses) {
+    registry.set(cls.name, cls)
+  }
+}
+
+/** Look up a job constructor by its class name */
+export function resolveJob(name: string): Constructor<Job> | undefined {
+  return registry.get(name)
+}
+
+/** Get all registered job classes (for debugging/testing) */
+export function getRegisteredJobs(): Map<string, Constructor<Job>> {
+  return new Map(registry)
+}
+
+/** Clear the registry (primarily for testing) */
+export function clearJobRegistry(): void {
+  registry.clear()
+}

package/src/PendingDispatch.ts

@@ -0,0 +1,80 @@
+import type { Job } from './Job.ts'
+import type { QueueManager } from './QueueManager.ts'
+
+/** Resolve the QueueManager lazily to avoid circular deps */
+let _resolveManager: (() => QueueManager) | null = null
+
+export function setPendingDispatchResolver(resolver: () => QueueManager): void {
+  _resolveManager = resolver
+}
+
+/**
+ * Fluent dispatch builder that is **thenable**.
+ *
+ * This allows `await dispatch(job).delay(60).onQueue('payments')` to work
+ * because PendingDispatch implements `then()` which triggers the actual push.
+ *
+ * @example
+ * ```ts
+ * // All of these work:
+ * await dispatch(new ProcessPayment(order))
+ * await dispatch(new ProcessPayment(order)).delay(60)
+ * await dispatch(new ProcessPayment(order)).onQueue('payments').delay(30)
+ * ```
+ */
+export class PendingDispatch {
+  private _delay = 0
+  private _queue: string | null = null
+  private _connection: string | null = null
+
+  constructor(private readonly job: Job) {}
+
+  /** Set the delay in seconds before the job becomes available */
+  delay(seconds: number): this {
+    this._delay = seconds
+    return this
+  }
+
+  /** Override the queue name for this dispatch */
+  onQueue(queue: string): this {
+    this._queue = queue
+    return this
+  }
+
+  /** Override the connection name for this dispatch */
+  onConnection(connection: string): this {
+    this._connection = connection
+    return this
+  }
+
+  /** Push the job to the queue. Called automatically by then(). */
+  async send(): Promise<void> {
+    if (!_resolveManager) {
+      throw new Error('QueueManager not initialized. Call setPendingDispatchResolver() first.')
+    }
+
+    const manager = _resolveManager()
+    const connection = this._connection ?? this.job.connection
+    const driver = manager.driver(connection ?? undefined)
+
+    const payload = this.job.serialize()
+    if (this._queue) payload.queue = this._queue
+    if (this._connection) payload.connection = this._connection
+
+    const queue = this._queue ?? this.job.queue
+    const delay = this._delay || this.job.delay
+
+    await driver.push(payload, queue, delay)
+  }
+
+  /**
+   * Makes PendingDispatch thenable so `await dispatch(job)` works.
+   * This is the magic that lets the fluent API work with await.
+   */
+  then<TResult1 = void, TResult2 = never>(
+    onfulfilled?: ((value: void) => TResult1 | PromiseLike<TResult1>) | null,
+    onrejected?: ((reason: any) => TResult2 | PromiseLike<TResult2>) | null,
+  ): PromiseLike<TResult1 | TResult2> {
+    return this.send().then(onfulfilled, onrejected)
+  }
+}