ingenium 0.0.1

package/src/jobs/queue.ts

@@ -0,0 +1,306 @@
+import { MemoryQueueStore } from './store-memory.ts'
+import type {
+  FailedJob,
+  QueueOptions,
+  QueueStore,
+  QueueWorker,
+  RetryPolicy,
+} from './types.ts'
+
+/** Default exponential backoff: 100ms, 400ms, 1.6s, 6.4s, ... (4^(n-1) * 100). */
+const DEFAULT_RETRIES: RetryPolicy = {
+  attempts: 3,
+  backoffMs: (attempt: number) => 100 * Math.pow(4, attempt - 1),
+}
+
+function normalizeRetries(retries: number | RetryPolicy | undefined): RetryPolicy {
+  if (retries === undefined) return DEFAULT_RETRIES
+  if (typeof retries === 'number') {
+    return { attempts: Math.max(1, retries), backoffMs: DEFAULT_RETRIES.backoffMs }
+  }
+  return retries
+}
+
+/**
+ * A single named background queue. Wraps a {@link QueueStore} with a worker
+ * pool, retry/backoff logic, pause/resume controls, and a `drain()` for
+ * graceful shutdown.
+ *
+ * The pool is event-driven: when a slot frees up, the queue immediately
+ * tries to pull another job; if none is ready (empty or all delayed), the
+ * pool sleeps until either:
+ *   - a new `add()` call wakes it, or
+ *   - the earliest delayed job's `notBefore` elapses (timer-based wake).
+ *
+ * All timers are `unref()`'d so the queue alone never keeps the event loop alive.
+ */
+export class IngeniumQueue<TData = unknown> {
+  readonly name: string
+  private readonly store: QueueStore<TData>
+  private readonly worker: QueueWorker<TData>
+  private readonly retries: RetryPolicy
+  private readonly concurrency: number
+  private readonly onFailed: ((job: FailedJob<TData>) => void | Promise<void>) | undefined
+
+  /** Active worker slot count. When `< concurrency`, pull more work. */
+  private active = 0
+  /** Whether `pause()` has been called and `resume()` not yet. */
+  private paused = false
+  /** Whether `drain()`/close has been called. No new jobs accepted. */
+  private closed = false
+  /** Timer for waking the pool when a delayed retry becomes due. */
+  private wakeTimer: NodeJS.Timeout | null = null
+  /** Resolvers waiting for `active` to hit 0 (used by `drain()`). */
+  private idleWaiters: (() => void)[] = []
+  /** Set when the pool is started — protects against double-start. */
+  private started = false
+
+  constructor(name: string, opts: QueueOptions<TData>, worker: QueueWorker<TData>) {
+    this.name = name
+    this.store = opts.store ?? new MemoryQueueStore<TData>()
+    this.worker = worker
+    this.retries = normalizeRetries(opts.retries)
+    this.concurrency = Math.max(1, opts.concurrency ?? 1)
+    this.onFailed = opts.onFailed
+  }
+
+  /**
+   * Start the worker pool. Idempotent — safe to call multiple times. The
+   * pool is also implicitly started by the first `add()` call so direct
+   * invocation is optional.
+   */
+  start(): void {
+    if (this.started) return
+    this.started = true
+    this.pump()
+  }
+
+  /** Enqueue a job. Returns the assigned id. */
+  async add(data: TData): Promise<{ id: string }> {
+    if (this.closed) {
+      throw new Error(`ingenium: queue "${this.name}" is closed (no new jobs accepted)`)
+    }
+    const handle = await this.store.enqueue(data)
+    // Lazy-start: first add() triggers worker pool boot if `start()` wasn't called.
+    if (!this.started) this.start()
+    else this.pump()
+    return handle
+  }
+
+  /** Approximate number of pending jobs. */
+  size(): Promise<number> {
+    return this.store.size()
+  }
+
+  /** Number of jobs in the dead-letter list. */
+  failedCount(): Promise<number> {
+    return this.store.failedCount()
+  }
+
+  /**
+   * Empty the dead-letter list. Only effective when the underlying store
+   * is the default {@link MemoryQueueStore}; custom stores should provide
+   * their own clearing surface.
+   */
+  clearFailed(): void {
+    if (this.store instanceof MemoryQueueStore) this.store.clearFailed()
+  }
+
+  /**
+   * Stop pulling new jobs from the store. In-flight jobs continue to run
+   * until they complete. Idempotent.
+   */
+  pause(): void {
+    this.paused = true
+  }
+
+  /** Resume pulling jobs. Wakes the pool. */
+  resume(): void {
+    if (!this.paused) return
+    this.paused = false
+    this.pump()
+  }
+
+  /**
+   * Wait for all in-flight jobs to complete, then stop accepting new ones.
+   * If `timeoutMs` elapses first, resolve anyway — the orphaned jobs keep
+   * running until they naturally finish (JS can't cancel a Promise), but
+   * the framework stops waiting.
+   *
+   * Returns `true` if the queue drained cleanly; `false` on timeout.
+   */
+  async drain(timeoutMs?: number): Promise<boolean> {
+    this.closed = true
+    this.paused = true // stop pulling new work even if currently active
+    if (this.active === 0) {
+      this.clearWakeTimer()
+      return true
+    }
+    return new Promise<boolean>((resolve) => {
+      let settled = false
+      const onIdle = (): void => {
+        if (settled) return
+        settled = true
+        if (timer) clearTimeout(timer)
+        this.clearWakeTimer()
+        resolve(true)
+      }
+      this.idleWaiters.push(onIdle)
+      const timer = timeoutMs !== undefined
+        ? setTimeout(() => {
+            if (settled) return
+            settled = true
+            // Drop our waiter so the pool doesn't re-resolve us.
+            const idx = this.idleWaiters.indexOf(onIdle)
+            if (idx >= 0) this.idleWaiters.splice(idx, 1)
+            this.clearWakeTimer()
+            resolve(false)
+          }, timeoutMs)
+        : null
+      timer?.unref?.()
+    })
+  }
+
+  /**
+   * Pump the pool: while we have free slots and we're not paused, pull jobs
+   * and dispatch them. No-op when paused / saturated. Re-entrant: every
+   * job completion calls `pump()` again to fill the slot.
+   */
+  private pump(): void {
+    if (this.paused) {
+      this.checkIdle()
+      return
+    }
+    while (this.active < this.concurrency) {
+      // Synchronously check store state via a microtask. The store API is
+      // async (so a Redis adapter works), so each pull is a Promise we don't
+      // await inline — we just kick it off and let `runOne` increment/decrement
+      // `active` around the actual worker invocation.
+      const slotOpen = this.tryFillSlot()
+      if (!slotOpen) break
+    }
+    this.checkIdle()
+  }
+
+  /**
+   * Attempts to take one job from the store and run it. Returns `false` if
+   * the store is empty (or all-delayed) so the caller stops looping.
+   *
+   * NOTE: we increment `active` BEFORE the async `next()` resolves so a
+   * burst of synchronous `pump()` calls doesn't over-subscribe the pool.
+   * We decrement on the `null` path.
+   */
+  private tryFillSlot(): boolean {
+    this.active++
+    void this.runOne().catch(() => {
+      // runOne handles all errors internally; this catch is a defensive
+      // backstop so an unexpected bug doesn't unhandle a rejection.
+    })
+    return true
+  }
+
+  private async runOne(): Promise<void> {
+    let job: { id: string; data: TData; attempt: number } | null = null
+    try {
+      job = await this.store.next()
+    } catch {
+      // Store failure pulling the next job — release slot and back off.
+      this.active--
+      this.checkIdle()
+      return
+    }
+    if (job === null) {
+      // Empty (or all delayed). Release the speculative slot, schedule a
+      // wake for the earliest delayed job (if any), and stop pulling.
+      this.active--
+      this.scheduleDelayedWake()
+      this.checkIdle()
+      return
+    }
+
+    let lastError: unknown = undefined
+    try {
+      await this.worker({ id: job.id, data: job.data, attempt: job.attempt })
+      await this.store.ack(job.id)
+    } catch (err) {
+      lastError = err
+      const attempt = job.attempt
+      if (attempt < this.retries.attempts) {
+        const delay = Math.max(0, this.retries.backoffMs(attempt))
+        try {
+          await this.store.retry(job.id, delay)
+        } catch {
+          // If retry bookkeeping fails, fall back to fail() so the job
+          // doesn't get stuck in-flight forever.
+          await this.safeFail(job, lastError)
+        }
+      } else {
+        await this.safeFail(job, lastError)
+      }
+    } finally {
+      this.active--
+      // Refill the freed slot on the next macrotask, not synchronously. A
+      // synchronous refill would chain pickup of the *next* job into the same
+      // microtask run as this job's completion, so the pool would never be
+      // observably idle between jobs (active would dip and immediately rise
+      // within one turn). Deferring to setImmediate yields a macrotask
+      // boundary where `active` reflects only genuinely-running jobs.
+      const t = setImmediate(() => this.pump())
+      t.unref?.()
+    }
+  }
+
+  private async safeFail(
+    job: { id: string; data: TData; attempt: number },
+    lastError: unknown,
+  ): Promise<void> {
+    try {
+      await this.store.fail(job.id)
+    } catch {
+      // If even fail() throws, we've done what we can — the job stays
+      // in-flight in the store, which is at-least-once-correct.
+      return
+    }
+    if (this.onFailed) {
+      try {
+        await this.onFailed({ id: job.id, data: job.data, attempt: job.attempt, lastError })
+      } catch {
+        // onFailed errors are observation-only; swallow.
+      }
+    }
+  }
+
+  /**
+   * If the store has only delayed entries (e.g. just-retried jobs whose
+   * backoff hasn't elapsed), schedule a one-shot wake when the earliest
+   * delay fires so we don't spin or sleep forever.
+   *
+   * Only the default in-memory store exposes `earliestPendingAt`; for
+   * custom stores we don't poll — we rely on the next `add()` to wake us.
+   */
+  private scheduleDelayedWake(): void {
+    if (!(this.store instanceof MemoryQueueStore)) return
+    const earliest = this.store.earliestPendingAt()
+    if (earliest === null) return
+    const delay = Math.max(1, earliest - Date.now())
+    this.clearWakeTimer()
+    this.wakeTimer = setTimeout(() => {
+      this.wakeTimer = null
+      this.pump()
+    }, delay)
+    this.wakeTimer.unref?.()
+  }
+
+  private clearWakeTimer(): void {
+    if (this.wakeTimer) {
+      clearTimeout(this.wakeTimer)
+      this.wakeTimer = null
+    }
+  }
+
+  private checkIdle(): void {
+    if (this.active !== 0 || this.idleWaiters.length === 0) return
+    const waiters = this.idleWaiters.splice(0, this.idleWaiters.length)
+    for (const w of waiters) w()
+  }
+}

package/src/jobs/registry.ts

@@ -0,0 +1,82 @@
+import { IngeniumQueue } from './queue.ts'
+import type { QueueOptions, QueueWorker } from './types.ts'
+
+/**
+ * Maps queue names → {@link IngeniumQueue} instances. Held by `IngeniumApp`,
+ * mirroring the shape of {@link CronRegistry}.
+ *
+ * Lookup is O(1). Names must be unique within an app — re-registering the
+ * same name throws (mirrors how `app.get('/users')` would conflict if you
+ * registered the same path twice with the same method).
+ */
+export class QueueRegistry {
+  private readonly queues: Map<string, IngeniumQueue<unknown>> = new Map()
+
+  /**
+   * Register a new queue. Returns the created instance. Throws if a queue
+   * with `name` is already registered.
+   */
+  register<TData>(
+    name: string,
+    opts: QueueOptions<TData>,
+    worker: QueueWorker<TData>,
+  ): IngeniumQueue<TData> {
+    if (this.queues.has(name)) {
+      throw new Error(`ingenium: queue "${name}" is already registered`)
+    }
+    const queue = new IngeniumQueue<TData>(name, opts, worker)
+    this.queues.set(name, queue as unknown as IngeniumQueue<unknown>)
+    return queue
+  }
+
+  /** Look up a queue. Throws if not registered (typos surface immediately). */
+  get<TData = unknown>(name: string): IngeniumQueue<TData> {
+    const queue = this.queues.get(name)
+    if (!queue) {
+      throw new Error(
+        `ingenium: queue "${name}" is not registered (call app.queue("${name}", ...) first)`,
+      )
+    }
+    return queue as unknown as IngeniumQueue<TData>
+  }
+
+  /** Has a queue with this name been registered? */
+  has(name: string): boolean {
+    return this.queues.has(name)
+  }
+
+  /** Number of registered queues. */
+  count(): number {
+    return this.queues.size
+  }
+
+  /** All registered queue names (insertion order). */
+  names(): string[] {
+    return [...this.queues.keys()]
+  }
+
+  /**
+   * Start the worker pool of every registered queue. Called by the app's
+   * composition step so workers don't process jobs before the app is ready.
+   */
+  startAll(): void {
+    for (const q of this.queues.values()) q.start()
+  }
+
+  /**
+   * Drain every queue concurrently. Resolves when all queues either finish
+   * their in-flight work or hit `timeoutMs`. The returned object reports
+   * which queues drained cleanly vs timed out — useful for shutdown logs.
+   */
+  async drainAll(timeoutMs?: number): Promise<{ clean: string[]; timedOut: string[] }> {
+    const entries = [...this.queues.entries()]
+    const results = await Promise.all(entries.map(([, q]) => q.drain(timeoutMs)))
+    const clean: string[] = []
+    const timedOut: string[] = []
+    entries.forEach(([name], i) => {
+      if (results[i]) clean.push(name)
+      else timedOut.push(name)
+    })
+    return { clean, timedOut }
+  }
+}

package/src/jobs/store-memory.ts

@@ -0,0 +1,113 @@
+import type { QueueStore } from './types.ts'
+
+interface PendingEntry<TData> {
+  id: string
+  data: TData
+  attempt: number
+  /** When `> Date.now()`, the job is delayed (used by retries). */
+  notBefore: number
+}
+
+interface InFlightEntry<TData> {
+  id: string
+  data: TData
+  attempt: number
+}
+
+/**
+ * In-process FIFO queue store. Backs {@link IngeniumQueue} when no custom
+ * store is supplied. Suitable for single-instance deployments and tests.
+ *
+ * Layout:
+ *   - `pending`: ordered list of jobs ready to be picked up. Delayed jobs
+ *      (post-retry backoff) sit here too — `next()` skips entries whose
+ *      `notBefore` hasn't elapsed yet, so callers should poll on a timer.
+ *   - `inFlight`: jobs that have been `next()`-ed but not yet `ack`/`retry`/`fail`-ed.
+ *   - `failed`: dead-letter list. Persists until `clearFailed()` is called.
+ *
+ * No background timers — purely event-driven via the queue worker pool.
+ */
+export class MemoryQueueStore<TData> implements QueueStore<TData> {
+  private readonly pending: PendingEntry<TData>[] = []
+  private readonly inFlight: Map<string, InFlightEntry<TData>> = new Map()
+  private readonly failed: { id: string; data: TData; attempt: number }[] = []
+  private nextId = 1
+
+  enqueue(data: TData): Promise<{ id: string }> {
+    const id = String(this.nextId++)
+    this.pending.push({ id, data, attempt: 1, notBefore: 0 })
+    return Promise.resolve({ id })
+  }
+
+  next(): Promise<{ id: string; data: TData; attempt: number } | null> {
+    const now = Date.now()
+    // Find the first pending entry whose delay has elapsed.
+    for (let i = 0; i < this.pending.length; i++) {
+      const entry = this.pending[i]!
+      if (entry.notBefore <= now) {
+        this.pending.splice(i, 1)
+        const inflight: InFlightEntry<TData> = {
+          id: entry.id,
+          data: entry.data,
+          attempt: entry.attempt,
+        }
+        this.inFlight.set(entry.id, inflight)
+        return Promise.resolve({ id: entry.id, data: entry.data, attempt: entry.attempt })
+      }
+    }
+    return Promise.resolve(null)
+  }
+
+  ack(id: string): Promise<void> {
+    this.inFlight.delete(id)
+    return Promise.resolve()
+  }
+
+  retry(id: string, delayMs: number): Promise<void> {
+    const entry = this.inFlight.get(id)
+    if (!entry) return Promise.resolve()
+    this.inFlight.delete(id)
+    this.pending.push({
+      id: entry.id,
+      data: entry.data,
+      attempt: entry.attempt + 1,
+      notBefore: Date.now() + Math.max(0, delayMs),
+    })
+    return Promise.resolve()
+  }
+
+  fail(id: string): Promise<void> {
+    const entry = this.inFlight.get(id)
+    if (!entry) return Promise.resolve()
+    this.inFlight.delete(id)
+    this.failed.push({ id: entry.id, data: entry.data, attempt: entry.attempt })
+    return Promise.resolve()
+  }
+
+  size(): Promise<number> {
+    return Promise.resolve(this.pending.length)
+  }
+
+  failedCount(): Promise<number> {
+    return Promise.resolve(this.failed.length)
+  }
+
+  /** @internal Used by `IngeniumQueue.clearFailed()`. */
+  clearFailed(): void {
+    this.failed.length = 0
+  }
+
+  /** @internal Used by `IngeniumQueue.drain()` to know if work is outstanding. */
+  inFlightCount(): number {
+    return this.inFlight.size
+  }
+
+  /** @internal Earliest `notBefore` of any pending entry, or `null` if none. */
+  earliestPendingAt(): number | null {
+    let min: number | null = null
+    for (const e of this.pending) {
+      if (min === null || e.notBefore < min) min = e.notBefore
+    }
+    return min
+  }
+}

package/src/jobs/types.ts

@@ -0,0 +1,135 @@
+/**
+ * Background-job type surface for Ingenium.
+ *
+ * The core abstraction is a {@link QueueStore} — a pluggable persistence layer
+ * for FIFO jobs with retries and a dead-letter list. The default implementation
+ * ({@link MemoryQueueStore}) keeps everything in process; a Redis adapter can
+ * land later by simply implementing this interface.
+ *
+ * A {@link IngeniumQueue} wraps a store with a worker pool, retry policy, and
+ * pause/resume/drain controls; a {@link QueueRegistry} (held by `IngeniumApp`)
+ * indexes named queues so route handlers can enqueue from any code path.
+ */
+
+/**
+ * Worker function for a registered queue. Throwing causes a retry per the
+ * configured {@link RetryPolicy}; resolving means the job is `ack`'d.
+ */
+export type QueueWorker<TData> = (job: {
+  /** Stable id assigned by the store at enqueue time. */
+  id: string
+  /** Job payload (the value passed to `add`). */
+  data: TData
+  /**
+   * 1-indexed attempt counter. `1` on first delivery; incremented before each
+   * retry. Use this in the worker to back off external calls or short-circuit
+   * non-recoverable failures.
+   */
+  attempt: number
+}) => unknown | Promise<unknown>
+
+/**
+ * Retry policy. The first attempt is included in the count: `attempts: 3`
+ * means one initial try + two retries.
+ */
+export interface RetryPolicy {
+  /** Total tries including the first delivery. Must be `>= 1`. */
+  attempts: number
+  /**
+   * Delay (ms) before the NEXT attempt, given the attempt that just failed
+   * (1-indexed). E.g. for `attempts: 3`, this is called with `1` then `2`.
+   */
+  backoffMs: (attempt: number) => number
+}
+
+/**
+ * Options for {@link IngeniumApp.queue}. All fields optional.
+ *
+ * @typeParam TData - shape of job payloads enqueued onto this queue
+ */
+export interface QueueOptions<TData> {
+  /**
+   * Max concurrent jobs processed in parallel by this queue's worker pool.
+   * Default `1` (strict FIFO). Bump this for I/O-bound work.
+   */
+  concurrency?: number
+  /**
+   * Retry policy on worker throw. Numeric shorthand `n` is equivalent to
+   * `{ attempts: n, backoffMs: exponential }`. Default: 3 attempts at
+   * 100ms / 400ms / 1.6s.
+   */
+  retries?: number | RetryPolicy
+  /**
+   * Custom store. Default {@link MemoryQueueStore}. Implement this to back
+   * the queue with Redis / Postgres / SQS / etc.
+   */
+  store?: QueueStore<TData>
+  /**
+   * Called once retries are exhausted, just before the job is moved to the
+   * dead-letter list. Throwing here is logged and swallowed — the job is
+   * still moved to the DLQ.
+   */
+  onFailed?: (job: FailedJob<TData>) => void | Promise<void>
+}
+
+/**
+ * Pluggable persistence layer. The default {@link MemoryQueueStore} keeps
+ * pending and failed jobs in arrays + an in-flight map. A Redis adapter
+ * would map this onto LPUSH / BRPOPLPUSH / a processing list / a DLQ list.
+ *
+ * Implementations MUST guarantee at-least-once delivery (a `next()`-ed
+ * job that is neither `ack`'d nor `retry`'d nor `fail`'d is considered
+ * stuck and may be re-delivered by the store on its own schedule).
+ */
+export interface QueueStore<TData> {
+  /** Append a job to the tail. Returns the assigned id. */
+  enqueue(data: TData): Promise<{ id: string }>
+  /**
+   * Pop the next pending job and move it to the in-flight set. Returns
+   * `null` when the queue is empty. The returned `attempt` reflects how
+   * many times this job has been delivered (1 on first delivery).
+   */
+  next(): Promise<{ id: string; data: TData; attempt: number } | null>
+  /** Mark an in-flight job as completed. Removes it from the store. */
+  ack(id: string): Promise<void>
+  /**
+   * Re-enqueue the in-flight job for another attempt after `delayMs`.
+   * The store MUST increment its internal attempt counter so the next
+   * `next()` returns it with the bumped count.
+   */
+  retry(id: string, delayMs: number): Promise<void>
+  /** Move the in-flight job to the dead-letter list. */
+  fail(id: string): Promise<void>
+  /** Number of pending (not in-flight, not failed) jobs. */
+  size(): Promise<number>
+  /** Number of jobs in the dead-letter list. */
+  failedCount(): Promise<number>
+}
+
+/** Payload passed to {@link QueueOptions.onFailed} when retries are exhausted. */
+export interface FailedJob<TData> {
+  id: string
+  data: TData
+  /** Final attempt number (== `retries.attempts`). */
+  attempt: number
+  /** Whatever the worker threw on its last attempt. */
+  lastError: unknown
+}
+
+/** Per-request handle returned by `ctx.queue(name)`. */
+export interface JobHandle<TData = unknown> {
+  /** Enqueue `data`. Resolves to the assigned job id. */
+  add(data: TData): Promise<{ id: string }>
+}
+
+/**
+ * Bookkeeping wrapper a {@link QueueRegistry} keeps for every registered
+ * queue. Mostly useful for introspection / tests.
+ */
+export interface RegisteredQueue<TData = unknown> {
+  name: string
+  options: Required<Pick<QueueOptions<TData>, 'concurrency'>> & {
+    retries: RetryPolicy
+    onFailed: ((job: FailedJob<TData>) => void | Promise<void>) | undefined
+  }
+}