@strav/queue 0.3.30 → 0.3.33

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@strav/queue",
-  "version": "0.3.30",
+  "version": "0.3.33",
   "type": "module",
   "description": "Background job processing and task scheduling for the Strav framework",
   "license": "MIT",
@@ -28,8 +28,8 @@
     "./providers/*": "./src/providers/*.ts"
   },
   "peerDependencies": {
-    "@strav/kernel": "0.3.30",
-    "@strav/database": "0.3.30"
+    "@strav/kernel": "0.3.33",
+    "@strav/database": "0.3.33"
   },
   "scripts": {
     "test": "bun test tests/",

package/src/queue/circuit_breaker.ts

@@ -0,0 +1,135 @@
+/**
+ * Per-handler circuit breaker. Tracks recent failure timestamps in
+ * memory; trips the circuit when the failure count within the window
+ * exceeds the threshold and pauses dispatch of that handler for
+ * `cooldownMs`. Dispatches are auto-resumed once the cooldown expires.
+ *
+ * Intended defense against retry storms — a handler that consistently
+ * fails (stale schema, downed dependency) shouldn't keep eating worker
+ * cycles and DB connections. Tripping pushes failed jobs back to the
+ * queue with a delay so they retry AFTER the cooldown.
+ *
+ * State is per-process (in-memory). Multi-worker deployments will each
+ * track independently — that's fine; each worker self-pauses without
+ * cross-talk, and a handler that's failing for a global reason will
+ * trip every worker quickly.
+ */
+
+import Emitter from '@strav/kernel/events/emitter'
+
+export interface CircuitBreakerOptions {
+  /** Number of failures within the window that trips the breaker. Default: 10. */
+  threshold?: number
+  /** Window in ms over which failures are counted. Default: 60_000 (1 min). */
+  windowMs?: number
+  /** Cooldown in ms after tripping before retry resumes. Default: 30_000 (30 s). */
+  cooldownMs?: number
+}
+
+export interface ResolvedBreakerOptions {
+  threshold: number
+  windowMs: number
+  cooldownMs: number
+}
+
+interface BreakerState {
+  options: ResolvedBreakerOptions
+  failures: number[] // unix-ms timestamps, recent-first not enforced
+  trippedUntil: number | null // unix-ms; null when closed
+}
+
+const DEFAULTS: ResolvedBreakerOptions = {
+  threshold: 10,
+  windowMs: 60_000,
+  cooldownMs: 30_000,
+}
+
+const breakers = new Map<string, BreakerState>()
+
+/** Register / update a breaker for a handler. */
+export function configureBreaker(handlerName: string, options: CircuitBreakerOptions): void {
+  breakers.set(handlerName, {
+    options: { ...DEFAULTS, ...options },
+    failures: [],
+    trippedUntil: null,
+  })
+}
+
+/** Forget all breaker state. Test-only. */
+export function resetBreakers(): void {
+  breakers.clear()
+}
+
+/**
+ * Check if a handler is currently tripped. Returns the remaining
+ * cooldown in ms (>= 0) when tripped, or `null` when the circuit is
+ * closed (handler is dispatchable). Auto-resets state when the
+ * cooldown has elapsed and emits `queue:circuit_reset` once on
+ * transition.
+ */
+export function checkBreaker(handlerName: string, now: number = Date.now()): number | null {
+  const state = breakers.get(handlerName)
+  if (!state) return null
+  if (state.trippedUntil === null) return null
+
+  if (now >= state.trippedUntil) {
+    // Cooldown expired — close the circuit. Reset failure history so
+    // the next set of failures starts a fresh window.
+    state.trippedUntil = null
+    state.failures = []
+    if (Emitter.listenerCount('queue:circuit_reset') > 0) {
+      void Emitter.emit('queue:circuit_reset', { handler: handlerName }).catch(() => {})
+    }
+    return null
+  }
+
+  return state.trippedUntil - now
+}
+
+/**
+ * Record a failure for a handler. Trips the circuit when the failure
+ * count within `windowMs` reaches `threshold`. Returns the new cooldown
+ * (ms) when tripping, or `null` when the threshold is not yet reached.
+ */
+export function recordFailure(handlerName: string, now: number = Date.now()): number | null {
+  const state = breakers.get(handlerName)
+  if (!state) return null
+
+  // Drop failures outside the window then push the new one.
+  const cutoff = now - state.options.windowMs
+  state.failures = state.failures.filter(t => t > cutoff)
+  state.failures.push(now)
+
+  if (state.trippedUntil !== null) {
+    // Already tripped — do nothing.
+    return state.trippedUntil - now
+  }
+
+  if (state.failures.length >= state.options.threshold) {
+    state.trippedUntil = now + state.options.cooldownMs
+    if (Emitter.listenerCount('queue:circuit_tripped') > 0) {
+      void Emitter.emit('queue:circuit_tripped', {
+        handler: handlerName,
+        threshold: state.options.threshold,
+        windowMs: state.options.windowMs,
+        cooldownMs: state.options.cooldownMs,
+        trippedUntil: state.trippedUntil,
+      }).catch(() => {})
+    }
+    return state.options.cooldownMs
+  }
+
+  return null
+}
+
+/**
+ * Record a success — clears the failure history for this handler so
+ * intermittent errors don't accumulate. Does NOT close a tripped
+ * circuit (only the cooldown expiry does).
+ */
+export function recordSuccess(handlerName: string): void {
+  const state = breakers.get(handlerName)
+  if (!state) return
+  if (state.trippedUntil !== null) return
+  state.failures = []
+}

package/src/queue/index.ts

@@ -7,5 +7,16 @@ export type {
   JobRecord,
   FailedJobRecord,
   JobHandler,
+  JobHandlerOptions,
+  JobHandlerRegistration,
+  JobPayloadSchema,
 } from './queue.ts'
 export type { WorkerOptions } from './worker.ts'
+export {
+  configureBreaker,
+  checkBreaker,
+  recordFailure,
+  recordSuccess,
+  resetBreakers,
+} from './circuit_breaker.ts'
+export type { CircuitBreakerOptions, ResolvedBreakerOptions } from './circuit_breaker.ts'

package/src/queue/queue.ts

@@ -3,6 +3,7 @@ import Configuration from '@strav/kernel/config/configuration'
 import Database from '@strav/database/database/database'
 import Emitter from '@strav/kernel/events/emitter'
 import { ConfigurationError } from '@strav/kernel/exceptions/errors'
+import { configureBreaker, type CircuitBreakerOptions } from './circuit_breaker.ts'
 
 export interface JobOptions {
   queue?: string
@@ -26,6 +27,29 @@ export interface JobMeta {
   job: string
   attempts: number
   maxAttempts: number
+  /**
+   * Report progress for a long-running job. `value` is `0..1`. The reported
+   * value is persisted to the job row so external consumers can poll via
+   * {@link Queue.progressOf}, and a `queue:progress` event is emitted for
+   * live consumers (e.g. SSE).
+   *
+   * Returns immediately after persisting; safe to call from a tight loop
+   * but throttle to avoid hammering the database (e.g. every N rows or
+   * every 1 s).
+   */
+  progress: (value: number, message?: string) => Promise<void>
+}
+
+/** Snapshot of a job's current progress, returned by {@link Queue.progressOf}. */
+export interface JobProgress {
+  /** Job id. */
+  id: number
+  /** 0..1, last reported by the handler. */
+  value: number
+  /** Optional human-readable message attached to the last update. */
+  message: string | null
+  /** Current attempt count. */
+  attempts: number
 }
 
 /** A raw job row from the _strav_jobs table. */
@@ -54,6 +78,46 @@ export interface FailedJobRecord {
 
 export type JobHandler<T = any> = (payload: T, meta: JobMeta) => void | Promise<void>
 
+/**
+ * Minimal "schema-like" shape — anything that exposes `parse(input)`
+ * (Zod, ArkType, Valibot, hand-written validators) works. The schema
+ * is invoked at dequeue time, BEFORE the handler runs, so a tampered
+ * row in the DB or a payload from an older code revision is rejected
+ * loudly instead of executing with a half-formed shape.
+ */
+export interface JobPayloadSchema<T = unknown> {
+  parse(input: unknown): T
+}
+
+/** Per-handler registration options. */
+export interface JobHandlerOptions<T = any> {
+  /**
+   * Optional payload schema. When set, the worker calls `schema.parse(payload)`
+   * before invoking the handler; a parse failure routes the job to
+   * `_strav_failed_jobs` with the validation error message.
+   *
+   * Recommended for any handler whose payload comes from an external
+   * source (HTTP webhook, customer upload) or whose code has churned
+   * since older jobs were enqueued — the parse is a fail-fast invariant
+   * that catches drift before the handler corrupts state.
+   */
+  schema?: JobPayloadSchema<T>
+  /**
+   * Per-handler circuit breaker. Trips when the failure count within
+   * `windowMs` reaches `threshold`, pausing dispatch for `cooldownMs`.
+   * Defends against retry storms — a stale-schema or downed-dependency
+   * handler shouldn't keep eating worker cycles. Defaults: threshold
+   * 10, windowMs 60_000, cooldownMs 30_000.
+   */
+  circuitBreaker?: CircuitBreakerOptions
+}
+
+/** Internal registration record stored in Queue._handlers. */
+export interface JobHandlerRegistration<T = any> {
+  handler: JobHandler<T>
+  schema?: JobPayloadSchema<T>
+}
+
 /**
  * PostgreSQL-backed job queue.
  *
@@ -72,7 +136,7 @@ export type JobHandler<T = any> = (payload: T, meta: JobMeta) => void | Promise<
 export default class Queue {
   private static _db: Database
   private static _config: QueueConfig
-  private static _handlers = new Map<string, JobHandler>()
+  private static _handlers = new Map<string, JobHandlerRegistration>()
 
   constructor(db: Database, config: Configuration) {
     Queue._db = db
@@ -97,7 +161,7 @@ export default class Queue {
     return Queue._config
   }
 
-  static get handlers(): Map<string, JobHandler> {
+  static get handlers(): Map<string, JobHandlerRegistration> {
     return Queue._handlers
   }
 
@@ -116,10 +180,23 @@ export default class Queue {
         "timeout" INT NOT NULL DEFAULT 60000,
         "available_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(),
         "reserved_at" TIMESTAMPTZ,
-        "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW()
+        "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        "progress" NUMERIC NOT NULL DEFAULT 0,
+        "progress_message" TEXT
       )
     `
 
+    // Additive migrations for progress columns — for tables that existed
+    // before progress reporting was introduced.
+    await sql`
+      ALTER TABLE "_strav_jobs"
+        ADD COLUMN IF NOT EXISTS "progress" NUMERIC NOT NULL DEFAULT 0
+    `
+    await sql`
+      ALTER TABLE "_strav_jobs"
+        ADD COLUMN IF NOT EXISTS "progress_message" TEXT
+    `
+
     await sql`
       CREATE INDEX IF NOT EXISTS "idx_strav_jobs_queue_available"
         ON "_strav_jobs" ("queue", "available_at")
@@ -138,9 +215,27 @@ export default class Queue {
     `
   }
 
-  /** Register a handler for a named job. */
-  static handle<T = any>(name: string, handler: JobHandler<T>): void {
-    Queue._handlers.set(name, handler)
+  /**
+   * Register a handler for a named job. Pass `options.schema` to have
+   * the worker validate the payload (Zod / ArkType / etc.) before
+   * invoking the handler — a parse failure routes the job to
+   * `_strav_failed_jobs` instead of running the handler with bad data.
+   *
+   * @example
+   * import { z } from 'zod'
+   * Queue.handle('send-email', async (payload) => { ... }, {
+   *   schema: z.object({ to: z.string().email(), subject: z.string() }),
+   * })
+   */
+  static handle<T = any>(
+    name: string,
+    handler: JobHandler<T>,
+    options?: JobHandlerOptions<T>
+  ): void {
+    Queue._handlers.set(name, { handler, schema: options?.schema })
+    if (options?.circuitBreaker) {
+      configureBreaker(name, options.circuitBreaker)
+    }
   }
 
   /**
@@ -168,6 +263,57 @@ export default class Queue {
     return id
   }
 
+  /**
+   * Persist progress for an in-flight job and emit a `queue:progress` event.
+   * Called by the `JobMeta.progress` callback that workers hand to handlers,
+   * but exposed statically so other code (e.g. retry replay tools) can update
+   * progress directly. `value` is clamped to `[0, 1]`.
+   */
+  static async reportProgress(
+    id: number,
+    value: number,
+    message?: string
+  ): Promise<void> {
+    const sql = Queue.db.sql
+    const clamped = Math.max(0, Math.min(1, value))
+    const msg = message ?? null
+    await sql`
+      UPDATE "_strav_jobs"
+      SET "progress" = ${clamped}, "progress_message" = ${msg}
+      WHERE "id" = ${id}
+    `
+    if (Emitter.listenerCount('queue:progress') > 0) {
+      Emitter.emit('queue:progress', {
+        id,
+        value: clamped,
+        message: msg,
+      }).catch(() => {})
+    }
+  }
+
+  /**
+   * Read the latest progress snapshot for a job. Returns `null` once the
+   * job has completed (the row is deleted on success) or if the id is
+   * unknown.
+   */
+  static async progressOf(id: number): Promise<JobProgress | null> {
+    const sql = Queue.db.sql
+    const rows = await sql`
+      SELECT "id", "progress", "progress_message", "attempts"
+      FROM "_strav_jobs"
+      WHERE "id" = ${id}
+      LIMIT 1
+    `
+    if (rows.length === 0) return null
+    const row = rows[0] as Record<string, unknown>
+    return {
+      id: Number(row.id),
+      value: Number(row.progress ?? 0),
+      message: (row.progress_message as string | null) ?? null,
+      attempts: Number(row.attempts ?? 0),
+    }
+  }
+
   /**
    * Create a listener function suitable for Emitter.on().
    * When the event fires, the payload is pushed onto the queue.

package/src/queue/worker.ts

@@ -1,5 +1,6 @@
 import Queue, { hydrateJob } from './queue.ts'
 import Emitter from '@strav/kernel/events/emitter'
+import { checkBreaker, recordFailure, recordSuccess } from './circuit_breaker.ts'
 import type { JobRecord, JobMeta } from './queue.ts'
 
 export interface WorkerOptions {
@@ -109,26 +110,55 @@ export default class Worker {
 
   /** Process a single job: run handler, handle success/failure. */
   private async process(job: JobRecord): Promise<void> {
-    const handler = Queue.handlers.get(job.job)
+    const registration = Queue.handlers.get(job.job)
 
-    if (!handler) {
+    if (!registration) {
       await this.fail(job, new Error(`No handler registered for job "${job.job}"`))
       return
     }
 
+    // Q-1: per-handler circuit breaker. If the handler has tripped its
+    // breaker (too many failures in the configured window), defer this
+    // job rather than running it. Push it back to the queue with
+    // `available_at = now + cooldown` so it retries AFTER the breaker
+    // resets — this clears the worker to drain unrelated jobs from the
+    // queue instead of compounding the failure storm.
+    const cooldownRemaining = checkBreaker(job.job)
+    if (cooldownRemaining !== null) {
+      await this.deferForCooldown(job, cooldownRemaining)
+      return
+    }
+
     const meta: JobMeta = {
       id: job.id,
       queue: job.queue,
       job: job.job,
       attempts: job.attempts,
       maxAttempts: job.maxAttempts,
+      progress: (value: number, message?: string) => Queue.reportProgress(job.id, value, message),
+    }
+
+    // Re-parse the payload through the registered schema (CC-5). Catches
+    // payloads that drifted from the handler's expected shape — older
+    // enqueues, manual DB edits, malicious tampering. A parse failure
+    // is fatal: the job goes straight to failed_jobs without retry,
+    // because retrying with the same bad payload won't help.
+    let payload = job.payload
+    if (registration.schema) {
+      try {
+        payload = registration.schema.parse(job.payload)
+      } catch (err) {
+        const detail = err instanceof Error ? err.message : String(err)
+        await this.fail(job, new Error(`Job "${job.job}" payload failed validation: ${detail}`))
+        return
+      }
     }
 
     const start = performance.now()
 
     try {
       await Promise.race([
-        Promise.resolve(handler(job.payload, meta)),
+        Promise.resolve(registration.handler(payload, meta)),
         new Promise<never>((_, reject) =>
           setTimeout(
             () => reject(new Error(`Job "${job.job}" timed out after ${job.timeout}ms`)),
@@ -137,6 +167,7 @@ export default class Worker {
         ),
       ])
       await this.complete(job)
+      recordSuccess(job.job)
 
       if (Emitter.listenerCount('queue:processed') > 0) {
         const duration = performance.now() - start
@@ -149,6 +180,9 @@ export default class Worker {
       }
     } catch (error) {
       const err = error instanceof Error ? error : new Error(String(error))
+      // Update breaker state regardless of retry decision so a job that
+      // exhausts its retries still counts toward the trip threshold.
+      recordFailure(job.job)
       if (job.attempts >= job.maxAttempts) {
         await this.fail(job, err)
 
@@ -197,6 +231,24 @@ export default class Worker {
     `
   }
 
+  /**
+   * Push a tripped-circuit job back to the queue with `available_at`
+   * scheduled past the cooldown. Also rolls back the attempts counter
+   * the fetcher incremented so a circuit trip doesn't eat retry
+   * budget — the job genuinely never executed.
+   */
+  private async deferForCooldown(job: JobRecord, cooldownMs: number): Promise<void> {
+    const availableAt = new Date(Date.now() + Math.max(cooldownMs, 1_000))
+
+    await Queue.db.sql`
+      UPDATE "_strav_jobs"
+      SET "reserved_at" = NULL,
+          "available_at" = ${availableAt},
+          "attempts" = GREATEST("attempts" - 1, 0)
+      WHERE "id" = ${job.id}
+    `
+  }
+
   /** Calculate backoff delay in ms based on attempt number. */
   backoffDelay(attempts: number): number {
     if (Queue.config.retryBackoff === 'linear') {