bunderstack 0.15.1 → 0.16.0

package/src/jobs/define.ts

@@ -9,6 +9,8 @@ import type { StorageFacade } from '../index'
 
 import { parseCron } from './cron'
 
+import { CRON_PREFIX, type CatchUp } from './slots'
+
 export const DEFAULT_RETRIES = 3
 export const DEFAULT_TIMEOUT_MS = 60_000
 
@@ -21,6 +23,15 @@ export type EnqueueOptions = {
   runAt?: Date | number
 }
 
+export type TickResult = {
+  /** Rows moved from pending to running this tick. */
+  claimed: number
+  /** Handlers that completed successfully. */
+  ran: number
+  /** Handlers that threw, whether or not they will be retried. */
+  failed: number
+}
+
 /**
  * The untyped runtime facade. Handler ctx and tRPC ctx expose this shape;
  * `app.jobs` narrows `enqueue` to the declared job names/payloads.
@@ -32,7 +43,7 @@ export type JobsRuntimeFacade = {
     opts?: EnqueueOptions,
   ): Promise<{ id: string }>
   /** Run one poll cycle deterministically (tests). `now` defaults to Date.now(). */
-  tick(now?: number): Promise<void>
+  tick(now?: number): Promise<TickResult>
 }
 
 import type { RealtimeFacade } from '../realtime/facade'
@@ -66,7 +77,7 @@ export type QueueJobDefinition<
   retries?: number
   /** Delay before retry N (1-based). Default exponential: 1s, 2s, 4s, … */
   backoff?: ((attempt: number) => number) | { baseMs?: number; factor?: number }
-  /** Max simultaneous `running` rows of this type, enforced cross-replica. */
+  /** Max simultaneous `running` rows of this type, enforced per worker. */
   concurrency?: number
   /** Lease duration in ms; an expired lease sends the job back to pending. */
   timeout?: number
@@ -91,10 +102,26 @@ export type CronDefinition<
 > = {
   kind: 'cron'
   schedule: TSchedule
+  /** Attempts after the first failure. Default 3 (so 4 total attempts). */
+  retries?: number
+  /** Delay before retry N (1-based). Default exponential: 1s, 2s, 4s, … */
+  backoff?: ((attempt: number) => number) | { baseMs?: number; factor?: number }
+  /** Lease duration in ms; an expired lease sends the slot back to pending. */
+  timeout?: number
+  /** How missed slots are handled on wake. Default 'latest'. */
+  catchUp?: CatchUp
+  /** How far back catch-up looks, in ms. Default 1 hour. */
+  catchUpWindow?: number
   handler: (
     invocation: CronInvocation,
     ctx: JobContext<TSchema, TEnvResult>,
   ) => Promise<void> | void
+  /** Fires once, after the final attempt fails. Errors here are logged, never retried. */
+  onFailed?: (
+    invocation: CronInvocation,
+    error: Error,
+    ctx: JobContext<TSchema, TEnvResult>,
+  ) => Promise<void> | void
 }
 
 export type BackgroundDefinition =
@@ -111,6 +138,10 @@ export type JobDefinition<
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export type AnyJobDefinition = QueueJobDefinition<any, any, any>
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type AnyBackgroundDefinition =
+  | QueueJobDefinition<any, any, any>
+  | CronDefinition<any, any, any>
 export type JobsDefs = BackgroundDefs
 
 export type QueueJobKeys<TDefs extends BackgroundDefs> = {
@@ -129,18 +160,38 @@ export function validateBackgroundDefs(defs: BackgroundDefs): void {
     if (typeof def.handler !== 'function') {
       throw new Error(`[bunderstack] background task "${name}" has no handler`)
     }
-    if (def.kind === 'cron') {
-      parseCron(def.schedule)
-      continue
+    if (def.kind === 'job' && name.startsWith(CRON_PREFIX)) {
+      throw new Error(
+        `[bunderstack] job "${name}": the "${CRON_PREFIX}" prefix is reserved for cron tasks`,
+      )
     }
     if (
       def.retries !== undefined &&
       (def.retries < 0 || !Number.isInteger(def.retries))
     ) {
       throw new Error(
-        `[bunderstack] job "${name}": retries must be a non-negative integer`,
+        `[bunderstack] background task "${name}": retries must be a non-negative integer`,
+      )
+    }
+    if (def.timeout !== undefined && def.timeout <= 0) {
+      throw new Error(
+        `[bunderstack] background task "${name}": timeout must be positive`,
       )
     }
+    if (def.kind === 'cron') {
+      parseCron(def.schedule)
+      if ((def as { concurrency?: number }).concurrency !== undefined) {
+        throw new Error(
+          `[bunderstack] cron "${name}": concurrency is not supported for cron tasks — slots are already unique`,
+        )
+      }
+      if (def.catchUpWindow !== undefined && def.catchUpWindow <= 0) {
+        throw new Error(
+          `[bunderstack] cron "${name}": catchUpWindow must be positive`,
+        )
+      }
+      continue
+    }
     if (
       def.concurrency !== undefined &&
       (def.concurrency < 1 || !Number.isInteger(def.concurrency))
@@ -149,22 +200,27 @@ export function validateBackgroundDefs(defs: BackgroundDefs): void {
         `[bunderstack] job "${name}": concurrency must be a positive integer`,
       )
     }
-    if (def.timeout !== undefined && def.timeout <= 0) {
-      throw new Error(`[bunderstack] job "${name}": timeout must be positive`)
-    }
   }
 }
 
 /** @deprecated Use validateBackgroundDefs. */
 export const validateJobsDefs = validateBackgroundDefs
 
-/** Delay in ms before retry `attempt` (1-based = the attempt that just failed). */
-export function backoffMs(def: AnyJobDefinition, attempt: number): number {
+/**
+ * Delay in ms before retry `attempt` (1-based = the attempt that just failed).
+ * Jittered by ±20% so a shared outage does not retry every job in lockstep.
+ * A caller-supplied backoff function is returned verbatim — the caller owns it.
+ */
+export function backoffMs(
+  def: AnyBackgroundDefinition,
+  attempt: number,
+): number {
   const b = def.backoff
   if (typeof b === 'function') return b(attempt)
   const baseMs = b?.baseMs ?? 1000
   const factor = b?.factor ?? 2
-  return baseMs * factor ** (attempt - 1)
+  const flat = baseMs * factor ** (attempt - 1)
+  return Math.round(flat * (0.8 + Math.random() * 0.4))
 }
 
 /**

package/src/jobs/index.ts

@@ -25,15 +25,13 @@ export type {
 } from './define'
 export { enqueueJob } from './queue'
 export { createJobRunner } from './worker'
-export { runCronSlot, runScheduledSlot } from './cron-runner'
-export type { CronRunResult } from './cron-runner'
-export { buildCronRouter } from './cron-router'
-export { signScheduleRequest, verifyScheduleRequest } from './cron-auth'
 export { startJobWorker } from './runtime'
-export type { StartWorkerOptions, RunWorkerOptions, WorkerHandle } from './runtime'
-export { startLocalCronScheduler } from './local-cron'
 export type {
-  LocalCronScheduler,
-  LocalCronSchedulerOptions,
-} from './local-cron'
+  StartWorkerOptions,
+  RunWorkerOptions,
+  WorkerHandle,
+} from './runtime'
 export { parseCron, cronMatches } from './cron'
+export { slotsDue, floorSlot, CRON_PREFIX, SLOT_MS } from './slots'
+export type { CatchUp } from './slots'
+export type { TickResult } from './define'

package/src/jobs/queue.ts

@@ -7,6 +7,8 @@ import type { EnqueueOptions, JobsDefs } from './define'
 import { jobsTableFor } from '../internal-tables'
 import { generate } from '../typeid'
 
+import { CRON_PREFIX } from './slots'
+
 export async function enqueueJob(
   db: AnyDb,
   defs: JobsDefs,
@@ -15,11 +17,13 @@ export async function enqueueJob(
   opts: EnqueueOptions = {},
 ): Promise<{ id: string }> {
   const def = defs[name]
-  if (!def || def.kind !== 'job') {
-    throw new Error(`[bunderstack] unknown queue job "${name}"`)
+  if (!def) {
+    throw new Error(`[bunderstack] unknown background task "${name}"`)
   }
-  // Fail fast: a bad payload should throw at the call site, not in the worker.
-  const parsed = def.input ? def.input.parse(input) : null
+  const isCron = def.kind === 'cron'
+  const type = isCron ? `${CRON_PREFIX}${name}` : name
+  // Cron slots carry no payload; queue jobs validate theirs at the call site.
+  const parsed = isCron ? null : def.input ? def.input.parse(input) : null
   const t = jobsTableFor(db)
   const now = Date.now()
   const runAt =
@@ -35,7 +39,7 @@ export async function enqueueJob(
       .insert(t)
       .values({
         id,
-        type: name,
+        type,
         payloadJson: JSON.stringify(parsed),
         status: 'pending',
         attempts: 0,
@@ -49,7 +53,7 @@ export async function enqueueJob(
     const existing = await db
       .select({ id: t.id })
       .from(t)
-      .where(and(eq(t.type, name), eq(t.dedupeKey, opts.dedupeKey ?? '')))
+      .where(and(eq(t.type, type), eq(t.dedupeKey, opts.dedupeKey ?? '')))
       .limit(1)
     if (existing[0]) return { id: String(existing[0].id) }
   }

package/src/jobs/slots.ts

@@ -0,0 +1,52 @@
+// src/jobs/slots.ts — cron slot enumeration. Pure; no db, no clock reads.
+import { cronMatches, type ParsedCron } from './cron'
+
+/** Slot granularity. Every slot timestamp satisfies `slot % SLOT_MS === 0`. */
+export const SLOT_MS = 60_000
+
+/** Reserved job-type prefix for cron occurrences. */
+export const CRON_PREFIX = 'cron:'
+
+/** How far back either catch-up mode will look. */
+export const DEFAULT_CATCH_UP_WINDOW_MS = 60 * SLOT_MS
+
+export type CatchUp = 'latest' | 'all'
+
+/** Aligns `ms` down to its containing slot. */
+export function floorSlot(ms: number): number {
+  return Math.floor(ms / SLOT_MS) * SLOT_MS
+}
+
+/**
+ * Slots matching `cron` in the half-open range `(from, to]`, oldest first.
+ *
+ * `from` is exclusive so a stored watermark is never re-emitted. Both modes are
+ * clamped to `catchUpWindowMs` — without it a watermark far in the past would
+ * make this iterate unbounded minutes.
+ */
+export function slotsDue(args: {
+  cron: ParsedCron
+  from: number
+  to: number
+  catchUp?: CatchUp
+  catchUpWindowMs?: number
+}): number[] {
+  const catchUp = args.catchUp ?? 'latest'
+  const windowMs = args.catchUpWindowMs ?? DEFAULT_CATCH_UP_WINDOW_MS
+  const to = floorSlot(args.to)
+  const from = Math.max(floorSlot(args.from), to - windowMs)
+  if (to <= from) return []
+
+  if (catchUp === 'latest') {
+    for (let s = to; s > from; s -= SLOT_MS) {
+      if (cronMatches(args.cron, s)) return [s]
+    }
+    return []
+  }
+
+  const slots: number[] = []
+  for (let s = from + SLOT_MS; s <= to; s += SLOT_MS) {
+    if (cronMatches(args.cron, s)) slots.push(s)
+  }
+  return slots
+}

package/src/jobs/worker.ts

@@ -1,36 +1,56 @@
 // src/jobs/worker.ts — the queue worker. One `tick()` is a full cycle:
 // recover expired leases → reap old succeeded rows → claim and run queue jobs.
-import { and, eq, inArray, is, isNotNull, lt, lte, sql } from 'drizzle-orm'
+import { and, eq, inArray, is, isNotNull, lt, lte, max, sql } from 'drizzle-orm'
 import { PgDatabase } from 'drizzle-orm/pg-core'
 
 import type { AnyDb } from '../dialect'
 import type {
-  AnyJobDefinition,
+  AnyBackgroundDefinition,
   JobsDefs,
   JobsRuntimeFacade,
+  TickResult,
 } from './define'
 
 import { jobsTableFor } from '../internal-tables'
+import { parseCron } from './cron'
 import { backoffMs, DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from './define'
+import { enqueueJob } from './queue'
+import { CRON_PREFIX, floorSlot, slotsDue, SLOT_MS } from './slots'
 
 const CLAIM_BATCH = 10
 const SUCCEEDED_RETENTION_MS = 24 * 60 * 60 * 1000
+const REAP_INTERVAL_MS = 60 * 60_000
 
 type JobRow = {
   id: string
   type: string
   payloadJson: string
   attempts: number
+  runAt: number
 }
 
 function toError(err: unknown): Error {
   return err instanceof Error ? err : new Error(String(err))
 }
 
-function maxAttempts(def: AnyJobDefinition): number {
+function maxAttempts(def: AnyBackgroundDefinition): number {
   return 1 + (def.retries ?? DEFAULT_RETRIES)
 }
 
+/** Resolve a stored row type back to its definition. Cron rows carry the
+ *  reserved prefix; queue rows use the definition key verbatim. */
+function definitionFor(
+  defs: JobsDefs,
+  type: string,
+): AnyBackgroundDefinition | undefined {
+  if (type.startsWith(CRON_PREFIX)) {
+    const def = defs[type.slice(CRON_PREFIX.length)]
+    return def?.kind === 'cron' ? def : undefined
+  }
+  const def = defs[type]
+  return def?.kind === 'job' ? def : undefined
+}
+
 /** Terminal queue rows release their dedupe key. */
 function terminalPatch() {
   return { dedupeKey: null }
@@ -45,21 +65,66 @@ export function createJobRunner(deps: {
   const { db, defs } = deps
   const t = jobsTableFor(db)
   const ctx = { ...deps.ctx } as Record<string, unknown>
+  let lastReapAt = 0
+
+  /** Cron rows carry no payload — their handler input is the slot itself. */
+  function resolveInput(def: AnyBackgroundDefinition, row: JobRow): unknown {
+    if (def.kind === 'cron') {
+      return { scheduledFor: new Date(Number(row.runAt)) }
+    }
+    const raw = JSON.parse(row.payloadJson)
+    return def.input ? def.input.parse(raw) : undefined
+  }
+
+  /**
+   * The watermark is the newest slot we already stored for this cron. When no
+   * rows exist — a newly declared cron, or one whose rows were reaped — anchor
+   * one slot before now so the current minute is eligible and nothing older is.
+   */
+  async function cronWatermark(type: string, now: number): Promise<number> {
+    const rows = await db
+      .select({ latest: max(t.runAt) })
+      .from(t)
+      .where(eq(t.type, type))
+    const latest = rows[0]?.latest
+    return latest == null ? floorSlot(now) - SLOT_MS : Number(latest)
+  }
+
+  /** Enqueue a row per due slot. The unique(type, dedupeKey) constraint makes
+   *  this safe to run concurrently in any number of processes. */
+  async function materializeCronSlots(now: number) {
+    for (const [name, def] of Object.entries(defs)) {
+      if (def.kind !== 'cron') continue
+      const type = `${CRON_PREFIX}${name}`
+      const from = await cronWatermark(type, now)
+      const slots = slotsDue({
+        cron: parseCron(def.schedule),
+        from,
+        to: now,
+        catchUp: def.catchUp,
+        catchUpWindowMs: def.catchUpWindow,
+      })
+      for (const slot of slots) {
+        await enqueueJob(db, defs, name, null, {
+          runAt: slot,
+          dedupeKey: String(slot),
+        })
+      }
+    }
+  }
+
   async function fireOnFailed(
-    def: AnyJobDefinition,
-    payloadJson: string,
+    def: AnyBackgroundDefinition,
+    input: unknown,
     error: Error,
   ) {
     if (!def.onFailed) return
-    let input: unknown
-    try {
-      const raw = JSON.parse(payloadJson)
-      input = def.input ? def.input.parse(raw) : undefined
-    } catch {
-      input = undefined // payload unusable; the hook still gets the error
-    }
     try {
-      await def.onFailed(input, error, ctx as never)
+      await (def.onFailed as (i: unknown, e: Error, c: unknown) => unknown)(
+        input,
+        error,
+        ctx,
+      )
     } catch (hookErr) {
       console.error('[bunderstack] onFailed hook threw:', hookErr)
     }
@@ -74,15 +139,20 @@ export function createJobRunner(deps: {
         payloadJson: t.payloadJson,
         attempts: t.attempts,
         lastError: t.lastError,
+        runAt: t.runAt,
       })
       .from(t)
       .where(
-        and(eq(t.status, 'running'), isNotNull(t.lockedUntil), lt(t.lockedUntil, now)),
+        and(
+          eq(t.status, 'running'),
+          isNotNull(t.lockedUntil),
+          lt(t.lockedUntil, now),
+        ),
       )
     for (const row of expired) {
-      const def = defs[row.type]
+      const def = definitionFor(defs, row.type)
       const error = new Error('lease expired (worker crashed or timed out)')
-      if (!def || def.kind !== 'job') {
+      if (!def) {
         await db
           .update(t)
           .set({
@@ -106,7 +176,7 @@ export function createJobRunner(deps: {
             ...terminalPatch(),
           })
           .where(eq(t.id, row.id))
-        await fireOnFailed(def, row.payloadJson, error)
+        await fireOnFailed(def, resolveInput(def, row), error)
       } else {
         await db
           .update(t)
@@ -148,8 +218,11 @@ export function createJobRunner(deps: {
     // PG: lock the selected rows so concurrent replicas skip them. SQLite's
     // single-writer model makes the one-statement UPDATE atomic on its own.
     const sub = is(db, PgDatabase)
-      ? (pendingIds as unknown as { for: (m: string, o: object) => typeof pendingIds })
-          .for('update', { skipLocked: true })
+      ? (
+          pendingIds as unknown as {
+            for: (m: string, o: object) => typeof pendingIds
+          }
+        ).for('update', { skipLocked: true })
       : pendingIds
     const rows: JobRow[] = await db
       .update(t)
@@ -164,6 +237,7 @@ export function createJobRunner(deps: {
         type: t.type,
         payloadJson: t.payloadJson,
         attempts: t.attempts,
+        runAt: t.runAt,
       })
     return rows
   }
@@ -171,15 +245,19 @@ export function createJobRunner(deps: {
   // `now` is the tick's injected clock: retry runAt math uses it so tests can
   // drive backoff deterministically. finishedAt uses the real clock (a handler
   // may run long past the tick's start).
-  async function runJob(row: JobRow, def: AnyJobDefinition, now: number) {
+  async function runJob(
+    row: JobRow,
+    def: AnyBackgroundDefinition,
+    now: number,
+    leaseUntil: number,
+  ): Promise<'ran' | 'failed' | 'lost'> {
     let input: unknown
     try {
-      const raw = JSON.parse(row.payloadJson)
-      input = def.input ? def.input.parse(raw) : undefined
+      input = resolveInput(def, row)
     } catch (err) {
       // Stored payload no longer parses (schema drift): retrying can't help.
       const e = toError(err)
-      await db
+      const updated = await db
         .update(t)
         .set({
           status: 'failed',
@@ -188,13 +266,15 @@ export function createJobRunner(deps: {
           lastError: e.message,
           ...terminalPatch(),
         })
-        .where(eq(t.id, row.id))
-      await fireOnFailed(def, row.payloadJson, e)
-      return
+        .where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
+        .returning({ id: t.id })
+      if (!updated[0]) return 'lost'
+      await fireOnFailed(def, undefined, e)
+      return 'failed'
     }
     try {
-      await def.handler(input, ctx as never)
-      await db
+      await (def.handler as (i: unknown, c: unknown) => unknown)(input, ctx)
+      const updated = await db
         .update(t)
         .set({
           status: 'succeeded',
@@ -202,11 +282,14 @@ export function createJobRunner(deps: {
           lockedUntil: null,
           ...terminalPatch(),
         })
-        .where(eq(t.id, row.id))
+        .where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
+        .returning({ id: t.id })
+      if (!updated[0]) return 'lost'
+      return 'ran'
     } catch (err) {
       const e = toError(err)
       if (Number(row.attempts) < maxAttempts(def)) {
-        await db
+        const updated = await db
           .update(t)
           .set({
             status: 'pending',
@@ -214,9 +297,11 @@ export function createJobRunner(deps: {
             runAt: now + backoffMs(def, Number(row.attempts)),
             lastError: e.message,
           })
-          .where(eq(t.id, row.id))
+          .where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
+          .returning({ id: t.id })
+        if (!updated[0]) return 'lost'
       } else {
-        await db
+        const updated = await db
           .update(t)
           .set({
             status: 'failed',
@@ -225,19 +310,22 @@ export function createJobRunner(deps: {
             lastError: e.message,
             ...terminalPatch(),
           })
-          .where(eq(t.id, row.id))
-        await fireOnFailed(def, row.payloadJson, e)
+          .where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
+          .returning({ id: t.id })
+        if (!updated[0]) return 'lost'
+        await fireOnFailed(def, input, e)
       }
+      return 'failed'
     }
   }
 
-  async function runClaimable(now: number) {
-    const work: Promise<void>[] = []
-    for (const [type, candidate] of Object.entries(defs)) {
-      if (candidate.kind !== 'job') continue
-      const def = candidate
+  async function runClaimable(now: number): Promise<TickResult> {
+    const work: Promise<'ran' | 'failed' | 'lost'>[] = []
+    let totalClaimed = 0
+    for (const [name, def] of Object.entries(defs)) {
+      const type = def.kind === 'cron' ? `${CRON_PREFIX}${name}` : name
       let limit = CLAIM_BATCH
-      if (def.concurrency !== undefined) {
+      if (def.kind === 'job' && def.concurrency !== undefined) {
         const runningRows = await db
           .select({ id: t.id })
           .from(t)
@@ -248,16 +336,28 @@ export function createJobRunner(deps: {
       }
       const leaseUntil = now + (def.timeout ?? DEFAULT_TIMEOUT_MS)
       const claimed = await claim(type, limit, now, leaseUntil)
-      for (const row of claimed) work.push(runJob(row, def, now))
+      totalClaimed += claimed.length
+      for (const row of claimed) work.push(runJob(row, def, now, leaseUntil))
+    }
+    const outcomes = await Promise.all(work)
+    let ran = 0
+    let failed = 0
+    for (const outcome of outcomes) {
+      if (outcome === 'ran') ran++
+      else if (outcome === 'failed') failed++
     }
-    await Promise.all(work)
+    return { claimed: totalClaimed, ran, failed }
   }
 
   return {
-    async tick(now: number = Date.now()) {
+    async tick(now: number = Date.now()): Promise<TickResult> {
+      await materializeCronSlots(now)
       await recoverExpiredLeases(now)
-      await reapSucceeded(now)
-      await runClaimable(now)
+      if (now - lastReapAt >= REAP_INTERVAL_MS) {
+        lastReapAt = now
+        await reapSucceeded(now)
+      }
+      return runClaimable(now)
     },
     setJobsFacade(f: JobsRuntimeFacade) {
       ctx.jobs = f

package/src/list-query.ts

@@ -16,8 +16,8 @@ import {
 } from 'drizzle-orm'
 import { PgTable } from 'drizzle-orm/pg-core'
 
-import type { AnyDb } from './dialect'
 import type { ResolvedTableAccess, SortOrder } from './access'
+import type { AnyDb } from './dialect'
 
 import { ErrorCode, ListQueryError } from './errors'
 
@@ -282,9 +282,7 @@ export function encodeCursor(payload: CursorPayload): string {
 
 export function decodeCursor(cursor: string): CursorPayload {
   try {
-    const parsed = JSON.parse(
-      Buffer.from(cursor, 'base64url').toString('utf8'),
-    )
+    const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'))
     if (!isCursorPayload(parsed)) {
       throw new Error('invalid cursor shape')
     }