bunderstack 0.15.2 → 0.17.0-beta.0

package/src/jobs/worker.ts

@@ -1,32 +1,57 @@
 // src/jobs/worker.ts — the queue worker. One `tick()` is a full cycle:
 // recover expired leases → reap old succeeded rows → claim and run queue jobs.
-import { and, eq, inArray, is, isNotNull, lt, lte, sql } from 'drizzle-orm'
+import { and, eq, inArray, is, isNotNull, lt, lte, max, sql } from 'drizzle-orm'
 import { PgDatabase } from 'drizzle-orm/pg-core'
 
 import type { AnyDb } from '../dialect'
-import type { AnyJobDefinition, JobsDefs, JobsRuntimeFacade } from './define'
+import type {
+  AnyBackgroundDefinition,
+  JobsDefs,
+  JobsRuntimeFacade,
+  TickResult,
+} from './define'
 
 import { jobsTableFor } from '../internal-tables'
+import { validateStandardSchema } from '../standard-schema'
+import { parseCron } from './cron'
 import { backoffMs, DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from './define'
+import { enqueueJob } from './queue'
+import { CRON_PREFIX, floorSlot, slotsDue, SLOT_MS } from './slots'
 
 const CLAIM_BATCH = 10
 const SUCCEEDED_RETENTION_MS = 24 * 60 * 60 * 1000
+const REAP_INTERVAL_MS = 60 * 60_000
 
 type JobRow = {
   id: string
   type: string
   payloadJson: string
   attempts: number
+  runAt: number
 }
 
 function toError(err: unknown): Error {
   return err instanceof Error ? err : new Error(String(err))
 }
 
-function maxAttempts(def: AnyJobDefinition): number {
+function maxAttempts(def: AnyBackgroundDefinition): number {
   return 1 + (def.retries ?? DEFAULT_RETRIES)
 }
 
+/** Resolve a stored row type back to its definition. Cron rows carry the
+ *  reserved prefix; queue rows use the definition key verbatim. */
+function definitionFor(
+  defs: JobsDefs,
+  type: string,
+): AnyBackgroundDefinition | undefined {
+  if (type.startsWith(CRON_PREFIX)) {
+    const def = defs[type.slice(CRON_PREFIX.length)]
+    return def?.kind === 'cron' ? def : undefined
+  }
+  const def = defs[type]
+  return def?.kind === 'job' ? def : undefined
+}
+
 /** Terminal queue rows release their dedupe key. */
 function terminalPatch() {
   return { dedupeKey: null }
@@ -41,21 +66,68 @@ export function createJobRunner(deps: {
   const { db, defs } = deps
   const t = jobsTableFor(db)
   const ctx = { ...deps.ctx } as Record<string, unknown>
+  let lastReapAt = 0
+
+  /** Cron rows carry no payload — their handler input is the slot itself. */
+  function resolveInput(def: AnyBackgroundDefinition, row: JobRow): unknown {
+    if (def.kind === 'cron') {
+      return { scheduledFor: new Date(Number(row.runAt)) }
+    }
+    const raw = JSON.parse(row.payloadJson)
+    return def.input
+      ? validateStandardSchema(def.input, raw, 'job payload')
+      : undefined
+  }
+
+  /**
+   * The watermark is the newest slot we already stored for this cron. When no
+   * rows exist — a newly declared cron, or one whose rows were reaped — anchor
+   * one slot before now so the current minute is eligible and nothing older is.
+   */
+  async function cronWatermark(type: string, now: number): Promise<number> {
+    const rows = await db
+      .select({ latest: max(t.runAt) })
+      .from(t)
+      .where(eq(t.type, type))
+    const latest = rows[0]?.latest
+    return latest == null ? floorSlot(now) - SLOT_MS : Number(latest)
+  }
+
+  /** Enqueue a row per due slot. The unique(type, dedupeKey) constraint makes
+   *  this safe to run concurrently in any number of processes. */
+  async function materializeCronSlots(now: number) {
+    for (const [name, def] of Object.entries(defs)) {
+      if (def.kind !== 'cron') continue
+      const type = `${CRON_PREFIX}${name}`
+      const from = await cronWatermark(type, now)
+      const slots = slotsDue({
+        cron: parseCron(def.schedule),
+        from,
+        to: now,
+        catchUp: def.catchUp,
+        catchUpWindowMs: def.catchUpWindow,
+      })
+      for (const slot of slots) {
+        await enqueueJob(db, defs, name, null, {
+          runAt: slot,
+          dedupeKey: String(slot),
+        })
+      }
+    }
+  }
+
   async function fireOnFailed(
-    def: AnyJobDefinition,
-    payloadJson: string,
+    def: AnyBackgroundDefinition,
+    input: unknown,
     error: Error,
   ) {
     if (!def.onFailed) return
-    let input: unknown
     try {
-      const raw = JSON.parse(payloadJson)
-      input = def.input ? def.input.parse(raw) : undefined
-    } catch {
-      input = undefined // payload unusable; the hook still gets the error
-    }
-    try {
-      await def.onFailed(input, error, ctx as never)
+      await (def.onFailed as (i: unknown, e: Error, c: unknown) => unknown)(
+        input,
+        error,
+        ctx,
+      )
     } catch (hookErr) {
       console.error('[bunderstack] onFailed hook threw:', hookErr)
     }
@@ -70,6 +142,7 @@ export function createJobRunner(deps: {
         payloadJson: t.payloadJson,
         attempts: t.attempts,
         lastError: t.lastError,
+        runAt: t.runAt,
       })
       .from(t)
       .where(
@@ -80,9 +153,9 @@ export function createJobRunner(deps: {
         ),
       )
     for (const row of expired) {
-      const def = defs[row.type]
+      const def = definitionFor(defs, row.type)
       const error = new Error('lease expired (worker crashed or timed out)')
-      if (!def || def.kind !== 'job') {
+      if (!def) {
         await db
           .update(t)
           .set({
@@ -106,7 +179,7 @@ export function createJobRunner(deps: {
             ...terminalPatch(),
           })
           .where(eq(t.id, row.id))
-        await fireOnFailed(def, row.payloadJson, error)
+        await fireOnFailed(def, resolveInput(def, row), error)
       } else {
         await db
           .update(t)
@@ -167,6 +240,7 @@ export function createJobRunner(deps: {
         type: t.type,
         payloadJson: t.payloadJson,
         attempts: t.attempts,
+        runAt: t.runAt,
       })
     return rows
   }
@@ -174,15 +248,19 @@ export function createJobRunner(deps: {
   // `now` is the tick's injected clock: retry runAt math uses it so tests can
   // drive backoff deterministically. finishedAt uses the real clock (a handler
   // may run long past the tick's start).
-  async function runJob(row: JobRow, def: AnyJobDefinition, now: number) {
+  async function runJob(
+    row: JobRow,
+    def: AnyBackgroundDefinition,
+    now: number,
+    leaseUntil: number,
+  ): Promise<'ran' | 'failed' | 'lost'> {
     let input: unknown
     try {
-      const raw = JSON.parse(row.payloadJson)
-      input = def.input ? def.input.parse(raw) : undefined
+      input = resolveInput(def, row)
     } catch (err) {
       // Stored payload no longer parses (schema drift): retrying can't help.
       const e = toError(err)
-      await db
+      const updated = await db
         .update(t)
         .set({
           status: 'failed',
@@ -191,13 +269,15 @@ export function createJobRunner(deps: {
           lastError: e.message,
           ...terminalPatch(),
         })
-        .where(eq(t.id, row.id))
-      await fireOnFailed(def, row.payloadJson, e)
-      return
+        .where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
+        .returning({ id: t.id })
+      if (!updated[0]) return 'lost'
+      await fireOnFailed(def, undefined, e)
+      return 'failed'
     }
     try {
-      await def.handler(input, ctx as never)
-      await db
+      await (def.handler as (i: unknown, c: unknown) => unknown)(input, ctx)
+      const updated = await db
         .update(t)
         .set({
           status: 'succeeded',
@@ -205,11 +285,14 @@ export function createJobRunner(deps: {
           lockedUntil: null,
           ...terminalPatch(),
         })
-        .where(eq(t.id, row.id))
+        .where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
+        .returning({ id: t.id })
+      if (!updated[0]) return 'lost'
+      return 'ran'
     } catch (err) {
       const e = toError(err)
       if (Number(row.attempts) < maxAttempts(def)) {
-        await db
+        const updated = await db
           .update(t)
           .set({
             status: 'pending',
@@ -217,9 +300,11 @@ export function createJobRunner(deps: {
             runAt: now + backoffMs(def, Number(row.attempts)),
             lastError: e.message,
           })
-          .where(eq(t.id, row.id))
+          .where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
+          .returning({ id: t.id })
+        if (!updated[0]) return 'lost'
       } else {
-        await db
+        const updated = await db
           .update(t)
           .set({
             status: 'failed',
@@ -228,19 +313,22 @@ export function createJobRunner(deps: {
             lastError: e.message,
             ...terminalPatch(),
           })
-          .where(eq(t.id, row.id))
-        await fireOnFailed(def, row.payloadJson, e)
+          .where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
+          .returning({ id: t.id })
+        if (!updated[0]) return 'lost'
+        await fireOnFailed(def, input, e)
       }
+      return 'failed'
     }
   }
 
-  async function runClaimable(now: number) {
-    const work: Promise<void>[] = []
-    for (const [type, candidate] of Object.entries(defs)) {
-      if (candidate.kind !== 'job') continue
-      const def = candidate
+  async function runClaimable(now: number): Promise<TickResult> {
+    const work: Promise<'ran' | 'failed' | 'lost'>[] = []
+    let totalClaimed = 0
+    for (const [name, def] of Object.entries(defs)) {
+      const type = def.kind === 'cron' ? `${CRON_PREFIX}${name}` : name
       let limit = CLAIM_BATCH
-      if (def.concurrency !== undefined) {
+      if (def.kind === 'job' && def.concurrency !== undefined) {
         const runningRows = await db
           .select({ id: t.id })
           .from(t)
@@ -251,16 +339,28 @@ export function createJobRunner(deps: {
       }
       const leaseUntil = now + (def.timeout ?? DEFAULT_TIMEOUT_MS)
       const claimed = await claim(type, limit, now, leaseUntil)
-      for (const row of claimed) work.push(runJob(row, def, now))
+      totalClaimed += claimed.length
+      for (const row of claimed) work.push(runJob(row, def, now, leaseUntil))
+    }
+    const outcomes = await Promise.all(work)
+    let ran = 0
+    let failed = 0
+    for (const outcome of outcomes) {
+      if (outcome === 'ran') ran++
+      else if (outcome === 'failed') failed++
     }
-    await Promise.all(work)
+    return { claimed: totalClaimed, ran, failed }
   }
 
   return {
-    async tick(now: number = Date.now()) {
+    async tick(now: number = Date.now()): Promise<TickResult> {
+      await materializeCronSlots(now)
       await recoverExpiredLeases(now)
-      await reapSucceeded(now)
-      await runClaimable(now)
+      if (now - lastReapAt >= REAP_INTERVAL_MS) {
+        lastReapAt = now
+        await reapSucceeded(now)
+      }
+      return runClaimable(now)
     },
     setJobsFacade(f: JobsRuntimeFacade) {
       ctx.jobs = f

package/src/manifest.ts

@@ -1,5 +1,7 @@
+import type { StandardSchemaV1 } from '@standard-schema/spec'
+
 import { getTableName, isTable } from 'drizzle-orm'
-import { z, type ZodType } from 'zod'
+import * as v from 'valibot'
 
 import type { Dialect } from './dialect'
 import type { EnvConfigInput } from './env'
@@ -7,12 +9,15 @@ import type { JobsDefs } from './jobs/define'
 import type { ResolvedBucket, ResolvedStorageBuckets } from './storage/buckets'
 
 import {
-  bunderstackCronRuns,
   bunderstackFiles,
   bunderstackIdempotency,
   bunderstackJobs,
 } from './internal-tables'
 import { parseCron } from './jobs/cron'
+import {
+  StandardSchemaValidationError,
+  validateStandardSchema,
+} from './standard-schema'
 
 export type ManifestEnvVar = {
   key: string
@@ -44,95 +49,77 @@ export type BunderstackManifest = {
   }
 }
 
-const nonEmpty = z.string().min(1)
-const migrationDirectory = nonEmpty.refine(
-  (value) =>
-    value.startsWith('/') ||
-    (!value.includes('\\') &&
-      value.split('/').every((part) => part !== '..' && part !== '')),
-  {
-    message:
-      'migrationsDirectory must be an absolute path or a relative path without traversal',
-  },
+const nonEmpty = v.pipe(v.string(), v.minLength(1))
+const migrationDirectory = v.pipe(
+  nonEmpty,
+  v.check(
+    (value) =>
+      value.startsWith('/') ||
+      (!value.includes('\\') &&
+        value.split('/').every((part) => part !== '..' && part !== '')),
+    'migrationsDirectory must be an absolute path or a relative path without traversal',
+  ),
 )
-const cronSchedule = nonEmpty.refine(
-  (value) => {
+const cronSchedule = v.pipe(
+  nonEmpty,
+  v.check((value) => {
     try {
       parseCron(value)
       return true
     } catch {
       return false
     }
-  },
-  { message: 'invalid cron schedule' },
+  }, 'invalid cron schedule'),
 )
 
-const manifestSchema = z
-  .object({
-    version: z.literal(3),
-    database: z
-      .object({
-        dialect: z.enum(['sqlite', 'pg']),
-        migrationsDirectory: migrationDirectory,
-        tables: z.array(
-          z
-            .object({
-              exportName: nonEmpty,
-              physicalName: nonEmpty,
-              system: z.boolean(),
-            })
-            .strict(),
-        ),
-      })
-      .strict(),
-    storage: z
-      .object({
-        defaultBucket: nonEmpty,
-        buckets: z.array(
-          z
-            .object({
-              name: nonEmpty,
-              visibility: z.enum(['public', 'private']),
-            })
-            .strict(),
-        ),
-      })
-      .strict(),
-    realtime: z.object({ required: z.boolean() }).strict(),
-    environment: z.array(
-      z
-        .object({
-          key: nonEmpty,
-          required: z.boolean(),
-          scope: z.enum(['server', 'client']),
-        })
-        .strict(),
+const manifestSchema = v.strictObject({
+  version: v.literal(3),
+  database: v.strictObject({
+    dialect: v.picklist(['sqlite', 'pg']),
+    migrationsDirectory: migrationDirectory,
+    tables: v.array(
+      v.strictObject({
+        exportName: nonEmpty,
+        physicalName: nonEmpty,
+        system: v.boolean(),
+      }),
     ),
-    background: z
-      .object({
-        jobs: z.array(z.object({ name: nonEmpty }).strict()),
-        cron: z.array(
-          z
-            .object({
-              name: nonEmpty,
-              schedule: cronSchedule,
-              timezone: z.literal('UTC'),
-            })
-            .strict(),
-        ),
-        maintenance: z.array(
-          z
-            .object({
-              name: z.literal('storage-sweep'),
-              schedule: cronSchedule,
-              timezone: z.literal('UTC'),
-            })
-            .strict(),
-        ),
-      })
-      .strict(),
-  })
-  .strict()
+  }),
+  storage: v.strictObject({
+    defaultBucket: nonEmpty,
+    buckets: v.array(
+      v.strictObject({
+        name: nonEmpty,
+        visibility: v.picklist(['public', 'private']),
+      }),
+    ),
+  }),
+  realtime: v.strictObject({ required: v.boolean() }),
+  environment: v.array(
+    v.strictObject({
+      key: nonEmpty,
+      required: v.boolean(),
+      scope: v.picklist(['server', 'client']),
+    }),
+  ),
+  background: v.strictObject({
+    jobs: v.array(v.strictObject({ name: nonEmpty })),
+    cron: v.array(
+      v.strictObject({
+        name: nonEmpty,
+        schedule: cronSchedule,
+        timezone: v.literal('UTC'),
+      }),
+    ),
+    maintenance: v.array(
+      v.strictObject({
+        name: v.literal('storage-sweep'),
+        schedule: cronSchedule,
+        timezone: v.literal('UTC'),
+      }),
+    ),
+  }),
+})
 
 function sortBy<T>(entries: readonly T[], key: (entry: T) => string): T[] {
   return [...entries].sort((left, right) => key(left).localeCompare(key(right)))
@@ -162,14 +149,19 @@ function describeTables(schema: Record<string, unknown>) {
 }
 
 function describeSection(
-  section: Record<string, ZodType> | undefined,
+  section: Record<string, StandardSchemaV1> | undefined,
   scope: ManifestEnvVar['scope'],
 ): ManifestEnvVar[] {
-  return Object.entries(section ?? {}).map(([key, schema]) => ({
-    key,
-    required: !schema.safeParse(undefined).success,
-    scope,
-  }))
+  return Object.entries(section ?? {}).map(([key, schema]) => {
+    let required = false
+    try {
+      validateStandardSchema(schema, undefined, 'env')
+    } catch (error) {
+      if (!(error instanceof StandardSchemaValidationError)) throw error
+      required = true
+    }
+    return { key, required, scope }
+  })
 }
 
 function systemTables() {
@@ -189,16 +181,15 @@ function systemTables() {
       physicalName: getTableName(bunderstackJobs),
       system: true,
     },
-    {
-      exportName: '_system.scheduledRuns',
-      physicalName: getTableName(bunderstackCronRuns),
-      system: true,
-    },
   ]
 }
 
 export function parseManifest(value: unknown): BunderstackManifest {
-  const manifest = manifestSchema.parse(value) as BunderstackManifest
+  const manifest = validateStandardSchema(
+    manifestSchema,
+    value,
+    'manifest',
+  ) as BunderstackManifest
   rejectDuplicates(
     'database physical table',
     manifest.database.tables.map((entry) => entry.physicalName),

package/src/realtime/facade.ts

@@ -1,6 +1,9 @@
 import { getTableName, type InferSelectModel, type Table } from 'drizzle-orm'
 
-import type { RealtimeAction, RealtimeBroker } from './index'
+import type {
+  RealtimeAction,
+  RealtimePublisher,
+} from './publisher'
 
 export type RealtimeTransport = 'disabled' | 'memory' | 'redis'
 
@@ -23,30 +26,30 @@ export interface RealtimeFacade<
 }
 
 export function createRealtimeFacade<TSchema extends Record<string, unknown>>(
-  broker?: RealtimeBroker,
-  transport: RealtimeTransport = broker ? 'memory' : 'disabled',
+  publisher?: RealtimePublisher,
+  transport: RealtimeTransport = publisher ? 'memory' : 'disabled',
 ): RealtimeFacade<TSchema> {
-  if (!broker && transport !== 'disabled') {
+  if (!publisher && transport !== 'disabled') {
     throw new Error(
-      '[bunderstack] an enabled realtime transport requires a broker',
+      '[bunderstack] an enabled realtime transport requires a publisher',
     )
   }
-  if (broker && transport === 'disabled') {
+  if (publisher && transport === 'disabled') {
     throw new Error(
-      '[bunderstack] a realtime broker cannot use the disabled transport',
+      '[bunderstack] a realtime publisher cannot use the disabled transport',
     )
   }
 
   return {
-    enabled: broker !== undefined,
+    enabled: publisher !== undefined,
     transport,
     async publish(table, action, record) {
-      if (!broker) return
-      await broker.publish(
-        getTableName(table),
+      if (!publisher) return
+      await publisher.publish('change', {
+        table: getTableName(table),
         action,
-        record as unknown as Record<string, unknown>,
-      )
+        record: record as unknown as Record<string, unknown>,
+      })
     },
   }
 }

package/src/realtime/filter.ts

@@ -0,0 +1,77 @@
+import { getEventMeta, withEventMeta } from '@orpc/server'
+
+import type {
+  AccessUser,
+  ResolvedAccess,
+} from '../access'
+import type { RealtimeChange } from './publisher'
+
+import {
+  checkAccess,
+  rowMatchesScope,
+  tableEntryForName,
+} from '../access'
+
+export interface FilterRealtimeChangesOptions {
+  subscriptions: readonly string[]
+  access: ResolvedAccess
+  request: Request
+  getSession: () => Promise<{
+    user: AccessUser | null
+    activeOrganizationId: string | null
+  }>
+}
+
+export async function* filterRealtimeChanges(
+  source: AsyncIterable<RealtimeChange>,
+  options: FilterRealtimeChangesOptions,
+): AsyncGenerator<RealtimeChange, void, void> {
+  const subscriptions = new Set(options.subscriptions)
+  let sessionPromise:
+    | ReturnType<FilterRealtimeChangesOptions['getSession']>
+    | undefined
+  const getSession = () => (sessionPromise ??= options.getSession())
+
+  for await (const change of source) {
+    const entry = tableEntryForName(options.access, change.table)
+    if (!entry?.enabled) continue
+
+    const recordId = change.record.id
+    if (
+      !subscriptions.has(change.table) &&
+      (recordId == null ||
+        !subscriptions.has(`${change.table}/${String(recordId)}`))
+    ) {
+      continue
+    }
+    if (entry.get === 'deny') continue
+
+    const needsSession = entry.get !== 'public' || entry.readScope !== undefined
+    const session = needsSession
+      ? await getSession()
+      : { user: null, activeOrganizationId: null }
+    const context = {
+      request: options.request,
+      user: session.user,
+      row: change.record,
+      session: { activeOrganizationId: session.activeOrganizationId },
+    }
+    if (!(await checkAccess(entry.get, context, entry.ownerColumn)).allowed) {
+      continue
+    }
+    if (
+      entry.readScope &&
+      !rowMatchesScope(change.record, entry.readScope(context))
+    ) {
+      continue
+    }
+
+    const projected: RealtimeChange = {
+      table: change.table,
+      action: change.action,
+      record: change.record,
+    }
+    const meta = getEventMeta(change)
+    yield meta ? withEventMeta(projected, meta) : projected
+  }
+}