@meith/drivers 0.16.0

package/src/queue/postgres-queue.ts

@@ -0,0 +1,218 @@
+import { randomUUID } from 'node:crypto'
+
+import { and, eq, inArray, isNull, lt, or, sql } from 'drizzle-orm'
+
+import { type EnqueueOptions, type Job, logger, type QueueDriver } from '@meith/core'
+import { type Database, getDb, jobs, resultRows } from '@meith/db'
+
+const LEASE_SECONDS = 300
+
+function backoffSeconds(attempt: number): number {
+  return Math.min(10 * attempt * attempt, 3600)
+}
+
+export class PostgresQueue implements QueueDriver {
+  private readonly workerId = `${process.pid}-${randomUUID().slice(0, 8)}`
+
+  constructor(private readonly db: Database = getDb()) {}
+
+  async enqueue<TPayload>(
+    kind: string,
+    payload: TPayload,
+    options: EnqueueOptions = {},
+  ): Promise<{ id: string; deduplicated: boolean }> {
+    const row = {
+      kind,
+      payload: payload as never,
+      runAt: options.runAt ?? new Date(),
+      maxAttempts: options.maxAttempts ?? 5,
+      idempotencyKey: options.dedupeKey ?? null,
+    }
+
+    if (options.dedupeKey) {
+      const inserted = await this.db
+        .insert(jobs)
+        .values(row)
+        .onConflictDoNothing()
+        .returning({ id: jobs.id })
+
+      const first = inserted[0]
+      if (!first) {
+        const existing = await this.db
+          .select({ id: jobs.id })
+          .from(jobs)
+          .where(and(eq(jobs.idempotencyKey, options.dedupeKey), eq(jobs.status, 'pending')))
+          .limit(1)
+
+        return { id: String(existing[0]?.id ?? ''), deduplicated: true }
+      }
+      return { id: String(first.id), deduplicated: false }
+    }
+
+    const inserted = await this.db.insert(jobs).values(row).returning({ id: jobs.id })
+    return { id: String(inserted[0]!.id), deduplicated: false }
+  }
+
+  async drain(
+    limit: number,
+    handler: (job: Job) => Promise<void>,
+    options: { readonly signal?: AbortSignal } = {},
+  ): Promise<{ processed: number; failed: number }> {
+    const claimed = await this.claim(limit)
+
+    let processed = 0
+    let failed = 0
+
+    for (const [index, job] of claimed.entries()) {
+      if (options.signal?.aborted === true) {
+        await this.unclaim(claimed.slice(index))
+        break
+      }
+
+      try {
+        await handler(job)
+        await this.db
+          .update(jobs)
+          .set({ status: 'done', completedAt: new Date(), lockedUntil: null, lockedBy: null })
+          .where(eq(jobs.id, Number(job.id)))
+        processed += 1
+      } catch (error) {
+        failed += 1
+        await this.recordFailure(job, error)
+      }
+    }
+
+    return { processed, failed }
+  }
+
+  private async claim(limit: number): Promise<Job[]> {
+    const now = new Date()
+    const leaseUntil = new Date(now.getTime() + LEASE_SECONDS * 1000)
+
+    const rows = await this.db.execute(sql`
+      UPDATE ${jobs}
+         SET status       = 'running',
+             attempts     = ${jobs.attempts} + 1,
+             locked_until = ${leaseUntil},
+             locked_by    = ${this.workerId}
+       WHERE id IN (
+         SELECT id
+           FROM ${jobs}
+          WHERE run_at <= ${now}
+            AND (
+              status = 'pending'
+              -- Reclaim rows whose lease expired: the worker holding them died.
+              OR (status = 'running' AND locked_until < ${now})
+            )
+            AND attempts < max_attempts
+          ORDER BY run_at
+          LIMIT ${limit}
+          FOR UPDATE SKIP LOCKED
+       )
+      RETURNING id, kind, payload, attempts, correlation_id
+    `)
+
+    return resultRows(rows).map((r) => {
+      const job: Job = {
+        id: String(r.id),
+        kind: String(r.kind),
+        payload: r.payload,
+        attempt: Number(r.attempts),
+      }
+      const correlationId = r.correlation_id
+      return typeof correlationId === 'string' ? { ...job, requestId: correlationId } : job
+    })
+  }
+
+  private async unclaim(jobs_: readonly Job[]): Promise<void> {
+    if (jobs_.length === 0) return
+
+    const ids = jobs_.map((job) => Number(job.id))
+
+    await this.db
+      .update(jobs)
+      .set({
+        status: 'pending',
+        attempts: sql`${jobs.attempts} - 1`,
+        lockedUntil: null,
+        lockedBy: null,
+      })
+      .where(inArray(jobs.id, ids))
+  }
+
+  private async recordFailure(job: Job, error: unknown): Promise<void> {
+    const message = error instanceof Error ? error.message : String(error)
+    const exhausted = await this.isExhausted(job)
+
+    await this.db
+      .update(jobs)
+      .set({
+        status: exhausted ? 'dead' : 'pending',
+        lastError: message.slice(0, 2000),
+        runAt: new Date(Date.now() + backoffSeconds(job.attempt) * 1000),
+        lockedUntil: null,
+        lockedBy: null,
+      })
+      .where(eq(jobs.id, Number(job.id)))
+
+    logger({ jobId: job.id, kind: job.kind, attempt: job.attempt }).error(
+      { err: message },
+      exhausted ? 'job dead-lettered' : 'job failed, will retry',
+    )
+  }
+
+  private async isExhausted(job: Job): Promise<boolean> {
+    const rows = await this.db
+      .select({ attempts: jobs.attempts, maxAttempts: jobs.maxAttempts })
+      .from(jobs)
+      .where(eq(jobs.id, Number(job.id)))
+      .limit(1)
+
+    const row = rows[0]
+    return row ? row.attempts >= row.maxAttempts : true
+  }
+
+  async deadLettered(limit: number): Promise<readonly Job[]> {
+    const rows = await this.db
+      .select({
+        id: jobs.id,
+        kind: jobs.kind,
+        payload: jobs.payload,
+        attempts: jobs.attempts,
+      })
+      .from(jobs)
+      .where(eq(jobs.status, 'dead'))
+      .limit(limit)
+
+    return rows.map((r) => ({
+      id: String(r.id),
+      kind: r.kind,
+      payload: r.payload,
+      attempt: r.attempts,
+    }))
+  }
+
+  async retry(jobId: string): Promise<boolean> {
+    const id = Number(jobId)
+    if (!Number.isSafeInteger(id)) return false
+
+    const requeued = await this.db
+      .update(jobs)
+      .set({
+        status: 'pending',
+        attempts: 0,
+        runAt: new Date(),
+        lastError: null,
+        lockedUntil: null,
+        lockedBy: null,
+      })
+      .where(and(eq(jobs.id, id), eq(jobs.status, 'dead')))
+      .returning({ id: jobs.id })
+
+    return requeued.length > 0
+  }
+
+  static completedBefore(cutoff: Date) {
+    return and(eq(jobs.status, 'done'), or(isNull(jobs.completedAt), lt(jobs.completedAt, cutoff)))
+  }
+}

package/src/resolve.ts

@@ -0,0 +1,89 @@
+import {
+  type CacheDriver,
+  ConfigurationError,
+  type Drivers,
+  env,
+  type FileStore,
+  type MailDriver,
+  type QueueDriver,
+} from '@meith/core'
+import { getDb, PostgresSettingsRepository } from '@meith/db'
+import {
+  type MailConfig,
+  mailConfigFromEnvironment,
+  mailConfigFromSettings,
+  NO_MAIL,
+  SettingsSnapshot,
+} from '@meith/settings'
+
+import { NextCacheDriver } from './cache/next-cache'
+import { RedisCacheDriver } from './cache/redis-cache'
+import { LocalFileStore } from './files/local-file-store'
+import { S3FileStore } from './files/s3-file-store'
+import { ConfiguredMailDriver } from './mail'
+import { MemoryQueue } from './queue/memory-queue'
+import { PostgresQueue } from './queue/postgres-queue'
+
+let bundle: Drivers | undefined
+
+function buildQueue(): QueueDriver {
+  switch (env.QUEUE_DRIVER) {
+    case 'postgres':
+      return new PostgresQueue()
+    case 'memory':
+      return new MemoryQueue()
+  }
+}
+
+function buildCache(): CacheDriver {
+  switch (env.CACHE_DRIVER) {
+    case 'next':
+    case 'memory':
+      return new NextCacheDriver()
+    case 'redis': {
+      if (!env.REDIS_URL) {
+        throw new ConfigurationError('CACHE_DRIVER=redis requires REDIS_URL to be set.')
+      }
+      return new RedisCacheDriver({ url: env.REDIS_URL })
+    }
+  }
+}
+
+function buildFiles(): FileStore {
+  switch (env.FILESTORE_DRIVER) {
+    case 'local':
+      return new LocalFileStore(env.UPLOADS_DIR)
+    case 's3':
+      return S3FileStore.fromEnv(env)
+  }
+}
+
+export async function currentMailConfig(): Promise<MailConfig> {
+  if (env.DEMO_MODE) return NO_MAIL
+
+  const fromEnvironment = mailConfigFromEnvironment(env)
+  if (fromEnvironment !== null) return fromEnvironment
+
+  if (env.DATA_SOURCE !== 'postgres') return NO_MAIL
+
+  const overrides = await new PostgresSettingsRepository(getDb()).loadAll()
+  return mailConfigFromSettings(SettingsSnapshot.fromOverrides(new Map(overrides)))
+}
+
+function buildMail(): MailDriver {
+  return new ConfiguredMailDriver(currentMailConfig)
+}
+
+export function drivers(): Drivers {
+  bundle ??= {
+    queue: buildQueue(),
+    cache: buildCache(),
+    files: buildFiles(),
+    mail: buildMail(),
+  }
+  return bundle
+}
+
+export function resetDriversForTests(): void {
+  bundle = undefined
+}