@lota-sdk/core 0.4.8 → 0.4.10

package/src/services/queue-job.service.ts

@@ -5,12 +5,19 @@ import {
   recordIdSchema,
 } from '@lota-sdk/shared'
 import type { QueueJobError, QueueJobStatus } from '@lota-sdk/shared'
-import { RecordId } from 'surrealdb'
+import { Context, Duration, Effect, Layer, Schedule } from 'effect'
+import type { RecordId } from 'surrealdb'
 import { z } from 'zod'
 
 import { recordIdToString } from '../db/record-id'
-import { databaseService } from '../db/service'
+import type { SurrealDBService } from '../db/service'
 import { TABLES } from '../db/tables'
+import { isRetriableTransactionConflict } from '../db/transaction-conflict'
+import { BadRequestError, DatabaseError } from '../effect/errors'
+import { makeEffectTryPromiseWithMessage } from '../effect/helpers'
+import { DatabaseServiceTag } from '../effect/services'
+import { createDeterministicRecordId } from '../utils/crypto'
+import { nowDate, unsafeDateFrom } from '../utils/date-time'
 import { compactRecord, readRecord, readString, readStringField, stringifyUnknown, truncateText } from '../utils/string'
 
 const QueueJobRowSchema = z.object({
@@ -52,7 +59,6 @@ const QueueJobAttemptRowSchema = z.object({
 
 const PERSISTENCE_MAX_ATTEMPTS = 4
 const PERSISTENCE_RETRY_BASE_DELAY_MS = 25
-const PERSISTENCE_RETRY_JITTER_MS = 25
 
 export interface TrackedBullJobLike {
   queueName: string
@@ -64,11 +70,6 @@ export interface TrackedBullJobLike {
   timestamp?: number
 }
 
-function buildDeterministicRecordId(table: string, key: string): RecordId {
-  const digest = new Bun.CryptoHasher('sha256').update(key).digest('hex')
-  return new RecordId(table, digest)
-}
-
 function sanitizeQueueValue(value: unknown, depth = 0): unknown {
   if (value === null || value === undefined) return value
   if (typeof value === 'string') return truncateText(value, 20_000)
@@ -117,21 +118,27 @@ function toQueueJobError(error: unknown): QueueJobError {
   return QueueJobErrorSchema.parse({ message: truncateText(stringifyUnknown(error) ?? 'Unknown error', 5_000) })
 }
 
-function getBullmqJobId(job: TrackedBullJobLike): string {
+function getBullmqJobId(job: TrackedBullJobLike): Effect.Effect<string, BadRequestError> {
   const id = job.id
-  if (typeof id === 'string' && id.length > 0) return id
-  if (typeof id === 'number') return String(id)
-  throw new Error(`BullMQ job for queue "${job.queueName}" is missing an id.`)
+  if (typeof id === 'string' && id.length > 0) return Effect.succeed(id)
+  if (typeof id === 'number') return Effect.succeed(String(id))
+  return Effect.fail(new BadRequestError({ message: `BullMQ job for queue "${job.queueName}" is missing an id.` }))
 }
 
-function getQueueJobRecordId(job: TrackedBullJobLike): RecordId {
-  return buildDeterministicRecordId(TABLES.QUEUE_JOB, `${job.queueName}:${getBullmqJobId(job)}`)
+function getQueueJobRecordId(job: TrackedBullJobLike): Effect.Effect<RecordId, BadRequestError> {
+  return getBullmqJobId(job).pipe(
+    Effect.map((bullmqJobId) => createDeterministicRecordId(TABLES.QUEUE_JOB, `${job.queueName}:${bullmqJobId}`)),
+  )
 }
 
-function getQueueJobAttemptRecordId(job: TrackedBullJobLike, attemptNumber: number): RecordId {
-  return buildDeterministicRecordId(
-    TABLES.QUEUE_JOB_ATTEMPT,
-    `${job.queueName}:${getBullmqJobId(job)}:${attemptNumber}`,
+function getQueueJobAttemptRecordId(
+  job: TrackedBullJobLike,
+  attemptNumber: number,
+): Effect.Effect<RecordId, BadRequestError> {
+  return getBullmqJobId(job).pipe(
+    Effect.map((bullmqJobId) =>
+      createDeterministicRecordId(TABLES.QUEUE_JOB_ATTEMPT, `${job.queueName}:${bullmqJobId}:${attemptNumber}`),
+    ),
   )
 }
 
@@ -178,225 +185,275 @@ function getQueuedStatus(job: TrackedBullJobLike): QueueJobStatus {
   return typeof delay === 'number' && delay > 0 ? 'delayed' : 'waiting'
 }
 
-function isRetriablePersistenceConflict(error: unknown): boolean {
-  if (!(error instanceof Error)) return false
-
-  const message = error.message.toLowerCase()
-  return (
-    message.includes('transaction conflict') ||
-    message.includes('transaction read conflict') ||
-    message.includes('read or write conflict') ||
-    message.includes('write conflict') ||
-    message.includes('resource busy') ||
-    message.includes('this transaction can be retried')
-  )
-}
-
-class QueueJobService {
-  private async withPersistenceRetry<T>(work: () => Promise<T>): Promise<T> {
-    let lastError: unknown = null
-
-    for (let attempt = 1; attempt <= PERSISTENCE_MAX_ATTEMPTS; attempt += 1) {
-      try {
-        return await work()
-      } catch (error) {
-        lastError = error
-        const hasMoreAttempts = attempt < PERSISTENCE_MAX_ATTEMPTS
-        if (!isRetriablePersistenceConflict(error) || !hasMoreAttempts) {
-          throw error
-        }
-
-        const backoffMs =
-          PERSISTENCE_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1) + Math.floor(Math.random() * PERSISTENCE_RETRY_JITTER_MS)
-        await Bun.sleep(backoffMs)
-      }
-    }
-
-    throw lastError instanceof Error ? lastError : new Error('Queue job persistence retry exhausted')
-  }
-
-  getQueueJobId(queueName: string, bullmqJobId: string): string {
-    return recordIdToString(
-      buildDeterministicRecordId(TABLES.QUEUE_JOB, `${queueName}:${bullmqJobId}`),
-      TABLES.QUEUE_JOB,
-    )
-  }
+const effectTryQueueJobPersistence = makeEffectTryPromiseWithMessage(
+  (message, cause) => new DatabaseError({ message, cause }),
+)
 
-  async recordEnqueued(job: TrackedBullJobLike, context?: Record<string, unknown>): Promise<string> {
-    await databaseService.connect()
-    return this.withPersistenceRetry(async () => {
-      const queueJobId = getQueueJobRecordId(job)
-      const queuedAt = typeof job.timestamp === 'number' ? new Date(job.timestamp) : new Date()
-      const mergedContext = compactRecord({ ...extractJobContext(job.data), ...context })
-
-      await databaseService.upsert(
-        TABLES.QUEUE_JOB,
-        queueJobId,
-        compactRecord({
-          queueName: job.queueName,
-          jobName: job.name,
-          bullmqJobId: getBullmqJobId(job),
-          status: getQueuedStatus(job),
-          data: wrapFlexibleValue(job.data),
-          options: sanitizeQueueValue(job.opts) as Record<string, unknown>,
-          context: Object.keys(mergedContext).length > 0 ? sanitizeQueueValue(mergedContext) : undefined,
-          deduplicationId: readDeduplicationId(job),
-          maxAttempts: readMaxAttempts(job),
-          queuedAt,
-        }),
-        QueueJobRowSchema,
-      )
-
-      return recordIdToString(queueJobId, TABLES.QUEUE_JOB)
-    })
-  }
-
-  async markAttemptStarted(job: TrackedBullJobLike): Promise<string> {
-    await databaseService.connect()
-    const attemptNumber = resolveAttemptNumber(job)
-    const queueJobId = getQueueJobRecordId(job)
-    const startedAt = new Date()
-
-    return this.withPersistenceRetry(async () => {
-      await databaseService.upsert(
-        TABLES.QUEUE_JOB,
-        queueJobId,
-        compactRecord({
-          queueName: job.queueName,
-          jobName: job.name,
-          bullmqJobId: getBullmqJobId(job),
-          status: 'active',
-          data: wrapFlexibleValue(job.data),
-          options: sanitizeQueueValue(job.opts) as Record<string, unknown>,
-          context: sanitizeQueueValue(extractJobContext(job.data)) as Record<string, unknown> | undefined,
-          deduplicationId: readDeduplicationId(job),
-          maxAttempts: readMaxAttempts(job),
-          attemptCount: attemptNumber,
-          queuedAt: typeof job.timestamp === 'number' ? new Date(job.timestamp) : startedAt,
-          startedAt,
-        }),
-        QueueJobRowSchema,
-      )
-
-      const attemptId = getQueueJobAttemptRecordId(job, attemptNumber)
-      await databaseService.upsert(
-        TABLES.QUEUE_JOB_ATTEMPT,
-        attemptId,
-        { queueJobId, attemptNumber, status: 'active', startedAt },
-        QueueJobAttemptRowSchema,
-      )
-
-      return recordIdToString(queueJobId, TABLES.QUEUE_JOB)
-    })
-  }
-
-  async markAttemptCompleted(job: TrackedBullJobLike, result: unknown): Promise<void> {
-    await databaseService.connect()
-    const attemptNumber = resolveAttemptNumber(job)
-    const queueJobId = getQueueJobRecordId(job)
-    const attemptId = getQueueJobAttemptRecordId(job, attemptNumber)
-    const completedAt = new Date()
-
-    await this.withPersistenceRetry(async () => {
-      const existingAttempt = await databaseService.findOne(
-        TABLES.QUEUE_JOB_ATTEMPT,
-        { id: attemptId },
-        QueueJobAttemptRowSchema,
-      )
-
-      await databaseService.upsert(
-        TABLES.QUEUE_JOB_ATTEMPT,
-        attemptId,
-        compactRecord({
-          queueJobId,
-          attemptNumber,
-          status: 'completed',
-          result: wrapFlexibleValue(result),
-          startedAt: existingAttempt?.startedAt ?? completedAt,
-          completedAt,
-          durationMs: existingAttempt ? Math.max(0, completedAt.getTime() - existingAttempt.startedAt.getTime()) : 0,
-        }),
-        QueueJobAttemptRowSchema,
-      )
-
-      await databaseService.upsert(
-        TABLES.QUEUE_JOB,
-        queueJobId,
-        compactRecord({
-          queueName: job.queueName,
-          jobName: job.name,
-          bullmqJobId: getBullmqJobId(job),
-          status: 'completed',
-          data: wrapFlexibleValue(job.data),
-          options: sanitizeQueueValue(job.opts) as Record<string, unknown>,
-          context: sanitizeQueueValue(extractJobContext(job.data)) as Record<string, unknown> | undefined,
-          deduplicationId: readDeduplicationId(job),
-          maxAttempts: readMaxAttempts(job),
-          attemptCount: attemptNumber,
-          queuedAt: typeof job.timestamp === 'number' ? new Date(job.timestamp) : completedAt,
-          completedAt,
-          result: wrapFlexibleValue(result),
-          lastError: undefined,
-        }),
-        QueueJobRowSchema,
-      )
-    })
-  }
+function tryQueueJobPersistence<A>(
+  message: string,
+  evaluate: () => PromiseLike<A> | Effect.Effect<A, unknown>,
+): Effect.Effect<A, DatabaseError> {
+  return effectTryQueueJobPersistence(evaluate, message)
+}
 
-  async markAttemptFailed(job: TrackedBullJobLike, error: unknown): Promise<void> {
-    await databaseService.connect()
-    const attemptNumber = resolveAttemptNumber(job)
-    const queueJobId = getQueueJobRecordId(job)
-    const attemptId = getQueueJobAttemptRecordId(job, attemptNumber)
-    const failedAt = new Date()
-    const normalizedError = toQueueJobError(error)
-    const maxAttempts = readMaxAttempts(job)
-    const terminal = typeof maxAttempts === 'number' ? attemptNumber >= maxAttempts : true
-
-    await this.withPersistenceRetry(async () => {
-      const existingAttempt = await databaseService.findOne(
-        TABLES.QUEUE_JOB_ATTEMPT,
-        { id: attemptId },
-        QueueJobAttemptRowSchema,
-      )
+function withPersistenceRetry<T>(work: Effect.Effect<T, DatabaseError>): Effect.Effect<T, DatabaseError> {
+  return Effect.retry(work, {
+    times: PERSISTENCE_MAX_ATTEMPTS - 1,
+    schedule: Schedule.jittered(Schedule.exponential(Duration.millis(PERSISTENCE_RETRY_BASE_DELAY_MS), 2)),
+    while: isRetriableTransactionConflict,
+  })
+}
 
-      await databaseService.upsert(
-        TABLES.QUEUE_JOB_ATTEMPT,
-        attemptId,
-        compactRecord({
-          queueJobId,
-          attemptNumber,
-          status: 'failed',
-          error: normalizedError,
-          startedAt: existingAttempt?.startedAt ?? failedAt,
-          completedAt: failedAt,
-          durationMs: existingAttempt ? Math.max(0, failedAt.getTime() - existingAttempt.startedAt.getTime()) : 0,
-        }),
-        QueueJobAttemptRowSchema,
-      )
+function toPersistenceIdError(message: string, cause: unknown): DatabaseError {
+  return new DatabaseError({ message, cause })
+}
 
-      await databaseService.upsert(
+export function makeQueueJobService(db: SurrealDBService) {
+  return {
+    getQueueJobId(queueName: string, bullmqJobId: string): string {
+      return recordIdToString(
+        createDeterministicRecordId(TABLES.QUEUE_JOB, `${queueName}:${bullmqJobId}`),
         TABLES.QUEUE_JOB,
-        queueJobId,
-        compactRecord({
-          queueName: job.queueName,
-          jobName: job.name,
-          bullmqJobId: getBullmqJobId(job),
-          status: terminal ? 'failed' : 'waiting',
-          data: wrapFlexibleValue(job.data),
-          options: sanitizeQueueValue(job.opts) as Record<string, unknown>,
-          context: sanitizeQueueValue(extractJobContext(job.data)) as Record<string, unknown> | undefined,
-          deduplicationId: readDeduplicationId(job),
-          maxAttempts,
-          attemptCount: attemptNumber,
-          queuedAt: typeof job.timestamp === 'number' ? new Date(job.timestamp) : failedAt,
-          failedAt: terminal ? failedAt : undefined,
-          lastError: normalizedError,
-        }),
-        QueueJobRowSchema,
       )
-    })
+    },
+
+    recordEnqueued(job: TrackedBullJobLike, context?: Record<string, unknown>) {
+      return Effect.gen(function* () {
+        yield* tryQueueJobPersistence('Failed to connect before recording queued job metadata', () => db.connect())
+
+        const bullmqJobId = yield* getBullmqJobId(job)
+        const queueJobId = yield* getQueueJobRecordId(job)
+        const queuedAt = typeof job.timestamp === 'number' ? unsafeDateFrom(job.timestamp) : nowDate()
+        const mergedContext = compactRecord({ ...extractJobContext(job.data), ...context })
+
+        return yield* withPersistenceRetry(
+          Effect.gen(function* () {
+            yield* tryQueueJobPersistence('Failed to upsert queued job metadata', () =>
+              db.upsert(
+                TABLES.QUEUE_JOB,
+                queueJobId,
+                compactRecord({
+                  queueName: job.queueName,
+                  jobName: job.name,
+                  bullmqJobId,
+                  status: getQueuedStatus(job),
+                  data: wrapFlexibleValue(job.data),
+                  options: sanitizeQueueValue(job.opts) as Record<string, unknown>,
+                  context: Object.keys(mergedContext).length > 0 ? sanitizeQueueValue(mergedContext) : undefined,
+                  deduplicationId: readDeduplicationId(job),
+                  maxAttempts: readMaxAttempts(job),
+                  queuedAt,
+                }),
+                QueueJobRowSchema,
+              ),
+            )
+
+            return recordIdToString(queueJobId, TABLES.QUEUE_JOB)
+          }),
+        )
+      })
+    },
+
+    markAttemptStarted(job: TrackedBullJobLike) {
+      return Effect.gen(function* () {
+        yield* tryQueueJobPersistence('Failed to connect before marking queue job attempt started', () => db.connect())
+
+        const attemptNumber = resolveAttemptNumber(job)
+        const bullmqJobId = yield* getBullmqJobId(job).pipe(
+          Effect.mapError((cause) => toPersistenceIdError('Failed to read BullMQ job id.', cause)),
+        )
+        const queueJobId = yield* getQueueJobRecordId(job).pipe(
+          Effect.mapError((cause) => toPersistenceIdError('Failed to derive queue job record id.', cause)),
+        )
+        const attemptId = yield* getQueueJobAttemptRecordId(job, attemptNumber).pipe(
+          Effect.mapError((cause) => toPersistenceIdError('Failed to derive queue job attempt record id.', cause)),
+        )
+        const startedAt = nowDate()
+        const queuedAt = typeof job.timestamp === 'number' ? unsafeDateFrom(job.timestamp) : startedAt
+
+        return yield* withPersistenceRetry(
+          Effect.gen(function* () {
+            yield* tryQueueJobPersistence('Failed to upsert queue job start metadata', () =>
+              db.upsert(
+                TABLES.QUEUE_JOB,
+                queueJobId,
+                compactRecord({
+                  queueName: job.queueName,
+                  jobName: job.name,
+                  bullmqJobId,
+                  status: 'active',
+                  data: wrapFlexibleValue(job.data),
+                  options: sanitizeQueueValue(job.opts) as Record<string, unknown>,
+                  context: sanitizeQueueValue(extractJobContext(job.data)) as Record<string, unknown> | undefined,
+                  deduplicationId: readDeduplicationId(job),
+                  maxAttempts: readMaxAttempts(job),
+                  attemptCount: attemptNumber,
+                  queuedAt,
+                  startedAt,
+                }),
+                QueueJobRowSchema,
+              ),
+            )
+
+            yield* tryQueueJobPersistence('Failed to upsert queue job attempt start metadata', () =>
+              db.upsert(
+                TABLES.QUEUE_JOB_ATTEMPT,
+                attemptId,
+                { queueJobId, attemptNumber, status: 'active', startedAt },
+                QueueJobAttemptRowSchema,
+              ),
+            )
+
+            return recordIdToString(queueJobId, TABLES.QUEUE_JOB)
+          }),
+        )
+      })
+    },
+
+    markAttemptCompleted(job: TrackedBullJobLike, result: unknown) {
+      return Effect.gen(function* () {
+        yield* tryQueueJobPersistence('Failed to connect before marking queue job attempt completed', () =>
+          db.connect(),
+        )
+
+        const attemptNumber = resolveAttemptNumber(job)
+        const bullmqJobId = yield* getBullmqJobId(job)
+        const queueJobId = yield* getQueueJobRecordId(job)
+        const attemptId = yield* getQueueJobAttemptRecordId(job, attemptNumber)
+        const completedAt = nowDate()
+        const queuedAt = typeof job.timestamp === 'number' ? unsafeDateFrom(job.timestamp) : completedAt
+
+        yield* withPersistenceRetry(
+          Effect.gen(function* () {
+            const existingAttempt = yield* tryQueueJobPersistence(
+              'Failed to load existing queue job attempt before completion update',
+              () => db.findOne(TABLES.QUEUE_JOB_ATTEMPT, { id: attemptId }, QueueJobAttemptRowSchema),
+            )
+
+            yield* tryQueueJobPersistence('Failed to upsert completed queue job attempt metadata', () =>
+              db.upsert(
+                TABLES.QUEUE_JOB_ATTEMPT,
+                attemptId,
+                compactRecord({
+                  queueJobId,
+                  attemptNumber,
+                  status: 'completed',
+                  result: wrapFlexibleValue(result),
+                  startedAt: existingAttempt?.startedAt ?? completedAt,
+                  completedAt,
+                  durationMs: existingAttempt
+                    ? Math.max(0, completedAt.getTime() - existingAttempt.startedAt.getTime())
+                    : 0,
+                }),
+                QueueJobAttemptRowSchema,
+              ),
+            )
+
+            yield* tryQueueJobPersistence('Failed to upsert completed queue job metadata', () =>
+              db.upsert(
+                TABLES.QUEUE_JOB,
+                queueJobId,
+                compactRecord({
+                  queueName: job.queueName,
+                  jobName: job.name,
+                  bullmqJobId,
+                  status: 'completed',
+                  data: wrapFlexibleValue(job.data),
+                  options: sanitizeQueueValue(job.opts) as Record<string, unknown>,
+                  context: sanitizeQueueValue(extractJobContext(job.data)) as Record<string, unknown> | undefined,
+                  deduplicationId: readDeduplicationId(job),
+                  maxAttempts: readMaxAttempts(job),
+                  attemptCount: attemptNumber,
+                  queuedAt,
+                  completedAt,
+                  result: wrapFlexibleValue(result),
+                  lastError: undefined,
+                }),
+                QueueJobRowSchema,
+              ),
+            )
+          }),
+        )
+      })
+    },
+
+    markAttemptFailed(job: TrackedBullJobLike, error: unknown) {
+      return Effect.gen(function* () {
+        yield* tryQueueJobPersistence('Failed to connect before marking queue job attempt failed', () => db.connect())
+
+        const attemptNumber = resolveAttemptNumber(job)
+        const bullmqJobId = yield* getBullmqJobId(job)
+        const queueJobId = yield* getQueueJobRecordId(job)
+        const attemptId = yield* getQueueJobAttemptRecordId(job, attemptNumber)
+        const failedAt = nowDate()
+        const queuedAt = typeof job.timestamp === 'number' ? unsafeDateFrom(job.timestamp) : failedAt
+        const normalizedError = toQueueJobError(error)
+        const maxAttempts = readMaxAttempts(job)
+        const terminal = typeof maxAttempts === 'number' ? attemptNumber >= maxAttempts : true
+
+        yield* withPersistenceRetry(
+          Effect.gen(function* () {
+            const existingAttempt = yield* tryQueueJobPersistence(
+              'Failed to load existing queue job attempt before failure update',
+              () => db.findOne(TABLES.QUEUE_JOB_ATTEMPT, { id: attemptId }, QueueJobAttemptRowSchema),
+            )
+
+            yield* tryQueueJobPersistence('Failed to upsert failed queue job attempt metadata', () =>
+              db.upsert(
+                TABLES.QUEUE_JOB_ATTEMPT,
+                attemptId,
+                compactRecord({
+                  queueJobId,
+                  attemptNumber,
+                  status: 'failed',
+                  error: normalizedError,
+                  startedAt: existingAttempt?.startedAt ?? failedAt,
+                  completedAt: failedAt,
+                  durationMs: existingAttempt
+                    ? Math.max(0, failedAt.getTime() - existingAttempt.startedAt.getTime())
+                    : 0,
+                }),
+                QueueJobAttemptRowSchema,
+              ),
+            )
+
+            yield* tryQueueJobPersistence('Failed to upsert failed queue job metadata', () =>
+              db.upsert(
+                TABLES.QUEUE_JOB,
+                queueJobId,
+                compactRecord({
+                  queueName: job.queueName,
+                  jobName: job.name,
+                  bullmqJobId,
+                  status: terminal ? 'failed' : 'waiting',
+                  data: wrapFlexibleValue(job.data),
+                  options: sanitizeQueueValue(job.opts) as Record<string, unknown>,
+                  context: sanitizeQueueValue(extractJobContext(job.data)) as Record<string, unknown> | undefined,
+                  deduplicationId: readDeduplicationId(job),
+                  maxAttempts,
+                  attemptCount: attemptNumber,
+                  queuedAt,
+                  failedAt: terminal ? failedAt : undefined,
+                  lastError: normalizedError,
+                }),
+                QueueJobRowSchema,
+              ),
+            )
+          }),
+        )
+      })
+    },
   }
 }
 
-export const queueJobService = new QueueJobService()
+export class QueueJobServiceTag extends Context.Service<QueueJobServiceTag, ReturnType<typeof makeQueueJobService>>()(
+  '@lota-sdk/core/QueueJobService',
+) {}
+
+export const QueueJobServiceLive = Layer.effect(
+  QueueJobServiceTag,
+  Effect.gen(function* () {
+    const db = yield* DatabaseServiceTag
+    return makeQueueJobService(db)
+  }),
+)