@lota-sdk/core 0.1.14 → 0.1.16

package/src/queues/recent-activity-title-refinement.queue.ts

@@ -1,69 +1,30 @@
-import { Queue, Worker } from 'bullmq'
 import type { Job } from 'bullmq'
 
-import { serverLogger } from '../config/logger'
 import { databaseService } from '../db/service'
-import { getRedisConnectionForBullMQ } from '../redis'
 import { recentActivityTitleService } from '../services/recent-activity-title.service'
-import {
-  attachWorkerEvents,
-  createTracedWorkerProcessor,
-  createWorkerShutdown,
-  registerShutdownSignals,
-} from '../workers/worker-utils'
-import type { WorkerHandle } from '../workers/worker-utils'
+import { createQueueFactory } from './queue-factory'
 
 interface RecentActivityTitleRefinementJob {
   activityId: string
 }
 
-const RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE = 'recent-activity-title-refinement'
-
-let _recentActivityTitleRefinementQueue: Queue<RecentActivityTitleRefinementJob> | null = null
-function getRecentActivityTitleRefinementQueue(): Queue<RecentActivityTitleRefinementJob> {
-  if (!_recentActivityTitleRefinementQueue) {
-    _recentActivityTitleRefinementQueue = new Queue<RecentActivityTitleRefinementJob>(
-      RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE,
-      {
-        connection: getRedisConnectionForBullMQ(),
-        defaultJobOptions: {
-          removeOnComplete: 200,
-          removeOnFail: 200,
-          attempts: 3,
-          backoff: { type: 'exponential', delay: 2_000 },
-        },
-      },
-    )
-  }
-  return _recentActivityTitleRefinementQueue
-}
-
-export async function enqueueRecentActivityTitleRefinement(job: RecentActivityTitleRefinementJob) {
-  return await getRecentActivityTitleRefinementQueue().add('refine-recent-activity-title', job, {
-    jobId: `recent-activity-title:${job.activityId}`,
-  })
-}
-
 async function processRecentActivityTitleRefinementJob(job: Job<RecentActivityTitleRefinementJob>): Promise<void> {
   await databaseService.connect()
   await recentActivityTitleService.refineRecentActivityTitle(job.data.activityId)
 }
 
-export function startRecentActivityTitleRefinementWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
-  const { registerSignals = import.meta.main } = options
-  const worker = new Worker(
-    RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE,
-    createTracedWorkerProcessor(RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE, processRecentActivityTitleRefinementJob),
-    { connection: getRedisConnectionForBullMQ(), concurrency: 2, lockDuration: 300_000 },
-  )
-
-  attachWorkerEvents(worker, 'Recent activity title refinement', serverLogger)
-
-  const shutdown = createWorkerShutdown(worker, 'Recent activity title refinement', serverLogger)
-
-  if (registerSignals) {
-    registerShutdownSignals({ name: 'Recent activity title refinement', shutdown, logger: serverLogger })
-  }
-
-  return { worker, shutdown }
+const recentActivityTitleRefinement = createQueueFactory<RecentActivityTitleRefinementJob>({
+  name: 'recent-activity-title-refinement',
+  displayName: 'Recent activity title refinement',
+  jobName: 'refine-recent-activity-title',
+  concurrency: 2,
+  lockDuration: 300_000,
+  defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
+  processor: processRecentActivityTitleRefinementJob,
+})
+
+export function enqueueRecentActivityTitleRefinement(job: RecentActivityTitleRefinementJob) {
+  return recentActivityTitleRefinement.enqueue(job, { jobId: `recent-activity-title:${job.activityId}` })
 }
+
+export const startRecentActivityTitleRefinementWorker = recentActivityTitleRefinement.startWorker

package/src/queues/regular-chat-memory-digest.queue.ts

@@ -1,14 +1,5 @@
-import { Queue, Worker } from 'bullmq'
-
-import { serverLogger } from '../config/logger'
-import { getRedisConnectionForBullMQ } from '../redis'
-import {
-  attachWorkerEvents,
-  getWorkerPath,
-  createWorkerShutdown,
-  registerShutdownSignals,
-} from '../workers/worker-utils'
-import type { WorkerHandle } from '../workers/worker-utils'
+import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
+import { createQueueFactory } from './queue-factory'
 import {
   buildRegularChatMemoryDigestDeduplicationId,
   buildRegularChatMemoryDigestJobOptions,
@@ -18,55 +9,25 @@ export interface RegularChatMemoryDigestJob {
   orgId: string
 }
 
-const REGULAR_CHAT_MEMORY_DIGEST_QUEUE = 'regular-chat-memory-digest'
-
-let _regularChatMemoryDigestQueue: Queue<RegularChatMemoryDigestJob> | null = null
-function getRegularChatMemoryDigestQueue(): Queue<RegularChatMemoryDigestJob> {
-  if (!_regularChatMemoryDigestQueue) {
-    _regularChatMemoryDigestQueue = new Queue<RegularChatMemoryDigestJob>(REGULAR_CHAT_MEMORY_DIGEST_QUEUE, {
-      connection: getRedisConnectionForBullMQ(),
-      defaultJobOptions: {
-        removeOnComplete: 200,
-        removeOnFail: 200,
-        attempts: 2,
-        backoff: { type: 'exponential', delay: 5000 },
-      },
-    })
-  }
-  return _regularChatMemoryDigestQueue
-}
-
-export async function enqueueRegularChatMemoryDigest(job: RegularChatMemoryDigestJob) {
-  return await getRegularChatMemoryDigestQueue().add(
-    'run-digest',
-    job,
-    buildRegularChatMemoryDigestJobOptions(job.orgId),
-  )
+const regularChatMemoryDigest = createQueueFactory<RegularChatMemoryDigestJob>({
+  name: 'regular-chat-memory-digest',
+  displayName: 'Regular chat memory digest',
+  jobName: 'run-digest',
+  concurrency: 1,
+  lockDuration: LONG_JOB_LOCK_DURATION_MS,
+  defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
+  processorPath: getWorkerPath('regular-chat-memory-digest.worker.ts'),
+})
+
+export function enqueueRegularChatMemoryDigest(job: RegularChatMemoryDigestJob) {
+  return regularChatMemoryDigest.enqueue(job, buildRegularChatMemoryDigestJobOptions(job.orgId))
 }
 
 export async function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
-  await getRegularChatMemoryDigestQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId))
+  await regularChatMemoryDigest.getQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId))
 }
 
-export function startRegularChatMemoryDigestWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
-  const { registerSignals = import.meta.main } = options
-  const processorPath = getWorkerPath('regular-chat-memory-digest.worker.ts')
-  const worker = new Worker(REGULAR_CHAT_MEMORY_DIGEST_QUEUE, processorPath, {
-    connection: getRedisConnectionForBullMQ(),
-    concurrency: 1,
-    lockDuration: 600_000,
-  })
-
-  attachWorkerEvents(worker, 'Regular chat memory digest', serverLogger)
-
-  const shutdown = createWorkerShutdown(worker, 'Regular chat memory digest', serverLogger)
-
-  if (registerSignals) {
-    registerShutdownSignals({ name: 'Regular chat memory digest', shutdown, logger: serverLogger })
-  }
-
-  return { worker, shutdown }
-}
+export const startRegularChatMemoryDigestWorker = regularChatMemoryDigest.startWorker
 
 if (import.meta.main) {
   startRegularChatMemoryDigestWorker()

package/src/queues/skill-extraction.queue.ts

@@ -1,61 +1,26 @@
-import { Queue, Worker } from 'bullmq'
-
-import { serverLogger } from '../config/logger'
-import { getRedisConnectionForBullMQ } from '../redis'
-import {
-  attachWorkerEvents,
-  getWorkerPath,
-  createWorkerShutdown,
-  registerShutdownSignals,
-} from '../workers/worker-utils'
-import type { WorkerHandle } from '../workers/worker-utils'
+import { getWorkerPath } from '../workers/worker-utils'
+import { createQueueFactory } from './queue-factory'
 import { buildSkillExtractionJobOptions } from './skill-extraction.config'
 
 export interface SkillExtractionJob {
   orgId: string
 }
 
-const SKILL_EXTRACTION_QUEUE = 'skill-extraction'
-
-let _skillExtractionQueue: Queue<SkillExtractionJob> | null = null
-function getSkillExtractionQueue(): Queue<SkillExtractionJob> {
-  if (!_skillExtractionQueue) {
-    _skillExtractionQueue = new Queue<SkillExtractionJob>(SKILL_EXTRACTION_QUEUE, {
-      connection: getRedisConnectionForBullMQ(),
-      defaultJobOptions: {
-        removeOnComplete: 200,
-        removeOnFail: 200,
-        attempts: 2,
-        backoff: { type: 'exponential', delay: 5000 },
-      },
-    })
-  }
-  return _skillExtractionQueue
+const skillExtraction = createQueueFactory<SkillExtractionJob>({
+  name: 'skill-extraction',
+  displayName: 'Skill extraction',
+  jobName: 'run-extraction',
+  concurrency: 1,
+  lockDuration: 600_000,
+  defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
+  processorPath: getWorkerPath('skill-extraction.worker.ts'),
+})
+
+export function enqueueSkillExtraction(job: SkillExtractionJob) {
+  return skillExtraction.enqueue(job, buildSkillExtractionJobOptions(job.orgId))
 }
 
-export async function enqueueSkillExtraction(job: SkillExtractionJob) {
-  return await getSkillExtractionQueue().add('run-extraction', job, buildSkillExtractionJobOptions(job.orgId))
-}
-
-export function startSkillExtractionWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
-  const { registerSignals = import.meta.main } = options
-  const processorPath = getWorkerPath('skill-extraction.worker.ts')
-  const worker = new Worker(SKILL_EXTRACTION_QUEUE, processorPath, {
-    connection: getRedisConnectionForBullMQ(),
-    concurrency: 1,
-    lockDuration: 600_000,
-  })
-
-  attachWorkerEvents(worker, 'Skill extraction', serverLogger)
-
-  const shutdown = createWorkerShutdown(worker, 'Skill extraction', serverLogger)
-
-  if (registerSignals) {
-    registerShutdownSignals({ name: 'Skill extraction', shutdown, logger: serverLogger })
-  }
-
-  return { worker, shutdown }
-}
+export const startSkillExtractionWorker = skillExtraction.startWorker
 
 if (import.meta.main) {
   startSkillExtractionWorker()

package/src/queues/workstream-title-generation.queue.ts

@@ -1,69 +1,33 @@
-import { Queue, Worker } from 'bullmq'
 import type { Job } from 'bullmq'
 
-import { serverLogger } from '../config/logger'
 import { ensureRecordId } from '../db/record-id'
 import { databaseService } from '../db/service'
-import { getRedisConnectionForBullMQ } from '../redis'
 import { workstreamTitleService } from '../services/workstream-title.service'
-import {
-  attachWorkerEvents,
-  createTracedWorkerProcessor,
-  createWorkerShutdown,
-  registerShutdownSignals,
-} from '../workers/worker-utils'
-import type { WorkerHandle } from '../workers/worker-utils'
+import { createQueueFactory } from './queue-factory'
 
 interface WorkstreamTitleGenerationJob {
   workstreamId: string
   sourceText: string
 }
 
-const WORKSTREAM_TITLE_GENERATION_QUEUE = 'workstream-title-generation'
-
-let _workstreamTitleGenerationQueue: Queue<WorkstreamTitleGenerationJob> | null = null
-function getWorkstreamTitleGenerationQueue(): Queue<WorkstreamTitleGenerationJob> {
-  if (!_workstreamTitleGenerationQueue) {
-    _workstreamTitleGenerationQueue = new Queue<WorkstreamTitleGenerationJob>(WORKSTREAM_TITLE_GENERATION_QUEUE, {
-      connection: getRedisConnectionForBullMQ(),
-      defaultJobOptions: {
-        removeOnComplete: 200,
-        removeOnFail: 200,
-        attempts: 2,
-        backoff: { type: 'exponential', delay: 2_000 },
-      },
-    })
-  }
-  return _workstreamTitleGenerationQueue
-}
-
-export async function enqueueWorkstreamTitleGeneration(job: WorkstreamTitleGenerationJob) {
-  return await getWorkstreamTitleGenerationQueue().add('generate-workstream-title', job, {
-    jobId: `workstream-title:${job.workstreamId}`,
-  })
-}
-
 async function processWorkstreamTitleGenerationJob(job: Job<WorkstreamTitleGenerationJob>): Promise<void> {
   await databaseService.connect()
   const workstreamRef = ensureRecordId(job.data.workstreamId)
   await workstreamTitleService.generateAndPersistTitle(workstreamRef, job.data.sourceText)
 }
 
-export function startWorkstreamTitleGenerationWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
-  const { registerSignals = import.meta.main } = options
-  const worker = new Worker(
-    WORKSTREAM_TITLE_GENERATION_QUEUE,
-    createTracedWorkerProcessor(WORKSTREAM_TITLE_GENERATION_QUEUE, processWorkstreamTitleGenerationJob),
-    { connection: getRedisConnectionForBullMQ(), concurrency: 2, lockDuration: 60_000 },
-  )
-
-  attachWorkerEvents(worker, 'Workstream title generation', serverLogger)
-
-  const shutdown = createWorkerShutdown(worker, 'Workstream title generation', serverLogger)
-
-  if (registerSignals) {
-    registerShutdownSignals({ name: 'Workstream title generation', shutdown, logger: serverLogger })
-  }
-
-  return { worker, shutdown }
+const workstreamTitleGeneration = createQueueFactory<WorkstreamTitleGenerationJob>({
+  name: 'workstream-title-generation',
+  displayName: 'Workstream title generation',
+  jobName: 'generate-workstream-title',
+  concurrency: 2,
+  lockDuration: 60_000,
+  defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 2_000 } },
+  processor: processWorkstreamTitleGenerationJob,
+})
+
+export function enqueueWorkstreamTitleGeneration(job: WorkstreamTitleGenerationJob) {
+  return workstreamTitleGeneration.enqueue(job, { jobId: `workstream-title:${job.workstreamId}` })
 }
+
+export const startWorkstreamTitleGenerationWorker = workstreamTitleGeneration.startWorker

package/src/redis/connection.ts

@@ -1,7 +1,7 @@
 import IORedis from 'ioredis'
 import type { RedisOptions } from 'ioredis'
 
-import { getErrorMessage } from '../utils/error'
+import { getErrorMessage } from '../utils/errors'
 
 export interface RedisConnectionLogger {
   debug?: (message: string) => void
@@ -25,20 +25,23 @@ export interface RedisConnectionManager {
 }
 
 const DEFAULT_HEALTH_CHECK_INTERVAL_MS = 30_000
+const REDIS_RETRY_STEP_MS = 50
+const REDIS_RETRY_MAX_DELAY_MS = 2000
+const REDIS_CONNECT_TIMEOUT_MS = 10_000
 
 export const DEFAULT_REDIS_OPTIONS: RedisOptions = {
   maxRetriesPerRequest: null,
   enableReadyCheck: true,
   enableOfflineQueue: true,
   retryStrategy: (times: number) => {
-    const delay = Math.min(times * 50, 2000)
+    const delay = Math.min(times * REDIS_RETRY_STEP_MS, REDIS_RETRY_MAX_DELAY_MS)
     return delay
   },
   reconnectOnError: (err: Error) => {
     const targetErrors = ['READONLY', 'ETIMEDOUT', 'ECONNREFUSED', 'EAI_AGAIN']
     return targetErrors.some((candidate) => err.message.includes(candidate))
   },
-  connectTimeout: 10000,
+  connectTimeout: REDIS_CONNECT_TIMEOUT_MS,
   lazyConnect: false,
 }
 
@@ -55,6 +58,7 @@ class RedisConnectionManagerImpl implements RedisConnectionManager {
   private healthCheckInterval: ReturnType<typeof setInterval> | null = null
   private isInitialized = false
   private isClosing = false
+  private isHealthCheckRunning = false
 
   constructor(private readonly options: CreateRedisConnectionManagerOptions) {
     this.initializeConnection()
@@ -125,6 +129,8 @@ class RedisConnectionManagerImpl implements RedisConnectionManager {
 
     const intervalMs = this.options.healthCheckIntervalMs ?? DEFAULT_HEALTH_CHECK_INTERVAL_MS
     this.healthCheckInterval = setInterval(async () => {
+      if (this.isHealthCheckRunning) return
+      this.isHealthCheckRunning = true
       try {
         if (this.redis && this.redis.status === 'ready') {
           await this.redis.ping()
@@ -133,8 +139,11 @@ class RedisConnectionManagerImpl implements RedisConnectionManager {
       } catch (error) {
         log(this.options.logger, 'warn', `Redis health check failed: ${getErrorMessage(error)}`)
         this.isHealthy = false
+      } finally {
+        this.isHealthCheckRunning = false
       }
     }, intervalMs)
+    this.healthCheckInterval.unref()
   }
 
   getConnection(): IORedis {

package/src/redis/index.ts

@@ -8,7 +8,8 @@ export {
   type RedisConnectionAccessor,
 } from './connection-accessor'
 export { withOrgMemoryLock } from './org-memory-lock'
-export { createWorkstreamResumableContext } from './stream-context'
+export { LeaseLockLostError, withRedisLeaseLock } from './redis-lease-lock'
+export { closeSharedSubscriber, createWorkstreamResumableContext } from './stream-context'
 
 export { createRedisConnectionManager }
 export type { RedisConnectionManager }

package/src/redis/org-memory-lock.ts

@@ -9,7 +9,7 @@ const ORG_MEMORY_LOCK_REFRESH_INTERVAL_MS = 30_000
 const ORG_MEMORY_LOCK_WAIT_LOG_INTERVAL_MS = 30_000
 const ORG_MEMORY_LOCK_MAX_WAIT_MS = 15 * 60 * 1000
 
-export async function withOrgMemoryLock<T>(orgId: string, fn: () => Promise<T>): Promise<T> {
+export async function withOrgMemoryLock<T>(orgId: string, fn: (signal: AbortSignal) => Promise<T>): Promise<T> {
   const normalizedOrgId = orgId.trim()
 
   if (!normalizedOrgId) {

package/src/redis/redis-lease-lock.ts

@@ -3,7 +3,7 @@ import { setTimeout as delay } from 'node:timers/promises'
 
 import type IORedis from 'ioredis'
 
-import { getErrorMessage } from '../utils/error'
+import { getErrorMessage } from '../utils/errors'
 
 interface RedisLeaseLockLogger {
   debug?: (message: string) => void
@@ -95,16 +95,30 @@ async function releaseLeaseLock(options: RedisLeaseLockOptions & { lockValue: st
   await options.redis.eval(RELEASE_LOCK_SCRIPT, 1, options.lockKey, options.lockValue)
 }
 
+export class LeaseLockLostError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'LeaseLockLostError'
+  }
+}
+
 function startLeaseLockRefreshLoop(
   options: Required<Pick<RedisLeaseLockOptions, 'refreshIntervalMs'>> &
     RedisLeaseLockOptions & { lockValue: string; label: string },
+  ac: AbortController,
 ): () => void {
   let stopped = false
 
   const timer = setInterval(() => {
     if (stopped) return
     void refreshLeaseLock(options).catch((error: unknown) => {
-      log(options.logger, 'warn', `Failed to refresh ${options.label} (${options.lockKey}): ${getErrorMessage(error)}`)
+      stopped = true
+      clearInterval(timer)
+      const message = `Failed to refresh ${options.label} (${options.lockKey}): ${getErrorMessage(error)}`
+      log(options.logger, 'warn', message)
+      if (!ac.signal.aborted) {
+        ac.abort(new LeaseLockLostError(message))
+      }
     })
   }, options.refreshIntervalMs)
 
@@ -116,7 +130,10 @@ function startLeaseLockRefreshLoop(
   }
 }
 
-export async function withRedisLeaseLock<T>(options: RedisLeaseLockOptions, fn: () => Promise<T>): Promise<T> {
+export async function withRedisLeaseLock<T>(
+  options: RedisLeaseLockOptions,
+  fn: (signal: AbortSignal) => Promise<T>,
+): Promise<T> {
   const lockKey = options.lockKey.trim()
   if (!lockKey) {
     throw new Error('Redis lease lock requires a non-empty lock key')
@@ -138,17 +155,33 @@ export async function withRedisLeaseLock<T>(options: RedisLeaseLockOptions, fn:
     log(options.logger, 'info', `Acquired ${label} (${lockKey}) after waiting waitedMs=${waitedMs}`)
   }
 
-  const stopRefreshLoop = startLeaseLockRefreshLoop({ ...options, lockKey, lockValue, label, refreshIntervalMs })
+  const ac = new AbortController()
+  const stopRefreshLoop = startLeaseLockRefreshLoop({ ...options, lockKey, lockValue, label, refreshIntervalMs }, ac)
   const holdStart = Date.now()
 
   try {
-    return await fn()
+    const abortPromise = new Promise<never>((_, reject) => {
+      if (ac.signal.aborted) {
+        reject(ac.signal.reason as Error)
+        return
+      }
+      ac.signal.addEventListener(
+        'abort',
+        () => {
+          reject(ac.signal.reason as Error)
+        },
+        { once: true },
+      )
+    })
+    return await Promise.race([fn(ac.signal), abortPromise])
   } finally {
     stopRefreshLoop()
     const heldMs = Date.now() - holdStart
-    await releaseLeaseLock({ ...options, lockKey, lockValue }).catch((error: unknown) => {
-      log(options.logger, 'warn', `Failed to release ${label} (${lockKey}): ${getErrorMessage(error)}`)
-    })
+    if (!ac.signal.aborted) {
+      await releaseLeaseLock({ ...options, lockKey, lockValue }).catch((error: unknown) => {
+        log(options.logger, 'warn', `Failed to release ${label} (${lockKey}): ${getErrorMessage(error)}`)
+      })
+    }
     if (heldMs >= heldInfoThresholdMs) {
       log(options.logger, 'info', `Released ${label} (${lockKey}) heldMs=${heldMs}`)
     } else {

package/src/redis/stream-context.ts

@@ -44,6 +44,17 @@ function getSharedSubscriber(): Subscriber {
   return sharedSubscriber.subscriber
 }
 
+export async function closeSharedSubscriber(): Promise<void> {
+  if (!sharedSubscriber) return
+  const { client } = sharedSubscriber
+  sharedSubscriber = undefined
+  try {
+    await client.quit()
+  } catch {
+    client.disconnect()
+  }
+}
+
 export function createWorkstreamResumableContext() {
   const redis = getRedisConnection()
   return createResumableStreamContext({