@lota-sdk/core 0.4.9 → 0.4.11

package/src/queues/delayed-node-promotion.queue.ts

@@ -3,12 +3,23 @@ import { Effect } from 'effect'
 import type { Context } from 'effect'
 import type IORedis from 'ioredis'
 
-import { ConfigurationError } from '../effect/errors'
 import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
-import { PlanExecutorServiceTag } from '../services/plan/plan-executor.service'
-import { createQueueFactory } from './queue-factory'
+import { QueueJobServiceTag } from '../services/queue-job.service'
+import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
+import { createQueueFactoryWithDeps } from './queue-factory'
 import { runStandaloneQueueWorker } from './standalone-worker'
 
+// Minimal service shape used by the worker processor. Declared structurally to
+// avoid importing the service tag — which would form a dependency cycle since
+// PlanExecutorServiceLive depends on LotaQueuesServiceTag.
+interface DelayedNodePromotionPlanExecutorShape {
+  promoteDelayedNode(params: {
+    runId: string
+    nodeId: string
+    emittedBy: string
+  }): Effect.Effect<unknown, unknown, unknown>
+}
+
 export interface DelayedNodePromotionJob {
   runId: string
   nodeId: string
@@ -17,23 +28,16 @@ export interface DelayedNodePromotionJob {
 
 export const DELAYED_NODE_PROMOTION_QUEUE = 'delayed-node-promotion'
 
-interface DelayedNodePromotionQueueDeps {
+export interface DelayedNodePromotionWorkerDeps {
   databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
-  planExecutorService: Context.Service.Shape<typeof PlanExecutorServiceTag>
+  planExecutorService: DelayedNodePromotionPlanExecutorShape
 }
 
-let _deps: DelayedNodePromotionQueueDeps | null = null
-function getDeps(): DelayedNodePromotionQueueDeps {
-  if (!_deps)
-    throw new ConfigurationError({
-      message: 'Delayed node promotion queue is not configured. Initialize the runtime before starting the worker.',
-      key: 'queue-deps',
-    })
-  return _deps
-}
-
-function processDelayedNodePromotionJob(job: Job<DelayedNodePromotionJob>): Promise<void> {
-  const { planExecutorService } = getDeps()
+function processDelayedNodePromotionJob(
+  deps: DelayedNodePromotionWorkerDeps,
+  job: Job<DelayedNodePromotionJob>,
+): Promise<void> {
+  const { planExecutorService } = deps
   return Promise.resolve(
     planExecutorService.promoteDelayedNode({
       runId: job.data.runId,
@@ -43,36 +47,51 @@ function processDelayedNodePromotionJob(job: Job<DelayedNodePromotionJob>): Prom
   ).then(() => undefined)
 }
 
-const delayedNodePromotion = createQueueFactory<DelayedNodePromotionJob>({
-  name: DELAYED_NODE_PROMOTION_QUEUE,
-  displayName: 'Delayed node promotion',
-  jobName: 'promote-node',
-  concurrency: 1,
-  defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
-  prepare: () => getDeps().databaseService.connect(),
-  processor: processDelayedNodePromotionJob,
-})
-
-export function enqueueDelayedNodePromotion(job: DelayedNodePromotionJob, delayMs: number) {
-  return delayedNodePromotion.enqueue(job, { delay: delayMs, jobId: `promote:${job.runId}:${job.nodeId}` })
+export interface DelayedNodePromotionQueueRuntime {
+  enqueueDelayedNodePromotion(job: DelayedNodePromotionJob, delayMs: number): Promise<void>
+  startWorker(options: { registerSignals?: boolean; deps: DelayedNodePromotionWorkerDeps }): WorkerHandle
 }
 
-export function startDelayedNodePromotionWorker(options: {
-  registerSignals?: boolean
+interface MakeDelayedNodePromotionQueueRuntimeParams {
   connectionProvider: () => IORedis
-  deps: DelayedNodePromotionQueueDeps
-}): ReturnType<typeof delayedNodePromotion.startWorker> {
-  _deps = options.deps
-  return delayedNodePromotion.startWorker({
-    registerSignals: options.registerSignals,
-    connectionProvider: options.connectionProvider,
+  queueJobService: QueueJobService
+}
+
+export function makeDelayedNodePromotionQueueRuntime(
+  params: MakeDelayedNodePromotionQueueRuntimeParams,
+): DelayedNodePromotionQueueRuntime {
+  const { connectionProvider, queueJobService } = params
+  const queue = createQueueFactoryWithDeps<DelayedNodePromotionJob, DelayedNodePromotionWorkerDeps>({
+    name: DELAYED_NODE_PROMOTION_QUEUE,
+    displayName: 'Delayed node promotion',
+    jobName: 'promote-node',
+    concurrency: 1,
+    defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
+    connectionProvider,
+    queueJobService,
+    prepare: ({ databaseService }) => Effect.runPromise(databaseService.connect()),
+    processor: processDelayedNodePromotionJob,
   })
+
+  return {
+    enqueueDelayedNodePromotion: (job, delayMs) =>
+      queue.enqueue(job, { delay: delayMs, jobId: `promote:${job.runId}:${job.nodeId}` }),
+    startWorker: (options) =>
+      queue.startWorker({ deps: options.deps, registerSignals: options.registerSignals, connectionProvider }),
+  }
 }
 
 runStandaloneQueueWorker((runtime) => {
-  const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
-  startDelayedNodePromotionWorker({
-    connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
-    deps: { databaseService: resolve(DatabaseServiceTag), planExecutorService: resolve(PlanExecutorServiceTag) },
-  })
+  void (async () => {
+    const { PlanExecutorServiceTag } = await import('../services/plan/plan-executor.service')
+    const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
+    const redis = resolve(RedisServiceTag)
+    const delayedNodePromotionQueue = makeDelayedNodePromotionQueueRuntime({
+      connectionProvider: () => redis.getConnectionForBullMQ(),
+      queueJobService: resolve(QueueJobServiceTag),
+    })
+    delayedNodePromotionQueue.startWorker({
+      deps: { databaseService: resolve(DatabaseServiceTag), planExecutorService: resolve(PlanExecutorServiceTag) },
+    })
+  })()
 })

package/src/queues/document-processor.queue.ts

@@ -3,8 +3,8 @@ import type IORedis from 'ioredis'
 
 import type { chatLogger } from '../config/logger'
 import { sha256Hex } from '../utils/crypto'
-import { DEFAULT_JOB_RETENTION, getQueueJobService } from '../workers/worker-utils'
-import type { WorkerHandle } from '../workers/worker-utils'
+import { DEFAULT_JOB_RETENTION } from '../workers/worker-utils'
+import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
 import { createQueueFactory } from './queue-factory'
 
 export type DocumentSourceChannel = string
@@ -56,6 +56,7 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
   getConnectionForBullMQ: () => IORedis
   getWorkerPath: () => string
   logger: typeof chatLogger
+  queueJobService: QueueJobService
   queueName?: string
   workerName?: string
   concurrency?: number
@@ -75,6 +76,7 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
     connectionProvider: params.getConnectionForBullMQ,
     defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
     processorPath: params.getWorkerPath(),
+    queueJobService: params.queueJobService,
   })
 
   return {
@@ -86,7 +88,7 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
           )
 
           yield* Effect.catch(
-            getQueueJobService().recordEnqueued({
+            params.queueJobService.recordEnqueued({
               queueName,
               id: queuedJob.id,
               name: queuedJob.name,

package/src/queues/index.ts

@@ -8,4 +8,5 @@ export * from './organization-learning.queue'
 export * from './plan-agent-heartbeat.queue'
 export * from './plan-scheduler.queue'
 export * from './post-chat-memory.queue'
+export * from './queues.service'
 export * from './title-generation.queue'

package/src/queues/memory-consolidation.queue.ts

@@ -1,12 +1,12 @@
 import { Effect } from 'effect'
+import type { Context } from 'effect'
+import type IORedis from 'ioredis'
 
 import { serverLogger } from '../config/logger'
-import {
-  getQueueJobService,
-  getWorkerPath,
-  LONG_JOB_LOCK_DURATION_MS,
-  LOW_JOB_RETENTION,
-} from '../workers/worker-utils'
+import { RedisServiceTag } from '../effect/services'
+import { QueueJobServiceTag } from '../services/queue-job.service'
+import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
+import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS, LOW_JOB_RETENTION } from '../workers/worker-utils'
 import { createQueueFactory } from './queue-factory'
 import { runStandaloneQueueWorker } from './standalone-worker'
 
@@ -17,58 +17,84 @@ export interface MemoryConsolidationJob {
 const MEMORY_CONSOLIDATION_INTERVAL_MS = 24 * 60 * 60 * 1000
 const MEMORY_CONSOLIDATION_SCHEDULER_ID = 'memory-consolidation-recurring'
 
-const memoryConsolidation = createQueueFactory<MemoryConsolidationJob>({
-  name: 'memory-consolidation',
-  displayName: 'Memory consolidation',
-  jobName: 'consolidate-turn',
-  concurrency: 1,
-  lockDuration: LONG_JOB_LOCK_DURATION_MS,
-  defaultJobOptions: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
-  processorPath: getWorkerPath('memory-consolidation.worker.ts'),
-})
+export interface MemoryConsolidationQueueRuntime {
+  enqueueMemoryConsolidation(job?: MemoryConsolidationJob): Promise<void>
+  scheduleRecurringConsolidation(): Promise<void>
+  startWorker(options?: { registerSignals?: boolean }): WorkerHandle
+}
 
-export function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
-  return memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
+interface MakeMemoryConsolidationQueueRuntimeParams {
+  connectionProvider: () => IORedis
+  queueJobService: QueueJobService
 }
 
-export function scheduleRecurringConsolidation() {
-  return Effect.runPromise(
-    Effect.gen(function* () {
-      const queuedJob = yield* Effect.tryPromise(() =>
-        memoryConsolidation
-          .getQueue()
-          .upsertJobScheduler(
-            MEMORY_CONSOLIDATION_SCHEDULER_ID,
-            { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
-            {
-              name: 'consolidate',
-              data: {},
-              opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
-            },
-          ),
-      )
+export function makeMemoryConsolidationQueueRuntime(
+  params: MakeMemoryConsolidationQueueRuntimeParams,
+): MemoryConsolidationQueueRuntime {
+  const { connectionProvider, queueJobService } = params
+  const queue = createQueueFactory<MemoryConsolidationJob>({
+    name: 'memory-consolidation',
+    displayName: 'Memory consolidation',
+    jobName: 'consolidate-turn',
+    concurrency: 1,
+    lockDuration: LONG_JOB_LOCK_DURATION_MS,
+    defaultJobOptions: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
+    processorPath: getWorkerPath('memory-consolidation.worker.ts'),
+    connectionProvider,
+    queueJobService,
+  })
+
+  const enqueueMemoryConsolidation: MemoryConsolidationQueueRuntime['enqueueMemoryConsolidation'] = (job = {}) =>
+    queue.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
 
-      yield* Effect.catch(
-        getQueueJobService().recordEnqueued({
-          queueName: 'memory-consolidation',
-          id: queuedJob.id,
-          name: queuedJob.name,
-          data: queuedJob.data,
-          opts: queuedJob.opts,
-          attemptsMade: queuedJob.attemptsMade,
-          timestamp: queuedJob.timestamp,
-        }),
-        (error) =>
-          Effect.sync(() => {
-            serverLogger.error`Failed to persist queued job metadata (queue=memory-consolidation, job=${queuedJob.id}): ${error}`
+  const scheduleRecurringConsolidation: MemoryConsolidationQueueRuntime['scheduleRecurringConsolidation'] = () =>
+    Effect.runPromise(
+      Effect.gen(function* () {
+        const queuedJob = yield* Effect.tryPromise(() =>
+          queue
+            .getQueue()
+            .upsertJobScheduler(
+              MEMORY_CONSOLIDATION_SCHEDULER_ID,
+              { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
+              {
+                name: 'consolidate',
+                data: {},
+                opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
+              },
+            ),
+        )
+
+        yield* Effect.catch(
+          queueJobService.recordEnqueued({
+            queueName: 'memory-consolidation',
+            id: queuedJob.id,
+            name: queuedJob.name,
+            data: queuedJob.data,
+            opts: queuedJob.opts,
+            attemptsMade: queuedJob.attemptsMade,
+            timestamp: queuedJob.timestamp,
           }),
-      )
-    }),
-  )
-}
+          (error) =>
+            Effect.sync(() => {
+              serverLogger.error`Failed to persist queued job metadata (queue=memory-consolidation, job=${queuedJob.id}): ${error}`
+            }),
+        )
+      }),
+    )
 
-export const startMemoryConsolidationWorker = memoryConsolidation.startWorker
+  return {
+    enqueueMemoryConsolidation,
+    scheduleRecurringConsolidation,
+    startWorker: (options = {}) => queue.startWorker({ registerSignals: options.registerSignals, connectionProvider }),
+  }
+}
 
-runStandaloneQueueWorker(() => {
-  startMemoryConsolidationWorker()
+runStandaloneQueueWorker((runtime) => {
+  const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
+  const redis = resolve(RedisServiceTag)
+  const memoryConsolidationQueue = makeMemoryConsolidationQueueRuntime({
+    connectionProvider: () => redis.getConnectionForBullMQ(),
+    queueJobService: resolve(QueueJobServiceTag),
+  })
+  memoryConsolidationQueue.startWorker()
 })

package/src/queues/organization-learning.queue.ts

@@ -1,9 +1,19 @@
-import { Effect } from 'effect'
+import { Effect, Schema } from 'effect'
+import type { Context } from 'effect'
+import type IORedis from 'ioredis'
 
+import { RedisServiceTag } from '../effect/services'
+import { QueueJobServiceTag } from '../services/queue-job.service'
+import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
 import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
 import { createQueueFactory } from './queue-factory'
 import { runStandaloneQueueWorker } from './standalone-worker'
 
+class OrganizationLearningQueueError extends Schema.TaggedErrorClass<OrganizationLearningQueueError>()(
+  '@lota-sdk/core/OrganizationLearningQueueError',
+  { message: Schema.String, cause: Schema.optional(Schema.Defect) },
+) {}
+
 export const ORGANIZATION_LEARNING_QUEUE = 'organization-learning'
 
 const ORGANIZATION_LEARNING_DELAY_MS = 15 * 60 * 1000
@@ -46,42 +56,69 @@ export function buildSkillExtractionJobOptions(orgId: string) {
   return { delay: ORGANIZATION_LEARNING_DELAY_MS, deduplication: { id: buildSkillExtractionDeduplicationId(orgId) } }
 }
 
-const organizationLearningQueue = createQueueFactory<OrganizationLearningJob>({
-  name: ORGANIZATION_LEARNING_QUEUE,
-  displayName: 'Organization learning',
-  jobName: 'organization-learning',
-  concurrency: 4,
-  lockDuration: LONG_JOB_LOCK_DURATION_MS,
-  defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 5_000 } },
-  processorPath: getWorkerPath('organization-learning.worker.ts'),
-})
-
-export function enqueueRegularChatMemoryDigest(job: Omit<RegularChatMemoryDigestJob, 'kind'>) {
-  return organizationLearningQueue.enqueue(
-    { kind: 'regular-chat-memory-digest', ...job },
-    buildRegularChatMemoryDigestJobOptions(job.orgId),
-  )
+export interface OrganizationLearningQueueRuntime {
+  enqueueRegularChatMemoryDigest(job: Omit<RegularChatMemoryDigestJob, 'kind'>): Promise<void>
+  enqueueSkillExtraction(job: Omit<SkillExtractionJob, 'kind'>): Promise<void>
+  clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void>
+  startWorker(options?: { registerSignals?: boolean }): WorkerHandle
 }
 
-export function enqueueSkillExtraction(job: Omit<SkillExtractionJob, 'kind'>) {
-  return organizationLearningQueue.enqueue(
-    { kind: 'skill-extraction', ...job },
-    buildSkillExtractionJobOptions(job.orgId),
-  )
+interface MakeOrganizationLearningQueueRuntimeParams {
+  connectionProvider: () => IORedis
+  queueJobService: QueueJobService
 }
 
-export function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
-  return Effect.runPromise(
-    Effect.asVoid(
-      Effect.tryPromise(() =>
-        organizationLearningQueue.getQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId)),
-      ),
-    ),
-  )
-}
+export function makeOrganizationLearningQueueRuntime(
+  params: MakeOrganizationLearningQueueRuntimeParams,
+): OrganizationLearningQueueRuntime {
+  const { connectionProvider, queueJobService } = params
+  const queue = createQueueFactory<OrganizationLearningJob>({
+    name: ORGANIZATION_LEARNING_QUEUE,
+    displayName: 'Organization learning',
+    jobName: 'organization-learning',
+    concurrency: 4,
+    lockDuration: LONG_JOB_LOCK_DURATION_MS,
+    defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 5_000 } },
+    processorPath: getWorkerPath('organization-learning.worker.ts'),
+    connectionProvider,
+    queueJobService,
+  })
+
+  const enqueueRegularChatMemoryDigest: OrganizationLearningQueueRuntime['enqueueRegularChatMemoryDigest'] = (job) =>
+    queue.enqueue({ kind: 'regular-chat-memory-digest', ...job }, buildRegularChatMemoryDigestJobOptions(job.orgId))
+
+  const enqueueSkillExtraction: OrganizationLearningQueueRuntime['enqueueSkillExtraction'] = (job) =>
+    queue.enqueue({ kind: 'skill-extraction', ...job }, buildSkillExtractionJobOptions(job.orgId))
+
+  const clearRegularChatMemoryDigestDeduplicationKey: OrganizationLearningQueueRuntime['clearRegularChatMemoryDigestDeduplicationKey'] =
+    (orgId) =>
+      Effect.runPromise(
+        Effect.asVoid(
+          Effect.tryPromise({
+            try: () => queue.getQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId)),
+            catch: (cause) =>
+              new OrganizationLearningQueueError({
+                message: `Failed to clear regular-chat memory digest deduplication key for ${orgId}.`,
+                cause,
+              }),
+          }),
+        ),
+      )
 
-export const startOrganizationLearningWorker = organizationLearningQueue.startWorker
+  return {
+    enqueueRegularChatMemoryDigest,
+    enqueueSkillExtraction,
+    clearRegularChatMemoryDigestDeduplicationKey,
+    startWorker: (options = {}) => queue.startWorker({ registerSignals: options.registerSignals, connectionProvider }),
+  }
+}
 
-runStandaloneQueueWorker(() => {
-  startOrganizationLearningWorker()
+runStandaloneQueueWorker((runtime) => {
+  const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
+  const redis = resolve(RedisServiceTag)
+  const organizationLearningQueue = makeOrganizationLearningQueueRuntime({
+    connectionProvider: () => redis.getConnectionForBullMQ(),
+    queueJobService: resolve(QueueJobServiceTag),
+  })
+  organizationLearningQueue.startWorker()
 })