@lota-sdk/core 0.4.10 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/package.json +2 -2
  2. package/src/ai-gateway/ai-gateway.ts +149 -95
  3. package/src/ai-gateway/index.ts +16 -1
  4. package/src/config/agent-defaults.ts +4 -120
  5. package/src/config/logger.ts +18 -34
  6. package/src/config/thread-defaults.ts +1 -18
  7. package/src/create-runtime.ts +90 -28
  8. package/src/db/base.service.ts +30 -38
  9. package/src/db/service.ts +489 -545
  10. package/src/effect/index.ts +0 -2
  11. package/src/effect/layers.ts +6 -13
  12. package/src/embeddings/provider.ts +2 -7
  13. package/src/index.ts +4 -5
  14. package/src/queues/autonomous-job.queue.ts +159 -113
  15. package/src/queues/context-compaction.queue.ts +39 -25
  16. package/src/queues/delayed-node-promotion.queue.ts +56 -29
  17. package/src/queues/document-processor.queue.ts +5 -3
  18. package/src/queues/index.ts +1 -0
  19. package/src/queues/memory-consolidation.queue.ts +79 -53
  20. package/src/queues/organization-learning.queue.ts +63 -39
  21. package/src/queues/plan-agent-heartbeat.queue.ts +104 -79
  22. package/src/queues/plan-scheduler.queue.ts +100 -84
  23. package/src/queues/post-chat-memory.queue.ts +55 -33
  24. package/src/queues/queue-factory.ts +40 -41
  25. package/src/queues/queues.service.ts +61 -0
  26. package/src/queues/title-generation.queue.ts +42 -31
  27. package/src/redis/org-memory-lock.ts +24 -9
  28. package/src/redis/redis-lease-lock.ts +8 -1
  29. package/src/runtime/agent-identity-overrides.ts +7 -3
  30. package/src/runtime/agent-runtime-policy.ts +9 -4
  31. package/src/runtime/agent-stream-helpers.ts +9 -4
  32. package/src/runtime/context-compaction/context-compaction-runtime.ts +28 -32
  33. package/src/runtime/context-compaction/context-compaction.ts +9 -7
  34. package/src/runtime/domain-layer.ts +15 -4
  35. package/src/runtime/execution-plan-visibility.ts +5 -2
  36. package/src/runtime/graph-designer.ts +0 -22
  37. package/src/runtime/index.ts +1 -0
  38. package/src/runtime/indexed-repositories-policy.ts +2 -6
  39. package/src/runtime/plugin-resolution.ts +29 -12
  40. package/src/runtime/post-turn-side-effects.ts +139 -141
  41. package/src/runtime/runtime-config.ts +0 -6
  42. package/src/runtime/runtime-extensions.ts +0 -54
  43. package/src/runtime/runtime-lifecycle.ts +4 -4
  44. package/src/runtime/runtime-services.ts +122 -53
  45. package/src/runtime/runtime-worker-registry.ts +113 -30
  46. package/src/runtime/social-chat/social-chat-agent-runner.ts +6 -3
  47. package/src/runtime/social-chat/social-chat-history.ts +3 -1
  48. package/src/runtime/social-chat/social-chat.ts +35 -20
  49. package/src/runtime/team-consultation/team-consultation-orchestrator.ts +6 -5
  50. package/src/runtime/team-consultation/team-consultation-prompts.ts +11 -6
  51. package/src/runtime/thread-chat-helpers.ts +18 -9
  52. package/src/runtime/thread-turn-context.ts +7 -47
  53. package/src/runtime/turn-lifecycle.ts +6 -14
  54. package/src/services/agent-activity.service.ts +168 -175
  55. package/src/services/agent-executor.service.ts +35 -16
  56. package/src/services/attachment.service.ts +4 -70
  57. package/src/services/autonomous-job.service.ts +53 -61
  58. package/src/services/context-compaction.service.ts +7 -9
  59. package/src/services/execution-plan/execution-plan-graph.ts +106 -115
  60. package/src/services/execution-plan/execution-plan-schedule.ts +1 -15
  61. package/src/services/execution-plan/execution-plan.service.ts +67 -50
  62. package/src/services/global-orchestrator.service.ts +18 -7
  63. package/src/services/graph-full-routing.ts +7 -6
  64. package/src/services/memory/memory-conversation.ts +10 -5
  65. package/src/services/memory/memory.service.ts +11 -8
  66. package/src/services/ownership-dispatcher.service.ts +16 -5
  67. package/src/services/plan/plan-agent-heartbeat.service.ts +29 -15
  68. package/src/services/plan/plan-agent-query.service.ts +12 -8
  69. package/src/services/plan/plan-completion-side-effects.ts +93 -101
  70. package/src/services/plan/plan-cycle.service.ts +7 -45
  71. package/src/services/plan/plan-deadline.service.ts +28 -17
  72. package/src/services/plan/plan-event-delivery.service.ts +47 -40
  73. package/src/services/plan/plan-executor-context.ts +2 -0
  74. package/src/services/plan/plan-executor-graph.ts +366 -391
  75. package/src/services/plan/plan-executor.service.ts +13 -91
  76. package/src/services/plan/plan-scheduler.service.ts +62 -49
  77. package/src/services/plan/plan-transaction-events.ts +1 -1
  78. package/src/services/recent-activity-title.service.ts +6 -2
  79. package/src/services/thread/thread-bootstrap.ts +11 -9
  80. package/src/services/thread/thread-message.service.ts +6 -5
  81. package/src/services/thread/thread-turn-execution.ts +86 -82
  82. package/src/services/thread/thread-turn-preparation.service.ts +47 -24
  83. package/src/services/thread/thread-turn-streaming.ts +20 -25
  84. package/src/services/thread/thread-turn.ts +25 -44
  85. package/src/services/thread/thread.service.ts +21 -6
  86. package/src/system-agents/recent-activity-title-refiner.agent.ts +8 -5
  87. package/src/system-agents/thread-router.agent.ts +23 -20
  88. package/src/tools/execution-plan.tool.ts +8 -3
  89. package/src/tools/fetch-webpage.tool.ts +10 -9
  90. package/src/tools/firecrawl-client.ts +0 -15
  91. package/src/tools/remember-memory.tool.ts +3 -6
  92. package/src/tools/research-topic.tool.ts +12 -3
  93. package/src/tools/search-web.tool.ts +10 -9
  94. package/src/tools/search.tool.ts +4 -5
  95. package/src/tools/team-think.tool.ts +139 -121
  96. package/src/workers/bootstrap.ts +9 -10
  97. package/src/workers/memory-consolidation.worker.ts +4 -1
  98. package/src/workers/organization-learning.worker.ts +15 -2
  99. package/src/workers/regular-chat-memory-digest.helpers.ts +3 -4
  100. package/src/workers/regular-chat-memory-digest.runner.ts +21 -14
  101. package/src/workers/skill-extraction.runner.ts +13 -15
  102. package/src/workers/worker-utils.ts +6 -18
  103. package/src/effect/awaitable-effect.ts +0 -96
  104. package/src/effect/runtime-ref.ts +0 -25
  105. package/src/effect/runtime.ts +0 -46
  106. package/src/redis/runtime-connection.ts +0 -20
  107. package/src/runtime/runtime-accessors.ts +0 -92
  108. package/src/runtime/runtime-token.ts +0 -47
@@ -9,8 +9,9 @@ import { PlanCycleServiceTag } from '../services/plan/plan-cycle.service'
9
9
  import { PlanDeadlineServiceTag } from '../services/plan/plan-deadline.service'
10
10
  import { PlanExecutorServiceTag } from '../services/plan/plan-executor.service'
11
11
  import { PlanSchedulerServiceTag } from '../services/plan/plan-scheduler.service'
12
+ import { QueueJobServiceTag } from '../services/queue-job.service'
12
13
  import { nowEpochMillis } from '../utils/date-time'
13
- import type { WorkerHandle } from '../workers/worker-utils'
14
+ import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
14
15
  import { createQueueFactoryWithDeps } from './queue-factory'
15
16
  import { runStandaloneQueueWorker } from './standalone-worker'
16
17
 
@@ -28,7 +29,7 @@ export type PlanSchedulerJob = PlanSchedulerFireJob | PlanSchedulerDeadlineJob
28
29
 
29
30
  export const PLAN_SCHEDULER_QUEUE = 'plan-scheduler'
30
31
 
31
- interface PlanSchedulerQueueDeps {
32
+ export interface PlanSchedulerWorkerDeps {
32
33
  databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
33
34
  planSchedulerService: Context.Service.Shape<typeof PlanSchedulerServiceTag>
34
35
  planDeadlineService: Context.Service.Shape<typeof PlanDeadlineServiceTag>
@@ -46,110 +47,125 @@ function toPlanSchedulerQueueError(stage: PlanSchedulerQueueError['stage'], caus
46
47
  return new PlanSchedulerQueueError({ stage, message: cause instanceof Error ? cause.message : String(cause), cause })
47
48
  }
48
49
 
49
- function processPlanSchedulerJob(deps: PlanSchedulerQueueDeps, job: Job<PlanSchedulerJob>): Promise<void> {
50
+ function processPlanSchedulerJob(deps: PlanSchedulerWorkerDeps, job: Job<PlanSchedulerJob>): Promise<void> {
50
51
  const { planSchedulerService, planDeadlineService, planExecutorService, planCycleService } = deps
52
+ const runWithResolvedContext = <A, E>(effect: Effect.Effect<A, E, unknown>): Promise<void> =>
53
+ // Service Effects carry their provided context through the managed runtime
54
+ // that resolved these service tags, so residual R collapses at runtime.
55
+ Effect.runPromise(Effect.asVoid(effect as Effect.Effect<A, E, never>))
56
+
51
57
  switch (job.data.type) {
52
58
  case 'fire-schedule':
53
- return Effect.runPromise(
54
- Effect.asVoid(
55
- planSchedulerService.fireScheduleById(job.data.scheduleId, {
56
- promoteDelayedNode: (params) => planExecutorService.promoteDelayedNode(params),
57
- advanceCycle: (cycleId) => planCycleService.advanceCycle(cycleId),
58
- recoverDeadlineChecks: () => Effect.runPromise(Effect.asVoid(planDeadlineService.recoverDeadlineChecks())),
59
- }),
60
- ),
59
+ return runWithResolvedContext(
60
+ planSchedulerService.fireScheduleById(job.data.scheduleId, {
61
+ promoteDelayedNode: (params) => runWithResolvedContext(planExecutorService.promoteDelayedNode(params)),
62
+ advanceCycle: (cycleId) => runWithResolvedContext(planCycleService.advanceCycle(cycleId)),
63
+ recoverDeadlineChecks: () => runWithResolvedContext(planDeadlineService.recoverDeadlineChecks()),
64
+ }),
61
65
  )
62
66
  case 'check-deadlines':
63
- return Effect.runPromise(Effect.asVoid(planDeadlineService.checkDeadlines()))
67
+ return runWithResolvedContext(planDeadlineService.checkDeadlines())
64
68
  }
65
69
  }
66
70
 
67
- const planScheduler = createQueueFactoryWithDeps<PlanSchedulerJob, PlanSchedulerQueueDeps>({
68
- name: PLAN_SCHEDULER_QUEUE,
69
- displayName: 'Plan scheduler',
70
- jobName: 'plan-scheduler-job',
71
- concurrency: 1,
72
- defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
73
- prepare: ({ databaseService }) => databaseService.connect(),
74
- processor: processPlanSchedulerJob,
75
- })
76
-
77
- /** Enqueue a delayed job that fires a specific schedule at its nextFireAt time. */
78
- export function enqueueScheduleFire(scheduleId: string, delayMs: number): Promise<void> {
79
- return planScheduler.enqueue(
80
- { type: 'fire-schedule', scheduleId },
81
- { delay: Math.max(0, delayMs), jobId: `schedule:${scheduleId}`, removeOnComplete: true, removeOnFail: 50 },
82
- )
83
- }
84
-
85
- /** Remove a pending fire job for a schedule (on cancel/pause/complete). */
86
- export function removeScheduleFireJob(scheduleId: string): Promise<void> {
87
- return Effect.runPromise(
88
- Effect.tryPromise({
89
- try: () => planScheduler.getQueue().remove(`schedule:${scheduleId}`),
90
- catch: (cause) => toPlanSchedulerQueueError('remove-schedule-fire-job', cause),
91
- }).pipe(Effect.asVoid),
92
- )
93
- }
94
-
95
71
  const DEADLINE_CHECK_JOB_PREFIX = 'deadline-check'
96
72
 
97
- export function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
98
- const delay = Math.max(0, scheduledFor.getTime() - nowEpochMillis())
99
- return planScheduler.enqueue(
100
- { type: 'check-deadlines', scheduledFor: scheduledFor.toISOString() },
101
- {
102
- delay,
103
- jobId: `${DEADLINE_CHECK_JOB_PREFIX}:${scheduledFor.getTime()}`,
104
- removeOnComplete: true,
105
- removeOnFail: 50,
106
- },
107
- )
73
+ export interface PlanSchedulerQueueRuntime {
74
+ enqueueScheduleFire(scheduleId: string, delayMs: number): Promise<void>
75
+ removeScheduleFireJob(scheduleId: string): Promise<void>
76
+ enqueueDeadlineCheck(scheduledFor: Date): Promise<void>
77
+ startWorker(options: { registerSignals?: boolean; deps: PlanSchedulerWorkerDeps }): WorkerHandle
108
78
  }
109
79
 
110
- export function startPlanSchedulerWorker(options: {
111
- registerSignals?: boolean
80
+ interface MakePlanSchedulerQueueRuntimeParams {
112
81
  connectionProvider: () => IORedis
113
- deps: PlanSchedulerQueueDeps
114
- }): WorkerHandle {
115
- const handle = planScheduler.startWorker({
116
- deps: options.deps,
117
- registerSignals: options.registerSignals,
118
- connectionProvider: options.connectionProvider,
82
+ queueJobService: QueueJobService
83
+ }
84
+
85
+ export function makePlanSchedulerQueueRuntime(params: MakePlanSchedulerQueueRuntimeParams): PlanSchedulerQueueRuntime {
86
+ const { connectionProvider, queueJobService } = params
87
+
88
+ const queue = createQueueFactoryWithDeps<PlanSchedulerJob, PlanSchedulerWorkerDeps>({
89
+ name: PLAN_SCHEDULER_QUEUE,
90
+ displayName: 'Plan scheduler',
91
+ jobName: 'plan-scheduler-job',
92
+ concurrency: 1,
93
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
94
+ connectionProvider,
95
+ queueJobService,
96
+ prepare: ({ databaseService }) => Effect.runPromise(databaseService.connect()),
97
+ processor: processPlanSchedulerJob,
119
98
  })
120
99
 
121
- // Recover active schedules on startup
122
- void Effect.runFork(
123
- options.deps.planSchedulerService.recoverActiveSchedules().pipe(
124
- Effect.mapError((cause) => toPlanSchedulerQueueError('recover-active-schedules', cause)),
125
- Effect.catchTag('PlanSchedulerQueueError', (error) =>
126
- Effect.sync(() => {
127
- serverLogger.error`Plan scheduler startup recovery failed: ${error.message}`
128
- }),
100
+ const enqueueScheduleFire: PlanSchedulerQueueRuntime['enqueueScheduleFire'] = (scheduleId, delayMs) =>
101
+ queue.enqueue(
102
+ { type: 'fire-schedule', scheduleId },
103
+ { delay: Math.max(0, delayMs), jobId: `schedule:${scheduleId}`, removeOnComplete: true, removeOnFail: 50 },
104
+ )
105
+
106
+ const removeScheduleFireJob: PlanSchedulerQueueRuntime['removeScheduleFireJob'] = (scheduleId) =>
107
+ Effect.runPromise(
108
+ Effect.tryPromise({
109
+ try: () => queue.getQueue().remove(`schedule:${scheduleId}`),
110
+ catch: (cause) => toPlanSchedulerQueueError('remove-schedule-fire-job', cause),
111
+ }).pipe(Effect.asVoid),
112
+ )
113
+
114
+ const enqueueDeadlineCheck: PlanSchedulerQueueRuntime['enqueueDeadlineCheck'] = (scheduledFor) => {
115
+ const delay = Math.max(0, scheduledFor.getTime() - nowEpochMillis())
116
+ return queue.enqueue(
117
+ { type: 'check-deadlines', scheduledFor: scheduledFor.toISOString() },
118
+ {
119
+ delay,
120
+ jobId: `${DEADLINE_CHECK_JOB_PREFIX}:${scheduledFor.getTime()}`,
121
+ removeOnComplete: true,
122
+ removeOnFail: 50,
123
+ },
124
+ )
125
+ }
126
+
127
+ const startWorker: PlanSchedulerQueueRuntime['startWorker'] = (options) => {
128
+ const { deps } = options
129
+ const handle = queue.startWorker({ deps, registerSignals: options.registerSignals, connectionProvider })
130
+
131
+ // Recover active schedules on startup
132
+ void Effect.runFork(
133
+ deps.planSchedulerService.recoverActiveSchedules().pipe(
134
+ Effect.mapError((cause) => toPlanSchedulerQueueError('recover-active-schedules', cause)),
135
+ Effect.catchTag('PlanSchedulerQueueError', (error) =>
136
+ Effect.sync(() => {
137
+ serverLogger.error`Plan scheduler startup recovery failed: ${error.message}`
138
+ }),
139
+ ),
129
140
  ),
130
- ),
131
- )
132
-
133
- // Seed deadline checks from current runtime state. Subsequent checks are
134
- // re-seeded by the queue job after each sweep and by schedule fire events.
135
- void Effect.runFork(
136
- options.deps.planDeadlineService.recoverDeadlineChecks().pipe(
137
- Effect.mapError((cause) => toPlanSchedulerQueueError('recover-deadline-checks', cause)),
138
- Effect.catchTag('PlanSchedulerQueueError', (error) =>
139
- Effect.sync(() => {
140
- serverLogger.error`Plan deadline recovery failed: ${error.message}`
141
- }),
141
+ )
142
+
143
+ // Seed deadline checks from current runtime state.
144
+ void Effect.runFork(
145
+ deps.planDeadlineService.recoverDeadlineChecks().pipe(
146
+ Effect.mapError((cause) => toPlanSchedulerQueueError('recover-deadline-checks', cause)),
147
+ Effect.catchTag('PlanSchedulerQueueError', (error) =>
148
+ Effect.sync(() => {
149
+ serverLogger.error`Plan deadline recovery failed: ${error.message}`
150
+ }),
151
+ ),
142
152
  ),
143
- ),
144
- )
153
+ )
145
154
 
146
- return handle
155
+ return handle
156
+ }
157
+
158
+ return { enqueueScheduleFire, removeScheduleFireJob, enqueueDeadlineCheck, startWorker }
147
159
  }
148
160
 
149
161
  runStandaloneQueueWorker((runtime) => {
150
162
  const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
151
- startPlanSchedulerWorker({
152
- connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
163
+ const redis = resolve(RedisServiceTag)
164
+ const planSchedulerQueue = makePlanSchedulerQueueRuntime({
165
+ connectionProvider: () => redis.getConnectionForBullMQ(),
166
+ queueJobService: resolve(QueueJobServiceTag),
167
+ })
168
+ planSchedulerQueue.startWorker({
153
169
  deps: {
154
170
  databaseService: resolve(DatabaseServiceTag),
155
171
  planSchedulerService: resolve(PlanSchedulerServiceTag),
@@ -3,9 +3,11 @@ import { Effect } from 'effect'
3
3
  import type { Context } from 'effect'
4
4
  import type IORedis from 'ioredis'
5
5
 
6
- import type { MaybeAwaitableService } from '../effect/awaitable-effect'
7
6
  import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
7
+ import type { RedisConnectionManager } from '../redis/connection'
8
8
  import { MemoryServiceTag } from '../services/memory/memory.service'
9
+ import { QueueJobServiceTag } from '../services/queue-job.service'
10
+ import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
9
11
  import { createQueueFactoryWithDeps } from './queue-factory'
10
12
  import { runStandaloneQueueWorker } from './standalone-worker'
11
13
 
@@ -15,7 +17,7 @@ interface PostChatMemoryMessage {
15
17
  agentName?: string
16
18
  }
17
19
 
18
- interface PostChatMemoryExtractionJob {
20
+ export interface PostChatMemoryExtractionJob {
19
21
  orgId: string
20
22
  threadId: string
21
23
  sourceId: string
@@ -29,13 +31,17 @@ interface PostChatMemoryExtractionJob {
29
31
  attachmentContext?: string
30
32
  }
31
33
 
32
- interface PostChatMemoryQueueDeps {
34
+ export interface PostChatMemoryWorkerDeps {
33
35
  databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
34
- memoryService: MaybeAwaitableService<Context.Service.Shape<typeof MemoryServiceTag>>
36
+ memoryService: Context.Service.Shape<typeof MemoryServiceTag>
37
+ redisManager: RedisConnectionManager
35
38
  }
36
39
 
37
- function processPostChatMemoryJob(deps: PostChatMemoryQueueDeps, job: Job<PostChatMemoryExtractionJob>): Promise<void> {
38
- const { memoryService } = deps
40
+ function processPostChatMemoryJob(
41
+ deps: PostChatMemoryWorkerDeps,
42
+ job: Job<PostChatMemoryExtractionJob>,
43
+ ): Promise<void> {
44
+ const { memoryService, redisManager } = deps
39
45
  const data = job.data
40
46
  const userMessage = data.userMessage.trim()
41
47
  const agentMessages = data.agentMessages
@@ -77,43 +83,59 @@ function processPostChatMemoryJob(deps: PostChatMemoryQueueDeps, job: Job<PostCh
77
83
  attachmentContext: data.attachmentContext,
78
84
  agentNames: uniqueAgentNames,
79
85
  }),
80
- ),
86
+ ).pipe(Effect.provideService(RedisServiceTag, redisManager)),
81
87
  )
82
88
  }
83
89
 
84
- const postChatMemory = createQueueFactoryWithDeps<PostChatMemoryExtractionJob, PostChatMemoryQueueDeps>({
85
- name: 'post-chat-memory',
86
- displayName: 'Post-chat memory',
87
- jobName: 'extract-memory',
88
- concurrency: 10,
89
- lockDuration: 900_000,
90
- maxStalledCount: 10,
91
- stalledInterval: 120_000,
92
- defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
93
- prepare: ({ databaseService }) => databaseService.connect(),
94
- processor: processPostChatMemoryJob,
95
- })
96
-
97
- export function enqueuePostChatMemory(job: PostChatMemoryExtractionJob, options?: { dedupeKey?: string }) {
98
- return postChatMemory.enqueue(job, options?.dedupeKey ? { jobId: options.dedupeKey } : undefined)
90
+ export interface PostChatMemoryQueueRuntime {
91
+ enqueuePostChatMemory(job: PostChatMemoryExtractionJob, options?: { dedupeKey?: string }): Promise<void>
92
+ startWorker(options: { registerSignals?: boolean; deps: PostChatMemoryWorkerDeps }): WorkerHandle
99
93
  }
100
94
 
101
- export function startPostChatMemoryWorker(options: {
102
- registerSignals?: boolean
95
+ interface MakePostChatMemoryQueueRuntimeParams {
103
96
  connectionProvider: () => IORedis
104
- deps: PostChatMemoryQueueDeps
105
- }) {
106
- return postChatMemory.startWorker({
107
- deps: options.deps,
108
- registerSignals: options.registerSignals,
109
- connectionProvider: options.connectionProvider,
97
+ queueJobService: QueueJobService
98
+ }
99
+
100
+ export function makePostChatMemoryQueueRuntime(
101
+ params: MakePostChatMemoryQueueRuntimeParams,
102
+ ): PostChatMemoryQueueRuntime {
103
+ const { connectionProvider, queueJobService } = params
104
+ const queue = createQueueFactoryWithDeps<PostChatMemoryExtractionJob, PostChatMemoryWorkerDeps>({
105
+ name: 'post-chat-memory',
106
+ displayName: 'Post-chat memory',
107
+ jobName: 'extract-memory',
108
+ concurrency: 10,
109
+ lockDuration: 900_000,
110
+ maxStalledCount: 10,
111
+ stalledInterval: 120_000,
112
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
113
+ connectionProvider,
114
+ queueJobService,
115
+ prepare: ({ databaseService }) => Effect.runPromise(databaseService.connect()),
116
+ processor: processPostChatMemoryJob,
110
117
  })
118
+
119
+ return {
120
+ enqueuePostChatMemory: (job, options) =>
121
+ queue.enqueue(job, options?.dedupeKey ? { jobId: options.dedupeKey } : undefined),
122
+ startWorker: (options) =>
123
+ queue.startWorker({ deps: options.deps, registerSignals: options.registerSignals, connectionProvider }),
124
+ }
111
125
  }
112
126
 
113
127
  runStandaloneQueueWorker((runtime) => {
114
128
  const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
115
- startPostChatMemoryWorker({
116
- connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
117
- deps: { databaseService: resolve(DatabaseServiceTag), memoryService: resolve(MemoryServiceTag) },
129
+ const redis = resolve(RedisServiceTag)
130
+ const postChatMemoryQueue = makePostChatMemoryQueueRuntime({
131
+ connectionProvider: () => redis.getConnectionForBullMQ(),
132
+ queueJobService: resolve(QueueJobServiceTag),
133
+ })
134
+ postChatMemoryQueue.startWorker({
135
+ deps: {
136
+ databaseService: resolve(DatabaseServiceTag),
137
+ memoryService: resolve(MemoryServiceTag),
138
+ redisManager: redis,
139
+ },
118
140
  })
119
141
  })
@@ -5,8 +5,6 @@ import type IORedis from 'ioredis'
5
5
 
6
6
  import type { LotaLogger } from '../config/logger'
7
7
  import { serverLogger } from '../config/logger'
8
- import { getCurrentRuntime } from '../effect/runtime-ref'
9
- import { RedisServiceTag } from '../effect/services'
10
8
  import type { TrackedBullJobLike } from '../services/queue-job.service'
11
9
  import {
12
10
  attachWorkerEvents,
@@ -14,20 +12,14 @@ import {
14
12
  createWorkerShutdown,
15
13
  DEFAULT_JOB_RETENTION,
16
14
  registerShutdownSignals,
17
- getQueueJobService,
18
15
  } from '../workers/worker-utils'
19
- import type { WorkerHandle } from '../workers/worker-utils'
16
+ import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
20
17
 
21
18
  class QueueFactoryError extends Schema.TaggedErrorClass<QueueFactoryError>()('@lota-sdk/core/QueueFactoryError', {
22
19
  message: Schema.String,
23
20
  cause: Schema.optional(Schema.Defect),
24
21
  }) {}
25
22
 
26
- function getDefaultQueueConnectionProvider(): () => IORedis {
27
- const redis = getCurrentRuntime().runSync(Effect.service(RedisServiceTag))
28
- return () => redis.getConnectionForBullMQ()
29
- }
30
-
31
23
  type QueueShape<TJob> = Queue<TJob, unknown, string, TJob, unknown, string>
32
24
  type QueueMethod = 'add' | 'close' | 'remove' | 'removeDeduplicationKey' | 'removeJobScheduler'
33
25
 
@@ -49,7 +41,8 @@ interface QueueFactoryConfigBase {
49
41
  stalledInterval?: number
50
42
  maxStalledCount?: number
51
43
  defaultJobOptions?: JobsOptions
52
- connectionProvider?: () => IORedis
44
+ connectionProvider: () => IORedis
45
+ queueJobService: QueueJobService
53
46
  }
54
47
 
55
48
  interface QueueFactoryConfigInline<TJob> extends QueueFactoryConfigBase {
@@ -103,19 +96,18 @@ export interface QueueFactoryWithDeps<TJob, TDeps> {
103
96
  export function recordEnqueuedJobMetadata(params: {
104
97
  queueName: string
105
98
  job: Omit<TrackedBullJobLike, 'queueName'>
99
+ queueJobService: QueueJobService
106
100
  logger?: LotaLogger
107
101
  }): Effect.Effect<string | undefined> {
108
102
  const logger = params.logger ?? serverLogger
109
- return getQueueJobService()
110
- .recordEnqueued({ queueName: params.queueName, ...params.job })
111
- .pipe(
112
- Effect.tapError((error) =>
113
- Effect.sync(() => {
114
- logger.error`Failed to persist queued job metadata (queue=${params.queueName}, job=${params.job.id}): ${error}`
115
- }),
116
- ),
117
- Effect.orElseSucceed(() => undefined),
118
- )
103
+ return params.queueJobService.recordEnqueued({ queueName: params.queueName, ...params.job }).pipe(
104
+ Effect.tapError((error) =>
105
+ Effect.sync(() => {
106
+ logger.error`Failed to persist queued job metadata (queue=${params.queueName}, job=${params.job.id}): ${error}`
107
+ }),
108
+ ),
109
+ Effect.orElseSucceed(() => undefined),
110
+ )
119
111
  }
120
112
 
121
113
  function createQueueFactoryRuntime<TJob>(config: QueueFactoryConfigBase): {
@@ -132,8 +124,7 @@ function createQueueFactoryRuntime<TJob>(config: QueueFactoryConfigBase): {
132
124
 
133
125
  let state: State = { queue: null, rawQueue: null, connection: null, pendingClose: null }
134
126
 
135
- const resolveConnectionProvider = (): (() => IORedis) =>
136
- config.connectionProvider ?? getDefaultQueueConnectionProvider()
127
+ const resolveConnectionProvider = (): (() => IORedis) => config.connectionProvider
137
128
 
138
129
  const getConnection = (): IORedis => resolveConnectionProvider()()
139
130
 
@@ -214,7 +205,12 @@ function createQueueFactoryRuntime<TJob>(config: QueueFactoryConfigBase): {
214
205
  catch: (cause) =>
215
206
  new QueueFactoryError({ message: `Failed to enqueue job on queue "${config.name}".`, cause }),
216
207
  })
217
- yield* recordEnqueuedJobMetadata({ queueName: config.name, job: queuedJob, logger: config.logger })
208
+ yield* recordEnqueuedJobMetadata({
209
+ queueName: config.name,
210
+ job: queuedJob,
211
+ queueJobService: config.queueJobService,
212
+ logger: config.logger,
213
+ })
218
214
  }),
219
215
  )
220
216
 
@@ -237,26 +233,29 @@ function createQueueFactoryRuntime<TJob>(config: QueueFactoryConfigBase): {
237
233
  ? new Worker(config.name, workerConfig.processorPath, workerOptions)
238
234
  : new Worker(
239
235
  config.name,
240
- createTracedWorkerProcessor(config.name, (job) =>
241
- Effect.runPromise(
242
- Effect.gen(function* () {
243
- const inlineWorkerConfig = workerConfig as QueueWorkerConfigInline<TJob>
244
- const typedJob = job as Job<TJob>
245
- const prepare = inlineWorkerConfig.prepare
246
- if (prepare) {
247
- yield* Effect.tryPromise({
248
- try: () => prepare(typedJob),
236
+ createTracedWorkerProcessor(
237
+ config.name,
238
+ (job) =>
239
+ Effect.runPromise(
240
+ Effect.gen(function* () {
241
+ const inlineWorkerConfig = workerConfig as QueueWorkerConfigInline<TJob>
242
+ const typedJob = job as Job<TJob>
243
+ const prepare = inlineWorkerConfig.prepare
244
+ if (prepare) {
245
+ yield* Effect.tryPromise({
246
+ try: () => prepare(typedJob),
247
+ catch: (cause) =>
248
+ new QueueFactoryError({ message: `Worker prepare failed for queue "${config.name}".`, cause }),
249
+ })
250
+ }
251
+ return yield* Effect.tryPromise({
252
+ try: () => inlineWorkerConfig.processor(typedJob),
249
253
  catch: (cause) =>
250
- new QueueFactoryError({ message: `Worker prepare failed for queue "${config.name}".`, cause }),
254
+ new QueueFactoryError({ message: `Worker processor failed for queue "${config.name}".`, cause }),
251
255
  })
252
- }
253
- return yield* Effect.tryPromise({
254
- try: () => inlineWorkerConfig.processor(typedJob),
255
- catch: (cause) =>
256
- new QueueFactoryError({ message: `Worker processor failed for queue "${config.name}".`, cause }),
257
- })
258
- }),
259
- ),
256
+ }),
257
+ ),
258
+ config.queueJobService,
260
259
  ),
261
260
  workerOptions,
262
261
  )
@@ -0,0 +1,61 @@
1
+ import { Context, Effect, Layer } from 'effect'
2
+ import type IORedis from 'ioredis'
3
+
4
+ import { RedisServiceTag } from '../effect/services'
5
+ import { QueueJobServiceTag } from '../services/queue-job.service'
6
+ import { makeAutonomousJobQueueRuntime } from './autonomous-job.queue'
7
+ import type { AutonomousJobQueueRuntime } from './autonomous-job.queue'
8
+ import { makeContextCompactionQueueRuntime } from './context-compaction.queue'
9
+ import type { ContextCompactionQueueRuntime } from './context-compaction.queue'
10
+ import { makeDelayedNodePromotionQueueRuntime } from './delayed-node-promotion.queue'
11
+ import type { DelayedNodePromotionQueueRuntime } from './delayed-node-promotion.queue'
12
+ import { makeMemoryConsolidationQueueRuntime } from './memory-consolidation.queue'
13
+ import type { MemoryConsolidationQueueRuntime } from './memory-consolidation.queue'
14
+ import { makeOrganizationLearningQueueRuntime } from './organization-learning.queue'
15
+ import type { OrganizationLearningQueueRuntime } from './organization-learning.queue'
16
+ import { makePlanAgentHeartbeatQueueRuntime } from './plan-agent-heartbeat.queue'
17
+ import type { PlanAgentHeartbeatQueueRuntime } from './plan-agent-heartbeat.queue'
18
+ import { makePlanSchedulerQueueRuntime } from './plan-scheduler.queue'
19
+ import type { PlanSchedulerQueueRuntime } from './plan-scheduler.queue'
20
+ import { makePostChatMemoryQueueRuntime } from './post-chat-memory.queue'
21
+ import type { PostChatMemoryQueueRuntime } from './post-chat-memory.queue'
22
+ import { makeTitleGenerationQueueRuntime } from './title-generation.queue'
23
+ import type { TitleGenerationQueueRuntime } from './title-generation.queue'
24
+
25
+ export interface LotaQueuesRuntime {
26
+ autonomousJob: AutonomousJobQueueRuntime
27
+ contextCompaction: ContextCompactionQueueRuntime
28
+ delayedNodePromotion: DelayedNodePromotionQueueRuntime
29
+ memoryConsolidation: MemoryConsolidationQueueRuntime
30
+ organizationLearning: OrganizationLearningQueueRuntime
31
+ planAgentHeartbeat: PlanAgentHeartbeatQueueRuntime
32
+ planScheduler: PlanSchedulerQueueRuntime
33
+ postChatMemory: PostChatMemoryQueueRuntime
34
+ titleGeneration: TitleGenerationQueueRuntime
35
+ }
36
+
37
+ export class LotaQueuesServiceTag extends Context.Service<LotaQueuesServiceTag, LotaQueuesRuntime>()(
38
+ '@lota-sdk/core/LotaQueuesService',
39
+ ) {}
40
+
41
+ export const LotaQueuesLive = Layer.effect(
42
+ LotaQueuesServiceTag,
43
+ Effect.gen(function* () {
44
+ const redis = yield* RedisServiceTag
45
+ const queueJobService = yield* QueueJobServiceTag
46
+ const connectionProvider: () => IORedis = () => redis.getConnectionForBullMQ()
47
+ const baseDeps = { connectionProvider, queueJobService }
48
+
49
+ return LotaQueuesServiceTag.of({
50
+ autonomousJob: makeAutonomousJobQueueRuntime(baseDeps),
51
+ contextCompaction: makeContextCompactionQueueRuntime(baseDeps),
52
+ delayedNodePromotion: makeDelayedNodePromotionQueueRuntime(baseDeps),
53
+ memoryConsolidation: makeMemoryConsolidationQueueRuntime(baseDeps),
54
+ organizationLearning: makeOrganizationLearningQueueRuntime(baseDeps),
55
+ planAgentHeartbeat: makePlanAgentHeartbeatQueueRuntime(baseDeps),
56
+ planScheduler: makePlanSchedulerQueueRuntime(baseDeps),
57
+ postChatMemory: makePostChatMemoryQueueRuntime(baseDeps),
58
+ titleGeneration: makeTitleGenerationQueueRuntime(baseDeps),
59
+ })
60
+ }),
61
+ )