@lota-sdk/core 0.1.15 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/infrastructure/schema/00_identity.surql +0 -2
  2. package/infrastructure/schema/01_memory.surql +1 -1
  3. package/infrastructure/schema/02_execution_plan.surql +62 -1
  4. package/infrastructure/schema/03_learned_skill.surql +1 -1
  5. package/infrastructure/schema/06_playbook.surql +25 -0
  6. package/infrastructure/schema/07_institutional_memory.surql +13 -0
  7. package/infrastructure/schema/08_quality_metrics.surql +17 -0
  8. package/package.json +8 -7
  9. package/src/ai/definitions.ts +80 -2
  10. package/src/ai/index.ts +0 -2
  11. package/src/bifrost/bifrost.ts +2 -7
  12. package/src/config/agent-defaults.ts +31 -21
  13. package/src/config/agent-types.ts +11 -0
  14. package/src/config/constants.ts +2 -14
  15. package/src/config/debug-logger.ts +5 -1
  16. package/src/config/index.ts +3 -0
  17. package/src/config/model-constants.ts +16 -34
  18. package/src/config/search.ts +1 -15
  19. package/src/create-runtime.ts +244 -178
  20. package/src/db/cursor-pagination.ts +3 -6
  21. package/src/db/index.ts +2 -0
  22. package/src/db/memory-store.rows.ts +7 -7
  23. package/src/db/memory-store.ts +14 -18
  24. package/src/db/memory.ts +13 -13
  25. package/src/db/service.ts +153 -79
  26. package/src/db/startup.ts +6 -10
  27. package/src/db/surreal-mutation.ts +43 -0
  28. package/src/db/tables.ts +7 -0
  29. package/src/db/workstream-message-row.ts +15 -0
  30. package/src/embeddings/provider.ts +1 -1
  31. package/src/queues/context-compaction.queue.ts +15 -46
  32. package/src/queues/delayed-node-promotion.queue.ts +41 -0
  33. package/src/queues/index.ts +3 -0
  34. package/src/queues/memory-consolidation.queue.ts +16 -51
  35. package/src/queues/plan-scheduler.queue.ts +97 -0
  36. package/src/queues/post-chat-memory.queue.ts +15 -56
  37. package/src/queues/queue-factory.ts +100 -0
  38. package/src/queues/recent-activity-title-refinement.queue.ts +15 -50
  39. package/src/queues/regular-chat-memory-digest.queue.ts +16 -52
  40. package/src/queues/skill-extraction.queue.ts +15 -47
  41. package/src/queues/workstream-title-generation.queue.ts +15 -47
  42. package/src/redis/connection.ts +6 -0
  43. package/src/redis/index.ts +1 -1
  44. package/src/redis/stream-context.ts +11 -0
  45. package/src/runtime/agent-runtime-policy.ts +106 -21
  46. package/src/runtime/approval-continuation.ts +12 -6
  47. package/src/runtime/context-compaction-runtime.ts +1 -1
  48. package/src/runtime/context-compaction.ts +22 -60
  49. package/src/runtime/execution-plan.ts +22 -18
  50. package/src/runtime/graph-designer.ts +15 -0
  51. package/src/runtime/helper-model.ts +9 -197
  52. package/src/runtime/index.ts +2 -0
  53. package/src/runtime/llm-content.ts +1 -1
  54. package/src/runtime/memory-block.ts +9 -11
  55. package/src/runtime/memory-pipeline.ts +6 -9
  56. package/src/runtime/plugin-resolution.ts +35 -0
  57. package/src/runtime/plugin-types.ts +72 -0
  58. package/src/runtime/retrieval-adapters.ts +1 -1
  59. package/src/runtime/runtime-config.ts +25 -12
  60. package/src/runtime/runtime-extensions.ts +2 -2
  61. package/src/runtime/runtime-worker-registry.ts +6 -0
  62. package/src/runtime/team-consultation-orchestrator.ts +45 -28
  63. package/src/runtime/team-consultation-prompts.ts +11 -2
  64. package/src/runtime/title-helpers.ts +2 -4
  65. package/src/runtime/workstream-chat-helpers.ts +1 -1
  66. package/src/services/adaptive-playbook.service.ts +152 -0
  67. package/src/services/agent-executor.service.ts +293 -0
  68. package/src/services/artifact-provenance.service.ts +172 -0
  69. package/src/services/attachment.service.ts +6 -11
  70. package/src/services/context-compaction.service.ts +72 -55
  71. package/src/services/context-enrichment.service.ts +33 -0
  72. package/src/services/coordination-registry.service.ts +117 -0
  73. package/src/services/document-chunk.service.ts +1 -1
  74. package/src/services/domain-agent-executor.service.ts +71 -0
  75. package/src/services/execution-plan.service.ts +269 -50
  76. package/src/services/feedback-loop.service.ts +96 -0
  77. package/src/services/global-orchestrator.service.ts +148 -0
  78. package/src/services/index.ts +26 -0
  79. package/src/services/institutional-memory.service.ts +145 -0
  80. package/src/services/learned-skill.service.ts +24 -5
  81. package/src/services/memory-assessment.service.ts +3 -2
  82. package/src/services/memory-utils.ts +3 -8
  83. package/src/services/memory.service.ts +42 -59
  84. package/src/services/monitoring-window.service.ts +86 -0
  85. package/src/services/mutating-approval.service.ts +1 -1
  86. package/src/services/node-workspace.service.ts +155 -0
  87. package/src/services/notification.service.ts +39 -0
  88. package/src/services/organization-member.service.ts +11 -4
  89. package/src/services/organization.service.ts +5 -5
  90. package/src/services/ownership-dispatcher.service.ts +403 -0
  91. package/src/services/plan-approval.service.ts +1 -1
  92. package/src/services/plan-builder.service.ts +1 -0
  93. package/src/services/plan-checkpoint.service.ts +30 -2
  94. package/src/services/plan-compiler.service.ts +5 -0
  95. package/src/services/plan-coordination.service.ts +152 -0
  96. package/src/services/plan-cycle.service.ts +284 -0
  97. package/src/services/plan-deadline.service.ts +287 -0
  98. package/src/services/plan-executor.service.ts +384 -40
  99. package/src/services/plan-run.service.ts +41 -7
  100. package/src/services/plan-scheduler.service.ts +240 -0
  101. package/src/services/plan-template.service.ts +117 -0
  102. package/src/services/plan-validator.service.ts +84 -2
  103. package/src/services/plan-workspace.service.ts +83 -0
  104. package/src/services/playbook-registry.service.ts +67 -0
  105. package/src/services/plugin-executor.service.ts +103 -0
  106. package/src/services/quality-metrics.service.ts +132 -0
  107. package/src/services/recent-activity.service.ts +27 -31
  108. package/src/services/skill-resolver.service.ts +19 -0
  109. package/src/services/system-executor.service.ts +105 -0
  110. package/src/services/workstream-message.service.ts +12 -34
  111. package/src/services/workstream-plan-registry.service.ts +22 -0
  112. package/src/services/workstream-title.service.ts +3 -1
  113. package/src/services/workstream-turn-preparation.service.ts +34 -66
  114. package/src/services/workstream.service.ts +33 -55
  115. package/src/services/workstream.types.ts +9 -9
  116. package/src/services/write-intent-validator.service.ts +81 -0
  117. package/src/storage/attachment-parser.ts +1 -1
  118. package/src/storage/attachment-utils.ts +1 -1
  119. package/src/storage/generated-document-storage.service.ts +3 -2
  120. package/src/system-agents/delegated-agent-factory.ts +2 -0
  121. package/src/tools/execution-plan.tool.ts +17 -23
  122. package/src/tools/index.ts +0 -1
  123. package/src/tools/team-think.tool.ts +6 -4
  124. package/src/utils/async.ts +2 -1
  125. package/src/utils/date-time.ts +4 -32
  126. package/src/utils/env.ts +8 -0
  127. package/src/utils/errors.ts +42 -10
  128. package/src/utils/index.ts +9 -0
  129. package/src/utils/string.ts +114 -1
  130. package/src/workers/index.ts +1 -0
  131. package/src/workers/regular-chat-memory-digest.runner.ts +2 -2
  132. package/src/workers/skill-extraction.runner.ts +1 -1
  133. package/src/workers/utils/file-section-chunker.ts +2 -1
  134. package/src/workers/utils/repomix-file-sections.ts +2 -2
  135. package/src/workers/utils/sandbox-error.ts +11 -2
  136. package/src/workers/utils/workstream-message-query.ts +11 -20
  137. package/src/workers/worker-utils.ts +2 -2
  138. package/src/tools/log-hello-world.tool.ts +0 -17
@@ -71,7 +71,7 @@ export class ProviderEmbeddings {
71
71
  const redisCache = this.getCache()
72
72
  if (!redisCache) return null
73
73
 
74
- return await redisCache.get(this.getModelId(), text)
74
+ return redisCache.get(this.getModelId(), text)
75
75
  }
76
76
 
77
77
  async embedQuery(text: string): Promise<number[]> {
@@ -1,21 +1,11 @@
1
- import { Queue, Worker } from 'bullmq'
2
1
  import type { Job } from 'bullmq'
3
2
 
4
- import { serverLogger } from '../config/logger'
5
3
  import { ensureRecordId } from '../db/record-id'
6
4
  import { databaseService } from '../db/service'
7
5
  import { TABLES } from '../db/tables'
8
- import { getRedisConnectionForBullMQ } from '../redis'
9
6
  import { contextCompactionService } from '../services/context-compaction.service'
10
7
  import { workstreamService } from '../services/workstream.service'
11
- import {
12
- attachWorkerEvents,
13
- createTracedWorkerProcessor,
14
- createWorkerShutdown,
15
- DEFAULT_JOB_RETENTION,
16
- registerShutdownSignals,
17
- } from '../workers/worker-utils'
18
- import type { WorkerHandle } from '../workers/worker-utils'
8
+ import { createQueueFactory } from './queue-factory'
19
9
 
20
10
  interface ContextCompactionJob {
21
11
  domain: 'workstream'
@@ -23,25 +13,6 @@ interface ContextCompactionJob {
23
13
  contextSize?: number
24
14
  }
25
15
 
26
- const CONTEXT_COMPACTION_QUEUE = 'context-compaction'
27
-
28
- let _contextCompactionQueue: Queue<ContextCompactionJob> | null = null
29
- function getContextCompactionQueue(): Queue<ContextCompactionJob> {
30
- if (!_contextCompactionQueue) {
31
- _contextCompactionQueue = new Queue<ContextCompactionJob>(CONTEXT_COMPACTION_QUEUE, {
32
- connection: getRedisConnectionForBullMQ(),
33
- defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 3_000 } },
34
- })
35
- }
36
- return _contextCompactionQueue
37
- }
38
-
39
- export async function enqueueContextCompaction(job: ContextCompactionJob) {
40
- return await getContextCompactionQueue().add('compact', job, {
41
- deduplication: { id: `compact:${job.domain}:${job.entityId}` },
42
- })
43
- }
44
-
45
16
  async function processContextCompactionJob(job: Job<ContextCompactionJob>): Promise<void> {
46
17
  await databaseService.connect()
47
18
 
@@ -55,24 +26,22 @@ async function processContextCompactionJob(job: Job<ContextCompactionJob>): Prom
55
26
  }
56
27
  }
57
28
 
58
- export function startContextCompactionWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
59
- const { registerSignals = import.meta.main } = options
60
- const worker = new Worker(
61
- CONTEXT_COMPACTION_QUEUE,
62
- createTracedWorkerProcessor(CONTEXT_COMPACTION_QUEUE, processContextCompactionJob),
63
- { connection: getRedisConnectionForBullMQ(), concurrency: 2, lockDuration: 300_000 },
64
- )
65
-
66
- attachWorkerEvents(worker, 'Context compaction', serverLogger)
67
- const shutdown = createWorkerShutdown(worker, 'Context compaction', serverLogger)
68
-
69
- if (registerSignals) {
70
- registerShutdownSignals({ name: 'Context compaction', shutdown, logger: serverLogger })
71
- }
72
-
73
- return { worker, shutdown }
29
+ const contextCompaction = createQueueFactory<ContextCompactionJob>({
30
+ name: 'context-compaction',
31
+ displayName: 'Context compaction',
32
+ jobName: 'compact',
33
+ concurrency: 2,
34
+ lockDuration: 300_000,
35
+ defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 3_000 } },
36
+ processor: processContextCompactionJob,
37
+ })
38
+
39
+ export function enqueueContextCompaction(job: ContextCompactionJob) {
40
+ return contextCompaction.enqueue(job, { deduplication: { id: `compact:${job.domain}:${job.entityId}` } })
74
41
  }
75
42
 
43
+ export const startContextCompactionWorker = contextCompaction.startWorker
44
+
76
45
  if (import.meta.main) {
77
46
  startContextCompactionWorker()
78
47
  }
@@ -0,0 +1,41 @@
1
+ import type { Job } from 'bullmq'
2
+
3
+ import { databaseService } from '../db/service'
4
+ import { planExecutorService } from '../services/plan-executor.service'
5
+ import { createQueueFactory } from './queue-factory'
6
+
7
+ export interface DelayedNodePromotionJob {
8
+ runId: string
9
+ nodeId: string
10
+ emittedBy: string
11
+ }
12
+
13
+ export const DELAYED_NODE_PROMOTION_QUEUE = 'delayed-node-promotion'
14
+
15
+ async function processDelayedNodePromotionJob(job: Job<DelayedNodePromotionJob>): Promise<void> {
16
+ await databaseService.connect()
17
+ await planExecutorService.promoteDelayedNode({
18
+ runId: job.data.runId,
19
+ nodeId: job.data.nodeId,
20
+ emittedBy: job.data.emittedBy,
21
+ })
22
+ }
23
+
24
+ const delayedNodePromotion = createQueueFactory<DelayedNodePromotionJob>({
25
+ name: DELAYED_NODE_PROMOTION_QUEUE,
26
+ displayName: 'Delayed node promotion',
27
+ jobName: 'promote-node',
28
+ concurrency: 1,
29
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
30
+ processor: processDelayedNodePromotionJob,
31
+ })
32
+
33
+ export async function enqueueDelayedNodePromotion(job: DelayedNodePromotionJob, delayMs: number) {
34
+ await delayedNodePromotion.enqueue(job, { delay: delayMs, jobId: `promote:${job.runId}:${job.nodeId}` })
35
+ }
36
+
37
+ export const startDelayedNodePromotionWorker = delayedNodePromotion.startWorker
38
+
39
+ if (import.meta.main) {
40
+ startDelayedNodePromotionWorker()
41
+ }
@@ -1,6 +1,9 @@
1
+ export * from './queue-factory'
1
2
  export * from './context-compaction.queue'
3
+ export * from './delayed-node-promotion.queue'
2
4
  export * from './document-processor.queue'
3
5
  export * from './memory-consolidation.queue'
6
+ export * from './plan-scheduler.queue'
4
7
  export * from './post-chat-memory.queue'
5
8
  export * from './recent-activity-title-refinement.queue'
6
9
  export * from './regular-chat-memory-digest.config'
@@ -1,69 +1,34 @@
1
- import { Queue, Worker } from 'bullmq'
2
-
3
- import { serverLogger } from '../config/logger'
4
- import { getRedisConnectionForBullMQ } from '../redis'
5
- import {
6
- attachWorkerEvents,
7
- getWorkerPath,
8
- LONG_JOB_LOCK_DURATION_MS,
9
- LOW_JOB_RETENTION,
10
- createWorkerShutdown,
11
- registerShutdownSignals,
12
- } from '../workers/worker-utils'
13
- import type { WorkerHandle } from '../workers/worker-utils'
1
+ import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS, LOW_JOB_RETENTION } from '../workers/worker-utils'
2
+ import { createQueueFactory } from './queue-factory'
14
3
 
15
4
  export interface MemoryConsolidationJob {
16
5
  scopeId?: string
17
6
  }
18
7
 
19
- const MEMORY_CONSOLIDATION_QUEUE = 'memory-consolidation'
20
8
  const MEMORY_CONSOLIDATION_INTERVAL_MS = 24 * 60 * 60 * 1000
21
9
  const MEMORY_CONSOLIDATION_JOB_ID = 'memory-consolidation-recurring'
22
10
 
23
- let _memoryConsolidationQueue: Queue<MemoryConsolidationJob> | null = null
24
- function getMemoryConsolidationQueue(): Queue<MemoryConsolidationJob> {
25
- if (!_memoryConsolidationQueue) {
26
- _memoryConsolidationQueue = new Queue<MemoryConsolidationJob>(MEMORY_CONSOLIDATION_QUEUE, {
27
- connection: getRedisConnectionForBullMQ(),
28
- defaultJobOptions: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
29
- })
30
- }
31
- return _memoryConsolidationQueue
32
- }
11
+ const memoryConsolidation = createQueueFactory<MemoryConsolidationJob>({
12
+ name: 'memory-consolidation',
13
+ displayName: 'Memory consolidation',
14
+ jobName: 'consolidate-turn',
15
+ concurrency: 1,
16
+ lockDuration: LONG_JOB_LOCK_DURATION_MS,
17
+ defaultJobOptions: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
18
+ processorPath: getWorkerPath('memory-consolidation.worker.ts'),
19
+ })
33
20
 
34
21
  export async function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
35
- await getMemoryConsolidationQueue().add('consolidate-turn', job, {
36
- jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined,
37
- })
22
+ await memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
38
23
  }
39
24
 
40
25
  export async function scheduleRecurringConsolidation() {
41
- await getMemoryConsolidationQueue().add(
42
- 'consolidate',
43
- {},
44
- { repeat: { every: MEMORY_CONSOLIDATION_INTERVAL_MS }, jobId: MEMORY_CONSOLIDATION_JOB_ID },
45
- )
26
+ await memoryConsolidation
27
+ .getQueue()
28
+ .add('consolidate', {}, { repeat: { every: MEMORY_CONSOLIDATION_INTERVAL_MS }, jobId: MEMORY_CONSOLIDATION_JOB_ID })
46
29
  }
47
30
 
48
- export function startMemoryConsolidationWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
49
- const { registerSignals = import.meta.main } = options
50
- const processorPath = getWorkerPath('memory-consolidation.worker.ts')
51
- const worker = new Worker(MEMORY_CONSOLIDATION_QUEUE, processorPath, {
52
- connection: getRedisConnectionForBullMQ(),
53
- lockDuration: LONG_JOB_LOCK_DURATION_MS,
54
- concurrency: 1,
55
- })
56
-
57
- attachWorkerEvents(worker, 'Memory consolidation', serverLogger)
58
-
59
- const shutdown = createWorkerShutdown(worker, 'Memory consolidation', serverLogger)
60
-
61
- if (registerSignals) {
62
- registerShutdownSignals({ name: 'Memory consolidation', shutdown, logger: serverLogger })
63
- }
64
-
65
- return { worker, shutdown }
66
- }
31
+ export const startMemoryConsolidationWorker = memoryConsolidation.startWorker
67
32
 
68
33
  if (import.meta.main) {
69
34
  startMemoryConsolidationWorker()
@@ -0,0 +1,97 @@
1
+ import type { Job } from 'bullmq'
2
+
3
+ import { serverLogger } from '../config/logger'
4
+ import { databaseService } from '../db/service'
5
+ import { planDeadlineService } from '../services/plan-deadline.service'
6
+ import { planSchedulerService } from '../services/plan-scheduler.service'
7
+ import type { WorkerHandle } from '../workers/worker-utils'
8
+ import { createQueueFactory } from './queue-factory'
9
+
10
+ export interface PlanSchedulerFireJob {
11
+ type: 'fire-schedule'
12
+ scheduleId: string
13
+ }
14
+
15
+ export interface PlanSchedulerDeadlineJob {
16
+ type: 'check-deadlines'
17
+ scheduledFor?: string
18
+ }
19
+
20
+ export type PlanSchedulerJob = PlanSchedulerFireJob | PlanSchedulerDeadlineJob
21
+
22
+ export const PLAN_SCHEDULER_QUEUE = 'plan-scheduler'
23
+
24
+ async function processPlanSchedulerJob(job: Job<PlanSchedulerJob>): Promise<void> {
25
+ await databaseService.connect()
26
+
27
+ switch (job.data.type) {
28
+ case 'fire-schedule':
29
+ await planSchedulerService.fireScheduleById(job.data.scheduleId)
30
+ break
31
+ case 'check-deadlines':
32
+ await planDeadlineService.checkDeadlines()
33
+ break
34
+ }
35
+ }
36
+
37
+ const planScheduler = createQueueFactory<PlanSchedulerJob>({
38
+ name: PLAN_SCHEDULER_QUEUE,
39
+ displayName: 'Plan scheduler',
40
+ jobName: 'plan-scheduler-job',
41
+ concurrency: 1,
42
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
43
+ processor: processPlanSchedulerJob,
44
+ })
45
+
46
+ /** Enqueue a delayed job that fires a specific schedule at its nextFireAt time. */
47
+ export async function enqueueScheduleFire(scheduleId: string, delayMs: number): Promise<void> {
48
+ await planScheduler.enqueue(
49
+ { type: 'fire-schedule', scheduleId },
50
+ { delay: Math.max(0, delayMs), jobId: `schedule:${scheduleId}`, removeOnComplete: true, removeOnFail: 50 },
51
+ )
52
+ }
53
+
54
+ /** Remove a pending fire job for a schedule (on cancel/pause/complete). */
55
+ export async function removeScheduleFireJob(scheduleId: string): Promise<void> {
56
+ try {
57
+ await planScheduler.getQueue().remove(`schedule:${scheduleId}`)
58
+ } catch {
59
+ // Job may not exist (already fired or never enqueued) — safe to ignore
60
+ }
61
+ }
62
+
63
+ const DEADLINE_CHECK_JOB_PREFIX = 'deadline-check'
64
+
65
+ export async function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
66
+ const delay = Math.max(0, scheduledFor.getTime() - Date.now())
67
+ await planScheduler.enqueue(
68
+ { type: 'check-deadlines', scheduledFor: scheduledFor.toISOString() },
69
+ {
70
+ delay,
71
+ jobId: `${DEADLINE_CHECK_JOB_PREFIX}:${scheduledFor.getTime()}`,
72
+ removeOnComplete: true,
73
+ removeOnFail: 50,
74
+ },
75
+ )
76
+ }
77
+
78
+ export function startPlanSchedulerWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
79
+ const handle = planScheduler.startWorker(options)
80
+
81
+ // Recover active schedules on startup
82
+ planSchedulerService.recoverActiveSchedules().catch((err: unknown) => {
83
+ serverLogger.error`Plan scheduler startup recovery failed: ${err}`
84
+ })
85
+
86
+ // Seed deadline checks from current runtime state. Subsequent checks are
87
+ // re-seeded by the queue job after each sweep and by schedule fire events.
88
+ planDeadlineService.recoverDeadlineChecks().catch((err: unknown) => {
89
+ serverLogger.error`Plan deadline recovery failed: ${err}`
90
+ })
91
+
92
+ return handle
93
+ }
94
+
95
+ if (import.meta.main) {
96
+ startPlanSchedulerWorker()
97
+ }
@@ -1,19 +1,9 @@
1
1
  type OrganizationOnboardStatus = string
2
- import { Queue, Worker } from 'bullmq'
3
2
  import type { Job } from 'bullmq'
4
3
 
5
- import { serverLogger } from '../config/logger'
6
4
  import { databaseService } from '../db/service'
7
- import { getRedisConnectionForBullMQ } from '../redis'
8
5
  import { memoryService } from '../services/memory.service'
9
- import {
10
- attachWorkerEvents,
11
- createTracedWorkerProcessor,
12
- createWorkerShutdown,
13
- DEFAULT_JOB_RETENTION,
14
- registerShutdownSignals,
15
- } from '../workers/worker-utils'
16
- import type { WorkerHandle } from '../workers/worker-utils'
6
+ import { createQueueFactory } from './queue-factory'
17
7
 
18
8
  interface PostChatMemoryMessage {
19
9
  role: 'user' | 'agent'
@@ -33,27 +23,6 @@ interface PostChatMemoryExtractionJob {
33
23
  attachmentContext?: string
34
24
  }
35
25
 
36
- const POST_CHAT_MEMORY_QUEUE = 'post-chat-memory'
37
- const POST_CHAT_MEMORY_CONCURRENCY = 3
38
- const POST_CHAT_MEMORY_LOCK_DURATION_MS = 900_000
39
- const POST_CHAT_MEMORY_MAX_STALLED_COUNT = 10
40
- const POST_CHAT_MEMORY_STALLED_INTERVAL_MS = 120_000
41
-
42
- let _postChatMemoryQueue: Queue<PostChatMemoryExtractionJob> | null = null
43
- function getPostChatMemoryQueue(): Queue<PostChatMemoryExtractionJob> {
44
- if (!_postChatMemoryQueue) {
45
- _postChatMemoryQueue = new Queue<PostChatMemoryExtractionJob>(POST_CHAT_MEMORY_QUEUE, {
46
- connection: getRedisConnectionForBullMQ(),
47
- defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
48
- })
49
- }
50
- return _postChatMemoryQueue
51
- }
52
-
53
- export async function enqueuePostChatMemory(job: PostChatMemoryExtractionJob) {
54
- return await getPostChatMemoryQueue().add('extract-memory', job)
55
- }
56
-
57
26
  async function processPostChatMemoryJob(job: Job<PostChatMemoryExtractionJob>): Promise<void> {
58
27
  await databaseService.connect()
59
28
 
@@ -94,30 +63,20 @@ async function processPostChatMemoryJob(job: Job<PostChatMemoryExtractionJob>):
94
63
  })
95
64
  }
96
65
 
97
- export function startPostChatMemoryWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
98
- const { registerSignals = import.meta.main } = options
99
- const worker = new Worker(
100
- POST_CHAT_MEMORY_QUEUE,
101
- createTracedWorkerProcessor(POST_CHAT_MEMORY_QUEUE, processPostChatMemoryJob),
102
- {
103
- connection: getRedisConnectionForBullMQ(),
104
- concurrency: POST_CHAT_MEMORY_CONCURRENCY,
105
- lockDuration: POST_CHAT_MEMORY_LOCK_DURATION_MS,
106
- maxStalledCount: POST_CHAT_MEMORY_MAX_STALLED_COUNT,
107
- stalledInterval: POST_CHAT_MEMORY_STALLED_INTERVAL_MS,
108
- },
109
- )
110
-
111
- attachWorkerEvents(worker, 'Post-chat memory', serverLogger)
112
-
113
- const shutdown = createWorkerShutdown(worker, 'Post-chat memory', serverLogger)
114
-
115
- if (registerSignals) {
116
- registerShutdownSignals({ name: 'Post-chat memory', shutdown, logger: serverLogger })
117
- }
118
-
119
- return { worker, shutdown }
120
- }
66
+ const postChatMemory = createQueueFactory<PostChatMemoryExtractionJob>({
67
+ name: 'post-chat-memory',
68
+ displayName: 'Post-chat memory',
69
+ jobName: 'extract-memory',
70
+ concurrency: 3,
71
+ lockDuration: 900_000,
72
+ maxStalledCount: 10,
73
+ stalledInterval: 120_000,
74
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
75
+ processor: processPostChatMemoryJob,
76
+ })
77
+
78
+ export const enqueuePostChatMemory = postChatMemory.enqueue
79
+ export const startPostChatMemoryWorker = postChatMemory.startWorker
121
80
 
122
81
  if (import.meta.main) {
123
82
  startPostChatMemoryWorker()
@@ -0,0 +1,100 @@
1
+ import { Queue, Worker } from 'bullmq'
2
+ import type { Job, JobsOptions, WorkerOptions } from 'bullmq'
3
+ import type IORedis from 'ioredis'
4
+
5
+ import { serverLogger } from '../config/logger'
6
+ import { getRedisConnectionForBullMQ } from '../redis'
7
+ import {
8
+ attachWorkerEvents,
9
+ createTracedWorkerProcessor,
10
+ createWorkerShutdown,
11
+ DEFAULT_JOB_RETENTION,
12
+ registerShutdownSignals,
13
+ } from '../workers/worker-utils'
14
+ import type { WorkerHandle } from '../workers/worker-utils'
15
+
16
+ interface QueueFactoryConfigBase {
17
+ name: string
18
+ displayName: string
19
+ jobName: string
20
+ concurrency: number
21
+ lockDuration?: number
22
+ stalledInterval?: number
23
+ maxStalledCount?: number
24
+ defaultJobOptions?: JobsOptions
25
+ connectionProvider?: () => IORedis
26
+ }
27
+
28
+ interface QueueFactoryConfigInline<TJob> extends QueueFactoryConfigBase {
29
+ processor: (job: Job<TJob>) => Promise<void>
30
+ processorPath?: never
31
+ }
32
+
33
+ interface QueueFactoryConfigFile extends QueueFactoryConfigBase {
34
+ processor?: never
35
+ processorPath: string
36
+ }
37
+
38
+ export type QueueFactoryConfig<TJob> = QueueFactoryConfigInline<TJob> | QueueFactoryConfigFile
39
+
40
+ export interface QueueFactory<TJob> {
41
+ getQueue: () => Queue<TJob, unknown, string>
42
+ enqueue: (job: TJob, options?: JobsOptions) => Promise<void>
43
+ startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle
44
+ }
45
+
46
+ export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): QueueFactory<TJob> {
47
+ let _queue: Queue<TJob, unknown, string> | null = null
48
+
49
+ const getConnection = () => config.connectionProvider?.() ?? getRedisConnectionForBullMQ()
50
+
51
+ const getQueue = (): Queue<TJob, unknown, string> => {
52
+ if (!_queue) {
53
+ _queue = new Queue<TJob, unknown, string>(config.name, {
54
+ connection: getConnection(),
55
+ defaultJobOptions: { ...DEFAULT_JOB_RETENTION, ...config.defaultJobOptions },
56
+ })
57
+ }
58
+ return _queue
59
+ }
60
+
61
+ type QueueAdd = Queue<TJob, unknown, string>['add']
62
+ const jobName = config.jobName as Parameters<QueueAdd>[0]
63
+ const toData = (job: TJob) => job as Parameters<QueueAdd>[1]
64
+
65
+ const enqueue = async (job: TJob, options?: JobsOptions): Promise<void> => {
66
+ await getQueue().add(jobName, toData(job), options)
67
+ }
68
+
69
+ const startWorker = (options: { registerSignals?: boolean } = {}): WorkerHandle => {
70
+ const { registerSignals = import.meta.main } = options
71
+
72
+ const workerOptions: WorkerOptions = {
73
+ connection: getConnection(),
74
+ concurrency: config.concurrency,
75
+ ...(config.lockDuration !== undefined ? { lockDuration: config.lockDuration } : {}),
76
+ ...(config.stalledInterval !== undefined ? { stalledInterval: config.stalledInterval } : {}),
77
+ ...(config.maxStalledCount !== undefined ? { maxStalledCount: config.maxStalledCount } : {}),
78
+ }
79
+
80
+ const worker = config.processorPath
81
+ ? new Worker(config.name, config.processorPath, workerOptions)
82
+ : new Worker(
83
+ config.name,
84
+ createTracedWorkerProcessor(config.name, (config as QueueFactoryConfigInline<TJob>).processor),
85
+ workerOptions,
86
+ )
87
+
88
+ attachWorkerEvents(worker, config.displayName, serverLogger)
89
+
90
+ const shutdown = createWorkerShutdown(worker, config.displayName, serverLogger)
91
+
92
+ if (registerSignals) {
93
+ registerShutdownSignals({ name: config.displayName, shutdown, logger: serverLogger })
94
+ }
95
+
96
+ return { worker, shutdown }
97
+ }
98
+
99
+ return { getQueue, enqueue, startWorker }
100
+ }
@@ -1,65 +1,30 @@
1
- import { Queue, Worker } from 'bullmq'
2
1
  import type { Job } from 'bullmq'
3
2
 
4
- import { serverLogger } from '../config/logger'
5
3
  import { databaseService } from '../db/service'
6
- import { getRedisConnectionForBullMQ } from '../redis'
7
4
  import { recentActivityTitleService } from '../services/recent-activity-title.service'
8
- import {
9
- attachWorkerEvents,
10
- createTracedWorkerProcessor,
11
- createWorkerShutdown,
12
- DEFAULT_JOB_RETENTION,
13
- registerShutdownSignals,
14
- } from '../workers/worker-utils'
15
- import type { WorkerHandle } from '../workers/worker-utils'
5
+ import { createQueueFactory } from './queue-factory'
16
6
 
17
7
  interface RecentActivityTitleRefinementJob {
18
8
  activityId: string
19
9
  }
20
10
 
21
- const RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE = 'recent-activity-title-refinement'
22
-
23
- let _recentActivityTitleRefinementQueue: Queue<RecentActivityTitleRefinementJob> | null = null
24
- function getRecentActivityTitleRefinementQueue(): Queue<RecentActivityTitleRefinementJob> {
25
- if (!_recentActivityTitleRefinementQueue) {
26
- _recentActivityTitleRefinementQueue = new Queue<RecentActivityTitleRefinementJob>(
27
- RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE,
28
- {
29
- connection: getRedisConnectionForBullMQ(),
30
- defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
31
- },
32
- )
33
- }
34
- return _recentActivityTitleRefinementQueue
35
- }
36
-
37
- export async function enqueueRecentActivityTitleRefinement(job: RecentActivityTitleRefinementJob) {
38
- return await getRecentActivityTitleRefinementQueue().add('refine-recent-activity-title', job, {
39
- jobId: `recent-activity-title:${job.activityId}`,
40
- })
41
- }
42
-
43
11
  async function processRecentActivityTitleRefinementJob(job: Job<RecentActivityTitleRefinementJob>): Promise<void> {
44
12
  await databaseService.connect()
45
13
  await recentActivityTitleService.refineRecentActivityTitle(job.data.activityId)
46
14
  }
47
15
 
48
- export function startRecentActivityTitleRefinementWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
49
- const { registerSignals = import.meta.main } = options
50
- const worker = new Worker(
51
- RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE,
52
- createTracedWorkerProcessor(RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE, processRecentActivityTitleRefinementJob),
53
- { connection: getRedisConnectionForBullMQ(), concurrency: 2, lockDuration: 300_000 },
54
- )
55
-
56
- attachWorkerEvents(worker, 'Recent activity title refinement', serverLogger)
57
-
58
- const shutdown = createWorkerShutdown(worker, 'Recent activity title refinement', serverLogger)
59
-
60
- if (registerSignals) {
61
- registerShutdownSignals({ name: 'Recent activity title refinement', shutdown, logger: serverLogger })
62
- }
63
-
64
- return { worker, shutdown }
16
+ const recentActivityTitleRefinement = createQueueFactory<RecentActivityTitleRefinementJob>({
17
+ name: 'recent-activity-title-refinement',
18
+ displayName: 'Recent activity title refinement',
19
+ jobName: 'refine-recent-activity-title',
20
+ concurrency: 2,
21
+ lockDuration: 300_000,
22
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
23
+ processor: processRecentActivityTitleRefinementJob,
24
+ })
25
+
26
+ export function enqueueRecentActivityTitleRefinement(job: RecentActivityTitleRefinementJob) {
27
+ return recentActivityTitleRefinement.enqueue(job, { jobId: `recent-activity-title:${job.activityId}` })
65
28
  }
29
+
30
+ export const startRecentActivityTitleRefinementWorker = recentActivityTitleRefinement.startWorker