@lota-sdk/core 0.1.14 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/infrastructure/schema/00_identity.surql +0 -2
  2. package/infrastructure/schema/01_memory.surql +1 -1
  3. package/infrastructure/schema/02_execution_plan.surql +62 -1
  4. package/infrastructure/schema/03_learned_skill.surql +1 -1
  5. package/infrastructure/schema/06_playbook.surql +25 -0
  6. package/infrastructure/schema/07_institutional_memory.surql +13 -0
  7. package/infrastructure/schema/08_quality_metrics.surql +17 -0
  8. package/package.json +9 -8
  9. package/src/ai/definitions.ts +80 -2
  10. package/src/ai/embedding-cache.ts +7 -6
  11. package/src/ai/index.ts +0 -1
  12. package/src/bifrost/bifrost.ts +14 -14
  13. package/src/config/agent-defaults.ts +32 -22
  14. package/src/config/agent-types.ts +11 -0
  15. package/src/config/constants.ts +2 -14
  16. package/src/config/debug-logger.ts +5 -1
  17. package/src/config/index.ts +3 -0
  18. package/src/config/logger.ts +7 -9
  19. package/src/config/model-constants.ts +16 -34
  20. package/src/config/search.ts +1 -15
  21. package/src/create-runtime.ts +453 -0
  22. package/src/db/cursor-pagination.ts +3 -6
  23. package/src/db/index.ts +2 -0
  24. package/src/db/memory-store.rows.ts +7 -7
  25. package/src/db/memory-store.ts +24 -24
  26. package/src/db/memory.ts +18 -16
  27. package/src/db/schema-fingerprint.ts +1 -0
  28. package/src/db/service.ts +193 -122
  29. package/src/db/startup.ts +9 -13
  30. package/src/db/surreal-mutation.ts +43 -0
  31. package/src/db/tables.ts +7 -0
  32. package/src/db/workstream-message-row.ts +15 -0
  33. package/src/embeddings/provider.ts +1 -1
  34. package/src/index.ts +1 -1
  35. package/src/queues/context-compaction.queue.ts +17 -52
  36. package/src/queues/delayed-node-promotion.queue.ts +41 -0
  37. package/src/queues/document-processor.queue.ts +7 -7
  38. package/src/queues/index.ts +3 -0
  39. package/src/queues/memory-consolidation.queue.ts +18 -54
  40. package/src/queues/plan-scheduler.queue.ts +97 -0
  41. package/src/queues/post-chat-memory.queue.ts +15 -60
  42. package/src/queues/queue-factory.ts +100 -0
  43. package/src/queues/recent-activity-title-refinement.queue.ts +15 -54
  44. package/src/queues/regular-chat-memory-digest.queue.ts +16 -55
  45. package/src/queues/skill-extraction.queue.ts +15 -50
  46. package/src/queues/workstream-title-generation.queue.ts +15 -51
  47. package/src/redis/connection.ts +12 -3
  48. package/src/redis/index.ts +2 -1
  49. package/src/redis/org-memory-lock.ts +1 -1
  50. package/src/redis/redis-lease-lock.ts +41 -8
  51. package/src/redis/stream-context.ts +11 -0
  52. package/src/runtime/agent-runtime-policy.ts +106 -21
  53. package/src/runtime/agent-stream-helpers.ts +2 -1
  54. package/src/runtime/approval-continuation.ts +12 -6
  55. package/src/runtime/context-compaction-constants.ts +1 -1
  56. package/src/runtime/context-compaction-runtime.ts +7 -5
  57. package/src/runtime/context-compaction.ts +40 -97
  58. package/src/runtime/execution-plan.ts +23 -19
  59. package/src/runtime/graph-designer.ts +15 -0
  60. package/src/runtime/helper-model.ts +10 -196
  61. package/src/runtime/index.ts +14 -1
  62. package/src/runtime/llm-content.ts +1 -1
  63. package/src/runtime/memory-block.ts +11 -12
  64. package/src/runtime/memory-pipeline.ts +26 -10
  65. package/src/runtime/plugin-resolution.ts +35 -0
  66. package/src/runtime/plugin-types.ts +73 -1
  67. package/src/runtime/retrieval-adapters.ts +1 -1
  68. package/src/runtime/runtime-config.ts +25 -12
  69. package/src/runtime/runtime-extensions.ts +91 -15
  70. package/src/runtime/runtime-worker-registry.ts +6 -0
  71. package/src/runtime/team-consultation-orchestrator.ts +45 -28
  72. package/src/runtime/team-consultation-prompts.ts +11 -2
  73. package/src/runtime/title-helpers.ts +11 -4
  74. package/src/runtime/workstream-chat-helpers.ts +6 -7
  75. package/src/runtime/workstream-routing-policy.ts +0 -30
  76. package/src/runtime/workstream-state.ts +17 -7
  77. package/src/services/adaptive-playbook.service.ts +152 -0
  78. package/src/services/agent-executor.service.ts +293 -0
  79. package/src/services/artifact-provenance.service.ts +172 -0
  80. package/src/services/attachment.service.ts +7 -12
  81. package/src/services/context-compaction.service.ts +75 -58
  82. package/src/services/context-enrichment.service.ts +33 -0
  83. package/src/services/coordination-registry.service.ts +117 -0
  84. package/src/services/document-chunk.service.ts +38 -33
  85. package/src/services/domain-agent-executor.service.ts +71 -0
  86. package/src/services/execution-plan.service.ts +271 -50
  87. package/src/services/feedback-loop.service.ts +96 -0
  88. package/src/services/global-orchestrator.service.ts +148 -0
  89. package/src/services/index.ts +26 -0
  90. package/src/services/institutional-memory.service.ts +145 -0
  91. package/src/services/learned-skill.service.ts +30 -15
  92. package/src/services/memory-assessment.service.ts +3 -2
  93. package/src/services/{memory.utils.ts → memory-utils.ts} +4 -13
  94. package/src/services/memory.service.ts +55 -69
  95. package/src/services/monitoring-window.service.ts +86 -0
  96. package/src/services/mutating-approval.service.ts +1 -1
  97. package/src/services/node-workspace.service.ts +155 -0
  98. package/src/services/notification.service.ts +39 -0
  99. package/src/services/organization-member.service.ts +12 -5
  100. package/src/services/organization.service.ts +5 -5
  101. package/src/services/ownership-dispatcher.service.ts +403 -0
  102. package/src/services/plan-approval.service.ts +1 -1
  103. package/src/services/plan-artifact.service.ts +1 -0
  104. package/src/services/plan-builder.service.ts +1 -0
  105. package/src/services/plan-checkpoint.service.ts +30 -2
  106. package/src/services/plan-compiler.service.ts +5 -0
  107. package/src/services/plan-coordination.service.ts +152 -0
  108. package/src/services/plan-cycle.service.ts +284 -0
  109. package/src/services/plan-deadline.service.ts +287 -0
  110. package/src/services/plan-executor.service.ts +386 -58
  111. package/src/services/plan-helpers.ts +15 -0
  112. package/src/services/plan-run.service.ts +41 -7
  113. package/src/services/plan-scheduler.service.ts +240 -0
  114. package/src/services/plan-template.service.ts +117 -0
  115. package/src/services/plan-validator.service.ts +87 -20
  116. package/src/services/plan-workspace.service.ts +83 -0
  117. package/src/services/playbook-registry.service.ts +67 -0
  118. package/src/services/plugin-executor.service.ts +103 -0
  119. package/src/services/quality-metrics.service.ts +132 -0
  120. package/src/services/recent-activity-title.service.ts +3 -10
  121. package/src/services/recent-activity.service.ts +33 -43
  122. package/src/services/skill-resolver.service.ts +19 -0
  123. package/src/services/system-executor.service.ts +105 -0
  124. package/src/services/workstream-message.service.ts +29 -41
  125. package/src/services/workstream-plan-registry.service.ts +22 -0
  126. package/src/services/workstream-title.service.ts +3 -9
  127. package/src/services/{workstream-turn-preparation.ts → workstream-turn-preparation.service.ts} +428 -373
  128. package/src/services/workstream-turn.ts +2 -2
  129. package/src/services/workstream.service.ts +55 -65
  130. package/src/services/workstream.types.ts +10 -19
  131. package/src/services/write-intent-validator.service.ts +81 -0
  132. package/src/storage/attachment-parser.ts +1 -1
  133. package/src/storage/attachment-storage.service.ts +4 -4
  134. package/src/storage/{attachments.utils.ts → attachment-utils.ts} +2 -5
  135. package/src/storage/generated-document-storage.service.ts +3 -2
  136. package/src/storage/index.ts +2 -2
  137. package/src/system-agents/{context-compacter.agent.ts → context-compaction.agent.ts} +4 -4
  138. package/src/system-agents/delegated-agent-factory.ts +5 -2
  139. package/src/system-agents/index.ts +8 -0
  140. package/src/system-agents/memory-reranker.agent.ts +1 -1
  141. package/src/system-agents/memory.agent.ts +1 -1
  142. package/src/system-agents/recent-activity-title-refiner.agent.ts +1 -1
  143. package/src/tools/execution-plan.tool.ts +17 -19
  144. package/src/tools/fetch-webpage.tool.ts +20 -18
  145. package/src/tools/index.ts +2 -3
  146. package/src/tools/read-file-parts.tool.ts +1 -1
  147. package/src/tools/search-web.tool.ts +18 -15
  148. package/src/tools/{search-tools.ts → search.tool.ts} +1 -1
  149. package/src/tools/team-think.tool.ts +14 -8
  150. package/src/tools/{tool-contract.ts → tool-contracts.ts} +9 -2
  151. package/src/utils/async.ts +3 -2
  152. package/src/utils/date-time.ts +4 -32
  153. package/src/utils/env.ts +8 -0
  154. package/src/utils/errors.ts +47 -0
  155. package/src/utils/hono-error-handler.ts +1 -2
  156. package/src/utils/index.ts +19 -2
  157. package/src/utils/string.ts +128 -1
  158. package/src/workers/bootstrap.ts +2 -2
  159. package/src/workers/index.ts +1 -0
  160. package/src/workers/memory-consolidation.worker.ts +12 -12
  161. package/src/workers/regular-chat-memory-digest.helpers.ts +2 -7
  162. package/src/workers/regular-chat-memory-digest.runner.ts +11 -105
  163. package/src/workers/skill-extraction.runner.ts +8 -102
  164. package/src/workers/utils/file-section-chunker.ts +6 -3
  165. package/src/workers/utils/repomix-file-sections.ts +2 -2
  166. package/src/workers/utils/sandbox-error.ts +11 -2
  167. package/src/workers/utils/workstream-message-query.ts +97 -0
  168. package/src/workers/worker-utils.ts +6 -2
  169. package/src/runtime/retrieval-pipeline.ts +0 -3
  170. package/src/runtime.ts +0 -387
  171. package/src/tools/log-hello-world.tool.ts +0 -17
  172. package/src/utils/error.ts +0 -10
  173. /package/src/services/{context-compaction-runtime.ts → context-compaction-runtime.singleton.ts} +0 -0
  174. /package/src/storage/{attachments.types.ts → attachment-types.ts} +0 -0
@@ -0,0 +1,43 @@
1
+ export type SurrealSetClause = { bindingEntries: Record<string, unknown>; statement: string }
2
+
3
+ export function buildRequiredSurrealSetClause(field: string, bindingName: string, value: unknown): SurrealSetClause {
4
+ return { statement: `${field} = $${bindingName}`, bindingEntries: { [bindingName]: value } }
5
+ }
6
+
7
+ export function buildOptionalSurrealSetClause(field: string, value: unknown): SurrealSetClause
8
+ export function buildOptionalSurrealSetClause(field: string, bindingName: string, value: unknown): SurrealSetClause
9
+ export function buildOptionalSurrealSetClause(
10
+ field: string,
11
+ bindingNameOrValue: unknown,
12
+ value?: unknown,
13
+ ): SurrealSetClause {
14
+ const hasExplicitBinding = arguments.length === 3
15
+ const resolvedValue = hasExplicitBinding ? value : bindingNameOrValue
16
+ const resolvedBindingName = hasExplicitBinding
17
+ ? (bindingNameOrValue as string)
18
+ : `resource_${field.replace(/[^a-zA-Z0-9]+/g, '_')}`
19
+
20
+ if (resolvedValue === null || resolvedValue === undefined) {
21
+ return { statement: `${field} = NONE`, bindingEntries: {} }
22
+ }
23
+
24
+ return buildRequiredSurrealSetClause(field, resolvedBindingName, resolvedValue)
25
+ }
26
+
27
+ export function combineSurrealSetClauses(...clauses: SurrealSetClause[]): {
28
+ bindings: Record<string, unknown>
29
+ statement: string
30
+ } {
31
+ const bindings: Record<string, unknown> = {}
32
+ for (const clause of clauses) {
33
+ for (const [key, value] of Object.entries(clause.bindingEntries)) {
34
+ bindings[key] = value
35
+ }
36
+ }
37
+
38
+ return { bindings, statement: clauses.map((clause) => clause.statement).join(', ') }
39
+ }
40
+
41
+ export function compactDefinedRecord<T extends Record<string, unknown>>(value: T): Partial<T> {
42
+ return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined)) as Partial<T>
43
+ }
package/src/db/tables.ts CHANGED
@@ -17,11 +17,18 @@ export const TABLES = {
17
17
  PLAN_CHECKPOINT: 'planCheckpoint',
18
18
  PLAN_VALIDATION_ISSUE: 'planValidationIssue',
19
19
  PLAN_EVENT: 'planEvent',
20
+ PLAN_SCHEDULE: 'planSchedule',
21
+ PLAN_TEMPLATE: 'planTemplate',
22
+ PLAN_CYCLE: 'planCycle',
20
23
  ORGANIZATION: 'organization',
21
24
  ORGANIZATION_MEMBER: 'organizationMember',
22
25
  USER: 'user',
23
26
  RECENT_ACTIVITY_EVENT: 'recentActivityEvent',
24
27
  RECENT_ACTIVITY: 'recentActivity',
28
+ PLAYBOOK: 'playbook',
29
+ PLAYBOOK_VERSION: 'playbookVersion',
30
+ INSTITUTIONAL_MEMORY: 'institutionalMemory',
31
+ QUALITY_METRIC: 'qualityMetric',
25
32
  } as const
26
33
 
27
34
  export type DatabaseTable = (typeof TABLES)[keyof typeof TABLES] | (string & {})
@@ -0,0 +1,15 @@
1
+ import { recordIdSchema } from '@lota-sdk/shared'
2
+ import { z } from 'zod'
3
+
4
+ export const WorkstreamMessageRowSchema = z.object({
5
+ id: recordIdSchema,
6
+ workstreamId: recordIdSchema,
7
+ messageId: z.string(),
8
+ role: z.enum(['system', 'user', 'assistant']),
9
+ parts: z.array(z.record(z.string(), z.unknown())).optional(),
10
+ metadata: z.record(z.string(), z.unknown()).nullish(),
11
+ createdAt: z.coerce.date(),
12
+ updatedAt: z.coerce.date().optional(),
13
+ })
14
+
15
+ export type WorkstreamMessageRow = z.infer<typeof WorkstreamMessageRowSchema>
@@ -71,7 +71,7 @@ export class ProviderEmbeddings {
71
71
  const redisCache = this.getCache()
72
72
  if (!redisCache) return null
73
73
 
74
- return await redisCache.get(this.getModelId(), text)
74
+ return redisCache.get(this.getModelId(), text)
75
75
  }
76
76
 
77
77
  async embedQuery(text: string): Promise<number[]> {
package/src/index.ts CHANGED
@@ -1,4 +1,4 @@
1
- export * from './runtime'
1
+ export * from './create-runtime'
2
2
  export * from './ai'
3
3
  export * from './bifrost'
4
4
  export * from './config'
@@ -1,20 +1,11 @@
1
- import { Queue, Worker } from 'bullmq'
2
1
  import type { Job } from 'bullmq'
3
2
 
4
- import { serverLogger } from '../config/logger'
5
3
  import { ensureRecordId } from '../db/record-id'
6
4
  import { databaseService } from '../db/service'
7
5
  import { TABLES } from '../db/tables'
8
- import { getRedisConnectionForBullMQ } from '../redis'
9
6
  import { contextCompactionService } from '../services/context-compaction.service'
10
7
  import { workstreamService } from '../services/workstream.service'
11
- import {
12
- attachWorkerEvents,
13
- createTracedWorkerProcessor,
14
- createWorkerShutdown,
15
- registerShutdownSignals,
16
- } from '../workers/worker-utils'
17
- import type { WorkerHandle } from '../workers/worker-utils'
8
+ import { createQueueFactory } from './queue-factory'
18
9
 
19
10
  interface ContextCompactionJob {
20
11
  domain: 'workstream'
@@ -22,61 +13,35 @@ interface ContextCompactionJob {
22
13
  contextSize?: number
23
14
  }
24
15
 
25
- const CONTEXT_COMPACTION_QUEUE = 'context-compaction'
26
-
27
- let _contextCompactionQueue: Queue<ContextCompactionJob> | null = null
28
- function getContextCompactionQueue(): Queue<ContextCompactionJob> {
29
- if (!_contextCompactionQueue) {
30
- _contextCompactionQueue = new Queue<ContextCompactionJob>(CONTEXT_COMPACTION_QUEUE, {
31
- connection: getRedisConnectionForBullMQ(),
32
- defaultJobOptions: {
33
- removeOnComplete: 200,
34
- removeOnFail: 200,
35
- attempts: 2,
36
- backoff: { type: 'exponential', delay: 3_000 },
37
- },
38
- })
39
- }
40
- return _contextCompactionQueue
41
- }
42
-
43
- export async function enqueueContextCompaction(job: ContextCompactionJob) {
44
- return await getContextCompactionQueue().add('compact', job, {
45
- deduplication: { id: `compact:${job.domain}:${job.entityId}` },
46
- })
47
- }
48
-
49
16
  async function processContextCompactionJob(job: Job<ContextCompactionJob>): Promise<void> {
50
17
  await databaseService.connect()
51
18
 
52
19
  const { entityId, contextSize } = job.data
53
20
  const workstreamRef = ensureRecordId(entityId, TABLES.WORKSTREAM)
54
- await workstreamService.setCompacting(workstreamRef, true)
21
+ await workstreamService.markCompacting(workstreamRef)
55
22
  try {
56
23
  await contextCompactionService.compactWorkstreamHistory({ workstreamId: workstreamRef, contextSize })
57
24
  } finally {
58
- await workstreamService.setCompacting(workstreamRef, false)
25
+ await workstreamService.clearCompacting(workstreamRef)
59
26
  }
60
27
  }
61
28
 
62
- export function startContextCompactionWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
63
- const { registerSignals = import.meta.main } = options
64
- const worker = new Worker(
65
- CONTEXT_COMPACTION_QUEUE,
66
- createTracedWorkerProcessor(CONTEXT_COMPACTION_QUEUE, processContextCompactionJob),
67
- { connection: getRedisConnectionForBullMQ(), concurrency: 2, lockDuration: 300_000 },
68
- )
69
-
70
- attachWorkerEvents(worker, 'Context compaction', serverLogger)
71
- const shutdown = createWorkerShutdown(worker, 'Context compaction', serverLogger)
72
-
73
- if (registerSignals) {
74
- registerShutdownSignals({ name: 'Context compaction', shutdown, logger: serverLogger })
75
- }
76
-
77
- return { worker, shutdown }
29
+ const contextCompaction = createQueueFactory<ContextCompactionJob>({
30
+ name: 'context-compaction',
31
+ displayName: 'Context compaction',
32
+ jobName: 'compact',
33
+ concurrency: 2,
34
+ lockDuration: 300_000,
35
+ defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 3_000 } },
36
+ processor: processContextCompactionJob,
37
+ })
38
+
39
+ export function enqueueContextCompaction(job: ContextCompactionJob) {
40
+ return contextCompaction.enqueue(job, { deduplication: { id: `compact:${job.domain}:${job.entityId}` } })
78
41
  }
79
42
 
43
+ export const startContextCompactionWorker = contextCompaction.startWorker
44
+
80
45
  if (import.meta.main) {
81
46
  startContextCompactionWorker()
82
47
  }
@@ -0,0 +1,41 @@
1
+ import type { Job } from 'bullmq'
2
+
3
+ import { databaseService } from '../db/service'
4
+ import { planExecutorService } from '../services/plan-executor.service'
5
+ import { createQueueFactory } from './queue-factory'
6
+
7
+ export interface DelayedNodePromotionJob {
8
+ runId: string
9
+ nodeId: string
10
+ emittedBy: string
11
+ }
12
+
13
+ export const DELAYED_NODE_PROMOTION_QUEUE = 'delayed-node-promotion'
14
+
15
+ async function processDelayedNodePromotionJob(job: Job<DelayedNodePromotionJob>): Promise<void> {
16
+ await databaseService.connect()
17
+ await planExecutorService.promoteDelayedNode({
18
+ runId: job.data.runId,
19
+ nodeId: job.data.nodeId,
20
+ emittedBy: job.data.emittedBy,
21
+ })
22
+ }
23
+
24
+ const delayedNodePromotion = createQueueFactory<DelayedNodePromotionJob>({
25
+ name: DELAYED_NODE_PROMOTION_QUEUE,
26
+ displayName: 'Delayed node promotion',
27
+ jobName: 'promote-node',
28
+ concurrency: 1,
29
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
30
+ processor: processDelayedNodePromotionJob,
31
+ })
32
+
33
+ export async function enqueueDelayedNodePromotion(job: DelayedNodePromotionJob, delayMs: number) {
34
+ await delayedNodePromotion.enqueue(job, { delay: delayMs, jobId: `promote:${job.runId}:${job.nodeId}` })
35
+ }
36
+
37
+ export const startDelayedNodePromotionWorker = delayedNodePromotion.startWorker
38
+
39
+ if (import.meta.main) {
40
+ startDelayedNodePromotionWorker()
41
+ }
@@ -4,7 +4,12 @@ import { Queue, Worker } from 'bullmq'
4
4
  import type IORedis from 'ioredis'
5
5
 
6
6
  import type { chatLogger } from '../config/logger'
7
- import { attachWorkerEvents, createWorkerShutdown, registerShutdownSignals } from '../workers/worker-utils'
7
+ import {
8
+ attachWorkerEvents,
9
+ createWorkerShutdown,
10
+ DEFAULT_JOB_RETENTION,
11
+ registerShutdownSignals,
12
+ } from '../workers/worker-utils'
8
13
  import type { WorkerHandle } from '../workers/worker-utils'
9
14
 
10
15
  export type DocumentSourceChannel = string
@@ -82,12 +87,7 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
82
87
 
83
88
  queue = new Queue<TJob, unknown, string>(queueName, {
84
89
  connection: params.getConnectionForBullMQ(),
85
- defaultJobOptions: {
86
- removeOnComplete: 200,
87
- removeOnFail: 200,
88
- attempts: 3,
89
- backoff: { type: 'exponential', delay: 1000 },
90
- },
90
+ defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
91
91
  })
92
92
 
93
93
  return queue
@@ -1,6 +1,9 @@
1
+ export * from './queue-factory'
1
2
  export * from './context-compaction.queue'
3
+ export * from './delayed-node-promotion.queue'
2
4
  export * from './document-processor.queue'
3
5
  export * from './memory-consolidation.queue'
6
+ export * from './plan-scheduler.queue'
4
7
  export * from './post-chat-memory.queue'
5
8
  export * from './recent-activity-title-refinement.queue'
6
9
  export * from './regular-chat-memory-digest.config'
@@ -1,70 +1,34 @@
1
- import { Queue, Worker } from 'bullmq'
2
-
3
- import { serverLogger } from '../config/logger'
4
- import { getRedisConnectionForBullMQ } from '../redis'
5
- import {
6
- attachWorkerEvents,
7
- getWorkerPath,
8
- createWorkerShutdown,
9
- registerShutdownSignals,
10
- } from '../workers/worker-utils'
11
- import type { WorkerHandle } from '../workers/worker-utils'
1
+ import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS, LOW_JOB_RETENTION } from '../workers/worker-utils'
2
+ import { createQueueFactory } from './queue-factory'
12
3
 
13
4
  export interface MemoryConsolidationJob {
14
5
  scopeId?: string
15
6
  }
16
7
 
17
- const MEMORY_CONSOLIDATION_QUEUE = 'memory-consolidation'
8
+ const MEMORY_CONSOLIDATION_INTERVAL_MS = 24 * 60 * 60 * 1000
9
+ const MEMORY_CONSOLIDATION_JOB_ID = 'memory-consolidation-recurring'
18
10
 
19
- let _memoryConsolidationQueue: Queue<MemoryConsolidationJob> | null = null
20
- function getMemoryConsolidationQueue(): Queue<MemoryConsolidationJob> {
21
- if (!_memoryConsolidationQueue) {
22
- _memoryConsolidationQueue = new Queue<MemoryConsolidationJob>(MEMORY_CONSOLIDATION_QUEUE, {
23
- connection: getRedisConnectionForBullMQ(),
24
- defaultJobOptions: {
25
- removeOnComplete: 50,
26
- removeOnFail: 50,
27
- attempts: 2,
28
- backoff: { type: 'exponential', delay: 5000 },
29
- },
30
- })
31
- }
32
- return _memoryConsolidationQueue
33
- }
11
+ const memoryConsolidation = createQueueFactory<MemoryConsolidationJob>({
12
+ name: 'memory-consolidation',
13
+ displayName: 'Memory consolidation',
14
+ jobName: 'consolidate-turn',
15
+ concurrency: 1,
16
+ lockDuration: LONG_JOB_LOCK_DURATION_MS,
17
+ defaultJobOptions: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
18
+ processorPath: getWorkerPath('memory-consolidation.worker.ts'),
19
+ })
34
20
 
35
21
  export async function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
36
- await getMemoryConsolidationQueue().add('consolidate-turn', job, {
37
- jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined,
38
- })
22
+ await memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
39
23
  }
40
24
 
41
25
  export async function scheduleRecurringConsolidation() {
42
- await getMemoryConsolidationQueue().add(
43
- 'consolidate',
44
- {},
45
- { repeat: { every: 24 * 60 * 60 * 1000 }, jobId: 'memory-consolidation-recurring' },
46
- )
26
+ await memoryConsolidation
27
+ .getQueue()
28
+ .add('consolidate', {}, { repeat: { every: MEMORY_CONSOLIDATION_INTERVAL_MS }, jobId: MEMORY_CONSOLIDATION_JOB_ID })
47
29
  }
48
30
 
49
- export function startMemoryConsolidationWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
50
- const { registerSignals = import.meta.main } = options
51
- const processorPath = getWorkerPath('memory-consolidation.worker.ts')
52
- const worker = new Worker(MEMORY_CONSOLIDATION_QUEUE, processorPath, {
53
- connection: getRedisConnectionForBullMQ(),
54
- lockDuration: 600_000,
55
- concurrency: 1,
56
- })
57
-
58
- attachWorkerEvents(worker, 'Memory consolidation', serverLogger)
59
-
60
- const shutdown = createWorkerShutdown(worker, 'Memory consolidation', serverLogger)
61
-
62
- if (registerSignals) {
63
- registerShutdownSignals({ name: 'Memory consolidation', shutdown, logger: serverLogger })
64
- }
65
-
66
- return { worker, shutdown }
67
- }
31
+ export const startMemoryConsolidationWorker = memoryConsolidation.startWorker
68
32
 
69
33
  if (import.meta.main) {
70
34
  startMemoryConsolidationWorker()
@@ -0,0 +1,97 @@
1
+ import type { Job } from 'bullmq'
2
+
3
+ import { serverLogger } from '../config/logger'
4
+ import { databaseService } from '../db/service'
5
+ import { planDeadlineService } from '../services/plan-deadline.service'
6
+ import { planSchedulerService } from '../services/plan-scheduler.service'
7
+ import type { WorkerHandle } from '../workers/worker-utils'
8
+ import { createQueueFactory } from './queue-factory'
9
+
10
+ export interface PlanSchedulerFireJob {
11
+ type: 'fire-schedule'
12
+ scheduleId: string
13
+ }
14
+
15
+ export interface PlanSchedulerDeadlineJob {
16
+ type: 'check-deadlines'
17
+ scheduledFor?: string
18
+ }
19
+
20
+ export type PlanSchedulerJob = PlanSchedulerFireJob | PlanSchedulerDeadlineJob
21
+
22
+ export const PLAN_SCHEDULER_QUEUE = 'plan-scheduler'
23
+
24
+ async function processPlanSchedulerJob(job: Job<PlanSchedulerJob>): Promise<void> {
25
+ await databaseService.connect()
26
+
27
+ switch (job.data.type) {
28
+ case 'fire-schedule':
29
+ await planSchedulerService.fireScheduleById(job.data.scheduleId)
30
+ break
31
+ case 'check-deadlines':
32
+ await planDeadlineService.checkDeadlines()
33
+ break
34
+ }
35
+ }
36
+
37
+ const planScheduler = createQueueFactory<PlanSchedulerJob>({
38
+ name: PLAN_SCHEDULER_QUEUE,
39
+ displayName: 'Plan scheduler',
40
+ jobName: 'plan-scheduler-job',
41
+ concurrency: 1,
42
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
43
+ processor: processPlanSchedulerJob,
44
+ })
45
+
46
+ /** Enqueue a delayed job that fires a specific schedule at its nextFireAt time. */
47
+ export async function enqueueScheduleFire(scheduleId: string, delayMs: number): Promise<void> {
48
+ await planScheduler.enqueue(
49
+ { type: 'fire-schedule', scheduleId },
50
+ { delay: Math.max(0, delayMs), jobId: `schedule:${scheduleId}`, removeOnComplete: true, removeOnFail: 50 },
51
+ )
52
+ }
53
+
54
+ /** Remove a pending fire job for a schedule (on cancel/pause/complete). */
55
+ export async function removeScheduleFireJob(scheduleId: string): Promise<void> {
56
+ try {
57
+ await planScheduler.getQueue().remove(`schedule:${scheduleId}`)
58
+ } catch {
59
+ // Job may not exist (already fired or never enqueued) — safe to ignore
60
+ }
61
+ }
62
+
63
+ const DEADLINE_CHECK_JOB_PREFIX = 'deadline-check'
64
+
65
+ export async function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
66
+ const delay = Math.max(0, scheduledFor.getTime() - Date.now())
67
+ await planScheduler.enqueue(
68
+ { type: 'check-deadlines', scheduledFor: scheduledFor.toISOString() },
69
+ {
70
+ delay,
71
+ jobId: `${DEADLINE_CHECK_JOB_PREFIX}:${scheduledFor.getTime()}`,
72
+ removeOnComplete: true,
73
+ removeOnFail: 50,
74
+ },
75
+ )
76
+ }
77
+
78
+ export function startPlanSchedulerWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
79
+ const handle = planScheduler.startWorker(options)
80
+
81
+ // Recover active schedules on startup
82
+ planSchedulerService.recoverActiveSchedules().catch((err: unknown) => {
83
+ serverLogger.error`Plan scheduler startup recovery failed: ${err}`
84
+ })
85
+
86
+ // Seed deadline checks from current runtime state. Subsequent checks are
87
+ // re-seeded by the queue job after each sweep and by schedule fire events.
88
+ planDeadlineService.recoverDeadlineChecks().catch((err: unknown) => {
89
+ serverLogger.error`Plan deadline recovery failed: ${err}`
90
+ })
91
+
92
+ return handle
93
+ }
94
+
95
+ if (import.meta.main) {
96
+ startPlanSchedulerWorker()
97
+ }
@@ -1,18 +1,9 @@
1
1
  type OrganizationOnboardStatus = string
2
- import { Queue, Worker } from 'bullmq'
3
2
  import type { Job } from 'bullmq'
4
3
 
5
- import { serverLogger } from '../config/logger'
6
4
  import { databaseService } from '../db/service'
7
- import { getRedisConnectionForBullMQ } from '../redis'
8
5
  import { memoryService } from '../services/memory.service'
9
- import {
10
- attachWorkerEvents,
11
- createTracedWorkerProcessor,
12
- createWorkerShutdown,
13
- registerShutdownSignals,
14
- } from '../workers/worker-utils'
15
- import type { WorkerHandle } from '../workers/worker-utils'
6
+ import { createQueueFactory } from './queue-factory'
16
7
 
17
8
  interface PostChatMemoryMessage {
18
9
  role: 'user' | 'agent'
@@ -32,32 +23,6 @@ interface PostChatMemoryExtractionJob {
32
23
  attachmentContext?: string
33
24
  }
34
25
 
35
- const POST_CHAT_MEMORY_QUEUE = 'post-chat-memory'
36
- const POST_CHAT_MEMORY_CONCURRENCY = 3
37
- const POST_CHAT_MEMORY_LOCK_DURATION_MS = 900_000
38
- const POST_CHAT_MEMORY_MAX_STALLED_COUNT = 10
39
- const POST_CHAT_MEMORY_STALLED_INTERVAL_MS = 120_000
40
-
41
- let _postChatMemoryQueue: Queue<PostChatMemoryExtractionJob> | null = null
42
- function getPostChatMemoryQueue(): Queue<PostChatMemoryExtractionJob> {
43
- if (!_postChatMemoryQueue) {
44
- _postChatMemoryQueue = new Queue<PostChatMemoryExtractionJob>(POST_CHAT_MEMORY_QUEUE, {
45
- connection: getRedisConnectionForBullMQ(),
46
- defaultJobOptions: {
47
- removeOnComplete: 200,
48
- removeOnFail: 200,
49
- attempts: 3,
50
- backoff: { type: 'exponential', delay: 2_000 },
51
- },
52
- })
53
- }
54
- return _postChatMemoryQueue
55
- }
56
-
57
- export async function enqueuePostChatMemory(job: PostChatMemoryExtractionJob) {
58
- return await getPostChatMemoryQueue().add('extract-memory', job)
59
- }
60
-
61
26
  async function processPostChatMemoryJob(job: Job<PostChatMemoryExtractionJob>): Promise<void> {
62
27
  await databaseService.connect()
63
28
 
@@ -98,30 +63,20 @@ async function processPostChatMemoryJob(job: Job<PostChatMemoryExtractionJob>):
98
63
  })
99
64
  }
100
65
 
101
- export function startPostChatMemoryWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
102
- const { registerSignals = import.meta.main } = options
103
- const worker = new Worker(
104
- POST_CHAT_MEMORY_QUEUE,
105
- createTracedWorkerProcessor(POST_CHAT_MEMORY_QUEUE, processPostChatMemoryJob),
106
- {
107
- connection: getRedisConnectionForBullMQ(),
108
- concurrency: POST_CHAT_MEMORY_CONCURRENCY,
109
- lockDuration: POST_CHAT_MEMORY_LOCK_DURATION_MS,
110
- maxStalledCount: POST_CHAT_MEMORY_MAX_STALLED_COUNT,
111
- stalledInterval: POST_CHAT_MEMORY_STALLED_INTERVAL_MS,
112
- },
113
- )
114
-
115
- attachWorkerEvents(worker, 'Post-chat memory', serverLogger)
116
-
117
- const shutdown = createWorkerShutdown(worker, 'Post-chat memory', serverLogger)
118
-
119
- if (registerSignals) {
120
- registerShutdownSignals({ name: 'Post-chat memory', shutdown, logger: serverLogger })
121
- }
122
-
123
- return { worker, shutdown }
124
- }
66
+ const postChatMemory = createQueueFactory<PostChatMemoryExtractionJob>({
67
+ name: 'post-chat-memory',
68
+ displayName: 'Post-chat memory',
69
+ jobName: 'extract-memory',
70
+ concurrency: 3,
71
+ lockDuration: 900_000,
72
+ maxStalledCount: 10,
73
+ stalledInterval: 120_000,
74
+ defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
75
+ processor: processPostChatMemoryJob,
76
+ })
77
+
78
+ export const enqueuePostChatMemory = postChatMemory.enqueue
79
+ export const startPostChatMemoryWorker = postChatMemory.startWorker
125
80
 
126
81
  if (import.meta.main) {
127
82
  startPostChatMemoryWorker()
@@ -0,0 +1,100 @@
1
+ import { Queue, Worker } from 'bullmq'
2
+ import type { Job, JobsOptions, WorkerOptions } from 'bullmq'
3
+ import type IORedis from 'ioredis'
4
+
5
+ import { serverLogger } from '../config/logger'
6
+ import { getRedisConnectionForBullMQ } from '../redis'
7
+ import {
8
+ attachWorkerEvents,
9
+ createTracedWorkerProcessor,
10
+ createWorkerShutdown,
11
+ DEFAULT_JOB_RETENTION,
12
+ registerShutdownSignals,
13
+ } from '../workers/worker-utils'
14
+ import type { WorkerHandle } from '../workers/worker-utils'
15
+
16
+ interface QueueFactoryConfigBase {
17
+ name: string
18
+ displayName: string
19
+ jobName: string
20
+ concurrency: number
21
+ lockDuration?: number
22
+ stalledInterval?: number
23
+ maxStalledCount?: number
24
+ defaultJobOptions?: JobsOptions
25
+ connectionProvider?: () => IORedis
26
+ }
27
+
28
+ interface QueueFactoryConfigInline<TJob> extends QueueFactoryConfigBase {
29
+ processor: (job: Job<TJob>) => Promise<void>
30
+ processorPath?: never
31
+ }
32
+
33
+ interface QueueFactoryConfigFile extends QueueFactoryConfigBase {
34
+ processor?: never
35
+ processorPath: string
36
+ }
37
+
38
+ export type QueueFactoryConfig<TJob> = QueueFactoryConfigInline<TJob> | QueueFactoryConfigFile
39
+
40
+ export interface QueueFactory<TJob> {
41
+ getQueue: () => Queue<TJob, unknown, string>
42
+ enqueue: (job: TJob, options?: JobsOptions) => Promise<void>
43
+ startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle
44
+ }
45
+
46
+ export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): QueueFactory<TJob> {
47
+ let _queue: Queue<TJob, unknown, string> | null = null
48
+
49
+ const getConnection = () => config.connectionProvider?.() ?? getRedisConnectionForBullMQ()
50
+
51
+ const getQueue = (): Queue<TJob, unknown, string> => {
52
+ if (!_queue) {
53
+ _queue = new Queue<TJob, unknown, string>(config.name, {
54
+ connection: getConnection(),
55
+ defaultJobOptions: { ...DEFAULT_JOB_RETENTION, ...config.defaultJobOptions },
56
+ })
57
+ }
58
+ return _queue
59
+ }
60
+
61
+ type QueueAdd = Queue<TJob, unknown, string>['add']
62
+ const jobName = config.jobName as Parameters<QueueAdd>[0]
63
+ const toData = (job: TJob) => job as Parameters<QueueAdd>[1]
64
+
65
+ const enqueue = async (job: TJob, options?: JobsOptions): Promise<void> => {
66
+ await getQueue().add(jobName, toData(job), options)
67
+ }
68
+
69
+ const startWorker = (options: { registerSignals?: boolean } = {}): WorkerHandle => {
70
+ const { registerSignals = import.meta.main } = options
71
+
72
+ const workerOptions: WorkerOptions = {
73
+ connection: getConnection(),
74
+ concurrency: config.concurrency,
75
+ ...(config.lockDuration !== undefined ? { lockDuration: config.lockDuration } : {}),
76
+ ...(config.stalledInterval !== undefined ? { stalledInterval: config.stalledInterval } : {}),
77
+ ...(config.maxStalledCount !== undefined ? { maxStalledCount: config.maxStalledCount } : {}),
78
+ }
79
+
80
+ const worker = config.processorPath
81
+ ? new Worker(config.name, config.processorPath, workerOptions)
82
+ : new Worker(
83
+ config.name,
84
+ createTracedWorkerProcessor(config.name, (config as QueueFactoryConfigInline<TJob>).processor),
85
+ workerOptions,
86
+ )
87
+
88
+ attachWorkerEvents(worker, config.displayName, serverLogger)
89
+
90
+ const shutdown = createWorkerShutdown(worker, config.displayName, serverLogger)
91
+
92
+ if (registerSignals) {
93
+ registerShutdownSignals({ name: config.displayName, shutdown, logger: serverLogger })
94
+ }
95
+
96
+ return { worker, shutdown }
97
+ }
98
+
99
+ return { getQueue, enqueue, startWorker }
100
+ }