@lota-sdk/core 0.4.8 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/package.json +11 -12
  2. package/src/ai/embedding-cache.ts +94 -22
  3. package/src/ai-gateway/ai-gateway.ts +738 -223
  4. package/src/config/agent-defaults.ts +176 -75
  5. package/src/config/agent-types.ts +54 -4
  6. package/src/config/constants.ts +8 -2
  7. package/src/config/logger.ts +286 -19
  8. package/src/config/thread-defaults.ts +33 -21
  9. package/src/create-runtime.ts +725 -387
  10. package/src/db/base.service.ts +52 -28
  11. package/src/db/cursor-pagination.ts +71 -30
  12. package/src/db/memory-store.helpers.ts +4 -7
  13. package/src/db/memory-store.ts +856 -598
  14. package/src/db/memory.ts +398 -275
  15. package/src/db/record-id.ts +32 -10
  16. package/src/db/schema-fingerprint.ts +30 -12
  17. package/src/db/service-normalization.ts +255 -0
  18. package/src/db/service.ts +726 -761
  19. package/src/db/startup.ts +140 -66
  20. package/src/db/transaction-conflict.ts +15 -0
  21. package/src/effect/awaitable-effect.ts +87 -0
  22. package/src/effect/errors.ts +121 -0
  23. package/src/effect/helpers.ts +98 -0
  24. package/src/effect/index.ts +22 -0
  25. package/src/effect/layers.ts +228 -0
  26. package/src/effect/runtime-ref.ts +25 -0
  27. package/src/effect/runtime.ts +31 -0
  28. package/src/effect/services.ts +57 -0
  29. package/src/effect/zod.ts +43 -0
  30. package/src/embeddings/provider.ts +122 -76
  31. package/src/index.ts +46 -1
  32. package/src/openrouter/direct-provider.ts +11 -35
  33. package/src/queues/autonomous-job.queue.ts +130 -74
  34. package/src/queues/context-compaction.queue.ts +60 -15
  35. package/src/queues/delayed-node-promotion.queue.ts +52 -15
  36. package/src/queues/document-processor.queue.ts +52 -77
  37. package/src/queues/memory-consolidation.queue.ts +47 -32
  38. package/src/queues/organization-learning.queue.ts +13 -4
  39. package/src/queues/plan-agent-heartbeat.queue.ts +65 -21
  40. package/src/queues/plan-scheduler.queue.ts +107 -31
  41. package/src/queues/post-chat-memory.queue.ts +66 -24
  42. package/src/queues/queue-factory.ts +142 -52
  43. package/src/queues/standalone-worker.ts +39 -0
  44. package/src/queues/title-generation.queue.ts +54 -9
  45. package/src/redis/connection.ts +84 -32
  46. package/src/redis/index.ts +6 -8
  47. package/src/redis/org-memory-lock.ts +60 -27
  48. package/src/redis/redis-lease-lock.ts +200 -121
  49. package/src/redis/runtime-connection.ts +10 -0
  50. package/src/redis/stream-context.ts +84 -46
  51. package/src/runtime/agent-identity-overrides.ts +2 -2
  52. package/src/runtime/agent-runtime-policy.ts +4 -1
  53. package/src/runtime/agent-stream-helpers.ts +20 -9
  54. package/src/runtime/chat-run-orchestration.ts +102 -19
  55. package/src/runtime/chat-run-registry.ts +36 -2
  56. package/src/runtime/context-compaction/context-compaction-runtime.ts +107 -0
  57. package/src/runtime/{context-compaction.ts → context-compaction/context-compaction.ts} +114 -91
  58. package/src/runtime/execution-plan-visibility.ts +2 -2
  59. package/src/runtime/execution-plan.ts +42 -15
  60. package/src/runtime/graph-designer.ts +11 -7
  61. package/src/runtime/helper-model.ts +135 -48
  62. package/src/runtime/index.ts +7 -7
  63. package/src/runtime/indexed-repositories-policy.ts +3 -3
  64. package/src/runtime/{memory-block.ts → memory/memory-block.ts} +40 -36
  65. package/src/runtime/{memory-digest-policy.ts → memory/memory-digest-policy.ts} +1 -1
  66. package/src/runtime/{memory-pipeline.ts → memory/memory-pipeline.ts} +1 -1
  67. package/src/runtime/{memory-prompts-fact.ts → memory/memory-prompts-fact.ts} +2 -2
  68. package/src/runtime/{memory-scope.ts → memory/memory-scope.ts} +12 -6
  69. package/src/runtime/plugin-resolution.ts +144 -24
  70. package/src/runtime/plugin-types.ts +9 -1
  71. package/src/runtime/post-turn-side-effects.ts +197 -130
  72. package/src/runtime/retrieval-adapters.ts +38 -4
  73. package/src/runtime/runtime-config.ts +150 -61
  74. package/src/runtime/runtime-extensions.ts +21 -34
  75. package/src/runtime/social-chat/social-chat-agent-runner.ts +157 -0
  76. package/src/runtime/{social-chat-history.ts → social-chat/social-chat-history.ts} +42 -20
  77. package/src/runtime/social-chat/social-chat.ts +594 -0
  78. package/src/runtime/specialist-runner.ts +36 -10
  79. package/src/runtime/team-consultation/team-consultation-orchestrator.ts +427 -0
  80. package/src/runtime/{team-consultation-prompts.ts → team-consultation/team-consultation-prompts.ts} +6 -2
  81. package/src/runtime/thread-chat-helpers.ts +2 -2
  82. package/src/runtime/thread-plan-turn.ts +2 -1
  83. package/src/runtime/thread-turn-context.ts +172 -94
  84. package/src/runtime/turn-lifecycle.ts +93 -27
  85. package/src/services/agent-activity.service.ts +287 -203
  86. package/src/services/agent-executor.service.ts +329 -217
  87. package/src/services/artifact.service.ts +225 -148
  88. package/src/services/attachment.service.ts +137 -115
  89. package/src/services/autonomous-job.service.ts +888 -491
  90. package/src/services/chat-run-registry.service.ts +11 -1
  91. package/src/services/context-compaction.service.ts +136 -86
  92. package/src/services/document-chunk.service.ts +162 -90
  93. package/src/services/execution-plan/execution-plan-approval.ts +26 -0
  94. package/src/services/execution-plan/execution-plan-context.ts +29 -0
  95. package/src/services/execution-plan/execution-plan-graph.ts +256 -0
  96. package/src/services/execution-plan/execution-plan-schedule.ts +84 -0
  97. package/src/services/execution-plan/execution-plan-spec.ts +75 -0
  98. package/src/services/execution-plan/execution-plan.service.ts +1041 -0
  99. package/src/services/feedback-loop.service.ts +132 -76
  100. package/src/services/global-orchestrator.service.ts +80 -170
  101. package/src/services/graph-full-routing.ts +182 -0
  102. package/src/services/index.ts +18 -21
  103. package/src/services/institutional-memory.service.ts +220 -123
  104. package/src/services/learned-skill.service.ts +364 -259
  105. package/src/services/memory/memory-conversation.ts +95 -0
  106. package/src/services/memory/memory-org-memory.ts +39 -0
  107. package/src/services/memory/memory-preseeded.ts +80 -0
  108. package/src/services/memory/memory-rerank.ts +297 -0
  109. package/src/services/{memory-utils.ts → memory/memory-utils.ts} +5 -5
  110. package/src/services/memory/memory.service.ts +692 -0
  111. package/src/services/memory/rerank.service.ts +209 -0
  112. package/src/services/monitoring-window.service.ts +92 -70
  113. package/src/services/mutating-approval.service.ts +62 -53
  114. package/src/services/node-workspace.service.ts +141 -98
  115. package/src/services/notification.service.ts +17 -16
  116. package/src/services/organization-member.service.ts +120 -66
  117. package/src/services/organization.service.ts +144 -51
  118. package/src/services/ownership-dispatcher.service.ts +415 -264
  119. package/src/services/plan/plan-agent-heartbeat.service.ts +234 -0
  120. package/src/services/plan/plan-agent-query.service.ts +322 -0
  121. package/src/services/plan/plan-approval.service.ts +102 -0
  122. package/src/services/plan/plan-artifact.service.ts +60 -0
  123. package/src/services/plan/plan-builder.service.ts +76 -0
  124. package/src/services/plan/plan-checkpoint.service.ts +103 -0
  125. package/src/services/{plan-compiler.service.ts → plan/plan-compiler.service.ts} +26 -9
  126. package/src/services/plan/plan-completion-side-effects.ts +175 -0
  127. package/src/services/plan/plan-coordination.service.ts +181 -0
  128. package/src/services/plan/plan-cycle.service.ts +398 -0
  129. package/src/services/plan/plan-deadline.service.ts +547 -0
  130. package/src/services/plan/plan-event-delivery.service.ts +261 -0
  131. package/src/services/plan/plan-executor-context.ts +35 -0
  132. package/src/services/plan/plan-executor-graph.ts +475 -0
  133. package/src/services/plan/plan-executor-helpers.ts +322 -0
  134. package/src/services/plan/plan-executor-persistence.ts +209 -0
  135. package/src/services/plan/plan-executor.service.ts +1654 -0
  136. package/src/services/{plan-helpers.ts → plan/plan-helpers.ts} +1 -1
  137. package/src/services/{plan-run-data.ts → plan/plan-run-data.ts} +4 -4
  138. package/src/services/plan/plan-run-serialization.ts +15 -0
  139. package/src/services/plan/plan-run.service.ts +644 -0
  140. package/src/services/plan/plan-scheduler.service.ts +385 -0
  141. package/src/services/plan/plan-template.service.ts +224 -0
  142. package/src/services/plan/plan-transaction-events.ts +33 -0
  143. package/src/services/plan/plan-validator.service.ts +907 -0
  144. package/src/services/plan/plan-workspace.service.ts +125 -0
  145. package/src/services/plugin-executor.service.ts +97 -68
  146. package/src/services/quality-metrics.service.ts +112 -94
  147. package/src/services/queue-job.service.ts +296 -230
  148. package/src/services/recent-activity-title.service.ts +65 -36
  149. package/src/services/recent-activity.service.ts +274 -259
  150. package/src/services/skill-resolver.service.ts +38 -12
  151. package/src/services/social-chat-history.service.ts +176 -125
  152. package/src/services/system-executor.service.ts +91 -61
  153. package/src/services/thread/thread-active-run.ts +203 -0
  154. package/src/services/thread/thread-bootstrap.ts +369 -0
  155. package/src/services/thread/thread-listing.ts +198 -0
  156. package/src/services/thread/thread-memory-block.ts +117 -0
  157. package/src/services/thread/thread-message.service.ts +363 -0
  158. package/src/services/thread/thread-record-store.ts +155 -0
  159. package/src/services/thread/thread-title.service.ts +74 -0
  160. package/src/services/thread/thread-turn-execution.ts +280 -0
  161. package/src/services/thread/thread-turn-message-context.ts +73 -0
  162. package/src/services/thread/thread-turn-preparation.service.ts +1146 -0
  163. package/src/services/thread/thread-turn-streaming.ts +402 -0
  164. package/src/services/thread/thread-turn-tracing.ts +35 -0
  165. package/src/services/thread/thread-turn.ts +343 -0
  166. package/src/services/thread/thread.service.ts +335 -0
  167. package/src/services/user.service.ts +82 -32
  168. package/src/services/write-intent-validator.service.ts +63 -51
  169. package/src/storage/attachment-parser.ts +69 -27
  170. package/src/storage/attachment-storage.service.ts +331 -275
  171. package/src/storage/generated-document-storage.service.ts +66 -34
  172. package/src/system-agents/agent-result.ts +3 -1
  173. package/src/system-agents/context-compaction.agent.ts +2 -2
  174. package/src/system-agents/delegated-agent-factory.ts +159 -90
  175. package/src/system-agents/memory-reranker.agent.ts +2 -2
  176. package/src/system-agents/memory.agent.ts +2 -2
  177. package/src/system-agents/recent-activity-title-refiner.agent.ts +2 -2
  178. package/src/system-agents/regular-chat-memory-digest.agent.ts +2 -2
  179. package/src/system-agents/skill-extractor.agent.ts +2 -2
  180. package/src/system-agents/skill-manager.agent.ts +2 -2
  181. package/src/system-agents/thread-router.agent.ts +157 -113
  182. package/src/system-agents/title-generator.agent.ts +2 -2
  183. package/src/tools/execution-plan.tool.ts +220 -161
  184. package/src/tools/fetch-webpage.tool.ts +21 -17
  185. package/src/tools/firecrawl-client.ts +16 -6
  186. package/src/tools/index.ts +1 -0
  187. package/src/tools/memory-block.tool.ts +14 -6
  188. package/src/tools/plan-approval.tool.ts +49 -47
  189. package/src/tools/read-file-parts.tool.ts +44 -33
  190. package/src/tools/remember-memory.tool.ts +65 -45
  191. package/src/tools/search-web.tool.ts +26 -22
  192. package/src/tools/search.tool.ts +41 -29
  193. package/src/tools/team-think.tool.ts +124 -83
  194. package/src/tools/user-questions.tool.ts +4 -3
  195. package/src/tools/web-tool-shared.ts +6 -0
  196. package/src/utils/async.ts +17 -23
  197. package/src/utils/crypto.ts +21 -0
  198. package/src/utils/date-time.ts +40 -1
  199. package/src/utils/errors.ts +95 -16
  200. package/src/utils/hono-error-handler.ts +24 -39
  201. package/src/utils/index.ts +2 -1
  202. package/src/utils/null-proto-record.ts +41 -0
  203. package/src/utils/sse-keepalive.ts +124 -21
  204. package/src/workers/bootstrap.ts +186 -51
  205. package/src/workers/memory-consolidation.worker.ts +325 -237
  206. package/src/workers/organization-learning.worker.ts +50 -16
  207. package/src/workers/regular-chat-memory-digest.helpers.ts +28 -27
  208. package/src/workers/regular-chat-memory-digest.runner.ts +175 -114
  209. package/src/workers/skill-extraction.runner.ts +176 -93
  210. package/src/workers/utils/file-section-chunker.ts +8 -10
  211. package/src/workers/utils/repo-structure-extractor.ts +349 -260
  212. package/src/workers/utils/repomix-file-sections.ts +2 -2
  213. package/src/workers/utils/thread-message-query.ts +97 -38
  214. package/src/workers/worker-utils.ts +56 -31
  215. package/src/config/debug-logger.ts +0 -47
  216. package/src/redis/connection-accessor.ts +0 -26
  217. package/src/runtime/context-compaction-runtime.ts +0 -87
  218. package/src/runtime/social-chat-agent-runner.ts +0 -118
  219. package/src/runtime/social-chat.ts +0 -516
  220. package/src/runtime/team-consultation-orchestrator.ts +0 -272
  221. package/src/services/adaptive-playbook.service.ts +0 -152
  222. package/src/services/artifact-provenance.service.ts +0 -172
  223. package/src/services/chat-attachments.service.ts +0 -17
  224. package/src/services/context-compaction-runtime.singleton.ts +0 -13
  225. package/src/services/execution-plan.service.ts +0 -1118
  226. package/src/services/memory.service.ts +0 -914
  227. package/src/services/plan-agent-heartbeat.service.ts +0 -136
  228. package/src/services/plan-agent-query.service.ts +0 -267
  229. package/src/services/plan-approval.service.ts +0 -83
  230. package/src/services/plan-artifact.service.ts +0 -50
  231. package/src/services/plan-builder.service.ts +0 -67
  232. package/src/services/plan-checkpoint.service.ts +0 -81
  233. package/src/services/plan-completion-side-effects.ts +0 -80
  234. package/src/services/plan-coordination.service.ts +0 -157
  235. package/src/services/plan-cycle.service.ts +0 -284
  236. package/src/services/plan-deadline.service.ts +0 -430
  237. package/src/services/plan-event-delivery.service.ts +0 -166
  238. package/src/services/plan-executor.service.ts +0 -1950
  239. package/src/services/plan-run.service.ts +0 -515
  240. package/src/services/plan-scheduler.service.ts +0 -240
  241. package/src/services/plan-template.service.ts +0 -177
  242. package/src/services/plan-validator.service.ts +0 -818
  243. package/src/services/plan-workspace.service.ts +0 -83
  244. package/src/services/rerank.service.ts +0 -156
  245. package/src/services/thread-message.service.ts +0 -275
  246. package/src/services/thread-plan-registry.service.ts +0 -22
  247. package/src/services/thread-title.service.ts +0 -39
  248. package/src/services/thread-turn-preparation.service.ts +0 -1147
  249. package/src/services/thread-turn.ts +0 -172
  250. package/src/services/thread.service.ts +0 -869
  251. package/src/utils/env.ts +0 -8
  252. /package/src/runtime/{context-compaction-constants.ts → context-compaction/context-compaction-constants.ts} +0 -0
  253. /package/src/runtime/{memory-format.ts → memory/memory-format.ts} +0 -0
  254. /package/src/runtime/{memory-prompts-parse.ts → memory/memory-prompts-parse.ts} +0 -0
  255. /package/src/runtime/{memory-prompts-update.ts → memory/memory-prompts-update.ts} +0 -0
  256. /package/src/runtime/{social-chat-prompts.ts → social-chat/social-chat-prompts.ts} +0 -0
  257. /package/src/services/{plan-node-spec.ts → plan/plan-node-spec.ts} +0 -0
  258. /package/src/services/{thread-constants.ts → thread/thread-constants.ts} +0 -0
  259. /package/src/services/{thread.types.ts → thread/thread.types.ts} +0 -0
@@ -1,16 +1,11 @@
1
- import { Queue, Worker } from 'bullmq'
2
- import type { QueueOptions } from 'bullmq'
1
+ import { Effect } from 'effect'
3
2
  import type IORedis from 'ioredis'
4
3
 
5
4
  import type { chatLogger } from '../config/logger'
6
- import { queueJobService } from '../services/queue-job.service'
7
- import {
8
- attachWorkerEvents,
9
- createWorkerShutdown,
10
- DEFAULT_JOB_RETENTION,
11
- registerShutdownSignals,
12
- } from '../workers/worker-utils'
5
+ import { sha256Hex } from '../utils/crypto'
6
+ import { DEFAULT_JOB_RETENTION, getQueueJobService } from '../workers/worker-utils'
13
7
  import type { WorkerHandle } from '../workers/worker-utils'
8
+ import { createQueueFactory } from './queue-factory'
14
9
 
15
10
  export type DocumentSourceChannel = string
16
11
 
@@ -43,18 +38,16 @@ export function buildDocumentProcessorJobId(
43
38
  'orgId' | 'source' | 'sourceId' | 'sourceCanonicalKey' | 'sourceVersionKey' | 'title'
44
39
  >,
45
40
  ): string {
46
- const digest = new Bun.CryptoHasher('sha256')
47
- .update(
48
- JSON.stringify({
49
- orgId: job.orgId,
50
- source: job.source,
51
- sourceId: job.sourceId,
52
- sourceCanonicalKey: job.sourceCanonicalKey,
53
- sourceVersionKey: job.sourceVersionKey,
54
- title: job.title,
55
- }),
56
- )
57
- .digest('hex')
41
+ const digest = sha256Hex(
42
+ JSON.stringify({
43
+ orgId: job.orgId,
44
+ source: job.source,
45
+ sourceId: job.sourceId,
46
+ sourceCanonicalKey: job.sourceCanonicalKey,
47
+ sourceVersionKey: job.sourceVersionKey,
48
+ title: job.title,
49
+ }),
50
+ )
58
51
 
59
52
  return `doc__${digest}`
60
53
  }
@@ -67,68 +60,50 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
67
60
  workerName?: string
68
61
  concurrency?: number
69
62
  lockDuration?: number
70
- }): {
71
- enqueue: (job: TJob) => Promise<unknown>
72
- startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle
73
- } {
74
- type QueueShape = Queue<TJob, unknown, string, TJob, unknown, string>
75
-
63
+ }): { enqueue: (job: TJob) => Promise<void>; startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle } {
76
64
  const queueName = params.queueName ?? DEFAULT_DOCUMENT_PROCESSOR_QUEUE
77
65
  const workerName = params.workerName ?? DEFAULT_WORKER_NAME
78
66
  const concurrency = params.concurrency ?? 10
79
67
  const lockDuration = params.lockDuration ?? 300_000
80
- const jobName = 'process-document' as Parameters<QueueShape['add']>[0]
81
- const toQueueData = (job: TJob): Parameters<QueueShape['add']>[1] => job
82
- let queue: QueueShape | null = null
83
- const getConnection = (): IORedis => params.getConnectionForBullMQ()
84
-
85
- const getQueue = (): QueueShape => {
86
- if (queue) {
87
- return queue
88
- }
89
-
90
- queue = new Queue<TJob, unknown, string, TJob, unknown, string>(queueName, {
91
- connection: getConnection() as QueueOptions['connection'],
92
- defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
93
- })
94
-
95
- return queue
96
- }
68
+ const queueRuntime = createQueueFactory<TJob>({
69
+ name: queueName,
70
+ displayName: workerName,
71
+ jobName: 'process-document',
72
+ concurrency,
73
+ lockDuration,
74
+ logger: params.logger,
75
+ connectionProvider: params.getConnectionForBullMQ,
76
+ defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
77
+ processorPath: params.getWorkerPath(),
78
+ })
97
79
 
98
80
  return {
99
- enqueue: async (job) => {
100
- const queuedJob = await getQueue().add(jobName, toQueueData(job), { jobId: buildDocumentProcessorJobId(job) })
101
- try {
102
- await queueJobService.recordEnqueued({
103
- queueName,
104
- id: queuedJob.id,
105
- name: queuedJob.name,
106
- data: queuedJob.data,
107
- opts: queuedJob.opts,
108
- attemptsMade: queuedJob.attemptsMade,
109
- timestamp: queuedJob.timestamp,
110
- })
111
- } catch (error) {
112
- params.logger.error`Failed to persist queued job metadata (queue=${queueName}, job=${queuedJob.id}): ${error}`
113
- }
114
- },
115
- startWorker: (options = {}) => {
116
- const { registerSignals = import.meta.main } = options
117
- const worker = new Worker(queueName, params.getWorkerPath(), {
118
- connection: getConnection() as QueueOptions['connection'],
119
- concurrency,
120
- lockDuration,
121
- })
122
-
123
- attachWorkerEvents(worker, workerName, params.logger)
124
-
125
- const shutdown = createWorkerShutdown(worker, workerName, params.logger)
126
-
127
- if (registerSignals) {
128
- registerShutdownSignals({ name: workerName, shutdown, logger: params.logger })
129
- }
130
-
131
- return { worker, shutdown }
81
+ enqueue: (job) => {
82
+ return Effect.runPromise(
83
+ Effect.gen(function* () {
84
+ const queuedJob = yield* Effect.tryPromise(() =>
85
+ queueRuntime.getQueue().add('process-document', job, { jobId: buildDocumentProcessorJobId(job) }),
86
+ )
87
+
88
+ yield* Effect.catch(
89
+ getQueueJobService().recordEnqueued({
90
+ queueName,
91
+ id: queuedJob.id,
92
+ name: queuedJob.name,
93
+ data: queuedJob.data,
94
+ opts: queuedJob.opts,
95
+ attemptsMade: queuedJob.attemptsMade,
96
+ timestamp: queuedJob.timestamp,
97
+ }),
98
+ (error) =>
99
+ Effect.sync(() => {
100
+ params.logger
101
+ .error`Failed to persist queued job metadata (queue=${queueName}, job=${queuedJob.id}): ${error}`
102
+ }),
103
+ )
104
+ }),
105
+ )
132
106
  },
107
+ startWorker: (options = {}) => queueRuntime.startWorker(options),
133
108
  }
134
109
  }
@@ -1,7 +1,14 @@
1
+ import { Effect } from 'effect'
2
+
1
3
  import { serverLogger } from '../config/logger'
2
- import { queueJobService } from '../services/queue-job.service'
3
- import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS, LOW_JOB_RETENTION } from '../workers/worker-utils'
4
+ import {
5
+ getQueueJobService,
6
+ getWorkerPath,
7
+ LONG_JOB_LOCK_DURATION_MS,
8
+ LOW_JOB_RETENTION,
9
+ } from '../workers/worker-utils'
4
10
  import { createQueueFactory } from './queue-factory'
11
+ import { runStandaloneQueueWorker } from './standalone-worker'
5
12
 
6
13
  export interface MemoryConsolidationJob {
7
14
  scopeId?: string
@@ -20,40 +27,48 @@ const memoryConsolidation = createQueueFactory<MemoryConsolidationJob>({
20
27
  processorPath: getWorkerPath('memory-consolidation.worker.ts'),
21
28
  })
22
29
 
23
- export async function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
24
- await memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
30
+ export function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
31
+ return memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
25
32
  }
26
33
 
27
- export async function scheduleRecurringConsolidation() {
28
- const queuedJob = await memoryConsolidation
29
- .getQueue()
30
- .upsertJobScheduler(
31
- MEMORY_CONSOLIDATION_SCHEDULER_ID,
32
- { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
33
- {
34
- name: 'consolidate',
35
- data: {},
36
- opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
37
- },
38
- )
39
-
40
- try {
41
- await queueJobService.recordEnqueued({
42
- queueName: 'memory-consolidation',
43
- id: queuedJob.id,
44
- name: queuedJob.name,
45
- data: queuedJob.data,
46
- opts: queuedJob.opts,
47
- attemptsMade: queuedJob.attemptsMade,
48
- timestamp: queuedJob.timestamp,
49
- })
50
- } catch (error) {
51
- serverLogger.error`Failed to persist queued job metadata (queue=memory-consolidation, job=${queuedJob.id}): ${error}`
52
- }
34
+ export function scheduleRecurringConsolidation() {
35
+ return Effect.runPromise(
36
+ Effect.gen(function* () {
37
+ const queuedJob = yield* Effect.tryPromise(() =>
38
+ memoryConsolidation
39
+ .getQueue()
40
+ .upsertJobScheduler(
41
+ MEMORY_CONSOLIDATION_SCHEDULER_ID,
42
+ { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
43
+ {
44
+ name: 'consolidate',
45
+ data: {},
46
+ opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
47
+ },
48
+ ),
49
+ )
50
+
51
+ yield* Effect.catch(
52
+ getQueueJobService().recordEnqueued({
53
+ queueName: 'memory-consolidation',
54
+ id: queuedJob.id,
55
+ name: queuedJob.name,
56
+ data: queuedJob.data,
57
+ opts: queuedJob.opts,
58
+ attemptsMade: queuedJob.attemptsMade,
59
+ timestamp: queuedJob.timestamp,
60
+ }),
61
+ (error) =>
62
+ Effect.sync(() => {
63
+ serverLogger.error`Failed to persist queued job metadata (queue=memory-consolidation, job=${queuedJob.id}): ${error}`
64
+ }),
65
+ )
66
+ }),
67
+ )
53
68
  }
54
69
 
55
70
  export const startMemoryConsolidationWorker = memoryConsolidation.startWorker
56
71
 
57
- if (import.meta.main) {
72
+ runStandaloneQueueWorker(() => {
58
73
  startMemoryConsolidationWorker()
59
- }
74
+ })
@@ -1,5 +1,8 @@
1
+ import { Effect } from 'effect'
2
+
1
3
  import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
2
4
  import { createQueueFactory } from './queue-factory'
5
+ import { runStandaloneQueueWorker } from './standalone-worker'
3
6
 
4
7
  export const ORGANIZATION_LEARNING_QUEUE = 'organization-learning'
5
8
 
@@ -67,12 +70,18 @@ export function enqueueSkillExtraction(job: Omit<SkillExtractionJob, 'kind'>) {
67
70
  )
68
71
  }
69
72
 
70
- export async function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
71
- await organizationLearningQueue.getQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId))
73
+ export function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
74
+ return Effect.runPromise(
75
+ Effect.asVoid(
76
+ Effect.tryPromise(() =>
77
+ organizationLearningQueue.getQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId)),
78
+ ),
79
+ ),
80
+ )
72
81
  }
73
82
 
74
83
  export const startOrganizationLearningWorker = organizationLearningQueue.startWorker
75
84
 
76
- if (import.meta.main) {
85
+ runStandaloneQueueWorker(() => {
77
86
  startOrganizationLearningWorker()
78
- }
87
+ })
@@ -1,11 +1,16 @@
1
1
  import type { Job } from 'bullmq'
2
+ import { Effect } from 'effect'
3
+ import type { Context } from 'effect'
4
+ import type IORedis from 'ioredis'
2
5
 
3
6
  import { serverLogger } from '../config/logger'
4
- import { databaseService } from '../db/service'
5
- import { planAgentHeartbeatService } from '../services/plan-agent-heartbeat.service'
7
+ import { ConfigurationError } from '../effect/errors'
8
+ import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
9
+ import { PlanAgentHeartbeatServiceTag } from '../services/plan/plan-agent-heartbeat.service'
6
10
  import type { WorkerHandle } from '../workers/worker-utils'
7
11
  import { DEFAULT_JOB_RETENTION, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
8
12
  import { createQueueFactory } from './queue-factory'
13
+ import { runStandaloneQueueWorker } from './standalone-worker'
9
14
 
10
15
  export interface PlanAgentHeartbeatWakeJob {
11
16
  type: 'wake-node'
@@ -28,22 +33,45 @@ export const PLAN_AGENT_HEARTBEAT_QUEUE = 'plan-agent-heartbeat'
28
33
  const PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS = 30_000
29
34
  const PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID = 'plan-agent-heartbeat-sweep'
30
35
 
31
- async function enqueueDelayedPlanAgentHeartbeatSweep(delayMs = PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS): Promise<void> {
32
- await planAgentHeartbeatQueue.enqueue({ type: 'sweep' }, { delay: delayMs, jobId: PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID })
36
+ interface PlanAgentHeartbeatQueueDeps {
37
+ databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
38
+ planAgentHeartbeatService: Context.Service.Shape<typeof PlanAgentHeartbeatServiceTag>
33
39
  }
34
40
 
35
- async function processPlanAgentHeartbeatJob(job: Job<PlanAgentHeartbeatJob>): Promise<void> {
36
- await databaseService.connect()
41
+ let _deps: PlanAgentHeartbeatQueueDeps | null = null
42
+ function getDeps(): PlanAgentHeartbeatQueueDeps {
43
+ if (!_deps)
44
+ throw new ConfigurationError({
45
+ message: 'Plan agent heartbeat queue is not configured. Initialize the runtime before starting the worker.',
46
+ key: 'queue-deps',
47
+ })
48
+ return _deps
49
+ }
37
50
 
38
- if (job.data.type === 'wake-node') {
39
- await planAgentHeartbeatService.wakeNode(job.data)
40
- return
41
- }
51
+ function enqueueDelayedPlanAgentHeartbeatSweep(delayMs = PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS): Promise<void> {
52
+ return planAgentHeartbeatQueue.enqueue(
53
+ { type: 'sweep' },
54
+ { delay: delayMs, jobId: PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID },
55
+ )
56
+ }
42
57
 
43
- await planAgentHeartbeatService.sweep({ organizationId: job.data.organizationId })
44
- if (!job.data.organizationId) {
45
- await enqueueDelayedPlanAgentHeartbeatSweep()
46
- }
58
+ function processPlanAgentHeartbeatJob(job: Job<PlanAgentHeartbeatJob>): Promise<void> {
59
+ const { planAgentHeartbeatService } = getDeps()
60
+
61
+ return Effect.runPromise(
62
+ Effect.gen(function* () {
63
+ if (job.data.type === 'wake-node') {
64
+ const wakeJob = job.data
65
+ yield* planAgentHeartbeatService.wakeNode(wakeJob)
66
+ return
67
+ }
68
+
69
+ yield* planAgentHeartbeatService.sweep({ organizationId: job.data.organizationId })
70
+ if (!job.data.organizationId) {
71
+ yield* Effect.tryPromise(() => enqueueDelayedPlanAgentHeartbeatSweep())
72
+ }
73
+ }),
74
+ )
47
75
  }
48
76
 
49
77
  const planAgentHeartbeatQueue = createQueueFactory<PlanAgentHeartbeatJob>({
@@ -53,6 +81,7 @@ const planAgentHeartbeatQueue = createQueueFactory<PlanAgentHeartbeatJob>({
53
81
  concurrency: 2,
54
82
  lockDuration: LONG_JOB_LOCK_DURATION_MS,
55
83
  defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 5_000 } },
84
+ prepare: () => getDeps().databaseService.connect(),
56
85
  processor: processPlanAgentHeartbeatJob,
57
86
  })
58
87
 
@@ -68,7 +97,7 @@ function buildWakeJobId(params: {
68
97
  return `plan-agent-wake__${encode(params.runId)}__${encode(params.nodeId)}__${encode(params.agentId)}`
69
98
  }
70
99
 
71
- export async function enqueuePlanAgentHeartbeatWake(params: {
100
+ export function enqueuePlanAgentHeartbeatWake(params: {
72
101
  organizationId: string
73
102
  threadId: string
74
103
  runId: string
@@ -76,11 +105,19 @@ export async function enqueuePlanAgentHeartbeatWake(params: {
76
105
  agentId: string
77
106
  reason: string
78
107
  }): Promise<void> {
79
- await planAgentHeartbeatQueue.enqueue({ type: 'wake-node', ...params }, { jobId: buildWakeJobId(params) })
108
+ return planAgentHeartbeatQueue.enqueue({ type: 'wake-node', ...params }, { jobId: buildWakeJobId(params) })
80
109
  }
81
110
 
82
- export function startPlanAgentHeartbeatWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
83
- const handle = planAgentHeartbeatQueue.startWorker(options)
111
+ export function startPlanAgentHeartbeatWorker(options: {
112
+ registerSignals?: boolean
113
+ connectionProvider: () => IORedis
114
+ deps: PlanAgentHeartbeatQueueDeps
115
+ }): WorkerHandle {
116
+ _deps = options.deps
117
+ const handle = planAgentHeartbeatQueue.startWorker({
118
+ registerSignals: options.registerSignals,
119
+ connectionProvider: options.connectionProvider,
120
+ })
84
121
 
85
122
  enqueueDelayedPlanAgentHeartbeatSweep().catch((error: unknown) => {
86
123
  serverLogger.error`Plan agent heartbeat scheduler setup failed: ${error}`
@@ -89,6 +126,13 @@ export function startPlanAgentHeartbeatWorker(options: { registerSignals?: boole
89
126
  return handle
90
127
  }
91
128
 
92
- if (import.meta.main) {
93
- startPlanAgentHeartbeatWorker()
94
- }
129
+ runStandaloneQueueWorker((runtime) => {
130
+ const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
131
+ startPlanAgentHeartbeatWorker({
132
+ connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
133
+ deps: {
134
+ databaseService: resolve(DatabaseServiceTag),
135
+ planAgentHeartbeatService: resolve(PlanAgentHeartbeatServiceTag),
136
+ },
137
+ })
138
+ })
@@ -1,11 +1,19 @@
1
1
  import type { Job } from 'bullmq'
2
+ import { Schema, Effect } from 'effect'
3
+ import type { Context } from 'effect'
4
+ import type IORedis from 'ioredis'
2
5
 
3
6
  import { serverLogger } from '../config/logger'
4
- import { databaseService } from '../db/service'
5
- import { planDeadlineService } from '../services/plan-deadline.service'
6
- import { planSchedulerService } from '../services/plan-scheduler.service'
7
+ import { ConfigurationError } from '../effect/errors'
8
+ import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
9
+ import { PlanCycleServiceTag } from '../services/plan/plan-cycle.service'
10
+ import { PlanDeadlineServiceTag } from '../services/plan/plan-deadline.service'
11
+ import { PlanExecutorServiceTag } from '../services/plan/plan-executor.service'
12
+ import { PlanSchedulerServiceTag } from '../services/plan/plan-scheduler.service'
13
+ import { nowEpochMillis } from '../utils/date-time'
7
14
  import type { WorkerHandle } from '../workers/worker-utils'
8
15
  import { createQueueFactory } from './queue-factory'
16
+ import { runStandaloneQueueWorker } from './standalone-worker'
9
17
 
10
18
  export interface PlanSchedulerFireJob {
11
19
  type: 'fire-schedule'
@@ -21,16 +29,50 @@ export type PlanSchedulerJob = PlanSchedulerFireJob | PlanSchedulerDeadlineJob
21
29
 
22
30
  export const PLAN_SCHEDULER_QUEUE = 'plan-scheduler'
23
31
 
24
- async function processPlanSchedulerJob(job: Job<PlanSchedulerJob>): Promise<void> {
25
- await databaseService.connect()
32
+ interface PlanSchedulerQueueDeps {
33
+ databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
34
+ planSchedulerService: Context.Service.Shape<typeof PlanSchedulerServiceTag>
35
+ planDeadlineService: Context.Service.Shape<typeof PlanDeadlineServiceTag>
36
+ planExecutorService: Context.Service.Shape<typeof PlanExecutorServiceTag>
37
+ planCycleService: Context.Service.Shape<typeof PlanCycleServiceTag>
38
+ }
39
+
40
+ let _deps: PlanSchedulerQueueDeps | null = null
41
+ function getDeps(): PlanSchedulerQueueDeps {
42
+ if (!_deps)
43
+ throw new ConfigurationError({
44
+ message: 'Plan scheduler queue is not configured. Initialize the runtime before starting the worker.',
45
+ key: 'queue-deps',
46
+ })
47
+ return _deps
48
+ }
49
+
50
+ class PlanSchedulerQueueError extends Schema.TaggedErrorClass<PlanSchedulerQueueError>()('PlanSchedulerQueueError', {
51
+ stage: Schema.Literals(['remove-schedule-fire-job', 'recover-active-schedules', 'recover-deadline-checks']),
52
+ message: Schema.String,
53
+ cause: Schema.Defect,
54
+ }) {}
55
+
56
+ function toPlanSchedulerQueueError(stage: PlanSchedulerQueueError['stage'], cause: unknown): PlanSchedulerQueueError {
57
+ return new PlanSchedulerQueueError({ stage, message: cause instanceof Error ? cause.message : String(cause), cause })
58
+ }
59
+
60
+ function processPlanSchedulerJob(job: Job<PlanSchedulerJob>): Promise<void> {
61
+ const { planSchedulerService, planDeadlineService, planExecutorService, planCycleService } = getDeps()
26
62
 
27
63
  switch (job.data.type) {
28
64
  case 'fire-schedule':
29
- await planSchedulerService.fireScheduleById(job.data.scheduleId)
30
- break
65
+ return Effect.runPromise(
66
+ Effect.asVoid(
67
+ planSchedulerService.fireScheduleById(job.data.scheduleId, {
68
+ promoteDelayedNode: (params) => planExecutorService.promoteDelayedNode(params),
69
+ advanceCycle: (cycleId) => planCycleService.advanceCycle(cycleId),
70
+ recoverDeadlineChecks: () => Effect.runPromise(Effect.asVoid(planDeadlineService.recoverDeadlineChecks())),
71
+ }),
72
+ ),
73
+ )
31
74
  case 'check-deadlines':
32
- await planDeadlineService.checkDeadlines()
33
- break
75
+ return Effect.runPromise(Effect.asVoid(planDeadlineService.checkDeadlines()))
34
76
  }
35
77
  }
36
78
 
@@ -40,31 +82,33 @@ const planScheduler = createQueueFactory<PlanSchedulerJob>({
40
82
  jobName: 'plan-scheduler-job',
41
83
  concurrency: 1,
42
84
  defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
85
+ prepare: () => getDeps().databaseService.connect(),
43
86
  processor: processPlanSchedulerJob,
44
87
  })
45
88
 
46
89
  /** Enqueue a delayed job that fires a specific schedule at its nextFireAt time. */
47
- export async function enqueueScheduleFire(scheduleId: string, delayMs: number): Promise<void> {
48
- await planScheduler.enqueue(
90
+ export function enqueueScheduleFire(scheduleId: string, delayMs: number): Promise<void> {
91
+ return planScheduler.enqueue(
49
92
  { type: 'fire-schedule', scheduleId },
50
93
  { delay: Math.max(0, delayMs), jobId: `schedule:${scheduleId}`, removeOnComplete: true, removeOnFail: 50 },
51
94
  )
52
95
  }
53
96
 
54
97
  /** Remove a pending fire job for a schedule (on cancel/pause/complete). */
55
- export async function removeScheduleFireJob(scheduleId: string): Promise<void> {
56
- try {
57
- await planScheduler.getQueue().remove(`schedule:${scheduleId}`)
58
- } catch {
59
- // Job may not exist (already fired or never enqueued) — safe to ignore
60
- }
98
+ export function removeScheduleFireJob(scheduleId: string): Promise<void> {
99
+ return Effect.runPromise(
100
+ Effect.tryPromise({
101
+ try: () => planScheduler.getQueue().remove(`schedule:${scheduleId}`),
102
+ catch: (cause) => toPlanSchedulerQueueError('remove-schedule-fire-job', cause),
103
+ }).pipe(Effect.asVoid),
104
+ )
61
105
  }
62
106
 
63
107
  const DEADLINE_CHECK_JOB_PREFIX = 'deadline-check'
64
108
 
65
- export async function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
66
- const delay = Math.max(0, scheduledFor.getTime() - Date.now())
67
- await planScheduler.enqueue(
109
+ export function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
110
+ const delay = Math.max(0, scheduledFor.getTime() - nowEpochMillis())
111
+ return planScheduler.enqueue(
68
112
  { type: 'check-deadlines', scheduledFor: scheduledFor.toISOString() },
69
113
  {
70
114
  delay,
@@ -75,23 +119,55 @@ export async function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
75
119
  )
76
120
  }
77
121
 
78
- export function startPlanSchedulerWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
79
- const handle = planScheduler.startWorker(options)
122
+ export function startPlanSchedulerWorker(options: {
123
+ registerSignals?: boolean
124
+ connectionProvider: () => IORedis
125
+ deps: PlanSchedulerQueueDeps
126
+ }): WorkerHandle {
127
+ _deps = options.deps
128
+ const handle = planScheduler.startWorker({
129
+ registerSignals: options.registerSignals,
130
+ connectionProvider: options.connectionProvider,
131
+ })
80
132
 
81
133
  // Recover active schedules on startup
82
- planSchedulerService.recoverActiveSchedules().catch((err: unknown) => {
83
- serverLogger.error`Plan scheduler startup recovery failed: ${err}`
84
- })
134
+ void Effect.runFork(
135
+ options.deps.planSchedulerService.recoverActiveSchedules().pipe(
136
+ Effect.mapError((cause) => toPlanSchedulerQueueError('recover-active-schedules', cause)),
137
+ Effect.catchTag('PlanSchedulerQueueError', (error) =>
138
+ Effect.sync(() => {
139
+ serverLogger.error`Plan scheduler startup recovery failed: ${error.message}`
140
+ }),
141
+ ),
142
+ ),
143
+ )
85
144
 
86
145
  // Seed deadline checks from current runtime state. Subsequent checks are
87
146
  // re-seeded by the queue job after each sweep and by schedule fire events.
88
- planDeadlineService.recoverDeadlineChecks().catch((err: unknown) => {
89
- serverLogger.error`Plan deadline recovery failed: ${err}`
90
- })
147
+ void Effect.runFork(
148
+ options.deps.planDeadlineService.recoverDeadlineChecks().pipe(
149
+ Effect.mapError((cause) => toPlanSchedulerQueueError('recover-deadline-checks', cause)),
150
+ Effect.catchTag('PlanSchedulerQueueError', (error) =>
151
+ Effect.sync(() => {
152
+ serverLogger.error`Plan deadline recovery failed: ${error.message}`
153
+ }),
154
+ ),
155
+ ),
156
+ )
91
157
 
92
158
  return handle
93
159
  }
94
160
 
95
- if (import.meta.main) {
96
- startPlanSchedulerWorker()
97
- }
161
+ runStandaloneQueueWorker((runtime) => {
162
+ const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
163
+ startPlanSchedulerWorker({
164
+ connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
165
+ deps: {
166
+ databaseService: resolve(DatabaseServiceTag),
167
+ planSchedulerService: resolve(PlanSchedulerServiceTag),
168
+ planDeadlineService: resolve(PlanDeadlineServiceTag),
169
+ planExecutorService: resolve(PlanExecutorServiceTag),
170
+ planCycleService: resolve(PlanCycleServiceTag),
171
+ },
172
+ })
173
+ })