@lota-sdk/core 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/package.json +11 -12
  2. package/src/ai/embedding-cache.ts +94 -22
  3. package/src/ai-gateway/ai-gateway.ts +738 -223
  4. package/src/config/agent-defaults.ts +176 -75
  5. package/src/config/agent-types.ts +54 -4
  6. package/src/config/constants.ts +8 -2
  7. package/src/config/logger.ts +286 -19
  8. package/src/config/model-constants.ts +1 -0
  9. package/src/config/thread-defaults.ts +33 -21
  10. package/src/create-runtime.ts +725 -383
  11. package/src/db/base.service.ts +52 -28
  12. package/src/db/cursor-pagination.ts +71 -30
  13. package/src/db/memory-store.helpers.ts +4 -7
  14. package/src/db/memory-store.ts +856 -598
  15. package/src/db/memory.ts +398 -275
  16. package/src/db/record-id.ts +32 -10
  17. package/src/db/schema-fingerprint.ts +30 -12
  18. package/src/db/service-normalization.ts +255 -0
  19. package/src/db/service.ts +726 -761
  20. package/src/db/startup.ts +140 -66
  21. package/src/db/transaction-conflict.ts +15 -0
  22. package/src/effect/awaitable-effect.ts +87 -0
  23. package/src/effect/errors.ts +121 -0
  24. package/src/effect/helpers.ts +98 -0
  25. package/src/effect/index.ts +22 -0
  26. package/src/effect/layers.ts +228 -0
  27. package/src/effect/runtime-ref.ts +25 -0
  28. package/src/effect/runtime.ts +31 -0
  29. package/src/effect/services.ts +57 -0
  30. package/src/effect/zod.ts +43 -0
  31. package/src/embeddings/provider.ts +122 -71
  32. package/src/index.ts +46 -1
  33. package/src/openrouter/direct-provider.ts +29 -0
  34. package/src/queues/autonomous-job.queue.ts +130 -74
  35. package/src/queues/context-compaction.queue.ts +60 -15
  36. package/src/queues/delayed-node-promotion.queue.ts +52 -15
  37. package/src/queues/document-processor.queue.ts +52 -77
  38. package/src/queues/memory-consolidation.queue.ts +47 -32
  39. package/src/queues/organization-learning.queue.ts +13 -4
  40. package/src/queues/plan-agent-heartbeat.queue.ts +65 -21
  41. package/src/queues/plan-scheduler.queue.ts +107 -31
  42. package/src/queues/post-chat-memory.queue.ts +66 -24
  43. package/src/queues/queue-factory.ts +142 -52
  44. package/src/queues/standalone-worker.ts +39 -0
  45. package/src/queues/title-generation.queue.ts +54 -9
  46. package/src/redis/connection.ts +84 -32
  47. package/src/redis/index.ts +6 -8
  48. package/src/redis/org-memory-lock.ts +60 -27
  49. package/src/redis/redis-lease-lock.ts +200 -121
  50. package/src/redis/runtime-connection.ts +10 -0
  51. package/src/redis/stream-context.ts +84 -46
  52. package/src/runtime/agent-identity-overrides.ts +2 -2
  53. package/src/runtime/agent-runtime-policy.ts +4 -1
  54. package/src/runtime/agent-stream-helpers.ts +20 -9
  55. package/src/runtime/chat-run-orchestration.ts +102 -19
  56. package/src/runtime/chat-run-registry.ts +36 -2
  57. package/src/runtime/context-compaction/context-compaction-runtime.ts +107 -0
  58. package/src/runtime/{context-compaction.ts → context-compaction/context-compaction.ts} +114 -91
  59. package/src/runtime/execution-plan-visibility.ts +2 -2
  60. package/src/runtime/execution-plan.ts +42 -15
  61. package/src/runtime/graph-designer.ts +11 -7
  62. package/src/runtime/helper-model.ts +135 -48
  63. package/src/runtime/index.ts +7 -7
  64. package/src/runtime/indexed-repositories-policy.ts +3 -3
  65. package/src/runtime/{memory-block.ts → memory/memory-block.ts} +40 -36
  66. package/src/runtime/{memory-digest-policy.ts → memory/memory-digest-policy.ts} +1 -1
  67. package/src/runtime/{memory-pipeline.ts → memory/memory-pipeline.ts} +1 -1
  68. package/src/runtime/{memory-prompts-fact.ts → memory/memory-prompts-fact.ts} +2 -2
  69. package/src/runtime/{memory-scope.ts → memory/memory-scope.ts} +12 -6
  70. package/src/runtime/plugin-resolution.ts +144 -24
  71. package/src/runtime/plugin-types.ts +9 -1
  72. package/src/runtime/post-turn-side-effects.ts +197 -130
  73. package/src/runtime/retrieval-adapters.ts +38 -4
  74. package/src/runtime/runtime-config.ts +165 -61
  75. package/src/runtime/runtime-extensions.ts +21 -34
  76. package/src/runtime/social-chat/social-chat-agent-runner.ts +157 -0
  77. package/src/runtime/{social-chat-history.ts → social-chat/social-chat-history.ts} +42 -20
  78. package/src/runtime/social-chat/social-chat.ts +594 -0
  79. package/src/runtime/specialist-runner.ts +36 -10
  80. package/src/runtime/team-consultation/team-consultation-orchestrator.ts +427 -0
  81. package/src/runtime/{team-consultation-prompts.ts → team-consultation/team-consultation-prompts.ts} +6 -2
  82. package/src/runtime/thread-chat-helpers.ts +2 -2
  83. package/src/runtime/thread-plan-turn.ts +2 -1
  84. package/src/runtime/thread-turn-context.ts +172 -94
  85. package/src/runtime/turn-lifecycle.ts +93 -27
  86. package/src/services/agent-activity.service.ts +287 -203
  87. package/src/services/agent-executor.service.ts +329 -217
  88. package/src/services/artifact.service.ts +225 -148
  89. package/src/services/attachment.service.ts +137 -115
  90. package/src/services/autonomous-job.service.ts +888 -491
  91. package/src/services/chat-run-registry.service.ts +11 -1
  92. package/src/services/context-compaction.service.ts +136 -86
  93. package/src/services/document-chunk.service.ts +162 -90
  94. package/src/services/execution-plan/execution-plan-approval.ts +26 -0
  95. package/src/services/execution-plan/execution-plan-context.ts +29 -0
  96. package/src/services/execution-plan/execution-plan-graph.ts +256 -0
  97. package/src/services/execution-plan/execution-plan-schedule.ts +84 -0
  98. package/src/services/execution-plan/execution-plan-spec.ts +75 -0
  99. package/src/services/execution-plan/execution-plan.service.ts +1041 -0
  100. package/src/services/feedback-loop.service.ts +132 -76
  101. package/src/services/global-orchestrator.service.ts +80 -170
  102. package/src/services/graph-full-routing.ts +182 -0
  103. package/src/services/index.ts +18 -20
  104. package/src/services/institutional-memory.service.ts +220 -123
  105. package/src/services/learned-skill.service.ts +364 -259
  106. package/src/services/memory/memory-conversation.ts +95 -0
  107. package/src/services/memory/memory-org-memory.ts +39 -0
  108. package/src/services/memory/memory-preseeded.ts +80 -0
  109. package/src/services/memory/memory-rerank.ts +297 -0
  110. package/src/services/{memory-utils.ts → memory/memory-utils.ts} +5 -5
  111. package/src/services/memory/memory.service.ts +692 -0
  112. package/src/services/memory/rerank.service.ts +209 -0
  113. package/src/services/monitoring-window.service.ts +92 -70
  114. package/src/services/mutating-approval.service.ts +62 -53
  115. package/src/services/node-workspace.service.ts +141 -98
  116. package/src/services/notification.service.ts +17 -16
  117. package/src/services/organization-member.service.ts +120 -66
  118. package/src/services/organization.service.ts +144 -51
  119. package/src/services/ownership-dispatcher.service.ts +415 -264
  120. package/src/services/plan/plan-agent-heartbeat.service.ts +234 -0
  121. package/src/services/plan/plan-agent-query.service.ts +322 -0
  122. package/src/services/plan/plan-approval.service.ts +102 -0
  123. package/src/services/plan/plan-artifact.service.ts +60 -0
  124. package/src/services/plan/plan-builder.service.ts +76 -0
  125. package/src/services/plan/plan-checkpoint.service.ts +103 -0
  126. package/src/services/{plan-compiler.service.ts → plan/plan-compiler.service.ts} +26 -9
  127. package/src/services/plan/plan-completion-side-effects.ts +175 -0
  128. package/src/services/plan/plan-coordination.service.ts +181 -0
  129. package/src/services/plan/plan-cycle.service.ts +398 -0
  130. package/src/services/plan/plan-deadline.service.ts +547 -0
  131. package/src/services/plan/plan-event-delivery.service.ts +261 -0
  132. package/src/services/plan/plan-executor-context.ts +35 -0
  133. package/src/services/plan/plan-executor-graph.ts +475 -0
  134. package/src/services/plan/plan-executor-helpers.ts +322 -0
  135. package/src/services/plan/plan-executor-persistence.ts +209 -0
  136. package/src/services/plan/plan-executor.service.ts +1654 -0
  137. package/src/services/{plan-helpers.ts → plan/plan-helpers.ts} +1 -1
  138. package/src/services/{plan-run-data.ts → plan/plan-run-data.ts} +4 -4
  139. package/src/services/plan/plan-run-serialization.ts +15 -0
  140. package/src/services/plan/plan-run.service.ts +644 -0
  141. package/src/services/plan/plan-scheduler.service.ts +385 -0
  142. package/src/services/plan/plan-template.service.ts +224 -0
  143. package/src/services/plan/plan-transaction-events.ts +33 -0
  144. package/src/services/plan/plan-validator.service.ts +907 -0
  145. package/src/services/plan/plan-workspace.service.ts +125 -0
  146. package/src/services/plugin-executor.service.ts +97 -68
  147. package/src/services/quality-metrics.service.ts +112 -94
  148. package/src/services/queue-job.service.ts +296 -230
  149. package/src/services/recent-activity-title.service.ts +65 -36
  150. package/src/services/recent-activity.service.ts +274 -259
  151. package/src/services/skill-resolver.service.ts +38 -12
  152. package/src/services/social-chat-history.service.ts +176 -125
  153. package/src/services/system-executor.service.ts +91 -61
  154. package/src/services/thread/thread-active-run.ts +203 -0
  155. package/src/services/thread/thread-bootstrap.ts +369 -0
  156. package/src/services/thread/thread-listing.ts +198 -0
  157. package/src/services/thread/thread-memory-block.ts +117 -0
  158. package/src/services/thread/thread-message.service.ts +363 -0
  159. package/src/services/thread/thread-record-store.ts +155 -0
  160. package/src/services/thread/thread-title.service.ts +74 -0
  161. package/src/services/thread/thread-turn-execution.ts +280 -0
  162. package/src/services/thread/thread-turn-message-context.ts +73 -0
  163. package/src/services/thread/thread-turn-preparation.service.ts +1146 -0
  164. package/src/services/thread/thread-turn-streaming.ts +402 -0
  165. package/src/services/thread/thread-turn-tracing.ts +35 -0
  166. package/src/services/thread/thread-turn.ts +343 -0
  167. package/src/services/thread/thread.service.ts +335 -0
  168. package/src/services/user.service.ts +82 -32
  169. package/src/services/write-intent-validator.service.ts +63 -51
  170. package/src/storage/attachment-parser.ts +69 -27
  171. package/src/storage/attachment-storage.service.ts +331 -275
  172. package/src/storage/generated-document-storage.service.ts +66 -34
  173. package/src/system-agents/agent-result.ts +3 -1
  174. package/src/system-agents/context-compaction.agent.ts +2 -2
  175. package/src/system-agents/delegated-agent-factory.ts +159 -90
  176. package/src/system-agents/memory-reranker.agent.ts +2 -2
  177. package/src/system-agents/memory.agent.ts +2 -2
  178. package/src/system-agents/recent-activity-title-refiner.agent.ts +2 -2
  179. package/src/system-agents/regular-chat-memory-digest.agent.ts +2 -2
  180. package/src/system-agents/skill-extractor.agent.ts +2 -2
  181. package/src/system-agents/skill-manager.agent.ts +2 -2
  182. package/src/system-agents/thread-router.agent.ts +157 -113
  183. package/src/system-agents/title-generator.agent.ts +2 -2
  184. package/src/tools/execution-plan.tool.ts +220 -161
  185. package/src/tools/fetch-webpage.tool.ts +21 -17
  186. package/src/tools/firecrawl-client.ts +16 -6
  187. package/src/tools/index.ts +1 -0
  188. package/src/tools/memory-block.tool.ts +14 -6
  189. package/src/tools/plan-approval.tool.ts +49 -47
  190. package/src/tools/read-file-parts.tool.ts +44 -33
  191. package/src/tools/remember-memory.tool.ts +65 -45
  192. package/src/tools/search-web.tool.ts +26 -22
  193. package/src/tools/search.tool.ts +41 -29
  194. package/src/tools/team-think.tool.ts +124 -83
  195. package/src/tools/user-questions.tool.ts +4 -3
  196. package/src/tools/web-tool-shared.ts +6 -0
  197. package/src/utils/async.ts +17 -23
  198. package/src/utils/crypto.ts +21 -0
  199. package/src/utils/date-time.ts +40 -1
  200. package/src/utils/errors.ts +95 -16
  201. package/src/utils/hono-error-handler.ts +24 -39
  202. package/src/utils/index.ts +2 -1
  203. package/src/utils/null-proto-record.ts +41 -0
  204. package/src/utils/sse-keepalive.ts +124 -21
  205. package/src/workers/bootstrap.ts +186 -51
  206. package/src/workers/memory-consolidation.worker.ts +325 -237
  207. package/src/workers/organization-learning.worker.ts +50 -16
  208. package/src/workers/regular-chat-memory-digest.helpers.ts +28 -27
  209. package/src/workers/regular-chat-memory-digest.runner.ts +175 -114
  210. package/src/workers/skill-extraction.runner.ts +176 -93
  211. package/src/workers/utils/file-section-chunker.ts +8 -10
  212. package/src/workers/utils/repo-structure-extractor.ts +349 -260
  213. package/src/workers/utils/repomix-file-sections.ts +2 -2
  214. package/src/workers/utils/thread-message-query.ts +97 -38
  215. package/src/workers/worker-utils.ts +56 -31
  216. package/src/config/debug-logger.ts +0 -47
  217. package/src/redis/connection-accessor.ts +0 -26
  218. package/src/runtime/context-compaction-runtime.ts +0 -87
  219. package/src/runtime/social-chat-agent-runner.ts +0 -118
  220. package/src/runtime/social-chat.ts +0 -516
  221. package/src/runtime/team-consultation-orchestrator.ts +0 -272
  222. package/src/services/adaptive-playbook.service.ts +0 -152
  223. package/src/services/artifact-provenance.service.ts +0 -172
  224. package/src/services/chat-attachments.service.ts +0 -17
  225. package/src/services/context-compaction-runtime.singleton.ts +0 -13
  226. package/src/services/execution-plan.service.ts +0 -1118
  227. package/src/services/memory.service.ts +0 -844
  228. package/src/services/plan-agent-heartbeat.service.ts +0 -136
  229. package/src/services/plan-agent-query.service.ts +0 -267
  230. package/src/services/plan-approval.service.ts +0 -83
  231. package/src/services/plan-artifact.service.ts +0 -50
  232. package/src/services/plan-builder.service.ts +0 -67
  233. package/src/services/plan-checkpoint.service.ts +0 -81
  234. package/src/services/plan-completion-side-effects.ts +0 -80
  235. package/src/services/plan-coordination.service.ts +0 -157
  236. package/src/services/plan-cycle.service.ts +0 -284
  237. package/src/services/plan-deadline.service.ts +0 -430
  238. package/src/services/plan-event-delivery.service.ts +0 -166
  239. package/src/services/plan-executor.service.ts +0 -1950
  240. package/src/services/plan-run.service.ts +0 -515
  241. package/src/services/plan-scheduler.service.ts +0 -240
  242. package/src/services/plan-template.service.ts +0 -177
  243. package/src/services/plan-validator.service.ts +0 -818
  244. package/src/services/plan-workspace.service.ts +0 -83
  245. package/src/services/thread-message.service.ts +0 -275
  246. package/src/services/thread-plan-registry.service.ts +0 -22
  247. package/src/services/thread-title.service.ts +0 -39
  248. package/src/services/thread-turn-preparation.service.ts +0 -1147
  249. package/src/services/thread-turn.ts +0 -172
  250. package/src/services/thread.service.ts +0 -869
  251. package/src/utils/env.ts +0 -8
  252. /package/src/runtime/{context-compaction-constants.ts → context-compaction/context-compaction-constants.ts} +0 -0
  253. /package/src/runtime/{memory-format.ts → memory/memory-format.ts} +0 -0
  254. /package/src/runtime/{memory-prompts-parse.ts → memory/memory-prompts-parse.ts} +0 -0
  255. /package/src/runtime/{memory-prompts-update.ts → memory/memory-prompts-update.ts} +0 -0
  256. /package/src/runtime/{social-chat-prompts.ts → social-chat/social-chat-prompts.ts} +0 -0
  257. /package/src/services/{plan-node-spec.ts → plan/plan-node-spec.ts} +0 -0
  258. /package/src/services/{thread-constants.ts → thread/thread-constants.ts} +0 -0
  259. /package/src/services/{thread.types.ts → thread/thread.types.ts} +0 -0
@@ -1,16 +1,11 @@
1
- import { Queue, Worker } from 'bullmq'
2
- import type { QueueOptions } from 'bullmq'
1
+ import { Effect } from 'effect'
3
2
  import type IORedis from 'ioredis'
4
3
 
5
4
  import type { chatLogger } from '../config/logger'
6
- import { queueJobService } from '../services/queue-job.service'
7
- import {
8
- attachWorkerEvents,
9
- createWorkerShutdown,
10
- DEFAULT_JOB_RETENTION,
11
- registerShutdownSignals,
12
- } from '../workers/worker-utils'
5
+ import { sha256Hex } from '../utils/crypto'
6
+ import { DEFAULT_JOB_RETENTION, getQueueJobService } from '../workers/worker-utils'
13
7
  import type { WorkerHandle } from '../workers/worker-utils'
8
+ import { createQueueFactory } from './queue-factory'
14
9
 
15
10
  export type DocumentSourceChannel = string
16
11
 
@@ -43,18 +38,16 @@ export function buildDocumentProcessorJobId(
43
38
  'orgId' | 'source' | 'sourceId' | 'sourceCanonicalKey' | 'sourceVersionKey' | 'title'
44
39
  >,
45
40
  ): string {
46
- const digest = new Bun.CryptoHasher('sha256')
47
- .update(
48
- JSON.stringify({
49
- orgId: job.orgId,
50
- source: job.source,
51
- sourceId: job.sourceId,
52
- sourceCanonicalKey: job.sourceCanonicalKey,
53
- sourceVersionKey: job.sourceVersionKey,
54
- title: job.title,
55
- }),
56
- )
57
- .digest('hex')
41
+ const digest = sha256Hex(
42
+ JSON.stringify({
43
+ orgId: job.orgId,
44
+ source: job.source,
45
+ sourceId: job.sourceId,
46
+ sourceCanonicalKey: job.sourceCanonicalKey,
47
+ sourceVersionKey: job.sourceVersionKey,
48
+ title: job.title,
49
+ }),
50
+ )
58
51
 
59
52
  return `doc__${digest}`
60
53
  }
@@ -67,68 +60,50 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
67
60
  workerName?: string
68
61
  concurrency?: number
69
62
  lockDuration?: number
70
- }): {
71
- enqueue: (job: TJob) => Promise<unknown>
72
- startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle
73
- } {
74
- type QueueShape = Queue<TJob, unknown, string, TJob, unknown, string>
75
-
63
+ }): { enqueue: (job: TJob) => Promise<void>; startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle } {
76
64
  const queueName = params.queueName ?? DEFAULT_DOCUMENT_PROCESSOR_QUEUE
77
65
  const workerName = params.workerName ?? DEFAULT_WORKER_NAME
78
66
  const concurrency = params.concurrency ?? 10
79
67
  const lockDuration = params.lockDuration ?? 300_000
80
- const jobName = 'process-document' as Parameters<QueueShape['add']>[0]
81
- const toQueueData = (job: TJob): Parameters<QueueShape['add']>[1] => job
82
- let queue: QueueShape | null = null
83
- const getConnection = (): IORedis => params.getConnectionForBullMQ()
84
-
85
- const getQueue = (): QueueShape => {
86
- if (queue) {
87
- return queue
88
- }
89
-
90
- queue = new Queue<TJob, unknown, string, TJob, unknown, string>(queueName, {
91
- connection: getConnection() as QueueOptions['connection'],
92
- defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
93
- })
94
-
95
- return queue
96
- }
68
+ const queueRuntime = createQueueFactory<TJob>({
69
+ name: queueName,
70
+ displayName: workerName,
71
+ jobName: 'process-document',
72
+ concurrency,
73
+ lockDuration,
74
+ logger: params.logger,
75
+ connectionProvider: params.getConnectionForBullMQ,
76
+ defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
77
+ processorPath: params.getWorkerPath(),
78
+ })
97
79
 
98
80
  return {
99
- enqueue: async (job) => {
100
- const queuedJob = await getQueue().add(jobName, toQueueData(job), { jobId: buildDocumentProcessorJobId(job) })
101
- try {
102
- await queueJobService.recordEnqueued({
103
- queueName,
104
- id: queuedJob.id,
105
- name: queuedJob.name,
106
- data: queuedJob.data,
107
- opts: queuedJob.opts,
108
- attemptsMade: queuedJob.attemptsMade,
109
- timestamp: queuedJob.timestamp,
110
- })
111
- } catch (error) {
112
- params.logger.error`Failed to persist queued job metadata (queue=${queueName}, job=${queuedJob.id}): ${error}`
113
- }
114
- },
115
- startWorker: (options = {}) => {
116
- const { registerSignals = import.meta.main } = options
117
- const worker = new Worker(queueName, params.getWorkerPath(), {
118
- connection: getConnection() as QueueOptions['connection'],
119
- concurrency,
120
- lockDuration,
121
- })
122
-
123
- attachWorkerEvents(worker, workerName, params.logger)
124
-
125
- const shutdown = createWorkerShutdown(worker, workerName, params.logger)
126
-
127
- if (registerSignals) {
128
- registerShutdownSignals({ name: workerName, shutdown, logger: params.logger })
129
- }
130
-
131
- return { worker, shutdown }
81
+ enqueue: (job) => {
82
+ return Effect.runPromise(
83
+ Effect.gen(function* () {
84
+ const queuedJob = yield* Effect.tryPromise(() =>
85
+ queueRuntime.getQueue().add('process-document', job, { jobId: buildDocumentProcessorJobId(job) }),
86
+ )
87
+
88
+ yield* Effect.catch(
89
+ getQueueJobService().recordEnqueued({
90
+ queueName,
91
+ id: queuedJob.id,
92
+ name: queuedJob.name,
93
+ data: queuedJob.data,
94
+ opts: queuedJob.opts,
95
+ attemptsMade: queuedJob.attemptsMade,
96
+ timestamp: queuedJob.timestamp,
97
+ }),
98
+ (error) =>
99
+ Effect.sync(() => {
100
+ params.logger
101
+ .error`Failed to persist queued job metadata (queue=${queueName}, job=${queuedJob.id}): ${error}`
102
+ }),
103
+ )
104
+ }),
105
+ )
132
106
  },
107
+ startWorker: (options = {}) => queueRuntime.startWorker(options),
133
108
  }
134
109
  }
@@ -1,7 +1,14 @@
1
+ import { Effect } from 'effect'
2
+
1
3
  import { serverLogger } from '../config/logger'
2
- import { queueJobService } from '../services/queue-job.service'
3
- import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS, LOW_JOB_RETENTION } from '../workers/worker-utils'
4
+ import {
5
+ getQueueJobService,
6
+ getWorkerPath,
7
+ LONG_JOB_LOCK_DURATION_MS,
8
+ LOW_JOB_RETENTION,
9
+ } from '../workers/worker-utils'
4
10
  import { createQueueFactory } from './queue-factory'
11
+ import { runStandaloneQueueWorker } from './standalone-worker'
5
12
 
6
13
  export interface MemoryConsolidationJob {
7
14
  scopeId?: string
@@ -20,40 +27,48 @@ const memoryConsolidation = createQueueFactory<MemoryConsolidationJob>({
20
27
  processorPath: getWorkerPath('memory-consolidation.worker.ts'),
21
28
  })
22
29
 
23
- export async function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
24
- await memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
30
+ export function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
31
+ return memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
25
32
  }
26
33
 
27
- export async function scheduleRecurringConsolidation() {
28
- const queuedJob = await memoryConsolidation
29
- .getQueue()
30
- .upsertJobScheduler(
31
- MEMORY_CONSOLIDATION_SCHEDULER_ID,
32
- { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
33
- {
34
- name: 'consolidate',
35
- data: {},
36
- opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
37
- },
38
- )
39
-
40
- try {
41
- await queueJobService.recordEnqueued({
42
- queueName: 'memory-consolidation',
43
- id: queuedJob.id,
44
- name: queuedJob.name,
45
- data: queuedJob.data,
46
- opts: queuedJob.opts,
47
- attemptsMade: queuedJob.attemptsMade,
48
- timestamp: queuedJob.timestamp,
49
- })
50
- } catch (error) {
51
- serverLogger.error`Failed to persist queued job metadata (queue=memory-consolidation, job=${queuedJob.id}): ${error}`
52
- }
34
+ export function scheduleRecurringConsolidation() {
35
+ return Effect.runPromise(
36
+ Effect.gen(function* () {
37
+ const queuedJob = yield* Effect.tryPromise(() =>
38
+ memoryConsolidation
39
+ .getQueue()
40
+ .upsertJobScheduler(
41
+ MEMORY_CONSOLIDATION_SCHEDULER_ID,
42
+ { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
43
+ {
44
+ name: 'consolidate',
45
+ data: {},
46
+ opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
47
+ },
48
+ ),
49
+ )
50
+
51
+ yield* Effect.catch(
52
+ getQueueJobService().recordEnqueued({
53
+ queueName: 'memory-consolidation',
54
+ id: queuedJob.id,
55
+ name: queuedJob.name,
56
+ data: queuedJob.data,
57
+ opts: queuedJob.opts,
58
+ attemptsMade: queuedJob.attemptsMade,
59
+ timestamp: queuedJob.timestamp,
60
+ }),
61
+ (error) =>
62
+ Effect.sync(() => {
63
+ serverLogger.error`Failed to persist queued job metadata (queue=memory-consolidation, job=${queuedJob.id}): ${error}`
64
+ }),
65
+ )
66
+ }),
67
+ )
53
68
  }
54
69
 
55
70
  export const startMemoryConsolidationWorker = memoryConsolidation.startWorker
56
71
 
57
- if (import.meta.main) {
72
+ runStandaloneQueueWorker(() => {
58
73
  startMemoryConsolidationWorker()
59
- }
74
+ })
@@ -1,5 +1,8 @@
1
+ import { Effect } from 'effect'
2
+
1
3
  import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
2
4
  import { createQueueFactory } from './queue-factory'
5
+ import { runStandaloneQueueWorker } from './standalone-worker'
3
6
 
4
7
  export const ORGANIZATION_LEARNING_QUEUE = 'organization-learning'
5
8
 
@@ -67,12 +70,18 @@ export function enqueueSkillExtraction(job: Omit<SkillExtractionJob, 'kind'>) {
67
70
  )
68
71
  }
69
72
 
70
- export async function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
71
- await organizationLearningQueue.getQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId))
73
+ export function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
74
+ return Effect.runPromise(
75
+ Effect.asVoid(
76
+ Effect.tryPromise(() =>
77
+ organizationLearningQueue.getQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId)),
78
+ ),
79
+ ),
80
+ )
72
81
  }
73
82
 
74
83
  export const startOrganizationLearningWorker = organizationLearningQueue.startWorker
75
84
 
76
- if (import.meta.main) {
85
+ runStandaloneQueueWorker(() => {
77
86
  startOrganizationLearningWorker()
78
- }
87
+ })
@@ -1,11 +1,16 @@
1
1
  import type { Job } from 'bullmq'
2
+ import { Effect } from 'effect'
3
+ import type { Context } from 'effect'
4
+ import type IORedis from 'ioredis'
2
5
 
3
6
  import { serverLogger } from '../config/logger'
4
- import { databaseService } from '../db/service'
5
- import { planAgentHeartbeatService } from '../services/plan-agent-heartbeat.service'
7
+ import { ConfigurationError } from '../effect/errors'
8
+ import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
9
+ import { PlanAgentHeartbeatServiceTag } from '../services/plan/plan-agent-heartbeat.service'
6
10
  import type { WorkerHandle } from '../workers/worker-utils'
7
11
  import { DEFAULT_JOB_RETENTION, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
8
12
  import { createQueueFactory } from './queue-factory'
13
+ import { runStandaloneQueueWorker } from './standalone-worker'
9
14
 
10
15
  export interface PlanAgentHeartbeatWakeJob {
11
16
  type: 'wake-node'
@@ -28,22 +33,45 @@ export const PLAN_AGENT_HEARTBEAT_QUEUE = 'plan-agent-heartbeat'
28
33
  const PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS = 30_000
29
34
  const PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID = 'plan-agent-heartbeat-sweep'
30
35
 
31
- async function enqueueDelayedPlanAgentHeartbeatSweep(delayMs = PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS): Promise<void> {
32
- await planAgentHeartbeatQueue.enqueue({ type: 'sweep' }, { delay: delayMs, jobId: PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID })
36
+ interface PlanAgentHeartbeatQueueDeps {
37
+ databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
38
+ planAgentHeartbeatService: Context.Service.Shape<typeof PlanAgentHeartbeatServiceTag>
33
39
  }
34
40
 
35
- async function processPlanAgentHeartbeatJob(job: Job<PlanAgentHeartbeatJob>): Promise<void> {
36
- await databaseService.connect()
41
+ let _deps: PlanAgentHeartbeatQueueDeps | null = null
42
+ function getDeps(): PlanAgentHeartbeatQueueDeps {
43
+ if (!_deps)
44
+ throw new ConfigurationError({
45
+ message: 'Plan agent heartbeat queue is not configured. Initialize the runtime before starting the worker.',
46
+ key: 'queue-deps',
47
+ })
48
+ return _deps
49
+ }
37
50
 
38
- if (job.data.type === 'wake-node') {
39
- await planAgentHeartbeatService.wakeNode(job.data)
40
- return
41
- }
51
+ function enqueueDelayedPlanAgentHeartbeatSweep(delayMs = PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS): Promise<void> {
52
+ return planAgentHeartbeatQueue.enqueue(
53
+ { type: 'sweep' },
54
+ { delay: delayMs, jobId: PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID },
55
+ )
56
+ }
42
57
 
43
- await planAgentHeartbeatService.sweep({ organizationId: job.data.organizationId })
44
- if (!job.data.organizationId) {
45
- await enqueueDelayedPlanAgentHeartbeatSweep()
46
- }
58
+ function processPlanAgentHeartbeatJob(job: Job<PlanAgentHeartbeatJob>): Promise<void> {
59
+ const { planAgentHeartbeatService } = getDeps()
60
+
61
+ return Effect.runPromise(
62
+ Effect.gen(function* () {
63
+ if (job.data.type === 'wake-node') {
64
+ const wakeJob = job.data
65
+ yield* planAgentHeartbeatService.wakeNode(wakeJob)
66
+ return
67
+ }
68
+
69
+ yield* planAgentHeartbeatService.sweep({ organizationId: job.data.organizationId })
70
+ if (!job.data.organizationId) {
71
+ yield* Effect.tryPromise(() => enqueueDelayedPlanAgentHeartbeatSweep())
72
+ }
73
+ }),
74
+ )
47
75
  }
48
76
 
49
77
  const planAgentHeartbeatQueue = createQueueFactory<PlanAgentHeartbeatJob>({
@@ -53,6 +81,7 @@ const planAgentHeartbeatQueue = createQueueFactory<PlanAgentHeartbeatJob>({
53
81
  concurrency: 2,
54
82
  lockDuration: LONG_JOB_LOCK_DURATION_MS,
55
83
  defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 5_000 } },
84
+ prepare: () => getDeps().databaseService.connect(),
56
85
  processor: processPlanAgentHeartbeatJob,
57
86
  })
58
87
 
@@ -68,7 +97,7 @@ function buildWakeJobId(params: {
68
97
  return `plan-agent-wake__${encode(params.runId)}__${encode(params.nodeId)}__${encode(params.agentId)}`
69
98
  }
70
99
 
71
- export async function enqueuePlanAgentHeartbeatWake(params: {
100
+ export function enqueuePlanAgentHeartbeatWake(params: {
72
101
  organizationId: string
73
102
  threadId: string
74
103
  runId: string
@@ -76,11 +105,19 @@ export async function enqueuePlanAgentHeartbeatWake(params: {
76
105
  agentId: string
77
106
  reason: string
78
107
  }): Promise<void> {
79
- await planAgentHeartbeatQueue.enqueue({ type: 'wake-node', ...params }, { jobId: buildWakeJobId(params) })
108
+ return planAgentHeartbeatQueue.enqueue({ type: 'wake-node', ...params }, { jobId: buildWakeJobId(params) })
80
109
  }
81
110
 
82
- export function startPlanAgentHeartbeatWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
83
- const handle = planAgentHeartbeatQueue.startWorker(options)
111
+ export function startPlanAgentHeartbeatWorker(options: {
112
+ registerSignals?: boolean
113
+ connectionProvider: () => IORedis
114
+ deps: PlanAgentHeartbeatQueueDeps
115
+ }): WorkerHandle {
116
+ _deps = options.deps
117
+ const handle = planAgentHeartbeatQueue.startWorker({
118
+ registerSignals: options.registerSignals,
119
+ connectionProvider: options.connectionProvider,
120
+ })
84
121
 
85
122
  enqueueDelayedPlanAgentHeartbeatSweep().catch((error: unknown) => {
86
123
  serverLogger.error`Plan agent heartbeat scheduler setup failed: ${error}`
@@ -89,6 +126,13 @@ export function startPlanAgentHeartbeatWorker(options: { registerSignals?: boole
89
126
  return handle
90
127
  }
91
128
 
92
- if (import.meta.main) {
93
- startPlanAgentHeartbeatWorker()
94
- }
129
+ runStandaloneQueueWorker((runtime) => {
130
+ const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
131
+ startPlanAgentHeartbeatWorker({
132
+ connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
133
+ deps: {
134
+ databaseService: resolve(DatabaseServiceTag),
135
+ planAgentHeartbeatService: resolve(PlanAgentHeartbeatServiceTag),
136
+ },
137
+ })
138
+ })
@@ -1,11 +1,19 @@
1
1
  import type { Job } from 'bullmq'
2
+ import { Schema, Effect } from 'effect'
3
+ import type { Context } from 'effect'
4
+ import type IORedis from 'ioredis'
2
5
 
3
6
  import { serverLogger } from '../config/logger'
4
- import { databaseService } from '../db/service'
5
- import { planDeadlineService } from '../services/plan-deadline.service'
6
- import { planSchedulerService } from '../services/plan-scheduler.service'
7
+ import { ConfigurationError } from '../effect/errors'
8
+ import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
9
+ import { PlanCycleServiceTag } from '../services/plan/plan-cycle.service'
10
+ import { PlanDeadlineServiceTag } from '../services/plan/plan-deadline.service'
11
+ import { PlanExecutorServiceTag } from '../services/plan/plan-executor.service'
12
+ import { PlanSchedulerServiceTag } from '../services/plan/plan-scheduler.service'
13
+ import { nowEpochMillis } from '../utils/date-time'
7
14
  import type { WorkerHandle } from '../workers/worker-utils'
8
15
  import { createQueueFactory } from './queue-factory'
16
+ import { runStandaloneQueueWorker } from './standalone-worker'
9
17
 
10
18
  export interface PlanSchedulerFireJob {
11
19
  type: 'fire-schedule'
@@ -21,16 +29,50 @@ export type PlanSchedulerJob = PlanSchedulerFireJob | PlanSchedulerDeadlineJob
21
29
 
22
30
  export const PLAN_SCHEDULER_QUEUE = 'plan-scheduler'
23
31
 
24
- async function processPlanSchedulerJob(job: Job<PlanSchedulerJob>): Promise<void> {
25
- await databaseService.connect()
32
+ interface PlanSchedulerQueueDeps {
33
+ databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
34
+ planSchedulerService: Context.Service.Shape<typeof PlanSchedulerServiceTag>
35
+ planDeadlineService: Context.Service.Shape<typeof PlanDeadlineServiceTag>
36
+ planExecutorService: Context.Service.Shape<typeof PlanExecutorServiceTag>
37
+ planCycleService: Context.Service.Shape<typeof PlanCycleServiceTag>
38
+ }
39
+
40
+ let _deps: PlanSchedulerQueueDeps | null = null
41
+ function getDeps(): PlanSchedulerQueueDeps {
42
+ if (!_deps)
43
+ throw new ConfigurationError({
44
+ message: 'Plan scheduler queue is not configured. Initialize the runtime before starting the worker.',
45
+ key: 'queue-deps',
46
+ })
47
+ return _deps
48
+ }
49
+
50
+ class PlanSchedulerQueueError extends Schema.TaggedErrorClass<PlanSchedulerQueueError>()('PlanSchedulerQueueError', {
51
+ stage: Schema.Literals(['remove-schedule-fire-job', 'recover-active-schedules', 'recover-deadline-checks']),
52
+ message: Schema.String,
53
+ cause: Schema.Defect,
54
+ }) {}
55
+
56
+ function toPlanSchedulerQueueError(stage: PlanSchedulerQueueError['stage'], cause: unknown): PlanSchedulerQueueError {
57
+ return new PlanSchedulerQueueError({ stage, message: cause instanceof Error ? cause.message : String(cause), cause })
58
+ }
59
+
60
+ function processPlanSchedulerJob(job: Job<PlanSchedulerJob>): Promise<void> {
61
+ const { planSchedulerService, planDeadlineService, planExecutorService, planCycleService } = getDeps()
26
62
 
27
63
  switch (job.data.type) {
28
64
  case 'fire-schedule':
29
- await planSchedulerService.fireScheduleById(job.data.scheduleId)
30
- break
65
+ return Effect.runPromise(
66
+ Effect.asVoid(
67
+ planSchedulerService.fireScheduleById(job.data.scheduleId, {
68
+ promoteDelayedNode: (params) => planExecutorService.promoteDelayedNode(params),
69
+ advanceCycle: (cycleId) => planCycleService.advanceCycle(cycleId),
70
+ recoverDeadlineChecks: () => Effect.runPromise(Effect.asVoid(planDeadlineService.recoverDeadlineChecks())),
71
+ }),
72
+ ),
73
+ )
31
74
  case 'check-deadlines':
32
- await planDeadlineService.checkDeadlines()
33
- break
75
+ return Effect.runPromise(Effect.asVoid(planDeadlineService.checkDeadlines()))
34
76
  }
35
77
  }
36
78
 
@@ -40,31 +82,33 @@ const planScheduler = createQueueFactory<PlanSchedulerJob>({
40
82
  jobName: 'plan-scheduler-job',
41
83
  concurrency: 1,
42
84
  defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
85
+ prepare: () => getDeps().databaseService.connect(),
43
86
  processor: processPlanSchedulerJob,
44
87
  })
45
88
 
46
89
  /** Enqueue a delayed job that fires a specific schedule at its nextFireAt time. */
47
- export async function enqueueScheduleFire(scheduleId: string, delayMs: number): Promise<void> {
48
- await planScheduler.enqueue(
90
+ export function enqueueScheduleFire(scheduleId: string, delayMs: number): Promise<void> {
91
+ return planScheduler.enqueue(
49
92
  { type: 'fire-schedule', scheduleId },
50
93
  { delay: Math.max(0, delayMs), jobId: `schedule:${scheduleId}`, removeOnComplete: true, removeOnFail: 50 },
51
94
  )
52
95
  }
53
96
 
54
97
  /** Remove a pending fire job for a schedule (on cancel/pause/complete). */
55
- export async function removeScheduleFireJob(scheduleId: string): Promise<void> {
56
- try {
57
- await planScheduler.getQueue().remove(`schedule:${scheduleId}`)
58
- } catch {
59
- // Job may not exist (already fired or never enqueued) — safe to ignore
60
- }
98
+ export function removeScheduleFireJob(scheduleId: string): Promise<void> {
99
+ return Effect.runPromise(
100
+ Effect.tryPromise({
101
+ try: () => planScheduler.getQueue().remove(`schedule:${scheduleId}`),
102
+ catch: (cause) => toPlanSchedulerQueueError('remove-schedule-fire-job', cause),
103
+ }).pipe(Effect.asVoid),
104
+ )
61
105
  }
62
106
 
63
107
  const DEADLINE_CHECK_JOB_PREFIX = 'deadline-check'
64
108
 
65
- export async function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
66
- const delay = Math.max(0, scheduledFor.getTime() - Date.now())
67
- await planScheduler.enqueue(
109
+ export function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
110
+ const delay = Math.max(0, scheduledFor.getTime() - nowEpochMillis())
111
+ return planScheduler.enqueue(
68
112
  { type: 'check-deadlines', scheduledFor: scheduledFor.toISOString() },
69
113
  {
70
114
  delay,
@@ -75,23 +119,55 @@ export async function enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
75
119
  )
76
120
  }
77
121
 
78
- export function startPlanSchedulerWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
79
- const handle = planScheduler.startWorker(options)
122
+ export function startPlanSchedulerWorker(options: {
123
+ registerSignals?: boolean
124
+ connectionProvider: () => IORedis
125
+ deps: PlanSchedulerQueueDeps
126
+ }): WorkerHandle {
127
+ _deps = options.deps
128
+ const handle = planScheduler.startWorker({
129
+ registerSignals: options.registerSignals,
130
+ connectionProvider: options.connectionProvider,
131
+ })
80
132
 
81
133
  // Recover active schedules on startup
82
- planSchedulerService.recoverActiveSchedules().catch((err: unknown) => {
83
- serverLogger.error`Plan scheduler startup recovery failed: ${err}`
84
- })
134
+ void Effect.runFork(
135
+ options.deps.planSchedulerService.recoverActiveSchedules().pipe(
136
+ Effect.mapError((cause) => toPlanSchedulerQueueError('recover-active-schedules', cause)),
137
+ Effect.catchTag('PlanSchedulerQueueError', (error) =>
138
+ Effect.sync(() => {
139
+ serverLogger.error`Plan scheduler startup recovery failed: ${error.message}`
140
+ }),
141
+ ),
142
+ ),
143
+ )
85
144
 
86
145
  // Seed deadline checks from current runtime state. Subsequent checks are
87
146
  // re-seeded by the queue job after each sweep and by schedule fire events.
88
- planDeadlineService.recoverDeadlineChecks().catch((err: unknown) => {
89
- serverLogger.error`Plan deadline recovery failed: ${err}`
90
- })
147
+ void Effect.runFork(
148
+ options.deps.planDeadlineService.recoverDeadlineChecks().pipe(
149
+ Effect.mapError((cause) => toPlanSchedulerQueueError('recover-deadline-checks', cause)),
150
+ Effect.catchTag('PlanSchedulerQueueError', (error) =>
151
+ Effect.sync(() => {
152
+ serverLogger.error`Plan deadline recovery failed: ${error.message}`
153
+ }),
154
+ ),
155
+ ),
156
+ )
91
157
 
92
158
  return handle
93
159
  }
94
160
 
95
- if (import.meta.main) {
96
- startPlanSchedulerWorker()
97
- }
161
+ runStandaloneQueueWorker((runtime) => {
162
+ const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
163
+ startPlanSchedulerWorker({
164
+ connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
165
+ deps: {
166
+ databaseService: resolve(DatabaseServiceTag),
167
+ planSchedulerService: resolve(PlanSchedulerServiceTag),
168
+ planDeadlineService: resolve(PlanDeadlineServiceTag),
169
+ planExecutorService: resolve(PlanExecutorServiceTag),
170
+ planCycleService: resolve(PlanCycleServiceTag),
171
+ },
172
+ })
173
+ })