@lota-sdk/core 0.4.8 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (272) hide show
  1. package/package.json +11 -12
  2. package/src/ai/embedding-cache.ts +96 -22
  3. package/src/ai-gateway/ai-gateway.ts +766 -223
  4. package/src/config/agent-defaults.ts +189 -75
  5. package/src/config/agent-types.ts +54 -4
  6. package/src/config/background-processing.ts +1 -1
  7. package/src/config/constants.ts +8 -2
  8. package/src/config/index.ts +0 -1
  9. package/src/config/logger.ts +299 -19
  10. package/src/config/thread-defaults.ts +40 -20
  11. package/src/create-runtime.ts +200 -449
  12. package/src/db/base.service.ts +52 -28
  13. package/src/db/cursor-pagination.ts +71 -30
  14. package/src/db/memory-query-builder.ts +2 -1
  15. package/src/db/memory-store.helpers.ts +4 -7
  16. package/src/db/memory-store.ts +868 -601
  17. package/src/db/memory.ts +396 -280
  18. package/src/db/record-id.ts +32 -10
  19. package/src/db/schema-fingerprint.ts +30 -12
  20. package/src/db/service-normalization.ts +288 -0
  21. package/src/db/service.ts +912 -779
  22. package/src/db/startup.ts +153 -68
  23. package/src/db/transaction-conflict.ts +15 -0
  24. package/src/effect/awaitable-effect.ts +96 -0
  25. package/src/effect/errors.ts +121 -0
  26. package/src/effect/helpers.ts +123 -0
  27. package/src/effect/index.ts +24 -0
  28. package/src/effect/layers.ts +238 -0
  29. package/src/effect/runtime-ref.ts +25 -0
  30. package/src/effect/runtime.ts +46 -0
  31. package/src/effect/services.ts +61 -0
  32. package/src/effect/zod.ts +43 -0
  33. package/src/embeddings/provider.ts +128 -83
  34. package/src/index.ts +48 -1
  35. package/src/openrouter/direct-provider.ts +11 -35
  36. package/src/queues/autonomous-job.queue.ts +117 -73
  37. package/src/queues/context-compaction.queue.ts +50 -17
  38. package/src/queues/delayed-node-promotion.queue.ts +46 -17
  39. package/src/queues/document-processor.queue.ts +52 -77
  40. package/src/queues/memory-consolidation.queue.ts +47 -32
  41. package/src/queues/organization-learning.queue.ts +26 -4
  42. package/src/queues/plan-agent-heartbeat.queue.ts +71 -24
  43. package/src/queues/plan-scheduler.queue.ts +97 -33
  44. package/src/queues/post-chat-memory.queue.ts +56 -26
  45. package/src/queues/queue-factory.ts +227 -59
  46. package/src/queues/standalone-worker.ts +39 -0
  47. package/src/queues/title-generation.queue.ts +45 -11
  48. package/src/redis/connection.ts +182 -113
  49. package/src/redis/index.ts +6 -8
  50. package/src/redis/org-memory-lock.ts +60 -27
  51. package/src/redis/redis-lease-lock.ts +200 -121
  52. package/src/redis/runtime-connection.ts +20 -0
  53. package/src/redis/stream-context.ts +92 -46
  54. package/src/runtime/agent-identity-overrides.ts +2 -2
  55. package/src/runtime/agent-runtime-policy.ts +5 -2
  56. package/src/runtime/agent-stream-helpers.ts +24 -9
  57. package/src/runtime/chat-run-orchestration.ts +102 -19
  58. package/src/runtime/chat-run-registry.ts +36 -2
  59. package/src/runtime/context-compaction/context-compaction-runtime.ts +107 -0
  60. package/src/runtime/{context-compaction.ts → context-compaction/context-compaction.ts} +161 -94
  61. package/src/runtime/domain-layer.ts +192 -0
  62. package/src/runtime/execution-plan-visibility.ts +2 -2
  63. package/src/runtime/execution-plan.ts +42 -15
  64. package/src/runtime/graph-designer.ts +16 -4
  65. package/src/runtime/helper-model.ts +139 -48
  66. package/src/runtime/index.ts +7 -8
  67. package/src/runtime/indexed-repositories-policy.ts +3 -3
  68. package/src/runtime/{memory-block.ts → memory/memory-block.ts} +50 -36
  69. package/src/runtime/{memory-digest-policy.ts → memory/memory-digest-policy.ts} +1 -1
  70. package/src/runtime/{memory-pipeline.ts → memory/memory-pipeline.ts} +54 -67
  71. package/src/runtime/{memory-prompts-fact.ts → memory/memory-prompts-fact.ts} +2 -2
  72. package/src/runtime/memory/memory-scope.ts +53 -0
  73. package/src/runtime/plugin-resolution.ts +124 -25
  74. package/src/runtime/plugin-types.ts +9 -1
  75. package/src/runtime/post-turn-side-effects.ts +177 -130
  76. package/src/runtime/retrieval-adapters.ts +40 -6
  77. package/src/runtime/runtime-accessors.ts +92 -0
  78. package/src/runtime/runtime-config.ts +150 -61
  79. package/src/runtime/runtime-extensions.ts +23 -25
  80. package/src/runtime/runtime-lifecycle.ts +124 -0
  81. package/src/runtime/runtime-services.ts +386 -0
  82. package/src/runtime/runtime-token.ts +47 -0
  83. package/src/runtime/social-chat/social-chat-agent-runner.ts +159 -0
  84. package/src/runtime/{social-chat-history.ts → social-chat/social-chat-history.ts} +51 -20
  85. package/src/runtime/social-chat/social-chat.ts +630 -0
  86. package/src/runtime/specialist-runner.ts +36 -10
  87. package/src/runtime/team-consultation/team-consultation-orchestrator.ts +433 -0
  88. package/src/runtime/{team-consultation-prompts.ts → team-consultation/team-consultation-prompts.ts} +6 -2
  89. package/src/runtime/thread-chat-helpers.ts +2 -2
  90. package/src/runtime/thread-plan-turn.ts +2 -1
  91. package/src/runtime/thread-turn-context.ts +183 -111
  92. package/src/runtime/turn-lifecycle.ts +93 -27
  93. package/src/services/agent-activity.service.ts +287 -203
  94. package/src/services/agent-executor.service.ts +253 -149
  95. package/src/services/artifact.service.ts +231 -149
  96. package/src/services/attachment.service.ts +171 -115
  97. package/src/services/autonomous-job.service.ts +890 -491
  98. package/src/services/background-work.service.ts +54 -0
  99. package/src/services/chat-run-registry.service.ts +13 -1
  100. package/src/services/context-compaction.service.ts +136 -86
  101. package/src/services/document-chunk.service.ts +151 -88
  102. package/src/services/execution-plan/execution-plan-approval.ts +26 -0
  103. package/src/services/execution-plan/execution-plan-context.ts +29 -0
  104. package/src/services/execution-plan/execution-plan-graph.ts +278 -0
  105. package/src/services/execution-plan/execution-plan-schedule.ts +84 -0
  106. package/src/services/execution-plan/execution-plan-spec.ts +75 -0
  107. package/src/services/execution-plan/execution-plan.service.ts +1041 -0
  108. package/src/services/feedback-loop.service.ts +132 -76
  109. package/src/services/global-orchestrator.service.ts +101 -168
  110. package/src/services/graph-full-routing.ts +193 -0
  111. package/src/services/index.ts +19 -21
  112. package/src/services/institutional-memory.service.ts +213 -125
  113. package/src/services/learned-skill.service.ts +368 -260
  114. package/src/services/memory/memory-conversation.ts +95 -0
  115. package/src/services/memory/memory-errors.ts +27 -0
  116. package/src/services/memory/memory-org-memory.ts +50 -0
  117. package/src/services/memory/memory-preseeded.ts +86 -0
  118. package/src/services/memory/memory-rerank.ts +297 -0
  119. package/src/services/{memory-utils.ts → memory/memory-utils.ts} +6 -5
  120. package/src/services/memory/memory.service.ts +674 -0
  121. package/src/services/memory/rerank.service.ts +201 -0
  122. package/src/services/monitoring-window.service.ts +92 -70
  123. package/src/services/mutating-approval.service.ts +62 -53
  124. package/src/services/node-workspace.service.ts +141 -98
  125. package/src/services/notification.service.ts +29 -16
  126. package/src/services/organization-member.service.ts +120 -66
  127. package/src/services/organization.service.ts +153 -77
  128. package/src/services/ownership-dispatcher.service.ts +456 -263
  129. package/src/services/plan/plan-agent-heartbeat.service.ts +234 -0
  130. package/src/services/plan/plan-agent-query.service.ts +322 -0
  131. package/src/services/{plan-approval.service.ts → plan/plan-approval.service.ts} +45 -22
  132. package/src/services/plan/plan-artifact.service.ts +60 -0
  133. package/src/services/plan/plan-builder.service.ts +76 -0
  134. package/src/services/plan/plan-checkpoint.service.ts +103 -0
  135. package/src/services/{plan-compiler.service.ts → plan/plan-compiler.service.ts} +26 -9
  136. package/src/services/plan/plan-completion-side-effects.ts +169 -0
  137. package/src/services/plan/plan-coordination.service.ts +181 -0
  138. package/src/services/plan/plan-cycle.service.ts +405 -0
  139. package/src/services/plan/plan-deadline.service.ts +533 -0
  140. package/src/services/plan/plan-event-delivery.service.ts +266 -0
  141. package/src/services/plan/plan-executor-context.ts +35 -0
  142. package/src/services/plan/plan-executor-graph.ts +522 -0
  143. package/src/services/plan/plan-executor-helpers.ts +307 -0
  144. package/src/services/plan/plan-executor-persistence.ts +209 -0
  145. package/src/services/plan/plan-executor.service.ts +1737 -0
  146. package/src/services/{plan-helpers.ts → plan/plan-helpers.ts} +1 -1
  147. package/src/services/{plan-run-data.ts → plan/plan-run-data.ts} +4 -4
  148. package/src/services/plan/plan-run-serialization.ts +15 -0
  149. package/src/services/plan/plan-run.service.ts +637 -0
  150. package/src/services/plan/plan-scheduler.service.ts +379 -0
  151. package/src/services/plan/plan-template.service.ts +224 -0
  152. package/src/services/plan/plan-transaction-events.ts +36 -0
  153. package/src/services/plan/plan-validator.service.ts +907 -0
  154. package/src/services/plan/plan-workspace.service.ts +131 -0
  155. package/src/services/plugin-executor.service.ts +102 -68
  156. package/src/services/quality-metrics.service.ts +112 -94
  157. package/src/services/queue-job.service.ts +288 -231
  158. package/src/services/recent-activity-title.service.ts +73 -36
  159. package/src/services/recent-activity.service.ts +274 -259
  160. package/src/services/skill-resolver.service.ts +38 -12
  161. package/src/services/social-chat-history.service.ts +190 -122
  162. package/src/services/system-executor.service.ts +96 -61
  163. package/src/services/thread/thread-active-run.ts +203 -0
  164. package/src/services/thread/thread-bootstrap.ts +385 -0
  165. package/src/services/thread/thread-listing.ts +199 -0
  166. package/src/services/thread/thread-memory-block.ts +130 -0
  167. package/src/services/thread/thread-message.service.ts +379 -0
  168. package/src/services/thread/thread-record-store.ts +155 -0
  169. package/src/services/thread/thread-title.service.ts +74 -0
  170. package/src/services/thread/thread-turn-execution.ts +280 -0
  171. package/src/services/thread/thread-turn-message-context.ts +73 -0
  172. package/src/services/thread/thread-turn-preparation.service.ts +1148 -0
  173. package/src/services/thread/thread-turn-streaming.ts +403 -0
  174. package/src/services/thread/thread-turn-tracing.ts +35 -0
  175. package/src/services/thread/thread-turn.ts +376 -0
  176. package/src/services/thread/thread.service.ts +344 -0
  177. package/src/services/user.service.ts +82 -32
  178. package/src/services/write-intent-validator.service.ts +63 -51
  179. package/src/storage/attachment-parser.ts +69 -27
  180. package/src/storage/attachment-storage.service.ts +334 -275
  181. package/src/storage/generated-document-storage.service.ts +66 -34
  182. package/src/system-agents/agent-result.ts +3 -1
  183. package/src/system-agents/context-compaction.agent.ts +3 -3
  184. package/src/system-agents/delegated-agent-factory.ts +159 -90
  185. package/src/system-agents/helper-agent-options.ts +1 -1
  186. package/src/system-agents/memory-reranker.agent.ts +3 -3
  187. package/src/system-agents/memory.agent.ts +3 -3
  188. package/src/system-agents/recent-activity-title-refiner.agent.ts +3 -3
  189. package/src/system-agents/regular-chat-memory-digest.agent.ts +3 -3
  190. package/src/system-agents/skill-extractor.agent.ts +3 -3
  191. package/src/system-agents/skill-manager.agent.ts +3 -3
  192. package/src/system-agents/thread-router.agent.ts +157 -113
  193. package/src/system-agents/title-generator.agent.ts +3 -3
  194. package/src/tools/execution-plan.tool.ts +241 -171
  195. package/src/tools/fetch-webpage.tool.ts +29 -18
  196. package/src/tools/firecrawl-client.ts +26 -6
  197. package/src/tools/index.ts +1 -0
  198. package/src/tools/memory-block.tool.ts +14 -6
  199. package/src/tools/plan-approval.tool.ts +57 -47
  200. package/src/tools/read-file-parts.tool.ts +44 -33
  201. package/src/tools/remember-memory.tool.ts +65 -45
  202. package/src/tools/search-web.tool.ts +33 -22
  203. package/src/tools/search.tool.ts +41 -29
  204. package/src/tools/team-think.tool.ts +125 -84
  205. package/src/tools/user-questions.tool.ts +4 -3
  206. package/src/tools/web-tool-shared.ts +6 -0
  207. package/src/utils/async.ts +25 -22
  208. package/src/utils/crypto.ts +21 -0
  209. package/src/utils/date-time.ts +40 -1
  210. package/src/utils/errors.ts +111 -20
  211. package/src/utils/hono-error-handler.ts +24 -39
  212. package/src/utils/index.ts +2 -1
  213. package/src/utils/null-proto-record.ts +41 -0
  214. package/src/utils/sse-keepalive.ts +124 -21
  215. package/src/workers/bootstrap.ts +164 -52
  216. package/src/workers/memory-consolidation.worker.ts +325 -237
  217. package/src/workers/organization-learning.worker.ts +50 -16
  218. package/src/workers/regular-chat-memory-digest.helpers.ts +28 -27
  219. package/src/workers/regular-chat-memory-digest.runner.ts +185 -114
  220. package/src/workers/skill-extraction.runner.ts +176 -93
  221. package/src/workers/utils/file-section-chunker.ts +8 -10
  222. package/src/workers/utils/repo-structure-extractor.ts +349 -260
  223. package/src/workers/utils/repomix-file-sections.ts +2 -2
  224. package/src/workers/utils/thread-message-query.ts +97 -38
  225. package/src/workers/worker-utils.ts +74 -31
  226. package/src/config/debug-logger.ts +0 -47
  227. package/src/config/search.ts +0 -3
  228. package/src/redis/connection-accessor.ts +0 -26
  229. package/src/runtime/agent-types.ts +0 -1
  230. package/src/runtime/context-compaction-runtime.ts +0 -87
  231. package/src/runtime/memory-scope.ts +0 -43
  232. package/src/runtime/social-chat-agent-runner.ts +0 -118
  233. package/src/runtime/social-chat.ts +0 -516
  234. package/src/runtime/team-consultation-orchestrator.ts +0 -272
  235. package/src/services/adaptive-playbook.service.ts +0 -152
  236. package/src/services/artifact-provenance.service.ts +0 -172
  237. package/src/services/chat-attachments.service.ts +0 -17
  238. package/src/services/context-compaction-runtime.singleton.ts +0 -13
  239. package/src/services/execution-plan.service.ts +0 -1118
  240. package/src/services/memory.service.ts +0 -914
  241. package/src/services/plan-agent-heartbeat.service.ts +0 -136
  242. package/src/services/plan-agent-query.service.ts +0 -267
  243. package/src/services/plan-artifact.service.ts +0 -50
  244. package/src/services/plan-builder.service.ts +0 -67
  245. package/src/services/plan-checkpoint.service.ts +0 -81
  246. package/src/services/plan-completion-side-effects.ts +0 -80
  247. package/src/services/plan-coordination.service.ts +0 -157
  248. package/src/services/plan-cycle.service.ts +0 -284
  249. package/src/services/plan-deadline.service.ts +0 -430
  250. package/src/services/plan-event-delivery.service.ts +0 -166
  251. package/src/services/plan-executor.service.ts +0 -1950
  252. package/src/services/plan-run.service.ts +0 -515
  253. package/src/services/plan-scheduler.service.ts +0 -240
  254. package/src/services/plan-template.service.ts +0 -177
  255. package/src/services/plan-validator.service.ts +0 -818
  256. package/src/services/plan-workspace.service.ts +0 -83
  257. package/src/services/rerank.service.ts +0 -156
  258. package/src/services/thread-message.service.ts +0 -275
  259. package/src/services/thread-plan-registry.service.ts +0 -22
  260. package/src/services/thread-title.service.ts +0 -39
  261. package/src/services/thread-turn-preparation.service.ts +0 -1147
  262. package/src/services/thread-turn.ts +0 -172
  263. package/src/services/thread.service.ts +0 -869
  264. package/src/utils/env.ts +0 -8
  265. /package/src/runtime/{context-compaction-constants.ts → context-compaction/context-compaction-constants.ts} +0 -0
  266. /package/src/runtime/{memory-format.ts → memory/memory-format.ts} +0 -0
  267. /package/src/runtime/{memory-prompts-parse.ts → memory/memory-prompts-parse.ts} +0 -0
  268. /package/src/runtime/{memory-prompts-update.ts → memory/memory-prompts-update.ts} +0 -0
  269. /package/src/runtime/{social-chat-prompts.ts → social-chat/social-chat-prompts.ts} +0 -0
  270. /package/src/services/{plan-node-spec.ts → plan/plan-node-spec.ts} +0 -0
  271. /package/src/services/{thread-constants.ts → thread/thread-constants.ts} +0 -0
  272. /package/src/services/{thread.types.ts → thread/thread.types.ts} +0 -0
@@ -1,11 +1,15 @@
1
1
  import type { Job } from 'bullmq'
2
+ import { Effect } from 'effect'
3
+ import type { Context } from 'effect'
4
+ import type IORedis from 'ioredis'
2
5
 
3
6
  import { ensureRecordId } from '../db/record-id'
4
- import { databaseService } from '../db/service'
5
7
  import { TABLES } from '../db/tables'
6
- import { contextCompactionService } from '../services/context-compaction.service'
7
- import { threadService } from '../services/thread.service'
8
- import { createQueueFactory } from './queue-factory'
8
+ import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
9
+ import { ContextCompactionServiceTag } from '../services/context-compaction.service'
10
+ import { ThreadServiceTag } from '../services/thread/thread.service'
11
+ import { createQueueFactoryWithDeps } from './queue-factory'
12
+ import { runStandaloneQueueWorker } from './standalone-worker'
9
13
 
10
14
  interface ContextCompactionJob {
11
15
  domain: 'thread'
@@ -13,26 +17,37 @@ interface ContextCompactionJob {
13
17
  contextSize?: number
14
18
  }
15
19
 
16
- async function processContextCompactionJob(job: Job<ContextCompactionJob>): Promise<void> {
17
- await databaseService.connect()
20
+ interface ContextCompactionQueueDeps {
21
+ databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
22
+ threadService: Context.Service.Shape<typeof ThreadServiceTag>
23
+ contextCompactionService: Context.Service.Shape<typeof ContextCompactionServiceTag>
24
+ }
18
25
 
26
+ function processContextCompactionJob(deps: ContextCompactionQueueDeps, job: Job<ContextCompactionJob>): Promise<void> {
27
+ const { threadService, contextCompactionService } = deps
19
28
  const { entityId, contextSize } = job.data
20
29
  const threadRef = ensureRecordId(entityId, TABLES.THREAD)
21
- await threadService.setCompacting(threadRef, true)
22
- try {
23
- await contextCompactionService.compactThreadHistory({ threadId: threadRef, contextSize })
24
- } finally {
25
- await threadService.setCompacting(threadRef, false)
26
- }
30
+ return Effect.runPromise(
31
+ Effect.ensuring(
32
+ Effect.asVoid(
33
+ Effect.gen(function* () {
34
+ yield* threadService.setCompacting(threadRef, true)
35
+ yield* contextCompactionService.compactThreadHistory({ threadId: threadRef, contextSize })
36
+ }),
37
+ ),
38
+ Effect.ignore(threadService.setCompacting(threadRef, false)),
39
+ ),
40
+ )
27
41
  }
28
42
 
29
- const contextCompaction = createQueueFactory<ContextCompactionJob>({
43
+ const contextCompaction = createQueueFactoryWithDeps<ContextCompactionJob, ContextCompactionQueueDeps>({
30
44
  name: 'context-compaction',
31
45
  displayName: 'Context compaction',
32
46
  jobName: 'compact',
33
47
  concurrency: 2,
34
48
  lockDuration: 300_000,
35
49
  defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 3_000 } },
50
+ prepare: ({ databaseService }) => databaseService.connect(),
36
51
  processor: processContextCompactionJob,
37
52
  })
38
53
 
@@ -40,8 +55,26 @@ export function enqueueContextCompaction(job: ContextCompactionJob) {
40
55
  return contextCompaction.enqueue(job, { deduplication: { id: `compact:${job.domain}:${job.entityId}` } })
41
56
  }
42
57
 
43
- export const startContextCompactionWorker = contextCompaction.startWorker
44
-
45
- if (import.meta.main) {
46
- startContextCompactionWorker()
58
+ export function startContextCompactionWorker(options: {
59
+ registerSignals?: boolean
60
+ connectionProvider: () => IORedis
61
+ deps: ContextCompactionQueueDeps
62
+ }): ReturnType<typeof contextCompaction.startWorker> {
63
+ return contextCompaction.startWorker({
64
+ deps: options.deps,
65
+ registerSignals: options.registerSignals,
66
+ connectionProvider: options.connectionProvider,
67
+ })
47
68
  }
69
+
70
+ runStandaloneQueueWorker((runtime) => {
71
+ const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
72
+ startContextCompactionWorker({
73
+ connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
74
+ deps: {
75
+ databaseService: resolve(DatabaseServiceTag),
76
+ threadService: resolve(ThreadServiceTag),
77
+ contextCompactionService: resolve(ContextCompactionServiceTag),
78
+ },
79
+ })
80
+ })
@@ -1,8 +1,12 @@
1
1
  import type { Job } from 'bullmq'
2
+ import { Effect } from 'effect'
3
+ import type { Context } from 'effect'
4
+ import type IORedis from 'ioredis'
2
5
 
3
- import { databaseService } from '../db/service'
4
- import { planExecutorService } from '../services/plan-executor.service'
5
- import { createQueueFactory } from './queue-factory'
6
+ import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
7
+ import { PlanExecutorServiceTag } from '../services/plan/plan-executor.service'
8
+ import { createQueueFactoryWithDeps } from './queue-factory'
9
+ import { runStandaloneQueueWorker } from './standalone-worker'
6
10
 
7
11
  export interface DelayedNodePromotionJob {
8
12
  runId: string
@@ -12,30 +16,55 @@ export interface DelayedNodePromotionJob {
12
16
 
13
17
  export const DELAYED_NODE_PROMOTION_QUEUE = 'delayed-node-promotion'
14
18
 
15
- async function processDelayedNodePromotionJob(job: Job<DelayedNodePromotionJob>): Promise<void> {
16
- await databaseService.connect()
17
- await planExecutorService.promoteDelayedNode({
18
- runId: job.data.runId,
19
- nodeId: job.data.nodeId,
20
- emittedBy: job.data.emittedBy,
21
- })
19
+ interface DelayedNodePromotionQueueDeps {
20
+ databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
21
+ planExecutorService: Context.Service.Shape<typeof PlanExecutorServiceTag>
22
+ }
23
+
24
+ function processDelayedNodePromotionJob(
25
+ deps: DelayedNodePromotionQueueDeps,
26
+ job: Job<DelayedNodePromotionJob>,
27
+ ): Promise<void> {
28
+ const { planExecutorService } = deps
29
+ return Promise.resolve(
30
+ planExecutorService.promoteDelayedNode({
31
+ runId: job.data.runId,
32
+ nodeId: job.data.nodeId,
33
+ emittedBy: job.data.emittedBy,
34
+ }),
35
+ ).then(() => undefined)
22
36
  }
23
37
 
24
- const delayedNodePromotion = createQueueFactory<DelayedNodePromotionJob>({
38
+ const delayedNodePromotion = createQueueFactoryWithDeps<DelayedNodePromotionJob, DelayedNodePromotionQueueDeps>({
25
39
  name: DELAYED_NODE_PROMOTION_QUEUE,
26
40
  displayName: 'Delayed node promotion',
27
41
  jobName: 'promote-node',
28
42
  concurrency: 1,
29
43
  defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
44
+ prepare: ({ databaseService }) => databaseService.connect(),
30
45
  processor: processDelayedNodePromotionJob,
31
46
  })
32
47
 
33
- export async function enqueueDelayedNodePromotion(job: DelayedNodePromotionJob, delayMs: number) {
34
- await delayedNodePromotion.enqueue(job, { delay: delayMs, jobId: `promote:${job.runId}:${job.nodeId}` })
48
+ export function enqueueDelayedNodePromotion(job: DelayedNodePromotionJob, delayMs: number) {
49
+ return delayedNodePromotion.enqueue(job, { delay: delayMs, jobId: `promote:${job.runId}:${job.nodeId}` })
35
50
  }
36
51
 
37
- export const startDelayedNodePromotionWorker = delayedNodePromotion.startWorker
38
-
39
- if (import.meta.main) {
40
- startDelayedNodePromotionWorker()
52
+ export function startDelayedNodePromotionWorker(options: {
53
+ registerSignals?: boolean
54
+ connectionProvider: () => IORedis
55
+ deps: DelayedNodePromotionQueueDeps
56
+ }): ReturnType<typeof delayedNodePromotion.startWorker> {
57
+ return delayedNodePromotion.startWorker({
58
+ deps: options.deps,
59
+ registerSignals: options.registerSignals,
60
+ connectionProvider: options.connectionProvider,
61
+ })
41
62
  }
63
+
64
+ runStandaloneQueueWorker((runtime) => {
65
+ const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
66
+ startDelayedNodePromotionWorker({
67
+ connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
68
+ deps: { databaseService: resolve(DatabaseServiceTag), planExecutorService: resolve(PlanExecutorServiceTag) },
69
+ })
70
+ })
@@ -1,16 +1,11 @@
1
- import { Queue, Worker } from 'bullmq'
2
- import type { QueueOptions } from 'bullmq'
1
+ import { Effect } from 'effect'
3
2
  import type IORedis from 'ioredis'
4
3
 
5
4
  import type { chatLogger } from '../config/logger'
6
- import { queueJobService } from '../services/queue-job.service'
7
- import {
8
- attachWorkerEvents,
9
- createWorkerShutdown,
10
- DEFAULT_JOB_RETENTION,
11
- registerShutdownSignals,
12
- } from '../workers/worker-utils'
5
+ import { sha256Hex } from '../utils/crypto'
6
+ import { DEFAULT_JOB_RETENTION, getQueueJobService } from '../workers/worker-utils'
13
7
  import type { WorkerHandle } from '../workers/worker-utils'
8
+ import { createQueueFactory } from './queue-factory'
14
9
 
15
10
  export type DocumentSourceChannel = string
16
11
 
@@ -43,18 +38,16 @@ export function buildDocumentProcessorJobId(
43
38
  'orgId' | 'source' | 'sourceId' | 'sourceCanonicalKey' | 'sourceVersionKey' | 'title'
44
39
  >,
45
40
  ): string {
46
- const digest = new Bun.CryptoHasher('sha256')
47
- .update(
48
- JSON.stringify({
49
- orgId: job.orgId,
50
- source: job.source,
51
- sourceId: job.sourceId,
52
- sourceCanonicalKey: job.sourceCanonicalKey,
53
- sourceVersionKey: job.sourceVersionKey,
54
- title: job.title,
55
- }),
56
- )
57
- .digest('hex')
41
+ const digest = sha256Hex(
42
+ JSON.stringify({
43
+ orgId: job.orgId,
44
+ source: job.source,
45
+ sourceId: job.sourceId,
46
+ sourceCanonicalKey: job.sourceCanonicalKey,
47
+ sourceVersionKey: job.sourceVersionKey,
48
+ title: job.title,
49
+ }),
50
+ )
58
51
 
59
52
  return `doc__${digest}`
60
53
  }
@@ -67,68 +60,50 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
67
60
  workerName?: string
68
61
  concurrency?: number
69
62
  lockDuration?: number
70
- }): {
71
- enqueue: (job: TJob) => Promise<unknown>
72
- startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle
73
- } {
74
- type QueueShape = Queue<TJob, unknown, string, TJob, unknown, string>
75
-
63
+ }): { enqueue: (job: TJob) => Promise<void>; startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle } {
76
64
  const queueName = params.queueName ?? DEFAULT_DOCUMENT_PROCESSOR_QUEUE
77
65
  const workerName = params.workerName ?? DEFAULT_WORKER_NAME
78
66
  const concurrency = params.concurrency ?? 10
79
67
  const lockDuration = params.lockDuration ?? 300_000
80
- const jobName = 'process-document' as Parameters<QueueShape['add']>[0]
81
- const toQueueData = (job: TJob): Parameters<QueueShape['add']>[1] => job
82
- let queue: QueueShape | null = null
83
- const getConnection = (): IORedis => params.getConnectionForBullMQ()
84
-
85
- const getQueue = (): QueueShape => {
86
- if (queue) {
87
- return queue
88
- }
89
-
90
- queue = new Queue<TJob, unknown, string, TJob, unknown, string>(queueName, {
91
- connection: getConnection() as QueueOptions['connection'],
92
- defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
93
- })
94
-
95
- return queue
96
- }
68
+ const queueRuntime = createQueueFactory<TJob>({
69
+ name: queueName,
70
+ displayName: workerName,
71
+ jobName: 'process-document',
72
+ concurrency,
73
+ lockDuration,
74
+ logger: params.logger,
75
+ connectionProvider: params.getConnectionForBullMQ,
76
+ defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
77
+ processorPath: params.getWorkerPath(),
78
+ })
97
79
 
98
80
  return {
99
- enqueue: async (job) => {
100
- const queuedJob = await getQueue().add(jobName, toQueueData(job), { jobId: buildDocumentProcessorJobId(job) })
101
- try {
102
- await queueJobService.recordEnqueued({
103
- queueName,
104
- id: queuedJob.id,
105
- name: queuedJob.name,
106
- data: queuedJob.data,
107
- opts: queuedJob.opts,
108
- attemptsMade: queuedJob.attemptsMade,
109
- timestamp: queuedJob.timestamp,
110
- })
111
- } catch (error) {
112
- params.logger.error`Failed to persist queued job metadata (queue=${queueName}, job=${queuedJob.id}): ${error}`
113
- }
114
- },
115
- startWorker: (options = {}) => {
116
- const { registerSignals = import.meta.main } = options
117
- const worker = new Worker(queueName, params.getWorkerPath(), {
118
- connection: getConnection() as QueueOptions['connection'],
119
- concurrency,
120
- lockDuration,
121
- })
122
-
123
- attachWorkerEvents(worker, workerName, params.logger)
124
-
125
- const shutdown = createWorkerShutdown(worker, workerName, params.logger)
126
-
127
- if (registerSignals) {
128
- registerShutdownSignals({ name: workerName, shutdown, logger: params.logger })
129
- }
130
-
131
- return { worker, shutdown }
81
+ enqueue: (job) => {
82
+ return Effect.runPromise(
83
+ Effect.gen(function* () {
84
+ const queuedJob = yield* Effect.tryPromise(() =>
85
+ queueRuntime.getQueue().add('process-document', job, { jobId: buildDocumentProcessorJobId(job) }),
86
+ )
87
+
88
+ yield* Effect.catch(
89
+ getQueueJobService().recordEnqueued({
90
+ queueName,
91
+ id: queuedJob.id,
92
+ name: queuedJob.name,
93
+ data: queuedJob.data,
94
+ opts: queuedJob.opts,
95
+ attemptsMade: queuedJob.attemptsMade,
96
+ timestamp: queuedJob.timestamp,
97
+ }),
98
+ (error) =>
99
+ Effect.sync(() => {
100
+ params.logger
101
+ .error`Failed to persist queued job metadata (queue=${queueName}, job=${queuedJob.id}): ${error}`
102
+ }),
103
+ )
104
+ }),
105
+ )
132
106
  },
107
+ startWorker: (options = {}) => queueRuntime.startWorker(options),
133
108
  }
134
109
  }
@@ -1,7 +1,14 @@
1
+ import { Effect } from 'effect'
2
+
1
3
  import { serverLogger } from '../config/logger'
2
- import { queueJobService } from '../services/queue-job.service'
3
- import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS, LOW_JOB_RETENTION } from '../workers/worker-utils'
4
+ import {
5
+ getQueueJobService,
6
+ getWorkerPath,
7
+ LONG_JOB_LOCK_DURATION_MS,
8
+ LOW_JOB_RETENTION,
9
+ } from '../workers/worker-utils'
4
10
  import { createQueueFactory } from './queue-factory'
11
+ import { runStandaloneQueueWorker } from './standalone-worker'
5
12
 
6
13
  export interface MemoryConsolidationJob {
7
14
  scopeId?: string
@@ -20,40 +27,48 @@ const memoryConsolidation = createQueueFactory<MemoryConsolidationJob>({
20
27
  processorPath: getWorkerPath('memory-consolidation.worker.ts'),
21
28
  })
22
29
 
23
- export async function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
24
- await memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
30
+ export function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
31
+ return memoryConsolidation.enqueue(job, { jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined })
25
32
  }
26
33
 
27
- export async function scheduleRecurringConsolidation() {
28
- const queuedJob = await memoryConsolidation
29
- .getQueue()
30
- .upsertJobScheduler(
31
- MEMORY_CONSOLIDATION_SCHEDULER_ID,
32
- { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
33
- {
34
- name: 'consolidate',
35
- data: {},
36
- opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
37
- },
38
- )
39
-
40
- try {
41
- await queueJobService.recordEnqueued({
42
- queueName: 'memory-consolidation',
43
- id: queuedJob.id,
44
- name: queuedJob.name,
45
- data: queuedJob.data,
46
- opts: queuedJob.opts,
47
- attemptsMade: queuedJob.attemptsMade,
48
- timestamp: queuedJob.timestamp,
49
- })
50
- } catch (error) {
51
- serverLogger.error`Failed to persist queued job metadata (queue=memory-consolidation, job=${queuedJob.id}): ${error}`
52
- }
34
+ export function scheduleRecurringConsolidation() {
35
+ return Effect.runPromise(
36
+ Effect.gen(function* () {
37
+ const queuedJob = yield* Effect.tryPromise(() =>
38
+ memoryConsolidation
39
+ .getQueue()
40
+ .upsertJobScheduler(
41
+ MEMORY_CONSOLIDATION_SCHEDULER_ID,
42
+ { every: MEMORY_CONSOLIDATION_INTERVAL_MS },
43
+ {
44
+ name: 'consolidate',
45
+ data: {},
46
+ opts: { ...LOW_JOB_RETENTION, attempts: 2, backoff: { type: 'exponential', delay: 5000 } },
47
+ },
48
+ ),
49
+ )
50
+
51
+ yield* Effect.catch(
52
+ getQueueJobService().recordEnqueued({
53
+ queueName: 'memory-consolidation',
54
+ id: queuedJob.id,
55
+ name: queuedJob.name,
56
+ data: queuedJob.data,
57
+ opts: queuedJob.opts,
58
+ attemptsMade: queuedJob.attemptsMade,
59
+ timestamp: queuedJob.timestamp,
60
+ }),
61
+ (error) =>
62
+ Effect.sync(() => {
63
+ serverLogger.error`Failed to persist queued job metadata (queue=memory-consolidation, job=${queuedJob.id}): ${error}`
64
+ }),
65
+ )
66
+ }),
67
+ )
53
68
  }
54
69
 
55
70
  export const startMemoryConsolidationWorker = memoryConsolidation.startWorker
56
71
 
57
- if (import.meta.main) {
72
+ runStandaloneQueueWorker(() => {
58
73
  startMemoryConsolidationWorker()
59
- }
74
+ })
@@ -1,5 +1,13 @@
1
+ import { Effect, Schema } from 'effect'
2
+
1
3
  import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
2
4
  import { createQueueFactory } from './queue-factory'
5
+ import { runStandaloneQueueWorker } from './standalone-worker'
6
+
7
+ class OrganizationLearningQueueError extends Schema.TaggedErrorClass<OrganizationLearningQueueError>()(
8
+ '@lota-sdk/core/OrganizationLearningQueueError',
9
+ { message: Schema.String, cause: Schema.optional(Schema.Defect) },
10
+ ) {}
3
11
 
4
12
  export const ORGANIZATION_LEARNING_QUEUE = 'organization-learning'
5
13
 
@@ -67,12 +75,26 @@ export function enqueueSkillExtraction(job: Omit<SkillExtractionJob, 'kind'>) {
67
75
  )
68
76
  }
69
77
 
70
- export async function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
71
- await organizationLearningQueue.getQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId))
78
+ export function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
79
+ return Effect.runPromise(
80
+ Effect.asVoid(
81
+ Effect.tryPromise({
82
+ try: () =>
83
+ organizationLearningQueue
84
+ .getQueue()
85
+ .removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId)),
86
+ catch: (cause) =>
87
+ new OrganizationLearningQueueError({
88
+ message: `Failed to clear regular-chat memory digest deduplication key for ${orgId}.`,
89
+ cause,
90
+ }),
91
+ }),
92
+ ),
93
+ )
72
94
  }
73
95
 
74
96
  export const startOrganizationLearningWorker = organizationLearningQueue.startWorker
75
97
 
76
- if (import.meta.main) {
98
+ runStandaloneQueueWorker(() => {
77
99
  startOrganizationLearningWorker()
78
- }
100
+ })
@@ -1,11 +1,20 @@
1
1
  import type { Job } from 'bullmq'
2
+ import { Effect, Schema } from 'effect'
3
+ import type { Context } from 'effect'
4
+ import type IORedis from 'ioredis'
2
5
 
3
6
  import { serverLogger } from '../config/logger'
4
- import { databaseService } from '../db/service'
5
- import { planAgentHeartbeatService } from '../services/plan-agent-heartbeat.service'
7
+ import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
8
+ import { PlanAgentHeartbeatServiceTag } from '../services/plan/plan-agent-heartbeat.service'
6
9
  import type { WorkerHandle } from '../workers/worker-utils'
7
10
  import { DEFAULT_JOB_RETENTION, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
8
- import { createQueueFactory } from './queue-factory'
11
+ import { createQueueFactoryWithDeps } from './queue-factory'
12
+ import { runStandaloneQueueWorker } from './standalone-worker'
13
+
14
+ class PlanAgentHeartbeatQueueError extends Schema.TaggedErrorClass<PlanAgentHeartbeatQueueError>()(
15
+ '@lota-sdk/core/PlanAgentHeartbeatQueueError',
16
+ { message: Schema.String, cause: Schema.optional(Schema.Defect) },
17
+ ) {}
9
18
 
10
19
  export interface PlanAgentHeartbeatWakeJob {
11
20
  type: 'wake-node'
@@ -28,31 +37,54 @@ export const PLAN_AGENT_HEARTBEAT_QUEUE = 'plan-agent-heartbeat'
28
37
  const PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS = 30_000
29
38
  const PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID = 'plan-agent-heartbeat-sweep'
30
39
 
31
- async function enqueueDelayedPlanAgentHeartbeatSweep(delayMs = PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS): Promise<void> {
32
- await planAgentHeartbeatQueue.enqueue({ type: 'sweep' }, { delay: delayMs, jobId: PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID })
40
+ interface PlanAgentHeartbeatQueueDeps {
41
+ databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
42
+ planAgentHeartbeatService: Context.Service.Shape<typeof PlanAgentHeartbeatServiceTag>
33
43
  }
34
44
 
35
- async function processPlanAgentHeartbeatJob(job: Job<PlanAgentHeartbeatJob>): Promise<void> {
36
- await databaseService.connect()
37
-
38
- if (job.data.type === 'wake-node') {
39
- await planAgentHeartbeatService.wakeNode(job.data)
40
- return
41
- }
45
+ function enqueueDelayedPlanAgentHeartbeatSweep(delayMs = PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS): Promise<void> {
46
+ return planAgentHeartbeatQueue.enqueue(
47
+ { type: 'sweep' },
48
+ { delay: delayMs, jobId: PLAN_AGENT_HEARTBEAT_SWEEP_JOB_ID },
49
+ )
50
+ }
42
51
 
43
- await planAgentHeartbeatService.sweep({ organizationId: job.data.organizationId })
44
- if (!job.data.organizationId) {
45
- await enqueueDelayedPlanAgentHeartbeatSweep()
46
- }
52
+ function processPlanAgentHeartbeatJob(
53
+ deps: PlanAgentHeartbeatQueueDeps,
54
+ job: Job<PlanAgentHeartbeatJob>,
55
+ ): Promise<void> {
56
+ const { planAgentHeartbeatService } = deps
57
+ return Effect.runPromise(
58
+ Effect.gen(function* () {
59
+ if (job.data.type === 'wake-node') {
60
+ const wakeJob = job.data
61
+ yield* planAgentHeartbeatService.wakeNode(wakeJob)
62
+ return
63
+ }
64
+
65
+ yield* planAgentHeartbeatService.sweep({ organizationId: job.data.organizationId })
66
+ if (!job.data.organizationId) {
67
+ yield* Effect.tryPromise({
68
+ try: () => enqueueDelayedPlanAgentHeartbeatSweep(),
69
+ catch: (cause) =>
70
+ new PlanAgentHeartbeatQueueError({
71
+ message: 'Failed to enqueue delayed plan-agent heartbeat sweep.',
72
+ cause,
73
+ }),
74
+ })
75
+ }
76
+ }),
77
+ )
47
78
  }
48
79
 
49
- const planAgentHeartbeatQueue = createQueueFactory<PlanAgentHeartbeatJob>({
80
+ const planAgentHeartbeatQueue = createQueueFactoryWithDeps<PlanAgentHeartbeatJob, PlanAgentHeartbeatQueueDeps>({
50
81
  name: PLAN_AGENT_HEARTBEAT_QUEUE,
51
82
  displayName: 'Plan agent heartbeat',
52
83
  jobName: 'plan-agent-heartbeat-job',
53
84
  concurrency: 2,
54
85
  lockDuration: LONG_JOB_LOCK_DURATION_MS,
55
86
  defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 5_000 } },
87
+ prepare: ({ databaseService }) => databaseService.connect(),
56
88
  processor: processPlanAgentHeartbeatJob,
57
89
  })
58
90
 
@@ -68,7 +100,7 @@ function buildWakeJobId(params: {
68
100
  return `plan-agent-wake__${encode(params.runId)}__${encode(params.nodeId)}__${encode(params.agentId)}`
69
101
  }
70
102
 
71
- export async function enqueuePlanAgentHeartbeatWake(params: {
103
+ export function enqueuePlanAgentHeartbeatWake(params: {
72
104
  organizationId: string
73
105
  threadId: string
74
106
  runId: string
@@ -76,11 +108,19 @@ export async function enqueuePlanAgentHeartbeatWake(params: {
76
108
  agentId: string
77
109
  reason: string
78
110
  }): Promise<void> {
79
- await planAgentHeartbeatQueue.enqueue({ type: 'wake-node', ...params }, { jobId: buildWakeJobId(params) })
111
+ return planAgentHeartbeatQueue.enqueue({ type: 'wake-node', ...params }, { jobId: buildWakeJobId(params) })
80
112
  }
81
113
 
82
- export function startPlanAgentHeartbeatWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
83
- const handle = planAgentHeartbeatQueue.startWorker(options)
114
+ export function startPlanAgentHeartbeatWorker(options: {
115
+ registerSignals?: boolean
116
+ connectionProvider: () => IORedis
117
+ deps: PlanAgentHeartbeatQueueDeps
118
+ }): WorkerHandle {
119
+ const handle = planAgentHeartbeatQueue.startWorker({
120
+ deps: options.deps,
121
+ registerSignals: options.registerSignals,
122
+ connectionProvider: options.connectionProvider,
123
+ })
84
124
 
85
125
  enqueueDelayedPlanAgentHeartbeatSweep().catch((error: unknown) => {
86
126
  serverLogger.error`Plan agent heartbeat scheduler setup failed: ${error}`
@@ -89,6 +129,13 @@ export function startPlanAgentHeartbeatWorker(options: { registerSignals?: boole
89
129
  return handle
90
130
  }
91
131
 
92
- if (import.meta.main) {
93
- startPlanAgentHeartbeatWorker()
94
- }
132
+ runStandaloneQueueWorker((runtime) => {
133
+ const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
134
+ startPlanAgentHeartbeatWorker({
135
+ connectionProvider: () => resolve(RedisServiceTag).getConnectionForBullMQ(),
136
+ deps: {
137
+ databaseService: resolve(DatabaseServiceTag),
138
+ planAgentHeartbeatService: resolve(PlanAgentHeartbeatServiceTag),
139
+ },
140
+ })
141
+ })