@lota-sdk/core 0.4.9 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/ai/embedding-cache.ts +3 -1
- package/src/ai-gateway/ai-gateway.ts +38 -10
- package/src/config/agent-defaults.ts +22 -9
- package/src/config/agent-types.ts +1 -1
- package/src/config/background-processing.ts +1 -1
- package/src/config/index.ts +0 -1
- package/src/config/logger.ts +20 -7
- package/src/config/thread-defaults.ts +12 -4
- package/src/create-runtime.ts +69 -656
- package/src/db/memory-query-builder.ts +2 -1
- package/src/db/memory-store.ts +29 -20
- package/src/db/memory.ts +188 -195
- package/src/db/service-normalization.ts +97 -64
- package/src/db/service.ts +706 -538
- package/src/db/startup.ts +30 -19
- package/src/effect/awaitable-effect.ts +46 -37
- package/src/effect/helpers.ts +30 -5
- package/src/effect/index.ts +7 -5
- package/src/effect/layers.ts +82 -72
- package/src/effect/runtime.ts +18 -3
- package/src/effect/services.ts +15 -11
- package/src/embeddings/provider.ts +65 -66
- package/src/index.ts +13 -11
- package/src/queues/autonomous-job.queue.ts +59 -71
- package/src/queues/context-compaction.queue.ts +6 -18
- package/src/queues/delayed-node-promotion.queue.ts +9 -17
- package/src/queues/organization-learning.queue.ts +17 -4
- package/src/queues/plan-agent-heartbeat.queue.ts +23 -20
- package/src/queues/plan-scheduler.queue.ts +6 -18
- package/src/queues/post-chat-memory.queue.ts +6 -18
- package/src/queues/queue-factory.ts +128 -50
- package/src/queues/title-generation.queue.ts +6 -17
- package/src/redis/connection.ts +181 -164
- package/src/redis/runtime-connection.ts +13 -3
- package/src/redis/stream-context.ts +17 -9
- package/src/runtime/agent-runtime-policy.ts +1 -1
- package/src/runtime/agent-stream-helpers.ts +15 -11
- package/src/runtime/chat-run-orchestration.ts +1 -1
- package/src/runtime/context-compaction/context-compaction-runtime.ts +1 -1
- package/src/runtime/context-compaction/context-compaction.ts +126 -82
- package/src/runtime/domain-layer.ts +192 -0
- package/src/runtime/graph-designer.ts +15 -7
- package/src/runtime/helper-model.ts +8 -4
- package/src/runtime/index.ts +0 -1
- package/src/runtime/memory/memory-block.ts +19 -9
- package/src/runtime/memory/memory-pipeline.ts +53 -66
- package/src/runtime/memory/memory-scope.ts +33 -29
- package/src/runtime/plugin-resolution.ts +33 -54
- package/src/runtime/post-turn-side-effects.ts +6 -26
- package/src/runtime/retrieval-adapters.ts +4 -4
- package/src/runtime/runtime-accessors.ts +92 -0
- package/src/runtime/runtime-config.ts +3 -3
- package/src/runtime/runtime-extensions.ts +20 -9
- package/src/runtime/runtime-lifecycle.ts +124 -0
- package/src/runtime/runtime-services.ts +386 -0
- package/src/runtime/runtime-token.ts +47 -0
- package/src/runtime/social-chat/social-chat-agent-runner.ts +7 -5
- package/src/runtime/social-chat/social-chat-history.ts +21 -12
- package/src/runtime/social-chat/social-chat.ts +401 -365
- package/src/runtime/team-consultation/team-consultation-orchestrator.ts +58 -52
- package/src/runtime/thread-turn-context.ts +21 -27
- package/src/services/agent-activity.service.ts +1 -1
- package/src/services/agent-executor.service.ts +179 -187
- package/src/services/artifact.service.ts +10 -5
- package/src/services/attachment.service.ts +35 -1
- package/src/services/autonomous-job.service.ts +58 -56
- package/src/services/background-work.service.ts +54 -0
- package/src/services/chat-run-registry.service.ts +3 -1
- package/src/services/context-compaction.service.ts +1 -1
- package/src/services/document-chunk.service.ts +8 -17
- package/src/services/execution-plan/execution-plan-graph.ts +74 -52
- package/src/services/execution-plan/execution-plan.service.ts +1 -1
- package/src/services/feedback-loop.service.ts +1 -1
- package/src/services/global-orchestrator.service.ts +33 -10
- package/src/services/graph-full-routing.ts +44 -33
- package/src/services/index.ts +1 -0
- package/src/services/institutional-memory.service.ts +8 -17
- package/src/services/learned-skill.service.ts +38 -35
- package/src/services/memory/memory-errors.ts +27 -0
- package/src/services/memory/memory-org-memory.ts +14 -3
- package/src/services/memory/memory-preseeded.ts +10 -4
- package/src/services/memory/memory-utils.ts +2 -1
- package/src/services/memory/memory.service.ts +26 -44
- package/src/services/memory/rerank.service.ts +3 -11
- package/src/services/monitoring-window.service.ts +1 -1
- package/src/services/mutating-approval.service.ts +1 -1
- package/src/services/node-workspace.service.ts +2 -2
- package/src/services/notification.service.ts +16 -4
- package/src/services/organization-member.service.ts +1 -1
- package/src/services/organization.service.ts +34 -51
- package/src/services/ownership-dispatcher.service.ts +132 -90
- package/src/services/plan/plan-agent-heartbeat.service.ts +1 -1
- package/src/services/plan/plan-agent-query.service.ts +1 -1
- package/src/services/plan/plan-approval.service.ts +52 -48
- package/src/services/plan/plan-artifact.service.ts +2 -2
- package/src/services/plan/plan-builder.service.ts +2 -2
- package/src/services/plan/plan-checkpoint.service.ts +1 -1
- package/src/services/plan/plan-compiler.service.ts +1 -1
- package/src/services/plan/plan-completion-side-effects.ts +18 -24
- package/src/services/plan/plan-coordination.service.ts +1 -1
- package/src/services/plan/plan-cycle.service.ts +171 -164
- package/src/services/plan/plan-deadline.service.ts +290 -304
- package/src/services/plan/plan-event-delivery.service.ts +44 -39
- package/src/services/plan/plan-executor-graph.ts +114 -67
- package/src/services/plan/plan-executor-helpers.ts +60 -75
- package/src/services/plan/plan-executor.service.ts +550 -467
- package/src/services/plan/plan-run.service.ts +12 -19
- package/src/services/plan/plan-scheduler.service.ts +27 -33
- package/src/services/plan/plan-template.service.ts +1 -1
- package/src/services/plan/plan-transaction-events.ts +8 -5
- package/src/services/plan/plan-validator.service.ts +1 -1
- package/src/services/plan/plan-workspace.service.ts +17 -11
- package/src/services/plugin-executor.service.ts +26 -21
- package/src/services/quality-metrics.service.ts +1 -1
- package/src/services/queue-job.service.ts +8 -17
- package/src/services/recent-activity-title.service.ts +17 -9
- package/src/services/recent-activity.service.ts +1 -1
- package/src/services/skill-resolver.service.ts +1 -1
- package/src/services/social-chat-history.service.ts +37 -20
- package/src/services/system-executor.service.ts +25 -20
- package/src/services/thread/thread-bootstrap.ts +26 -10
- package/src/services/thread/thread-listing.ts +2 -1
- package/src/services/thread/thread-memory-block.ts +18 -5
- package/src/services/thread/thread-message.service.ts +24 -8
- package/src/services/thread/thread-title.service.ts +1 -1
- package/src/services/thread/thread-turn-execution.ts +1 -1
- package/src/services/thread/thread-turn-preparation.service.ts +18 -16
- package/src/services/thread/thread-turn-streaming.ts +12 -11
- package/src/services/thread/thread-turn.ts +43 -10
- package/src/services/thread/thread.service.ts +11 -2
- package/src/services/user.service.ts +1 -1
- package/src/services/write-intent-validator.service.ts +1 -1
- package/src/storage/attachment-storage.service.ts +7 -4
- package/src/storage/generated-document-storage.service.ts +1 -1
- package/src/system-agents/context-compaction.agent.ts +1 -1
- package/src/system-agents/helper-agent-options.ts +1 -1
- package/src/system-agents/memory-reranker.agent.ts +1 -1
- package/src/system-agents/memory.agent.ts +1 -1
- package/src/system-agents/recent-activity-title-refiner.agent.ts +1 -1
- package/src/system-agents/regular-chat-memory-digest.agent.ts +1 -1
- package/src/system-agents/skill-extractor.agent.ts +1 -1
- package/src/system-agents/skill-manager.agent.ts +1 -1
- package/src/system-agents/title-generator.agent.ts +1 -1
- package/src/tools/execution-plan.tool.ts +28 -17
- package/src/tools/fetch-webpage.tool.ts +20 -13
- package/src/tools/firecrawl-client.ts +13 -3
- package/src/tools/plan-approval.tool.ts +9 -1
- package/src/tools/search-web.tool.ts +16 -9
- package/src/tools/team-think.tool.ts +2 -2
- package/src/utils/async.ts +15 -6
- package/src/utils/errors.ts +27 -15
- package/src/workers/bootstrap.ts +25 -48
- package/src/workers/organization-learning.worker.ts +1 -1
- package/src/workers/regular-chat-memory-digest.runner.ts +25 -15
- package/src/workers/worker-utils.ts +20 -2
- package/src/config/search.ts +0 -3
- package/src/runtime/agent-types.ts +0 -1
|
@@ -1,17 +1,21 @@
|
|
|
1
1
|
import type { Job } from 'bullmq'
|
|
2
|
-
import { Effect } from 'effect'
|
|
2
|
+
import { Effect, Schema } from 'effect'
|
|
3
3
|
import type { Context } from 'effect'
|
|
4
4
|
import type IORedis from 'ioredis'
|
|
5
5
|
|
|
6
6
|
import { serverLogger } from '../config/logger'
|
|
7
|
-
import { ConfigurationError } from '../effect/errors'
|
|
8
7
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
9
8
|
import { PlanAgentHeartbeatServiceTag } from '../services/plan/plan-agent-heartbeat.service'
|
|
10
9
|
import type { WorkerHandle } from '../workers/worker-utils'
|
|
11
10
|
import { DEFAULT_JOB_RETENTION, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
|
|
12
|
-
import {
|
|
11
|
+
import { createQueueFactoryWithDeps } from './queue-factory'
|
|
13
12
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
14
13
|
|
|
14
|
+
class PlanAgentHeartbeatQueueError extends Schema.TaggedErrorClass<PlanAgentHeartbeatQueueError>()(
|
|
15
|
+
'@lota-sdk/core/PlanAgentHeartbeatQueueError',
|
|
16
|
+
{ message: Schema.String, cause: Schema.optional(Schema.Defect) },
|
|
17
|
+
) {}
|
|
18
|
+
|
|
15
19
|
export interface PlanAgentHeartbeatWakeJob {
|
|
16
20
|
type: 'wake-node'
|
|
17
21
|
organizationId: string
|
|
@@ -38,16 +42,6 @@ interface PlanAgentHeartbeatQueueDeps {
|
|
|
38
42
|
planAgentHeartbeatService: Context.Service.Shape<typeof PlanAgentHeartbeatServiceTag>
|
|
39
43
|
}
|
|
40
44
|
|
|
41
|
-
let _deps: PlanAgentHeartbeatQueueDeps | null = null
|
|
42
|
-
function getDeps(): PlanAgentHeartbeatQueueDeps {
|
|
43
|
-
if (!_deps)
|
|
44
|
-
throw new ConfigurationError({
|
|
45
|
-
message: 'Plan agent heartbeat queue is not configured. Initialize the runtime before starting the worker.',
|
|
46
|
-
key: 'queue-deps',
|
|
47
|
-
})
|
|
48
|
-
return _deps
|
|
49
|
-
}
|
|
50
|
-
|
|
51
45
|
function enqueueDelayedPlanAgentHeartbeatSweep(delayMs = PLAN_AGENT_HEARTBEAT_SWEEP_INTERVAL_MS): Promise<void> {
|
|
52
46
|
return planAgentHeartbeatQueue.enqueue(
|
|
53
47
|
{ type: 'sweep' },
|
|
@@ -55,9 +49,11 @@ function enqueueDelayedPlanAgentHeartbeatSweep(delayMs = PLAN_AGENT_HEARTBEAT_SW
|
|
|
55
49
|
)
|
|
56
50
|
}
|
|
57
51
|
|
|
58
|
-
function processPlanAgentHeartbeatJob(
|
|
59
|
-
|
|
60
|
-
|
|
52
|
+
function processPlanAgentHeartbeatJob(
|
|
53
|
+
deps: PlanAgentHeartbeatQueueDeps,
|
|
54
|
+
job: Job<PlanAgentHeartbeatJob>,
|
|
55
|
+
): Promise<void> {
|
|
56
|
+
const { planAgentHeartbeatService } = deps
|
|
61
57
|
return Effect.runPromise(
|
|
62
58
|
Effect.gen(function* () {
|
|
63
59
|
if (job.data.type === 'wake-node') {
|
|
@@ -68,20 +64,27 @@ function processPlanAgentHeartbeatJob(job: Job<PlanAgentHeartbeatJob>): Promise<
|
|
|
68
64
|
|
|
69
65
|
yield* planAgentHeartbeatService.sweep({ organizationId: job.data.organizationId })
|
|
70
66
|
if (!job.data.organizationId) {
|
|
71
|
-
yield* Effect.tryPromise(
|
|
67
|
+
yield* Effect.tryPromise({
|
|
68
|
+
try: () => enqueueDelayedPlanAgentHeartbeatSweep(),
|
|
69
|
+
catch: (cause) =>
|
|
70
|
+
new PlanAgentHeartbeatQueueError({
|
|
71
|
+
message: 'Failed to enqueue delayed plan-agent heartbeat sweep.',
|
|
72
|
+
cause,
|
|
73
|
+
}),
|
|
74
|
+
})
|
|
72
75
|
}
|
|
73
76
|
}),
|
|
74
77
|
)
|
|
75
78
|
}
|
|
76
79
|
|
|
77
|
-
const planAgentHeartbeatQueue =
|
|
80
|
+
const planAgentHeartbeatQueue = createQueueFactoryWithDeps<PlanAgentHeartbeatJob, PlanAgentHeartbeatQueueDeps>({
|
|
78
81
|
name: PLAN_AGENT_HEARTBEAT_QUEUE,
|
|
79
82
|
displayName: 'Plan agent heartbeat',
|
|
80
83
|
jobName: 'plan-agent-heartbeat-job',
|
|
81
84
|
concurrency: 2,
|
|
82
85
|
lockDuration: LONG_JOB_LOCK_DURATION_MS,
|
|
83
86
|
defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 5_000 } },
|
|
84
|
-
prepare: () =>
|
|
87
|
+
prepare: ({ databaseService }) => databaseService.connect(),
|
|
85
88
|
processor: processPlanAgentHeartbeatJob,
|
|
86
89
|
})
|
|
87
90
|
|
|
@@ -113,8 +116,8 @@ export function startPlanAgentHeartbeatWorker(options: {
|
|
|
113
116
|
connectionProvider: () => IORedis
|
|
114
117
|
deps: PlanAgentHeartbeatQueueDeps
|
|
115
118
|
}): WorkerHandle {
|
|
116
|
-
_deps = options.deps
|
|
117
119
|
const handle = planAgentHeartbeatQueue.startWorker({
|
|
120
|
+
deps: options.deps,
|
|
118
121
|
registerSignals: options.registerSignals,
|
|
119
122
|
connectionProvider: options.connectionProvider,
|
|
120
123
|
})
|
|
@@ -4,7 +4,6 @@ import type { Context } from 'effect'
|
|
|
4
4
|
import type IORedis from 'ioredis'
|
|
5
5
|
|
|
6
6
|
import { serverLogger } from '../config/logger'
|
|
7
|
-
import { ConfigurationError } from '../effect/errors'
|
|
8
7
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
9
8
|
import { PlanCycleServiceTag } from '../services/plan/plan-cycle.service'
|
|
10
9
|
import { PlanDeadlineServiceTag } from '../services/plan/plan-deadline.service'
|
|
@@ -12,7 +11,7 @@ import { PlanExecutorServiceTag } from '../services/plan/plan-executor.service'
|
|
|
12
11
|
import { PlanSchedulerServiceTag } from '../services/plan/plan-scheduler.service'
|
|
13
12
|
import { nowEpochMillis } from '../utils/date-time'
|
|
14
13
|
import type { WorkerHandle } from '../workers/worker-utils'
|
|
15
|
-
import {
|
|
14
|
+
import { createQueueFactoryWithDeps } from './queue-factory'
|
|
16
15
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
17
16
|
|
|
18
17
|
export interface PlanSchedulerFireJob {
|
|
@@ -37,16 +36,6 @@ interface PlanSchedulerQueueDeps {
|
|
|
37
36
|
planCycleService: Context.Service.Shape<typeof PlanCycleServiceTag>
|
|
38
37
|
}
|
|
39
38
|
|
|
40
|
-
let _deps: PlanSchedulerQueueDeps | null = null
|
|
41
|
-
function getDeps(): PlanSchedulerQueueDeps {
|
|
42
|
-
if (!_deps)
|
|
43
|
-
throw new ConfigurationError({
|
|
44
|
-
message: 'Plan scheduler queue is not configured. Initialize the runtime before starting the worker.',
|
|
45
|
-
key: 'queue-deps',
|
|
46
|
-
})
|
|
47
|
-
return _deps
|
|
48
|
-
}
|
|
49
|
-
|
|
50
39
|
class PlanSchedulerQueueError extends Schema.TaggedErrorClass<PlanSchedulerQueueError>()('PlanSchedulerQueueError', {
|
|
51
40
|
stage: Schema.Literals(['remove-schedule-fire-job', 'recover-active-schedules', 'recover-deadline-checks']),
|
|
52
41
|
message: Schema.String,
|
|
@@ -57,9 +46,8 @@ function toPlanSchedulerQueueError(stage: PlanSchedulerQueueError['stage'], caus
|
|
|
57
46
|
return new PlanSchedulerQueueError({ stage, message: cause instanceof Error ? cause.message : String(cause), cause })
|
|
58
47
|
}
|
|
59
48
|
|
|
60
|
-
function processPlanSchedulerJob(job: Job<PlanSchedulerJob>): Promise<void> {
|
|
61
|
-
const { planSchedulerService, planDeadlineService, planExecutorService, planCycleService } =
|
|
62
|
-
|
|
49
|
+
function processPlanSchedulerJob(deps: PlanSchedulerQueueDeps, job: Job<PlanSchedulerJob>): Promise<void> {
|
|
50
|
+
const { planSchedulerService, planDeadlineService, planExecutorService, planCycleService } = deps
|
|
63
51
|
switch (job.data.type) {
|
|
64
52
|
case 'fire-schedule':
|
|
65
53
|
return Effect.runPromise(
|
|
@@ -76,13 +64,13 @@ function processPlanSchedulerJob(job: Job<PlanSchedulerJob>): Promise<void> {
|
|
|
76
64
|
}
|
|
77
65
|
}
|
|
78
66
|
|
|
79
|
-
const planScheduler =
|
|
67
|
+
const planScheduler = createQueueFactoryWithDeps<PlanSchedulerJob, PlanSchedulerQueueDeps>({
|
|
80
68
|
name: PLAN_SCHEDULER_QUEUE,
|
|
81
69
|
displayName: 'Plan scheduler',
|
|
82
70
|
jobName: 'plan-scheduler-job',
|
|
83
71
|
concurrency: 1,
|
|
84
72
|
defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
|
|
85
|
-
prepare: () =>
|
|
73
|
+
prepare: ({ databaseService }) => databaseService.connect(),
|
|
86
74
|
processor: processPlanSchedulerJob,
|
|
87
75
|
})
|
|
88
76
|
|
|
@@ -124,8 +112,8 @@ export function startPlanSchedulerWorker(options: {
|
|
|
124
112
|
connectionProvider: () => IORedis
|
|
125
113
|
deps: PlanSchedulerQueueDeps
|
|
126
114
|
}): WorkerHandle {
|
|
127
|
-
_deps = options.deps
|
|
128
115
|
const handle = planScheduler.startWorker({
|
|
116
|
+
deps: options.deps,
|
|
129
117
|
registerSignals: options.registerSignals,
|
|
130
118
|
connectionProvider: options.connectionProvider,
|
|
131
119
|
})
|
|
@@ -4,10 +4,9 @@ import type { Context } from 'effect'
|
|
|
4
4
|
import type IORedis from 'ioredis'
|
|
5
5
|
|
|
6
6
|
import type { MaybeAwaitableService } from '../effect/awaitable-effect'
|
|
7
|
-
import { ConfigurationError } from '../effect/errors'
|
|
8
7
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
9
8
|
import { MemoryServiceTag } from '../services/memory/memory.service'
|
|
10
|
-
import {
|
|
9
|
+
import { createQueueFactoryWithDeps } from './queue-factory'
|
|
11
10
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
12
11
|
|
|
13
12
|
interface PostChatMemoryMessage {
|
|
@@ -35,19 +34,8 @@ interface PostChatMemoryQueueDeps {
|
|
|
35
34
|
memoryService: MaybeAwaitableService<Context.Service.Shape<typeof MemoryServiceTag>>
|
|
36
35
|
}
|
|
37
36
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
if (!_deps)
|
|
41
|
-
throw new ConfigurationError({
|
|
42
|
-
message: 'Post-chat memory queue is not configured. Initialize the runtime before starting the worker.',
|
|
43
|
-
key: 'queue-deps',
|
|
44
|
-
})
|
|
45
|
-
return _deps
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function processPostChatMemoryJob(job: Job<PostChatMemoryExtractionJob>): Promise<void> {
|
|
49
|
-
const { memoryService } = getDeps()
|
|
50
|
-
|
|
37
|
+
function processPostChatMemoryJob(deps: PostChatMemoryQueueDeps, job: Job<PostChatMemoryExtractionJob>): Promise<void> {
|
|
38
|
+
const { memoryService } = deps
|
|
51
39
|
const data = job.data
|
|
52
40
|
const userMessage = data.userMessage.trim()
|
|
53
41
|
const agentMessages = data.agentMessages
|
|
@@ -93,7 +81,7 @@ function processPostChatMemoryJob(job: Job<PostChatMemoryExtractionJob>): Promis
|
|
|
93
81
|
)
|
|
94
82
|
}
|
|
95
83
|
|
|
96
|
-
const postChatMemory =
|
|
84
|
+
const postChatMemory = createQueueFactoryWithDeps<PostChatMemoryExtractionJob, PostChatMemoryQueueDeps>({
|
|
97
85
|
name: 'post-chat-memory',
|
|
98
86
|
displayName: 'Post-chat memory',
|
|
99
87
|
jobName: 'extract-memory',
|
|
@@ -102,7 +90,7 @@ const postChatMemory = createQueueFactory<PostChatMemoryExtractionJob>({
|
|
|
102
90
|
maxStalledCount: 10,
|
|
103
91
|
stalledInterval: 120_000,
|
|
104
92
|
defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
|
|
105
|
-
prepare: () =>
|
|
93
|
+
prepare: ({ databaseService }) => databaseService.connect(),
|
|
106
94
|
processor: processPostChatMemoryJob,
|
|
107
95
|
})
|
|
108
96
|
|
|
@@ -115,8 +103,8 @@ export function startPostChatMemoryWorker(options: {
|
|
|
115
103
|
connectionProvider: () => IORedis
|
|
116
104
|
deps: PostChatMemoryQueueDeps
|
|
117
105
|
}) {
|
|
118
|
-
_deps = options.deps
|
|
119
106
|
return postChatMemory.startWorker({
|
|
107
|
+
deps: options.deps,
|
|
120
108
|
registerSignals: options.registerSignals,
|
|
121
109
|
connectionProvider: options.connectionProvider,
|
|
122
110
|
})
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { Queue, Worker } from 'bullmq'
|
|
2
2
|
import type { Job, JobsOptions, QueueOptions, WorkerOptions } from 'bullmq'
|
|
3
|
-
import { Effect } from 'effect'
|
|
3
|
+
import { Effect, Schema } from 'effect'
|
|
4
4
|
import type IORedis from 'ioredis'
|
|
5
5
|
|
|
6
6
|
import type { LotaLogger } from '../config/logger'
|
|
7
7
|
import { serverLogger } from '../config/logger'
|
|
8
8
|
import { getCurrentRuntime } from '../effect/runtime-ref'
|
|
9
9
|
import { RedisServiceTag } from '../effect/services'
|
|
10
|
+
import type { TrackedBullJobLike } from '../services/queue-job.service'
|
|
10
11
|
import {
|
|
11
12
|
attachWorkerEvents,
|
|
12
13
|
createTracedWorkerProcessor,
|
|
@@ -17,11 +18,27 @@ import {
|
|
|
17
18
|
} from '../workers/worker-utils'
|
|
18
19
|
import type { WorkerHandle } from '../workers/worker-utils'
|
|
19
20
|
|
|
21
|
+
class QueueFactoryError extends Schema.TaggedErrorClass<QueueFactoryError>()('@lota-sdk/core/QueueFactoryError', {
|
|
22
|
+
message: Schema.String,
|
|
23
|
+
cause: Schema.optional(Schema.Defect),
|
|
24
|
+
}) {}
|
|
25
|
+
|
|
20
26
|
function getDefaultQueueConnectionProvider(): () => IORedis {
|
|
21
27
|
const redis = getCurrentRuntime().runSync(Effect.service(RedisServiceTag))
|
|
22
28
|
return () => redis.getConnectionForBullMQ()
|
|
23
29
|
}
|
|
24
30
|
|
|
31
|
+
type QueueShape<TJob> = Queue<TJob, unknown, string, TJob, unknown, string>
|
|
32
|
+
type QueueMethod = 'add' | 'close' | 'remove' | 'removeDeduplicationKey' | 'removeJobScheduler'
|
|
33
|
+
|
|
34
|
+
const queueMethodsThatWaitForClose = new Set<QueueMethod>([
|
|
35
|
+
'add',
|
|
36
|
+
'close',
|
|
37
|
+
'remove',
|
|
38
|
+
'removeDeduplicationKey',
|
|
39
|
+
'removeJobScheduler',
|
|
40
|
+
])
|
|
41
|
+
|
|
25
42
|
interface QueueFactoryConfigBase {
|
|
26
43
|
name: string
|
|
27
44
|
displayName: string
|
|
@@ -48,29 +65,72 @@ interface QueueFactoryConfigFile extends QueueFactoryConfigBase {
|
|
|
48
65
|
|
|
49
66
|
export type QueueFactoryConfig<TJob> = QueueFactoryConfigInline<TJob> | QueueFactoryConfigFile
|
|
50
67
|
|
|
68
|
+
interface QueueFactoryWithDepsConfig<TJob, TDeps> extends QueueFactoryConfigBase {
|
|
69
|
+
prepare?: (deps: TDeps, job: Job<TJob>) => Promise<void>
|
|
70
|
+
processor: (deps: TDeps, job: Job<TJob>) => Promise<unknown>
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
interface QueueWorkerConfigInline<TJob> {
|
|
74
|
+
prepare?: (job: Job<TJob>) => Promise<void>
|
|
75
|
+
processor: (job: Job<TJob>) => Promise<unknown>
|
|
76
|
+
processorPath?: never
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
interface QueueWorkerConfigFile {
|
|
80
|
+
processor?: never
|
|
81
|
+
processorPath: string
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
type QueueWorkerConfig<TJob> = QueueWorkerConfigInline<TJob> | QueueWorkerConfigFile
|
|
85
|
+
|
|
86
|
+
export interface QueueFactoryWorkerOptions {
|
|
87
|
+
registerSignals?: boolean
|
|
88
|
+
connectionProvider?: () => IORedis
|
|
89
|
+
}
|
|
90
|
+
|
|
51
91
|
export interface QueueFactory<TJob> {
|
|
52
|
-
getQueue: () =>
|
|
92
|
+
getQueue: () => QueueShape<TJob>
|
|
53
93
|
enqueue: (job: TJob, options?: JobsOptions) => Promise<void>
|
|
54
|
-
startWorker: (options?:
|
|
94
|
+
startWorker: (options?: QueueFactoryWorkerOptions) => WorkerHandle
|
|
55
95
|
}
|
|
56
96
|
|
|
57
|
-
export
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
97
|
+
export interface QueueFactoryWithDeps<TJob, TDeps> {
|
|
98
|
+
getQueue: () => QueueShape<TJob>
|
|
99
|
+
enqueue: (job: TJob, options?: JobsOptions) => Promise<void>
|
|
100
|
+
startWorker: (options: QueueFactoryWorkerOptions & { deps: TDeps }) => WorkerHandle
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function recordEnqueuedJobMetadata(params: {
|
|
104
|
+
queueName: string
|
|
105
|
+
job: Omit<TrackedBullJobLike, 'queueName'>
|
|
106
|
+
logger?: LotaLogger
|
|
107
|
+
}): Effect.Effect<string | undefined> {
|
|
108
|
+
const logger = params.logger ?? serverLogger
|
|
109
|
+
return getQueueJobService()
|
|
110
|
+
.recordEnqueued({ queueName: params.queueName, ...params.job })
|
|
111
|
+
.pipe(
|
|
112
|
+
Effect.tapError((error) =>
|
|
113
|
+
Effect.sync(() => {
|
|
114
|
+
logger.error`Failed to persist queued job metadata (queue=${params.queueName}, job=${params.job.id}): ${error}`
|
|
115
|
+
}),
|
|
116
|
+
),
|
|
117
|
+
Effect.orElseSucceed(() => undefined),
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function createQueueFactoryRuntime<TJob>(config: QueueFactoryConfigBase): {
|
|
122
|
+
getQueue: () => QueueShape<TJob>
|
|
123
|
+
enqueue: (job: TJob, options?: JobsOptions) => Promise<void>
|
|
124
|
+
startWorker: (workerConfig: QueueWorkerConfig<TJob>, options?: QueueFactoryWorkerOptions) => WorkerHandle
|
|
125
|
+
} {
|
|
126
|
+
type State = {
|
|
127
|
+
queue: QueueShape<TJob> | null
|
|
128
|
+
rawQueue: QueueShape<TJob> | null
|
|
71
129
|
connection: IORedis | null
|
|
72
130
|
pendingClose: Promise<void> | null
|
|
73
|
-
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
let state: State = { queue: null, rawQueue: null, connection: null, pendingClose: null }
|
|
74
134
|
|
|
75
135
|
const resolveConnectionProvider = (): (() => IORedis) =>
|
|
76
136
|
config.connectionProvider ?? getDefaultQueueConnectionProvider()
|
|
@@ -90,7 +150,7 @@ export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): Queu
|
|
|
90
150
|
})
|
|
91
151
|
}
|
|
92
152
|
|
|
93
|
-
const wrapQueue = (queue: QueueShape): QueueShape =>
|
|
153
|
+
const wrapQueue = (queue: QueueShape<TJob>): QueueShape<TJob> =>
|
|
94
154
|
new Proxy(queue, {
|
|
95
155
|
get(target, property, receiver) {
|
|
96
156
|
if (typeof property !== 'string') {
|
|
@@ -109,7 +169,7 @@ export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): Queu
|
|
|
109
169
|
},
|
|
110
170
|
})
|
|
111
171
|
|
|
112
|
-
const getQueue = (): QueueShape => {
|
|
172
|
+
const getQueue = (): QueueShape<TJob> => {
|
|
113
173
|
const connection = getConnection()
|
|
114
174
|
const isStale =
|
|
115
175
|
state.rawQueue === null ||
|
|
@@ -146,36 +206,21 @@ export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): Queu
|
|
|
146
206
|
return wrappedQueue
|
|
147
207
|
}
|
|
148
208
|
|
|
149
|
-
const jobName = config.jobName
|
|
150
|
-
const toData = (job: TJob) => job
|
|
151
|
-
|
|
152
209
|
const enqueue = (job: TJob, options?: JobsOptions): Promise<void> =>
|
|
153
210
|
Effect.runPromise(
|
|
154
211
|
Effect.gen(function* () {
|
|
155
|
-
const queuedJob = yield* Effect.tryPromise(
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
data: queuedJob.data,
|
|
162
|
-
opts: queuedJob.opts,
|
|
163
|
-
attemptsMade: queuedJob.attemptsMade,
|
|
164
|
-
timestamp: queuedJob.timestamp,
|
|
165
|
-
})
|
|
166
|
-
.pipe(
|
|
167
|
-
Effect.tapError((error) =>
|
|
168
|
-
Effect.sync(() => {
|
|
169
|
-
serverLogger.error`Failed to persist queued job metadata (queue=${config.name}, job=${queuedJob.id}): ${error}`
|
|
170
|
-
}),
|
|
171
|
-
),
|
|
172
|
-
Effect.orElseSucceed(() => undefined),
|
|
173
|
-
)
|
|
212
|
+
const queuedJob = yield* Effect.tryPromise({
|
|
213
|
+
try: () => getQueue().add(config.jobName, job, options),
|
|
214
|
+
catch: (cause) =>
|
|
215
|
+
new QueueFactoryError({ message: `Failed to enqueue job on queue "${config.name}".`, cause }),
|
|
216
|
+
})
|
|
217
|
+
yield* recordEnqueuedJobMetadata({ queueName: config.name, job: queuedJob, logger: config.logger })
|
|
174
218
|
}),
|
|
175
219
|
)
|
|
176
220
|
|
|
177
221
|
const startWorker = (
|
|
178
|
-
|
|
222
|
+
workerConfig: QueueWorkerConfig<TJob>,
|
|
223
|
+
options: QueueFactoryWorkerOptions = {},
|
|
179
224
|
): WorkerHandle => {
|
|
180
225
|
const { registerSignals = import.meta.main, connectionProvider } = options
|
|
181
226
|
const logger = config.logger ?? serverLogger
|
|
@@ -188,21 +233,28 @@ export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): Queu
|
|
|
188
233
|
...(config.maxStalledCount !== undefined ? { maxStalledCount: config.maxStalledCount } : {}),
|
|
189
234
|
}
|
|
190
235
|
|
|
191
|
-
const worker =
|
|
192
|
-
? new Worker(config.name,
|
|
236
|
+
const worker = workerConfig.processorPath
|
|
237
|
+
? new Worker(config.name, workerConfig.processorPath, workerOptions)
|
|
193
238
|
: new Worker(
|
|
194
239
|
config.name,
|
|
195
240
|
createTracedWorkerProcessor(config.name, (job) =>
|
|
196
241
|
Effect.runPromise(
|
|
197
242
|
Effect.gen(function* () {
|
|
198
|
-
const
|
|
243
|
+
const inlineWorkerConfig = workerConfig as QueueWorkerConfigInline<TJob>
|
|
199
244
|
const typedJob = job as Job<TJob>
|
|
200
|
-
const prepare =
|
|
245
|
+
const prepare = inlineWorkerConfig.prepare
|
|
201
246
|
if (prepare) {
|
|
202
|
-
yield* Effect.tryPromise(
|
|
247
|
+
yield* Effect.tryPromise({
|
|
248
|
+
try: () => prepare(typedJob),
|
|
249
|
+
catch: (cause) =>
|
|
250
|
+
new QueueFactoryError({ message: `Worker prepare failed for queue "${config.name}".`, cause }),
|
|
251
|
+
})
|
|
203
252
|
}
|
|
204
|
-
|
|
205
|
-
|
|
253
|
+
return yield* Effect.tryPromise({
|
|
254
|
+
try: () => inlineWorkerConfig.processor(typedJob),
|
|
255
|
+
catch: (cause) =>
|
|
256
|
+
new QueueFactoryError({ message: `Worker processor failed for queue "${config.name}".`, cause }),
|
|
257
|
+
})
|
|
206
258
|
}),
|
|
207
259
|
),
|
|
208
260
|
),
|
|
@@ -222,3 +274,29 @@ export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): Queu
|
|
|
222
274
|
|
|
223
275
|
return { getQueue, enqueue, startWorker }
|
|
224
276
|
}
|
|
277
|
+
|
|
278
|
+
export function createQueueFactory<TJob>(config: QueueFactoryConfig<TJob>): QueueFactory<TJob> {
|
|
279
|
+
const runtime = createQueueFactoryRuntime<TJob>(config)
|
|
280
|
+
return {
|
|
281
|
+
getQueue: runtime.getQueue,
|
|
282
|
+
enqueue: runtime.enqueue,
|
|
283
|
+
startWorker: (options) => runtime.startWorker(config, options),
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export function createQueueFactoryWithDeps<TJob, TDeps>(
|
|
288
|
+
config: QueueFactoryWithDepsConfig<TJob, TDeps>,
|
|
289
|
+
): QueueFactoryWithDeps<TJob, TDeps> {
|
|
290
|
+
const runtime = createQueueFactoryRuntime<TJob>(config)
|
|
291
|
+
const prepare = config.prepare
|
|
292
|
+
|
|
293
|
+
return {
|
|
294
|
+
getQueue: runtime.getQueue,
|
|
295
|
+
enqueue: runtime.enqueue,
|
|
296
|
+
startWorker: ({ deps, ...options }) =>
|
|
297
|
+
runtime.startWorker(
|
|
298
|
+
{ prepare: prepare ? (job) => prepare(deps, job) : undefined, processor: (job) => config.processor(deps, job) },
|
|
299
|
+
options,
|
|
300
|
+
),
|
|
301
|
+
}
|
|
302
|
+
}
|
|
@@ -4,11 +4,10 @@ import type { Context } from 'effect'
|
|
|
4
4
|
import type IORedis from 'ioredis'
|
|
5
5
|
|
|
6
6
|
import { ensureRecordId } from '../db/record-id'
|
|
7
|
-
import { ConfigurationError } from '../effect/errors'
|
|
8
7
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
9
8
|
import { RecentActivityTitleServiceTag } from '../services/recent-activity-title.service'
|
|
10
9
|
import { ThreadTitleServiceTag } from '../services/thread/thread-title.service'
|
|
11
|
-
import {
|
|
10
|
+
import { createQueueFactoryWithDeps } from './queue-factory'
|
|
12
11
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
13
12
|
|
|
14
13
|
export const TITLE_GENERATION_QUEUE = 'title-generation'
|
|
@@ -36,18 +35,8 @@ interface TitleGenerationQueueDeps {
|
|
|
36
35
|
recentActivityTitleService: Context.Service.Shape<typeof RecentActivityTitleServiceTag>
|
|
37
36
|
}
|
|
38
37
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
if (!_deps)
|
|
42
|
-
throw new ConfigurationError({
|
|
43
|
-
message: 'Title generation queue is not configured. Initialize the runtime before starting the worker.',
|
|
44
|
-
key: 'queue-deps',
|
|
45
|
-
})
|
|
46
|
-
return _deps
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
function processTitleGenerationJob(job: Job<TitleGenerationJob>): Promise<void> {
|
|
50
|
-
const { threadTitleService, recentActivityTitleService } = getDeps()
|
|
38
|
+
function processTitleGenerationJob(deps: TitleGenerationQueueDeps, job: Job<TitleGenerationJob>): Promise<void> {
|
|
39
|
+
const { threadTitleService, recentActivityTitleService } = deps
|
|
51
40
|
if (job.data.kind === 'thread-title') {
|
|
52
41
|
return Effect.runPromise(
|
|
53
42
|
Effect.asVoid(threadTitleService.generateAndPersistTitle(ensureRecordId(job.data.threadId), job.data.sourceText)),
|
|
@@ -57,14 +46,14 @@ function processTitleGenerationJob(job: Job<TitleGenerationJob>): Promise<void>
|
|
|
57
46
|
return Effect.runPromise(Effect.asVoid(recentActivityTitleService.refineRecentActivityTitle(job.data.activityId)))
|
|
58
47
|
}
|
|
59
48
|
|
|
60
|
-
const titleGeneration =
|
|
49
|
+
const titleGeneration = createQueueFactoryWithDeps<TitleGenerationJob, TitleGenerationQueueDeps>({
|
|
61
50
|
name: TITLE_GENERATION_QUEUE,
|
|
62
51
|
displayName: 'Title generation',
|
|
63
52
|
jobName: 'title-generation',
|
|
64
53
|
concurrency: 10,
|
|
65
54
|
lockDuration: 300_000,
|
|
66
55
|
defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 2_000 } },
|
|
67
|
-
prepare: () =>
|
|
56
|
+
prepare: ({ databaseService }) => databaseService.connect(),
|
|
68
57
|
processor: processTitleGenerationJob,
|
|
69
58
|
})
|
|
70
59
|
|
|
@@ -84,8 +73,8 @@ export function startTitleGenerationWorker(options: {
|
|
|
84
73
|
connectionProvider: () => IORedis
|
|
85
74
|
deps: TitleGenerationQueueDeps
|
|
86
75
|
}): ReturnType<typeof titleGeneration.startWorker> {
|
|
87
|
-
_deps = options.deps
|
|
88
76
|
return titleGeneration.startWorker({
|
|
77
|
+
deps: options.deps,
|
|
89
78
|
registerSignals: options.registerSignals,
|
|
90
79
|
connectionProvider: options.connectionProvider,
|
|
91
80
|
})
|