@lota-sdk/core 0.4.9 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/ai/embedding-cache.ts +3 -1
- package/src/ai-gateway/ai-gateway.ts +38 -10
- package/src/config/agent-defaults.ts +22 -9
- package/src/config/agent-types.ts +1 -1
- package/src/config/background-processing.ts +1 -1
- package/src/config/index.ts +0 -1
- package/src/config/logger.ts +20 -7
- package/src/config/thread-defaults.ts +12 -4
- package/src/create-runtime.ts +69 -656
- package/src/db/memory-query-builder.ts +2 -1
- package/src/db/memory-store.ts +29 -20
- package/src/db/memory.ts +188 -195
- package/src/db/service-normalization.ts +97 -64
- package/src/db/service.ts +706 -538
- package/src/db/startup.ts +30 -19
- package/src/effect/awaitable-effect.ts +46 -37
- package/src/effect/helpers.ts +30 -5
- package/src/effect/index.ts +7 -5
- package/src/effect/layers.ts +82 -72
- package/src/effect/runtime.ts +18 -3
- package/src/effect/services.ts +15 -11
- package/src/embeddings/provider.ts +65 -66
- package/src/index.ts +13 -11
- package/src/queues/autonomous-job.queue.ts +59 -71
- package/src/queues/context-compaction.queue.ts +6 -18
- package/src/queues/delayed-node-promotion.queue.ts +9 -17
- package/src/queues/organization-learning.queue.ts +17 -4
- package/src/queues/plan-agent-heartbeat.queue.ts +23 -20
- package/src/queues/plan-scheduler.queue.ts +6 -18
- package/src/queues/post-chat-memory.queue.ts +6 -18
- package/src/queues/queue-factory.ts +128 -50
- package/src/queues/title-generation.queue.ts +6 -17
- package/src/redis/connection.ts +181 -164
- package/src/redis/runtime-connection.ts +13 -3
- package/src/redis/stream-context.ts +17 -9
- package/src/runtime/agent-runtime-policy.ts +1 -1
- package/src/runtime/agent-stream-helpers.ts +15 -11
- package/src/runtime/chat-run-orchestration.ts +1 -1
- package/src/runtime/context-compaction/context-compaction-runtime.ts +1 -1
- package/src/runtime/context-compaction/context-compaction.ts +126 -82
- package/src/runtime/domain-layer.ts +192 -0
- package/src/runtime/graph-designer.ts +15 -7
- package/src/runtime/helper-model.ts +8 -4
- package/src/runtime/index.ts +0 -1
- package/src/runtime/memory/memory-block.ts +19 -9
- package/src/runtime/memory/memory-pipeline.ts +53 -66
- package/src/runtime/memory/memory-scope.ts +33 -29
- package/src/runtime/plugin-resolution.ts +33 -54
- package/src/runtime/post-turn-side-effects.ts +6 -26
- package/src/runtime/retrieval-adapters.ts +4 -4
- package/src/runtime/runtime-accessors.ts +92 -0
- package/src/runtime/runtime-config.ts +3 -3
- package/src/runtime/runtime-extensions.ts +20 -9
- package/src/runtime/runtime-lifecycle.ts +124 -0
- package/src/runtime/runtime-services.ts +386 -0
- package/src/runtime/runtime-token.ts +47 -0
- package/src/runtime/social-chat/social-chat-agent-runner.ts +7 -5
- package/src/runtime/social-chat/social-chat-history.ts +21 -12
- package/src/runtime/social-chat/social-chat.ts +401 -365
- package/src/runtime/team-consultation/team-consultation-orchestrator.ts +58 -52
- package/src/runtime/thread-turn-context.ts +21 -27
- package/src/services/agent-activity.service.ts +1 -1
- package/src/services/agent-executor.service.ts +179 -187
- package/src/services/artifact.service.ts +10 -5
- package/src/services/attachment.service.ts +35 -1
- package/src/services/autonomous-job.service.ts +58 -56
- package/src/services/background-work.service.ts +54 -0
- package/src/services/chat-run-registry.service.ts +3 -1
- package/src/services/context-compaction.service.ts +1 -1
- package/src/services/document-chunk.service.ts +8 -17
- package/src/services/execution-plan/execution-plan-graph.ts +74 -52
- package/src/services/execution-plan/execution-plan.service.ts +1 -1
- package/src/services/feedback-loop.service.ts +1 -1
- package/src/services/global-orchestrator.service.ts +33 -10
- package/src/services/graph-full-routing.ts +44 -33
- package/src/services/index.ts +1 -0
- package/src/services/institutional-memory.service.ts +8 -17
- package/src/services/learned-skill.service.ts +38 -35
- package/src/services/memory/memory-errors.ts +27 -0
- package/src/services/memory/memory-org-memory.ts +14 -3
- package/src/services/memory/memory-preseeded.ts +10 -4
- package/src/services/memory/memory-utils.ts +2 -1
- package/src/services/memory/memory.service.ts +26 -44
- package/src/services/memory/rerank.service.ts +3 -11
- package/src/services/monitoring-window.service.ts +1 -1
- package/src/services/mutating-approval.service.ts +1 -1
- package/src/services/node-workspace.service.ts +2 -2
- package/src/services/notification.service.ts +16 -4
- package/src/services/organization-member.service.ts +1 -1
- package/src/services/organization.service.ts +34 -51
- package/src/services/ownership-dispatcher.service.ts +132 -90
- package/src/services/plan/plan-agent-heartbeat.service.ts +1 -1
- package/src/services/plan/plan-agent-query.service.ts +1 -1
- package/src/services/plan/plan-approval.service.ts +52 -48
- package/src/services/plan/plan-artifact.service.ts +2 -2
- package/src/services/plan/plan-builder.service.ts +2 -2
- package/src/services/plan/plan-checkpoint.service.ts +1 -1
- package/src/services/plan/plan-compiler.service.ts +1 -1
- package/src/services/plan/plan-completion-side-effects.ts +18 -24
- package/src/services/plan/plan-coordination.service.ts +1 -1
- package/src/services/plan/plan-cycle.service.ts +171 -164
- package/src/services/plan/plan-deadline.service.ts +290 -304
- package/src/services/plan/plan-event-delivery.service.ts +44 -39
- package/src/services/plan/plan-executor-graph.ts +114 -67
- package/src/services/plan/plan-executor-helpers.ts +60 -75
- package/src/services/plan/plan-executor.service.ts +550 -467
- package/src/services/plan/plan-run.service.ts +12 -19
- package/src/services/plan/plan-scheduler.service.ts +27 -33
- package/src/services/plan/plan-template.service.ts +1 -1
- package/src/services/plan/plan-transaction-events.ts +8 -5
- package/src/services/plan/plan-validator.service.ts +1 -1
- package/src/services/plan/plan-workspace.service.ts +17 -11
- package/src/services/plugin-executor.service.ts +26 -21
- package/src/services/quality-metrics.service.ts +1 -1
- package/src/services/queue-job.service.ts +8 -17
- package/src/services/recent-activity-title.service.ts +17 -9
- package/src/services/recent-activity.service.ts +1 -1
- package/src/services/skill-resolver.service.ts +1 -1
- package/src/services/social-chat-history.service.ts +37 -20
- package/src/services/system-executor.service.ts +25 -20
- package/src/services/thread/thread-bootstrap.ts +26 -10
- package/src/services/thread/thread-listing.ts +2 -1
- package/src/services/thread/thread-memory-block.ts +18 -5
- package/src/services/thread/thread-message.service.ts +24 -8
- package/src/services/thread/thread-title.service.ts +1 -1
- package/src/services/thread/thread-turn-execution.ts +1 -1
- package/src/services/thread/thread-turn-preparation.service.ts +18 -16
- package/src/services/thread/thread-turn-streaming.ts +12 -11
- package/src/services/thread/thread-turn.ts +43 -10
- package/src/services/thread/thread.service.ts +11 -2
- package/src/services/user.service.ts +1 -1
- package/src/services/write-intent-validator.service.ts +1 -1
- package/src/storage/attachment-storage.service.ts +7 -4
- package/src/storage/generated-document-storage.service.ts +1 -1
- package/src/system-agents/context-compaction.agent.ts +1 -1
- package/src/system-agents/helper-agent-options.ts +1 -1
- package/src/system-agents/memory-reranker.agent.ts +1 -1
- package/src/system-agents/memory.agent.ts +1 -1
- package/src/system-agents/recent-activity-title-refiner.agent.ts +1 -1
- package/src/system-agents/regular-chat-memory-digest.agent.ts +1 -1
- package/src/system-agents/skill-extractor.agent.ts +1 -1
- package/src/system-agents/skill-manager.agent.ts +1 -1
- package/src/system-agents/title-generator.agent.ts +1 -1
- package/src/tools/execution-plan.tool.ts +28 -17
- package/src/tools/fetch-webpage.tool.ts +20 -13
- package/src/tools/firecrawl-client.ts +13 -3
- package/src/tools/plan-approval.tool.ts +9 -1
- package/src/tools/search-web.tool.ts +16 -9
- package/src/tools/team-think.tool.ts +2 -2
- package/src/utils/async.ts +15 -6
- package/src/utils/errors.ts +27 -15
- package/src/workers/bootstrap.ts +25 -48
- package/src/workers/organization-learning.worker.ts +1 -1
- package/src/workers/regular-chat-memory-digest.runner.ts +25 -15
- package/src/workers/worker-utils.ts +20 -2
- package/src/config/search.ts +0 -3
- package/src/runtime/agent-types.ts +0 -1
|
@@ -2,7 +2,7 @@ import { embed, embedMany } from 'ai'
|
|
|
2
2
|
import { Schema, Effect } from 'effect'
|
|
3
3
|
|
|
4
4
|
import { ConfigurationError } from '../effect/errors'
|
|
5
|
-
import {
|
|
5
|
+
import { runPromiseWithOptionalLotaRuntime } from '../effect/runtime'
|
|
6
6
|
import { getDirectOpenRouterProvider, normalizeDirectOpenRouterModelId } from '../openrouter/direct-provider'
|
|
7
7
|
|
|
8
8
|
const SUPPORTED_EMBEDDING_PREFIXES = ['openai/', 'openrouter/'] as const
|
|
@@ -93,8 +93,7 @@ export class ProviderEmbeddings {
|
|
|
93
93
|
}
|
|
94
94
|
|
|
95
95
|
private runEffect<A>(effect: Effect.Effect<A, EmbeddingProviderError>): Promise<A> {
|
|
96
|
-
|
|
97
|
-
return runtime ? runtime.runPromise(effect) : Effect.runPromise(effect)
|
|
96
|
+
return runPromiseWithOptionalLotaRuntime(effect)
|
|
98
97
|
}
|
|
99
98
|
|
|
100
99
|
embedQuery(text: string): Promise<number[]> {
|
|
@@ -113,28 +112,28 @@ export class ProviderEmbeddings {
|
|
|
113
112
|
}
|
|
114
113
|
|
|
115
114
|
private executeEmbedQueryEffect(input: string): Effect.Effect<number[], EmbeddingProviderError> {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
115
|
+
return Effect.gen(
|
|
116
|
+
function* (this: ProviderEmbeddings) {
|
|
117
|
+
const cached = yield* tryEmbeddingPromise('Failed to load cached query embedding.', () =>
|
|
118
|
+
this.loadCachedEmbedding(input),
|
|
119
|
+
)
|
|
120
|
+
if (cached) {
|
|
121
|
+
return cached
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const result = yield* tryEmbeddingPromise('Failed to generate query embedding.', () =>
|
|
125
|
+
this.embedFn({ model: this.getModel(), value: input, maxRetries: 2 }),
|
|
126
|
+
)
|
|
127
|
+
const embedding = normalizeEmbedding(result.embedding)
|
|
128
|
+
|
|
129
|
+
const redisCache = this.getCache()
|
|
130
|
+
if (redisCache) {
|
|
131
|
+
void redisCache.set(this.getModelId(), input, embedding)
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return embedding
|
|
135
|
+
}.bind(this),
|
|
136
|
+
).pipe(Effect.withSpan('ProviderEmbeddings.executeEmbedQuery'))
|
|
138
137
|
}
|
|
139
138
|
|
|
140
139
|
embedDocuments(values: string[]): Promise<number[][]> {
|
|
@@ -157,51 +156,51 @@ export class ProviderEmbeddings {
|
|
|
157
156
|
normalized: string[],
|
|
158
157
|
uniqueTexts: string[],
|
|
159
158
|
): Effect.Effect<number[][], EmbeddingProviderError> {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
)
|
|
173
|
-
)
|
|
174
|
-
)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
159
|
+
return Effect.gen(
|
|
160
|
+
function* (this: ProviderEmbeddings) {
|
|
161
|
+
const embeddingsByText = new Map<string, number[]>()
|
|
162
|
+
let missingTexts = [...uniqueTexts]
|
|
163
|
+
const redisCache = this.getCache()
|
|
164
|
+
const redisResults =
|
|
165
|
+
redisCache && missingTexts.length > 0
|
|
166
|
+
? yield* Effect.all(
|
|
167
|
+
missingTexts.map((text) =>
|
|
168
|
+
tryEmbeddingPromise('Failed to load cached document embedding.', () =>
|
|
169
|
+
redisCache.get(this.getModelId(), text),
|
|
170
|
+
).pipe(Effect.map((embedding) => ({ text, embedding }))),
|
|
171
|
+
),
|
|
172
|
+
)
|
|
173
|
+
: ([] as Array<{ text: string; embedding: number[] | null }>)
|
|
174
|
+
|
|
175
|
+
if (redisCache && missingTexts.length > 0) {
|
|
176
|
+
missingTexts = []
|
|
177
|
+
for (const result of redisResults) {
|
|
178
|
+
if (!result.embedding) {
|
|
179
|
+
missingTexts.push(result.text)
|
|
180
|
+
continue
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
embeddingsByText.set(result.text, result.embedding)
|
|
183
184
|
}
|
|
184
|
-
|
|
185
|
-
embeddingsByText.set(result.text, result.embedding)
|
|
186
185
|
}
|
|
187
|
-
}
|
|
188
186
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
const result = yield* tryEmbeddingPromise('Failed to generate document embeddings.', () =>
|
|
194
|
-
self.embedManyFn({ model: self.getModel(), values: missingTexts, maxRetries: 2 }),
|
|
195
|
-
)
|
|
196
|
-
missingTexts.forEach((text, index) => {
|
|
197
|
-
const embedding = normalizeEmbedding(result.embeddings[index] ?? [])
|
|
198
|
-
embeddingsByText.set(text, embedding)
|
|
199
|
-
if (redisCache) {
|
|
200
|
-
void redisCache.set(self.getModelId(), text, embedding)
|
|
187
|
+
if (missingTexts.length === 0) {
|
|
188
|
+
return normalized.map((text) => (text ? (embeddingsByText.get(text) ?? []) : []))
|
|
201
189
|
}
|
|
202
|
-
})
|
|
203
190
|
|
|
204
|
-
|
|
205
|
-
|
|
191
|
+
const result = yield* tryEmbeddingPromise('Failed to generate document embeddings.', () =>
|
|
192
|
+
this.embedManyFn({ model: this.getModel(), values: missingTexts, maxRetries: 2 }),
|
|
193
|
+
)
|
|
194
|
+
missingTexts.forEach((text, index) => {
|
|
195
|
+
const embedding = normalizeEmbedding(result.embeddings[index] ?? [])
|
|
196
|
+
embeddingsByText.set(text, embedding)
|
|
197
|
+
if (redisCache) {
|
|
198
|
+
void redisCache.set(this.getModelId(), text, embedding)
|
|
199
|
+
}
|
|
200
|
+
})
|
|
201
|
+
|
|
202
|
+
return normalized.map((text) => (text ? (embeddingsByText.get(text) ?? []) : []))
|
|
203
|
+
}.bind(this),
|
|
204
|
+
).pipe(Effect.withSpan('ProviderEmbeddings.embedDocuments'))
|
|
206
205
|
}
|
|
207
206
|
}
|
package/src/index.ts
CHANGED
|
@@ -15,25 +15,21 @@ export * from './utils'
|
|
|
15
15
|
export * from './workers'
|
|
16
16
|
export { Effect } from 'effect'
|
|
17
17
|
export {
|
|
18
|
+
ActiveThreadRunConflictError,
|
|
18
19
|
AgentConfigLive,
|
|
19
20
|
AgentFactoryLive,
|
|
20
|
-
|
|
21
|
+
AiGenerationError,
|
|
21
22
|
AppLoggerLive,
|
|
22
|
-
|
|
23
|
-
|
|
23
|
+
AppLoggerTag,
|
|
24
|
+
BaseServicePersistenceError,
|
|
24
25
|
ConfigurationError,
|
|
25
26
|
ConflictError,
|
|
26
27
|
DatabaseError,
|
|
27
28
|
DatabaseLive,
|
|
28
29
|
DatabaseServiceTag as EffectDatabaseService,
|
|
29
|
-
|
|
30
|
-
EffectThreadTurnError,
|
|
31
|
-
EffectValidationError,
|
|
32
|
-
getLotaSdkRuntime,
|
|
33
|
-
isEffectError,
|
|
30
|
+
ForbiddenError,
|
|
34
31
|
LockAcquisitionError,
|
|
35
32
|
LockLostError,
|
|
36
|
-
ActiveThreadRunConflictError,
|
|
37
33
|
RedisError,
|
|
38
34
|
RedisLive,
|
|
39
35
|
RedisServiceTag as EffectRedisService,
|
|
@@ -42,15 +38,21 @@ export {
|
|
|
42
38
|
RuntimeConfigServiceTag,
|
|
43
39
|
RuntimeExtensionsLive,
|
|
44
40
|
RuntimeWorkerExtensionsServiceTag,
|
|
45
|
-
|
|
41
|
+
ServiceError,
|
|
46
42
|
ThreadConfigLive,
|
|
47
43
|
ThreadConfigServiceTag,
|
|
44
|
+
ThreadTurnError,
|
|
48
45
|
TimeoutError,
|
|
49
46
|
ToolProvidersServiceTag,
|
|
50
47
|
TurnHooksServiceTag,
|
|
48
|
+
ValidationError,
|
|
49
|
+
clearLotaSdkRuntime,
|
|
50
|
+
getLotaSdkRuntime,
|
|
51
|
+
isEffectError,
|
|
52
|
+
setLotaSdkRuntime,
|
|
53
|
+
summarizeZodIssues,
|
|
51
54
|
toAwaitableEffect,
|
|
52
55
|
toAwaitableService,
|
|
53
|
-
summarizeZodIssues,
|
|
54
56
|
toValidationError,
|
|
55
57
|
toValidationIssues,
|
|
56
58
|
zodParse,
|
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
import type { AutonomousJobSchedule } from '@lota-sdk/shared'
|
|
2
2
|
import type { Job } from 'bullmq'
|
|
3
|
-
import { Effect } from 'effect'
|
|
3
|
+
import { Effect, Schema } from 'effect'
|
|
4
4
|
import type { Context } from 'effect'
|
|
5
5
|
import type IORedis from 'ioredis'
|
|
6
6
|
|
|
7
7
|
import { serverLogger } from '../config/logger'
|
|
8
|
-
import { ConfigurationError } from '../effect/errors'
|
|
9
8
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
10
9
|
import { AutonomousJobServiceTag } from '../services/autonomous-job.service'
|
|
11
10
|
import { buildAutonomousAtJobId } from '../utils/autonomous-job-ids'
|
|
12
11
|
import type { WorkerHandle } from '../workers/worker-utils'
|
|
13
|
-
import { DEFAULT_JOB_RETENTION
|
|
14
|
-
import {
|
|
12
|
+
import { DEFAULT_JOB_RETENTION } from '../workers/worker-utils'
|
|
13
|
+
import { createQueueFactoryWithDeps, recordEnqueuedJobMetadata } from './queue-factory'
|
|
15
14
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
16
15
|
|
|
16
|
+
class AutonomousJobQueueError extends Schema.TaggedErrorClass<AutonomousJobQueueError>()(
|
|
17
|
+
'@lota-sdk/core/AutonomousJobQueueError',
|
|
18
|
+
{ message: Schema.String, cause: Schema.optional(Schema.Defect) },
|
|
19
|
+
) {}
|
|
20
|
+
|
|
17
21
|
export interface AutonomousJobQueuePayload {
|
|
18
22
|
autonomousJobId: string
|
|
19
23
|
autonomousJobRunId?: string
|
|
@@ -27,33 +31,26 @@ interface AutonomousJobQueueDeps {
|
|
|
27
31
|
autonomousJobService: Context.Service.Shape<typeof AutonomousJobServiceTag>
|
|
28
32
|
}
|
|
29
33
|
|
|
30
|
-
let _deps: AutonomousJobQueueDeps | null = null
|
|
31
|
-
function getDeps(): AutonomousJobQueueDeps {
|
|
32
|
-
if (!_deps)
|
|
33
|
-
throw new ConfigurationError({
|
|
34
|
-
message: 'Autonomous job queue is not configured. Initialize the runtime before starting the worker.',
|
|
35
|
-
key: 'queue-deps',
|
|
36
|
-
})
|
|
37
|
-
return _deps
|
|
38
|
-
}
|
|
39
|
-
|
|
40
34
|
const DEFAULT_AUTONOMOUS_JOB_OPTIONS = {
|
|
41
35
|
...DEFAULT_JOB_RETENTION,
|
|
42
36
|
attempts: 3,
|
|
43
37
|
backoff: { type: 'exponential', delay: 5_000 },
|
|
44
38
|
} as const
|
|
45
39
|
|
|
46
|
-
function processAutonomousJob(
|
|
47
|
-
|
|
40
|
+
function processAutonomousJob(
|
|
41
|
+
deps: AutonomousJobQueueDeps,
|
|
42
|
+
job: Job<AutonomousJobQueuePayload>,
|
|
43
|
+
): Promise<{ status: string; summary?: string }> {
|
|
44
|
+
return Effect.runPromise(deps.autonomousJobService.executeQueuedRun(job))
|
|
48
45
|
}
|
|
49
46
|
|
|
50
|
-
const autonomousJobQueue =
|
|
47
|
+
const autonomousJobQueue = createQueueFactoryWithDeps<AutonomousJobQueuePayload, AutonomousJobQueueDeps>({
|
|
51
48
|
name: AUTONOMOUS_JOB_QUEUE,
|
|
52
49
|
displayName: 'Autonomous job',
|
|
53
50
|
jobName: 'run-autonomous-job',
|
|
54
51
|
concurrency: 2,
|
|
55
52
|
defaultJobOptions: DEFAULT_AUTONOMOUS_JOB_OPTIONS,
|
|
56
|
-
prepare: () =>
|
|
53
|
+
prepare: ({ databaseService }) => databaseService.connect(),
|
|
57
54
|
processor: processAutonomousJob,
|
|
58
55
|
})
|
|
59
56
|
|
|
@@ -68,32 +65,19 @@ export function enqueueAutonomousJobRun(params: {
|
|
|
68
65
|
}): Promise<{ bullmqJobId: string; queueJobId?: string }> {
|
|
69
66
|
return Effect.runPromise(
|
|
70
67
|
Effect.gen(function* () {
|
|
71
|
-
const queuedJob = yield* Effect.tryPromise(
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
68
|
+
const queuedJob = yield* Effect.tryPromise({
|
|
69
|
+
try: () =>
|
|
70
|
+
autonomousJobQueue
|
|
71
|
+
.getQueue()
|
|
72
|
+
.add('run-autonomous-job', params.payload, {
|
|
73
|
+
...(typeof params.delayMs === 'number' ? { delay: Math.max(0, params.delayMs) } : {}),
|
|
74
|
+
...(params.jobId ? { jobId: params.jobId } : {}),
|
|
75
|
+
}),
|
|
76
|
+
catch: (cause) => new AutonomousJobQueueError({ message: 'Failed to enqueue autonomous job run.', cause }),
|
|
77
|
+
})
|
|
79
78
|
|
|
80
79
|
const bullmqJobId = String(queuedJob.id)
|
|
81
|
-
const queueJobId = yield*
|
|
82
|
-
getQueueJobService().recordEnqueued({
|
|
83
|
-
queueName: AUTONOMOUS_JOB_QUEUE,
|
|
84
|
-
id: queuedJob.id,
|
|
85
|
-
name: queuedJob.name,
|
|
86
|
-
data: queuedJob.data,
|
|
87
|
-
opts: queuedJob.opts,
|
|
88
|
-
attemptsMade: queuedJob.attemptsMade,
|
|
89
|
-
timestamp: queuedJob.timestamp,
|
|
90
|
-
}),
|
|
91
|
-
(error) =>
|
|
92
|
-
Effect.sync(() => {
|
|
93
|
-
serverLogger.error`Failed to persist queued job metadata (queue=${AUTONOMOUS_JOB_QUEUE}, job=${queuedJob.id}): ${error}`
|
|
94
|
-
return undefined
|
|
95
|
-
}),
|
|
96
|
-
)
|
|
80
|
+
const queueJobId = yield* recordEnqueuedJobMetadata({ queueName: AUTONOMOUS_JOB_QUEUE, job: queuedJob })
|
|
97
81
|
return { bullmqJobId, queueJobId }
|
|
98
82
|
}),
|
|
99
83
|
)
|
|
@@ -107,31 +91,23 @@ export function upsertAutonomousJobScheduler(params: {
|
|
|
107
91
|
params.schedule.kind === 'cron' ? { pattern: params.schedule.cron } : { every: params.schedule.intervalMs }
|
|
108
92
|
return Effect.runPromise(
|
|
109
93
|
Effect.gen(function* () {
|
|
110
|
-
const queuedJob = yield* Effect.tryPromise(
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
94
|
+
const queuedJob = yield* Effect.tryPromise({
|
|
95
|
+
try: () =>
|
|
96
|
+
autonomousJobQueue
|
|
97
|
+
.getQueue()
|
|
98
|
+
.upsertJobScheduler(buildAutonomousSchedulerId(params.autonomousJobId), repeatOpts, {
|
|
99
|
+
name: 'run-autonomous-job',
|
|
100
|
+
data: { autonomousJobId: params.autonomousJobId, trigger: 'scheduled' },
|
|
101
|
+
opts: DEFAULT_AUTONOMOUS_JOB_OPTIONS,
|
|
102
|
+
}),
|
|
103
|
+
catch: (cause) =>
|
|
104
|
+
new AutonomousJobQueueError({
|
|
105
|
+
message: `Failed to upsert autonomous job scheduler for ${params.autonomousJobId}.`,
|
|
106
|
+
cause,
|
|
117
107
|
}),
|
|
118
|
-
)
|
|
119
|
-
|
|
120
|
-
yield*
|
|
121
|
-
getQueueJobService().recordEnqueued({
|
|
122
|
-
queueName: AUTONOMOUS_JOB_QUEUE,
|
|
123
|
-
id: queuedJob.id,
|
|
124
|
-
name: queuedJob.name,
|
|
125
|
-
data: queuedJob.data,
|
|
126
|
-
opts: queuedJob.opts,
|
|
127
|
-
attemptsMade: queuedJob.attemptsMade,
|
|
128
|
-
timestamp: queuedJob.timestamp,
|
|
129
|
-
}),
|
|
130
|
-
(error) =>
|
|
131
|
-
Effect.sync(() => {
|
|
132
|
-
serverLogger.error`Failed to persist queued job metadata (queue=${AUTONOMOUS_JOB_QUEUE}, job=${queuedJob.id}): ${error}`
|
|
133
|
-
}),
|
|
134
|
-
)
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
yield* recordEnqueuedJobMetadata({ queueName: AUTONOMOUS_JOB_QUEUE, job: queuedJob })
|
|
135
111
|
}),
|
|
136
112
|
)
|
|
137
113
|
}
|
|
@@ -139,9 +115,14 @@ export function upsertAutonomousJobScheduler(params: {
|
|
|
139
115
|
export function removeAutonomousJobScheduler(autonomousJobId: string): Promise<void> {
|
|
140
116
|
return Effect.runPromise(
|
|
141
117
|
Effect.asVoid(
|
|
142
|
-
Effect.tryPromise(
|
|
143
|
-
autonomousJobQueue.getQueue().removeJobScheduler(buildAutonomousSchedulerId(autonomousJobId)),
|
|
144
|
-
|
|
118
|
+
Effect.tryPromise({
|
|
119
|
+
try: () => autonomousJobQueue.getQueue().removeJobScheduler(buildAutonomousSchedulerId(autonomousJobId)),
|
|
120
|
+
catch: (cause) =>
|
|
121
|
+
new AutonomousJobQueueError({
|
|
122
|
+
message: `Failed to remove autonomous job scheduler for ${autonomousJobId}.`,
|
|
123
|
+
cause,
|
|
124
|
+
}),
|
|
125
|
+
}),
|
|
145
126
|
),
|
|
146
127
|
)
|
|
147
128
|
}
|
|
@@ -150,7 +131,14 @@ export function removeAutonomousAtJob(autonomousJobId: string): Promise<void> {
|
|
|
150
131
|
return Effect.runPromise(
|
|
151
132
|
Effect.catch(
|
|
152
133
|
Effect.asVoid(
|
|
153
|
-
Effect.tryPromise(
|
|
134
|
+
Effect.tryPromise({
|
|
135
|
+
try: () => autonomousJobQueue.getQueue().remove(buildAutonomousAtJobId(autonomousJobId)),
|
|
136
|
+
catch: (cause) =>
|
|
137
|
+
new AutonomousJobQueueError({
|
|
138
|
+
message: `Failed to remove autonomous-at job for ${autonomousJobId}.`,
|
|
139
|
+
cause,
|
|
140
|
+
}),
|
|
141
|
+
}),
|
|
154
142
|
),
|
|
155
143
|
() => Effect.void,
|
|
156
144
|
),
|
|
@@ -164,8 +152,8 @@ interface AutonomousJobWorkerOptions {
|
|
|
164
152
|
}
|
|
165
153
|
|
|
166
154
|
export function startAutonomousJobWorker(options: AutonomousJobWorkerOptions): WorkerHandle {
|
|
167
|
-
_deps = options.deps
|
|
168
155
|
const handle = autonomousJobQueue.startWorker({
|
|
156
|
+
deps: options.deps,
|
|
169
157
|
registerSignals: options.registerSignals,
|
|
170
158
|
connectionProvider: options.connectionProvider,
|
|
171
159
|
})
|
|
@@ -5,11 +5,10 @@ import type IORedis from 'ioredis'
|
|
|
5
5
|
|
|
6
6
|
import { ensureRecordId } from '../db/record-id'
|
|
7
7
|
import { TABLES } from '../db/tables'
|
|
8
|
-
import { ConfigurationError } from '../effect/errors'
|
|
9
8
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
10
9
|
import { ContextCompactionServiceTag } from '../services/context-compaction.service'
|
|
11
10
|
import { ThreadServiceTag } from '../services/thread/thread.service'
|
|
12
|
-
import {
|
|
11
|
+
import { createQueueFactoryWithDeps } from './queue-factory'
|
|
13
12
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
14
13
|
|
|
15
14
|
interface ContextCompactionJob {
|
|
@@ -24,19 +23,8 @@ interface ContextCompactionQueueDeps {
|
|
|
24
23
|
contextCompactionService: Context.Service.Shape<typeof ContextCompactionServiceTag>
|
|
25
24
|
}
|
|
26
25
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if (!_deps)
|
|
30
|
-
throw new ConfigurationError({
|
|
31
|
-
message: 'Context compaction queue is not configured. Initialize the runtime before starting the worker.',
|
|
32
|
-
key: 'queue-deps',
|
|
33
|
-
})
|
|
34
|
-
return _deps
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
function processContextCompactionJob(job: Job<ContextCompactionJob>): Promise<void> {
|
|
38
|
-
const { threadService, contextCompactionService } = getDeps()
|
|
39
|
-
|
|
26
|
+
function processContextCompactionJob(deps: ContextCompactionQueueDeps, job: Job<ContextCompactionJob>): Promise<void> {
|
|
27
|
+
const { threadService, contextCompactionService } = deps
|
|
40
28
|
const { entityId, contextSize } = job.data
|
|
41
29
|
const threadRef = ensureRecordId(entityId, TABLES.THREAD)
|
|
42
30
|
return Effect.runPromise(
|
|
@@ -52,14 +40,14 @@ function processContextCompactionJob(job: Job<ContextCompactionJob>): Promise<vo
|
|
|
52
40
|
)
|
|
53
41
|
}
|
|
54
42
|
|
|
55
|
-
const contextCompaction =
|
|
43
|
+
const contextCompaction = createQueueFactoryWithDeps<ContextCompactionJob, ContextCompactionQueueDeps>({
|
|
56
44
|
name: 'context-compaction',
|
|
57
45
|
displayName: 'Context compaction',
|
|
58
46
|
jobName: 'compact',
|
|
59
47
|
concurrency: 2,
|
|
60
48
|
lockDuration: 300_000,
|
|
61
49
|
defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 3_000 } },
|
|
62
|
-
prepare: () =>
|
|
50
|
+
prepare: ({ databaseService }) => databaseService.connect(),
|
|
63
51
|
processor: processContextCompactionJob,
|
|
64
52
|
})
|
|
65
53
|
|
|
@@ -72,8 +60,8 @@ export function startContextCompactionWorker(options: {
|
|
|
72
60
|
connectionProvider: () => IORedis
|
|
73
61
|
deps: ContextCompactionQueueDeps
|
|
74
62
|
}): ReturnType<typeof contextCompaction.startWorker> {
|
|
75
|
-
_deps = options.deps
|
|
76
63
|
return contextCompaction.startWorker({
|
|
64
|
+
deps: options.deps,
|
|
77
65
|
registerSignals: options.registerSignals,
|
|
78
66
|
connectionProvider: options.connectionProvider,
|
|
79
67
|
})
|
|
@@ -3,10 +3,9 @@ import { Effect } from 'effect'
|
|
|
3
3
|
import type { Context } from 'effect'
|
|
4
4
|
import type IORedis from 'ioredis'
|
|
5
5
|
|
|
6
|
-
import { ConfigurationError } from '../effect/errors'
|
|
7
6
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
8
7
|
import { PlanExecutorServiceTag } from '../services/plan/plan-executor.service'
|
|
9
|
-
import {
|
|
8
|
+
import { createQueueFactoryWithDeps } from './queue-factory'
|
|
10
9
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
11
10
|
|
|
12
11
|
export interface DelayedNodePromotionJob {
|
|
@@ -22,18 +21,11 @@ interface DelayedNodePromotionQueueDeps {
|
|
|
22
21
|
planExecutorService: Context.Service.Shape<typeof PlanExecutorServiceTag>
|
|
23
22
|
}
|
|
24
23
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
key: 'queue-deps',
|
|
31
|
-
})
|
|
32
|
-
return _deps
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function processDelayedNodePromotionJob(job: Job<DelayedNodePromotionJob>): Promise<void> {
|
|
36
|
-
const { planExecutorService } = getDeps()
|
|
24
|
+
function processDelayedNodePromotionJob(
|
|
25
|
+
deps: DelayedNodePromotionQueueDeps,
|
|
26
|
+
job: Job<DelayedNodePromotionJob>,
|
|
27
|
+
): Promise<void> {
|
|
28
|
+
const { planExecutorService } = deps
|
|
37
29
|
return Promise.resolve(
|
|
38
30
|
planExecutorService.promoteDelayedNode({
|
|
39
31
|
runId: job.data.runId,
|
|
@@ -43,13 +35,13 @@ function processDelayedNodePromotionJob(job: Job<DelayedNodePromotionJob>): Prom
|
|
|
43
35
|
).then(() => undefined)
|
|
44
36
|
}
|
|
45
37
|
|
|
46
|
-
const delayedNodePromotion =
|
|
38
|
+
const delayedNodePromotion = createQueueFactoryWithDeps<DelayedNodePromotionJob, DelayedNodePromotionQueueDeps>({
|
|
47
39
|
name: DELAYED_NODE_PROMOTION_QUEUE,
|
|
48
40
|
displayName: 'Delayed node promotion',
|
|
49
41
|
jobName: 'promote-node',
|
|
50
42
|
concurrency: 1,
|
|
51
43
|
defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
|
|
52
|
-
prepare: () =>
|
|
44
|
+
prepare: ({ databaseService }) => databaseService.connect(),
|
|
53
45
|
processor: processDelayedNodePromotionJob,
|
|
54
46
|
})
|
|
55
47
|
|
|
@@ -62,8 +54,8 @@ export function startDelayedNodePromotionWorker(options: {
|
|
|
62
54
|
connectionProvider: () => IORedis
|
|
63
55
|
deps: DelayedNodePromotionQueueDeps
|
|
64
56
|
}): ReturnType<typeof delayedNodePromotion.startWorker> {
|
|
65
|
-
_deps = options.deps
|
|
66
57
|
return delayedNodePromotion.startWorker({
|
|
58
|
+
deps: options.deps,
|
|
67
59
|
registerSignals: options.registerSignals,
|
|
68
60
|
connectionProvider: options.connectionProvider,
|
|
69
61
|
})
|
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
import { Effect } from 'effect'
|
|
1
|
+
import { Effect, Schema } from 'effect'
|
|
2
2
|
|
|
3
3
|
import { getWorkerPath, LONG_JOB_LOCK_DURATION_MS } from '../workers/worker-utils'
|
|
4
4
|
import { createQueueFactory } from './queue-factory'
|
|
5
5
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
6
6
|
|
|
7
|
+
class OrganizationLearningQueueError extends Schema.TaggedErrorClass<OrganizationLearningQueueError>()(
|
|
8
|
+
'@lota-sdk/core/OrganizationLearningQueueError',
|
|
9
|
+
{ message: Schema.String, cause: Schema.optional(Schema.Defect) },
|
|
10
|
+
) {}
|
|
11
|
+
|
|
7
12
|
export const ORGANIZATION_LEARNING_QUEUE = 'organization-learning'
|
|
8
13
|
|
|
9
14
|
const ORGANIZATION_LEARNING_DELAY_MS = 15 * 60 * 1000
|
|
@@ -73,9 +78,17 @@ export function enqueueSkillExtraction(job: Omit<SkillExtractionJob, 'kind'>) {
|
|
|
73
78
|
export function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
|
|
74
79
|
return Effect.runPromise(
|
|
75
80
|
Effect.asVoid(
|
|
76
|
-
Effect.tryPromise(
|
|
77
|
-
|
|
78
|
-
|
|
81
|
+
Effect.tryPromise({
|
|
82
|
+
try: () =>
|
|
83
|
+
organizationLearningQueue
|
|
84
|
+
.getQueue()
|
|
85
|
+
.removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId)),
|
|
86
|
+
catch: (cause) =>
|
|
87
|
+
new OrganizationLearningQueueError({
|
|
88
|
+
message: `Failed to clear regular-chat memory digest deduplication key for ${orgId}.`,
|
|
89
|
+
cause,
|
|
90
|
+
}),
|
|
91
|
+
}),
|
|
79
92
|
),
|
|
80
93
|
)
|
|
81
94
|
}
|