@lota-sdk/core 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/ai-gateway/ai-gateway.ts +149 -95
- package/src/ai-gateway/index.ts +16 -1
- package/src/config/agent-defaults.ts +4 -120
- package/src/config/logger.ts +18 -34
- package/src/config/thread-defaults.ts +1 -18
- package/src/create-runtime.ts +90 -28
- package/src/db/base.service.ts +30 -38
- package/src/db/service.ts +489 -545
- package/src/effect/index.ts +0 -2
- package/src/effect/layers.ts +6 -13
- package/src/embeddings/provider.ts +2 -7
- package/src/index.ts +4 -5
- package/src/queues/autonomous-job.queue.ts +159 -113
- package/src/queues/context-compaction.queue.ts +39 -25
- package/src/queues/delayed-node-promotion.queue.ts +56 -29
- package/src/queues/document-processor.queue.ts +5 -3
- package/src/queues/index.ts +1 -0
- package/src/queues/memory-consolidation.queue.ts +79 -53
- package/src/queues/organization-learning.queue.ts +63 -39
- package/src/queues/plan-agent-heartbeat.queue.ts +104 -79
- package/src/queues/plan-scheduler.queue.ts +100 -84
- package/src/queues/post-chat-memory.queue.ts +55 -33
- package/src/queues/queue-factory.ts +40 -41
- package/src/queues/queues.service.ts +61 -0
- package/src/queues/title-generation.queue.ts +42 -31
- package/src/redis/org-memory-lock.ts +24 -9
- package/src/redis/redis-lease-lock.ts +8 -1
- package/src/runtime/agent-identity-overrides.ts +7 -3
- package/src/runtime/agent-runtime-policy.ts +9 -4
- package/src/runtime/agent-stream-helpers.ts +9 -4
- package/src/runtime/context-compaction/context-compaction-runtime.ts +28 -32
- package/src/runtime/context-compaction/context-compaction.ts +9 -7
- package/src/runtime/domain-layer.ts +15 -4
- package/src/runtime/execution-plan-visibility.ts +5 -2
- package/src/runtime/graph-designer.ts +0 -22
- package/src/runtime/index.ts +1 -0
- package/src/runtime/indexed-repositories-policy.ts +2 -6
- package/src/runtime/plugin-resolution.ts +29 -12
- package/src/runtime/post-turn-side-effects.ts +139 -141
- package/src/runtime/runtime-config.ts +0 -6
- package/src/runtime/runtime-extensions.ts +0 -54
- package/src/runtime/runtime-lifecycle.ts +4 -4
- package/src/runtime/runtime-services.ts +122 -53
- package/src/runtime/runtime-worker-registry.ts +113 -30
- package/src/runtime/social-chat/social-chat-agent-runner.ts +6 -3
- package/src/runtime/social-chat/social-chat-history.ts +3 -1
- package/src/runtime/social-chat/social-chat.ts +35 -20
- package/src/runtime/team-consultation/team-consultation-orchestrator.ts +6 -5
- package/src/runtime/team-consultation/team-consultation-prompts.ts +11 -6
- package/src/runtime/thread-chat-helpers.ts +18 -9
- package/src/runtime/thread-turn-context.ts +7 -47
- package/src/runtime/turn-lifecycle.ts +6 -14
- package/src/services/agent-activity.service.ts +168 -175
- package/src/services/agent-executor.service.ts +35 -16
- package/src/services/attachment.service.ts +4 -70
- package/src/services/autonomous-job.service.ts +53 -61
- package/src/services/context-compaction.service.ts +7 -9
- package/src/services/execution-plan/execution-plan-graph.ts +106 -115
- package/src/services/execution-plan/execution-plan-schedule.ts +1 -15
- package/src/services/execution-plan/execution-plan.service.ts +67 -50
- package/src/services/global-orchestrator.service.ts +18 -7
- package/src/services/graph-full-routing.ts +7 -6
- package/src/services/memory/memory-conversation.ts +10 -5
- package/src/services/memory/memory.service.ts +11 -8
- package/src/services/ownership-dispatcher.service.ts +16 -5
- package/src/services/plan/plan-agent-heartbeat.service.ts +29 -15
- package/src/services/plan/plan-agent-query.service.ts +12 -8
- package/src/services/plan/plan-completion-side-effects.ts +93 -101
- package/src/services/plan/plan-cycle.service.ts +7 -45
- package/src/services/plan/plan-deadline.service.ts +28 -17
- package/src/services/plan/plan-event-delivery.service.ts +47 -40
- package/src/services/plan/plan-executor-context.ts +2 -0
- package/src/services/plan/plan-executor-graph.ts +366 -391
- package/src/services/plan/plan-executor.service.ts +13 -91
- package/src/services/plan/plan-scheduler.service.ts +62 -49
- package/src/services/plan/plan-transaction-events.ts +1 -1
- package/src/services/recent-activity-title.service.ts +6 -2
- package/src/services/thread/thread-bootstrap.ts +11 -9
- package/src/services/thread/thread-message.service.ts +6 -5
- package/src/services/thread/thread-turn-execution.ts +86 -82
- package/src/services/thread/thread-turn-preparation.service.ts +47 -24
- package/src/services/thread/thread-turn-streaming.ts +20 -25
- package/src/services/thread/thread-turn.ts +25 -44
- package/src/services/thread/thread.service.ts +21 -6
- package/src/system-agents/recent-activity-title-refiner.agent.ts +8 -5
- package/src/system-agents/thread-router.agent.ts +23 -20
- package/src/tools/execution-plan.tool.ts +8 -3
- package/src/tools/fetch-webpage.tool.ts +10 -9
- package/src/tools/firecrawl-client.ts +0 -15
- package/src/tools/remember-memory.tool.ts +3 -6
- package/src/tools/research-topic.tool.ts +12 -3
- package/src/tools/search-web.tool.ts +10 -9
- package/src/tools/search.tool.ts +4 -5
- package/src/tools/team-think.tool.ts +139 -121
- package/src/workers/bootstrap.ts +9 -10
- package/src/workers/memory-consolidation.worker.ts +4 -1
- package/src/workers/organization-learning.worker.ts +15 -2
- package/src/workers/regular-chat-memory-digest.helpers.ts +3 -4
- package/src/workers/regular-chat-memory-digest.runner.ts +21 -14
- package/src/workers/skill-extraction.runner.ts +13 -15
- package/src/workers/worker-utils.ts +6 -18
- package/src/effect/awaitable-effect.ts +0 -96
- package/src/effect/runtime-ref.ts +0 -25
- package/src/effect/runtime.ts +0 -46
- package/src/redis/runtime-connection.ts +0 -20
- package/src/runtime/runtime-accessors.ts +0 -92
- package/src/runtime/runtime-token.ts +0 -47
package/src/effect/index.ts
CHANGED
package/src/effect/layers.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { Otlp } from 'effect/unstable/observability'
|
|
|
7
7
|
import type { CoreThreadProfile } from '../config/agent-defaults'
|
|
8
8
|
import { resolveAgentConfig, resolveAgentFactoryConfig } from '../config/agent-defaults'
|
|
9
9
|
import type { AgentFactory, AgentRuntimeConfigProvider, AgentToolBuilder } from '../config/agent-types'
|
|
10
|
-
import {
|
|
10
|
+
import { getLotaLoggers, toEffectLogLevel } from '../config/logger'
|
|
11
11
|
import type { LotaLogLevel } from '../config/logger'
|
|
12
12
|
import { resolveThreadConfig } from '../config/thread-defaults'
|
|
13
13
|
import type { LotaThreadConfig } from '../config/thread-defaults'
|
|
@@ -67,19 +67,12 @@ export function RedisLive(config: CreateRedisConnectionManagerOptions) {
|
|
|
67
67
|
}
|
|
68
68
|
|
|
69
69
|
export function AppLoggerLive(level: LotaLogLevel = 'info') {
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
//
|
|
73
|
-
//
|
|
74
|
-
const AppLoggersLayer = Layer.effect(
|
|
75
|
-
AppLoggerTag,
|
|
76
|
-
Effect.sync(() => {
|
|
77
|
-
configureLotaLogger(level)
|
|
78
|
-
return getLotaLoggers()
|
|
79
|
-
}),
|
|
80
|
-
)
|
|
70
|
+
// The logger level used by `serverLogger`/`chatLogger`/`aiLogger` is read
|
|
71
|
+
// once from `LOG_LEVEL` at module load. This layer wires the Effect-side
|
|
72
|
+
// console logger and minimum level plus the `AppLoggerTag` for callers
|
|
73
|
+
// that want the logger set from context.
|
|
81
74
|
return Layer.mergeAll(
|
|
82
|
-
|
|
75
|
+
Layer.succeed(AppLoggerTag, getLotaLoggers()),
|
|
83
76
|
Logger.layer([Logger.consolePretty()]),
|
|
84
77
|
Layer.succeed(References.MinimumLogLevel, toEffectLogLevel(level)),
|
|
85
78
|
)
|
|
@@ -2,7 +2,6 @@ import { embed, embedMany } from 'ai'
|
|
|
2
2
|
import { Schema, Effect } from 'effect'
|
|
3
3
|
|
|
4
4
|
import { ConfigurationError } from '../effect/errors'
|
|
5
|
-
import { runPromiseWithOptionalLotaRuntime } from '../effect/runtime'
|
|
6
5
|
import { getDirectOpenRouterProvider, normalizeDirectOpenRouterModelId } from '../openrouter/direct-provider'
|
|
7
6
|
|
|
8
7
|
const SUPPORTED_EMBEDDING_PREFIXES = ['openai/', 'openrouter/'] as const
|
|
@@ -92,10 +91,6 @@ export class ProviderEmbeddings {
|
|
|
92
91
|
return redisCache.get(this.getModelId(), text)
|
|
93
92
|
}
|
|
94
93
|
|
|
95
|
-
private runEffect<A>(effect: Effect.Effect<A, EmbeddingProviderError>): Promise<A> {
|
|
96
|
-
return runPromiseWithOptionalLotaRuntime(effect)
|
|
97
|
-
}
|
|
98
|
-
|
|
99
94
|
embedQuery(text: string): Promise<number[]> {
|
|
100
95
|
const input = text.trim()
|
|
101
96
|
if (!input) return Promise.resolve([])
|
|
@@ -104,7 +99,7 @@ export class ProviderEmbeddings {
|
|
|
104
99
|
const pending = this.inflightEmbeddings.get(dedupKey)
|
|
105
100
|
if (pending) return pending
|
|
106
101
|
|
|
107
|
-
const promise =
|
|
102
|
+
const promise = Effect.runPromise(this.executeEmbedQueryEffect(input))
|
|
108
103
|
this.inflightEmbeddings.set(dedupKey, promise)
|
|
109
104
|
void promise.finally(() => this.inflightEmbeddings.delete(dedupKey))
|
|
110
105
|
|
|
@@ -149,7 +144,7 @@ export class ProviderEmbeddings {
|
|
|
149
144
|
}
|
|
150
145
|
|
|
151
146
|
const uniqueTexts = [...new Set(nonEmptyEntries.map((entry) => entry.value))]
|
|
152
|
-
return
|
|
147
|
+
return Effect.runPromise(this.embedDocumentsEffect(normalized, uniqueTexts))
|
|
153
148
|
}
|
|
154
149
|
|
|
155
150
|
private embedDocumentsEffect(
|
package/src/index.ts
CHANGED
|
@@ -17,7 +17,9 @@ export { Effect } from 'effect'
|
|
|
17
17
|
export {
|
|
18
18
|
ActiveThreadRunConflictError,
|
|
19
19
|
AgentConfigLive,
|
|
20
|
+
AgentConfigServiceTag,
|
|
20
21
|
AgentFactoryLive,
|
|
22
|
+
AgentFactoryServiceTag,
|
|
21
23
|
AiGenerationError,
|
|
22
24
|
AppLoggerLive,
|
|
23
25
|
AppLoggerTag,
|
|
@@ -26,12 +28,14 @@ export {
|
|
|
26
28
|
ConflictError,
|
|
27
29
|
DatabaseError,
|
|
28
30
|
DatabaseLive,
|
|
31
|
+
DatabaseServiceTag,
|
|
29
32
|
DatabaseServiceTag as EffectDatabaseService,
|
|
30
33
|
ForbiddenError,
|
|
31
34
|
LockAcquisitionError,
|
|
32
35
|
LockLostError,
|
|
33
36
|
RedisError,
|
|
34
37
|
RedisLive,
|
|
38
|
+
RedisServiceTag,
|
|
35
39
|
RedisServiceTag as EffectRedisService,
|
|
36
40
|
RuntimeAdaptersServiceTag,
|
|
37
41
|
RuntimeConfigLive,
|
|
@@ -46,13 +50,8 @@ export {
|
|
|
46
50
|
ToolProvidersServiceTag,
|
|
47
51
|
TurnHooksServiceTag,
|
|
48
52
|
ValidationError,
|
|
49
|
-
clearLotaSdkRuntime,
|
|
50
|
-
getLotaSdkRuntime,
|
|
51
53
|
isEffectError,
|
|
52
|
-
setLotaSdkRuntime,
|
|
53
54
|
summarizeZodIssues,
|
|
54
|
-
toAwaitableEffect,
|
|
55
|
-
toAwaitableService,
|
|
56
55
|
toValidationError,
|
|
57
56
|
toValidationIssues,
|
|
58
57
|
zodParse,
|
|
@@ -6,13 +6,25 @@ import type IORedis from 'ioredis'
|
|
|
6
6
|
|
|
7
7
|
import { serverLogger } from '../config/logger'
|
|
8
8
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
9
|
-
import {
|
|
9
|
+
import { QueueJobServiceTag } from '../services/queue-job.service'
|
|
10
10
|
import { buildAutonomousAtJobId } from '../utils/autonomous-job-ids'
|
|
11
|
-
import type { WorkerHandle } from '../workers/worker-utils'
|
|
11
|
+
import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
|
|
12
12
|
import { DEFAULT_JOB_RETENTION } from '../workers/worker-utils'
|
|
13
13
|
import { createQueueFactoryWithDeps, recordEnqueuedJobMetadata } from './queue-factory'
|
|
14
14
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
15
15
|
|
|
16
|
+
// Minimal service shape used by the queue's worker processor. We declare it
|
|
17
|
+
// structurally to avoid importing `AutonomousJobServiceTag` — that would form a
|
|
18
|
+
// dependency cycle since `AutonomousJobServiceLive` now depends on
|
|
19
|
+
// `LotaQueuesServiceTag` (defined in `queues.service.ts` which imports this
|
|
20
|
+
// module).
|
|
21
|
+
interface AutonomousJobWorkerServiceShape {
|
|
22
|
+
executeQueuedRun(
|
|
23
|
+
job: Job<AutonomousJobQueuePayload>,
|
|
24
|
+
): Effect.Effect<{ status: string; summary?: string }, unknown, unknown>
|
|
25
|
+
recoverActiveJobs(): Effect.Effect<unknown, unknown, unknown>
|
|
26
|
+
}
|
|
27
|
+
|
|
16
28
|
class AutonomousJobQueueError extends Schema.TaggedErrorClass<AutonomousJobQueueError>()(
|
|
17
29
|
'@lota-sdk/core/AutonomousJobQueueError',
|
|
18
30
|
{ message: Schema.String, cause: Schema.optional(Schema.Defect) },
|
|
@@ -26,9 +38,9 @@ export interface AutonomousJobQueuePayload {
|
|
|
26
38
|
|
|
27
39
|
export const AUTONOMOUS_JOB_QUEUE = 'autonomous-job'
|
|
28
40
|
|
|
29
|
-
interface
|
|
41
|
+
export interface AutonomousJobWorkerDeps {
|
|
30
42
|
databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
|
|
31
|
-
autonomousJobService:
|
|
43
|
+
autonomousJobService: AutonomousJobWorkerServiceShape
|
|
32
44
|
}
|
|
33
45
|
|
|
34
46
|
const DEFAULT_AUTONOMOUS_JOB_OPTIONS = {
|
|
@@ -38,141 +50,175 @@ const DEFAULT_AUTONOMOUS_JOB_OPTIONS = {
|
|
|
38
50
|
} as const
|
|
39
51
|
|
|
40
52
|
function processAutonomousJob(
|
|
41
|
-
deps:
|
|
53
|
+
deps: AutonomousJobWorkerDeps,
|
|
42
54
|
job: Job<AutonomousJobQueuePayload>,
|
|
43
55
|
): Promise<{ status: string; summary?: string }> {
|
|
44
|
-
|
|
56
|
+
// `executeQueuedRun` is typed with a wide residual context to account for
|
|
57
|
+
// the dynamic thread-turn import inside the service; the actual Effect
|
|
58
|
+
// resolves every tag via the managed runtime that produced this service.
|
|
59
|
+
return Effect.runPromise(
|
|
60
|
+
deps.autonomousJobService.executeQueuedRun(job) as Effect.Effect<
|
|
61
|
+
{ status: string; summary?: string },
|
|
62
|
+
never,
|
|
63
|
+
never
|
|
64
|
+
>,
|
|
65
|
+
)
|
|
45
66
|
}
|
|
46
67
|
|
|
47
|
-
const autonomousJobQueue = createQueueFactoryWithDeps<AutonomousJobQueuePayload, AutonomousJobQueueDeps>({
|
|
48
|
-
name: AUTONOMOUS_JOB_QUEUE,
|
|
49
|
-
displayName: 'Autonomous job',
|
|
50
|
-
jobName: 'run-autonomous-job',
|
|
51
|
-
concurrency: 2,
|
|
52
|
-
defaultJobOptions: DEFAULT_AUTONOMOUS_JOB_OPTIONS,
|
|
53
|
-
prepare: ({ databaseService }) => databaseService.connect(),
|
|
54
|
-
processor: processAutonomousJob,
|
|
55
|
-
})
|
|
56
|
-
|
|
57
68
|
function buildAutonomousSchedulerId(autonomousJobId: string): string {
|
|
58
69
|
return `autonomous:${autonomousJobId}`
|
|
59
70
|
}
|
|
60
71
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
}): Promise<{ bullmqJobId: string; queueJobId?: string }> {
|
|
66
|
-
return Effect.runPromise(
|
|
67
|
-
Effect.gen(function* () {
|
|
68
|
-
const queuedJob = yield* Effect.tryPromise({
|
|
69
|
-
try: () =>
|
|
70
|
-
autonomousJobQueue
|
|
71
|
-
.getQueue()
|
|
72
|
-
.add('run-autonomous-job', params.payload, {
|
|
73
|
-
...(typeof params.delayMs === 'number' ? { delay: Math.max(0, params.delayMs) } : {}),
|
|
74
|
-
...(params.jobId ? { jobId: params.jobId } : {}),
|
|
75
|
-
}),
|
|
76
|
-
catch: (cause) => new AutonomousJobQueueError({ message: 'Failed to enqueue autonomous job run.', cause }),
|
|
77
|
-
})
|
|
72
|
+
interface MakeAutonomousJobQueueRuntimeParams {
|
|
73
|
+
connectionProvider: () => IORedis
|
|
74
|
+
queueJobService: QueueJobService
|
|
75
|
+
}
|
|
78
76
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
77
|
+
export interface AutonomousJobQueueRuntime {
|
|
78
|
+
enqueueAutonomousJobRun(params: {
|
|
79
|
+
payload: AutonomousJobQueuePayload
|
|
80
|
+
delayMs?: number
|
|
81
|
+
jobId?: string
|
|
82
|
+
}): Promise<{ bullmqJobId: string; queueJobId?: string }>
|
|
83
|
+
upsertAutonomousJobScheduler(params: {
|
|
84
|
+
autonomousJobId: string
|
|
85
|
+
schedule: Extract<AutonomousJobSchedule, { kind: 'cron' | 'every' }>
|
|
86
|
+
}): Promise<void>
|
|
87
|
+
removeAutonomousJobScheduler(autonomousJobId: string): Promise<void>
|
|
88
|
+
removeAutonomousAtJob(autonomousJobId: string): Promise<void>
|
|
89
|
+
startWorker(options: { registerSignals?: boolean; deps: AutonomousJobWorkerDeps }): WorkerHandle
|
|
84
90
|
}
|
|
85
91
|
|
|
86
|
-
export function
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
name: 'run-autonomous-job',
|
|
100
|
-
data: { autonomousJobId: params.autonomousJobId, trigger: 'scheduled' },
|
|
101
|
-
opts: DEFAULT_AUTONOMOUS_JOB_OPTIONS,
|
|
102
|
-
}),
|
|
103
|
-
catch: (cause) =>
|
|
104
|
-
new AutonomousJobQueueError({
|
|
105
|
-
message: `Failed to upsert autonomous job scheduler for ${params.autonomousJobId}.`,
|
|
106
|
-
cause,
|
|
107
|
-
}),
|
|
108
|
-
})
|
|
92
|
+
export function makeAutonomousJobQueueRuntime(params: MakeAutonomousJobQueueRuntimeParams): AutonomousJobQueueRuntime {
|
|
93
|
+
const { connectionProvider, queueJobService } = params
|
|
94
|
+
const queue = createQueueFactoryWithDeps<AutonomousJobQueuePayload, AutonomousJobWorkerDeps>({
|
|
95
|
+
name: AUTONOMOUS_JOB_QUEUE,
|
|
96
|
+
displayName: 'Autonomous job',
|
|
97
|
+
jobName: 'run-autonomous-job',
|
|
98
|
+
concurrency: 2,
|
|
99
|
+
defaultJobOptions: DEFAULT_AUTONOMOUS_JOB_OPTIONS,
|
|
100
|
+
connectionProvider,
|
|
101
|
+
queueJobService,
|
|
102
|
+
prepare: ({ databaseService }) => Effect.runPromise(databaseService.connect()),
|
|
103
|
+
processor: processAutonomousJob,
|
|
104
|
+
})
|
|
109
105
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
106
|
+
const enqueueAutonomousJobRun: AutonomousJobQueueRuntime['enqueueAutonomousJobRun'] = (request) =>
|
|
107
|
+
Effect.runPromise(
|
|
108
|
+
Effect.gen(function* () {
|
|
109
|
+
const queuedJob = yield* Effect.tryPromise({
|
|
110
|
+
try: () =>
|
|
111
|
+
queue
|
|
112
|
+
.getQueue()
|
|
113
|
+
.add('run-autonomous-job', request.payload, {
|
|
114
|
+
...(typeof request.delayMs === 'number' ? { delay: Math.max(0, request.delayMs) } : {}),
|
|
115
|
+
...(request.jobId ? { jobId: request.jobId } : {}),
|
|
116
|
+
}),
|
|
117
|
+
catch: (cause) => new AutonomousJobQueueError({ message: 'Failed to enqueue autonomous job run.', cause }),
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
const bullmqJobId = String(queuedJob.id)
|
|
121
|
+
const queueJobId = yield* recordEnqueuedJobMetadata({
|
|
122
|
+
queueName: AUTONOMOUS_JOB_QUEUE,
|
|
123
|
+
job: queuedJob,
|
|
124
|
+
queueJobService,
|
|
125
|
+
})
|
|
126
|
+
return { bullmqJobId, queueJobId }
|
|
127
|
+
}),
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
const upsertAutonomousJobScheduler: AutonomousJobQueueRuntime['upsertAutonomousJobScheduler'] = (request) => {
|
|
131
|
+
const repeatOpts =
|
|
132
|
+
request.schedule.kind === 'cron' ? { pattern: request.schedule.cron } : { every: request.schedule.intervalMs }
|
|
133
|
+
return Effect.runPromise(
|
|
134
|
+
Effect.gen(function* () {
|
|
135
|
+
const queuedJob = yield* Effect.tryPromise({
|
|
136
|
+
try: () =>
|
|
137
|
+
queue
|
|
138
|
+
.getQueue()
|
|
139
|
+
.upsertJobScheduler(buildAutonomousSchedulerId(request.autonomousJobId), repeatOpts, {
|
|
140
|
+
name: 'run-autonomous-job',
|
|
141
|
+
data: { autonomousJobId: request.autonomousJobId, trigger: 'scheduled' },
|
|
142
|
+
opts: DEFAULT_AUTONOMOUS_JOB_OPTIONS,
|
|
143
|
+
}),
|
|
144
|
+
catch: (cause) =>
|
|
145
|
+
new AutonomousJobQueueError({
|
|
146
|
+
message: `Failed to upsert autonomous job scheduler for ${request.autonomousJobId}.`,
|
|
147
|
+
cause,
|
|
148
|
+
}),
|
|
149
|
+
})
|
|
114
150
|
|
|
115
|
-
|
|
116
|
-
return Effect.runPromise(
|
|
117
|
-
Effect.asVoid(
|
|
118
|
-
Effect.tryPromise({
|
|
119
|
-
try: () => autonomousJobQueue.getQueue().removeJobScheduler(buildAutonomousSchedulerId(autonomousJobId)),
|
|
120
|
-
catch: (cause) =>
|
|
121
|
-
new AutonomousJobQueueError({
|
|
122
|
-
message: `Failed to remove autonomous job scheduler for ${autonomousJobId}.`,
|
|
123
|
-
cause,
|
|
124
|
-
}),
|
|
151
|
+
yield* recordEnqueuedJobMetadata({ queueName: AUTONOMOUS_JOB_QUEUE, job: queuedJob, queueJobService })
|
|
125
152
|
}),
|
|
126
|
-
)
|
|
127
|
-
|
|
128
|
-
}
|
|
153
|
+
)
|
|
154
|
+
}
|
|
129
155
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
Effect.catch(
|
|
156
|
+
const removeAutonomousJobScheduler: AutonomousJobQueueRuntime['removeAutonomousJobScheduler'] = (autonomousJobId) =>
|
|
157
|
+
Effect.runPromise(
|
|
133
158
|
Effect.asVoid(
|
|
134
159
|
Effect.tryPromise({
|
|
135
|
-
try: () =>
|
|
160
|
+
try: () => queue.getQueue().removeJobScheduler(buildAutonomousSchedulerId(autonomousJobId)),
|
|
136
161
|
catch: (cause) =>
|
|
137
162
|
new AutonomousJobQueueError({
|
|
138
|
-
message: `Failed to remove autonomous
|
|
163
|
+
message: `Failed to remove autonomous job scheduler for ${autonomousJobId}.`,
|
|
139
164
|
cause,
|
|
140
165
|
}),
|
|
141
166
|
}),
|
|
142
167
|
),
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
const removeAutonomousAtJob: AutonomousJobQueueRuntime['removeAutonomousAtJob'] = (autonomousJobId) =>
|
|
171
|
+
Effect.runPromise(
|
|
172
|
+
Effect.catch(
|
|
173
|
+
Effect.asVoid(
|
|
174
|
+
Effect.tryPromise({
|
|
175
|
+
try: () => queue.getQueue().remove(buildAutonomousAtJobId(autonomousJobId)),
|
|
176
|
+
catch: (cause) =>
|
|
177
|
+
new AutonomousJobQueueError({
|
|
178
|
+
message: `Failed to remove autonomous-at job for ${autonomousJobId}.`,
|
|
179
|
+
cause,
|
|
180
|
+
}),
|
|
181
|
+
}),
|
|
182
|
+
),
|
|
183
|
+
() => Effect.void,
|
|
184
|
+
),
|
|
185
|
+
)
|
|
160
186
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
serverLogger.error`Autonomous job startup recovery failed: ${error}`
|
|
165
|
-
}),
|
|
166
|
-
),
|
|
167
|
-
)
|
|
187
|
+
const startWorker: AutonomousJobQueueRuntime['startWorker'] = (options) => {
|
|
188
|
+
const { deps } = options
|
|
189
|
+
const handle = queue.startWorker({ deps, registerSignals: options.registerSignals, connectionProvider })
|
|
168
190
|
|
|
169
|
-
|
|
191
|
+
void Effect.runPromise(
|
|
192
|
+
Effect.catch(deps.autonomousJobService.recoverActiveJobs(), (error) =>
|
|
193
|
+
Effect.sync(() => {
|
|
194
|
+
serverLogger.error`Autonomous job startup recovery failed: ${error}`
|
|
195
|
+
}),
|
|
196
|
+
) as Effect.Effect<unknown, never, never>,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
return handle
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
enqueueAutonomousJobRun,
|
|
204
|
+
upsertAutonomousJobScheduler,
|
|
205
|
+
removeAutonomousJobScheduler,
|
|
206
|
+
removeAutonomousAtJob,
|
|
207
|
+
startWorker,
|
|
208
|
+
}
|
|
170
209
|
}
|
|
171
210
|
|
|
172
211
|
runStandaloneQueueWorker((runtime) => {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
212
|
+
void (async () => {
|
|
213
|
+
const { AutonomousJobServiceTag } = await import('../services/autonomous-job.service')
|
|
214
|
+
const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
|
|
215
|
+
const redis = resolve(RedisServiceTag)
|
|
216
|
+
const autonomousJobQueueRuntime = makeAutonomousJobQueueRuntime({
|
|
217
|
+
connectionProvider: () => redis.getConnectionForBullMQ(),
|
|
218
|
+
queueJobService: resolve(QueueJobServiceTag),
|
|
219
|
+
})
|
|
220
|
+
autonomousJobQueueRuntime.startWorker({
|
|
221
|
+
deps: { databaseService: resolve(DatabaseServiceTag), autonomousJobService: resolve(AutonomousJobServiceTag) },
|
|
222
|
+
})
|
|
223
|
+
})()
|
|
178
224
|
})
|
|
@@ -7,7 +7,9 @@ import { ensureRecordId } from '../db/record-id'
|
|
|
7
7
|
import { TABLES } from '../db/tables'
|
|
8
8
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
9
9
|
import { ContextCompactionServiceTag } from '../services/context-compaction.service'
|
|
10
|
+
import { QueueJobServiceTag } from '../services/queue-job.service'
|
|
10
11
|
import { ThreadServiceTag } from '../services/thread/thread.service'
|
|
12
|
+
import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
|
|
11
13
|
import { createQueueFactoryWithDeps } from './queue-factory'
|
|
12
14
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
13
15
|
|
|
@@ -17,13 +19,13 @@ interface ContextCompactionJob {
|
|
|
17
19
|
contextSize?: number
|
|
18
20
|
}
|
|
19
21
|
|
|
20
|
-
interface
|
|
22
|
+
export interface ContextCompactionWorkerDeps {
|
|
21
23
|
databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
|
|
22
24
|
threadService: Context.Service.Shape<typeof ThreadServiceTag>
|
|
23
25
|
contextCompactionService: Context.Service.Shape<typeof ContextCompactionServiceTag>
|
|
24
26
|
}
|
|
25
27
|
|
|
26
|
-
function processContextCompactionJob(deps:
|
|
28
|
+
function processContextCompactionJob(deps: ContextCompactionWorkerDeps, job: Job<ContextCompactionJob>): Promise<void> {
|
|
27
29
|
const { threadService, contextCompactionService } = deps
|
|
28
30
|
const { entityId, contextSize } = job.data
|
|
29
31
|
const threadRef = ensureRecordId(entityId, TABLES.THREAD)
|
|
@@ -40,37 +42,49 @@ function processContextCompactionJob(deps: ContextCompactionQueueDeps, job: Job<
|
|
|
40
42
|
)
|
|
41
43
|
}
|
|
42
44
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
jobName: 'compact',
|
|
47
|
-
concurrency: 2,
|
|
48
|
-
lockDuration: 300_000,
|
|
49
|
-
defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 3_000 } },
|
|
50
|
-
prepare: ({ databaseService }) => databaseService.connect(),
|
|
51
|
-
processor: processContextCompactionJob,
|
|
52
|
-
})
|
|
53
|
-
|
|
54
|
-
export function enqueueContextCompaction(job: ContextCompactionJob) {
|
|
55
|
-
return contextCompaction.enqueue(job, { deduplication: { id: `compact:${job.domain}:${job.entityId}` } })
|
|
45
|
+
export interface ContextCompactionQueueRuntime {
|
|
46
|
+
enqueueContextCompaction(job: ContextCompactionJob): Promise<void>
|
|
47
|
+
startWorker(options: { registerSignals?: boolean; deps: ContextCompactionWorkerDeps }): WorkerHandle
|
|
56
48
|
}
|
|
57
49
|
|
|
58
|
-
|
|
59
|
-
registerSignals?: boolean
|
|
50
|
+
interface MakeContextCompactionQueueRuntimeParams {
|
|
60
51
|
connectionProvider: () => IORedis
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
52
|
+
queueJobService: QueueJobService
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function makeContextCompactionQueueRuntime(
|
|
56
|
+
params: MakeContextCompactionQueueRuntimeParams,
|
|
57
|
+
): ContextCompactionQueueRuntime {
|
|
58
|
+
const { connectionProvider, queueJobService } = params
|
|
59
|
+
const queue = createQueueFactoryWithDeps<ContextCompactionJob, ContextCompactionWorkerDeps>({
|
|
60
|
+
name: 'context-compaction',
|
|
61
|
+
displayName: 'Context compaction',
|
|
62
|
+
jobName: 'compact',
|
|
63
|
+
concurrency: 2,
|
|
64
|
+
lockDuration: 300_000,
|
|
65
|
+
defaultJobOptions: { attempts: 2, backoff: { type: 'exponential', delay: 3_000 } },
|
|
66
|
+
connectionProvider,
|
|
67
|
+
queueJobService,
|
|
68
|
+
prepare: ({ databaseService }) => Effect.runPromise(databaseService.connect()),
|
|
69
|
+
processor: processContextCompactionJob,
|
|
67
70
|
})
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
enqueueContextCompaction: (job) =>
|
|
74
|
+
queue.enqueue(job, { deduplication: { id: `compact:${job.domain}:${job.entityId}` } }),
|
|
75
|
+
startWorker: (options) =>
|
|
76
|
+
queue.startWorker({ deps: options.deps, registerSignals: options.registerSignals, connectionProvider }),
|
|
77
|
+
}
|
|
68
78
|
}
|
|
69
79
|
|
|
70
80
|
runStandaloneQueueWorker((runtime) => {
|
|
71
81
|
const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
|
|
72
|
-
|
|
73
|
-
|
|
82
|
+
const redis = resolve(RedisServiceTag)
|
|
83
|
+
const contextCompactionQueue = makeContextCompactionQueueRuntime({
|
|
84
|
+
connectionProvider: () => redis.getConnectionForBullMQ(),
|
|
85
|
+
queueJobService: resolve(QueueJobServiceTag),
|
|
86
|
+
})
|
|
87
|
+
contextCompactionQueue.startWorker({
|
|
74
88
|
deps: {
|
|
75
89
|
databaseService: resolve(DatabaseServiceTag),
|
|
76
90
|
threadService: resolve(ThreadServiceTag),
|
|
@@ -4,10 +4,22 @@ import type { Context } from 'effect'
|
|
|
4
4
|
import type IORedis from 'ioredis'
|
|
5
5
|
|
|
6
6
|
import { DatabaseServiceTag, RedisServiceTag } from '../effect/services'
|
|
7
|
-
import {
|
|
7
|
+
import { QueueJobServiceTag } from '../services/queue-job.service'
|
|
8
|
+
import type { QueueJobService, WorkerHandle } from '../workers/worker-utils'
|
|
8
9
|
import { createQueueFactoryWithDeps } from './queue-factory'
|
|
9
10
|
import { runStandaloneQueueWorker } from './standalone-worker'
|
|
10
11
|
|
|
12
|
+
// Minimal service shape used by the worker processor. Declared structurally to
|
|
13
|
+
// avoid importing the service tag — which would form a dependency cycle since
|
|
14
|
+
// PlanExecutorServiceLive depends on LotaQueuesServiceTag.
|
|
15
|
+
interface DelayedNodePromotionPlanExecutorShape {
|
|
16
|
+
promoteDelayedNode(params: {
|
|
17
|
+
runId: string
|
|
18
|
+
nodeId: string
|
|
19
|
+
emittedBy: string
|
|
20
|
+
}): Effect.Effect<unknown, unknown, unknown>
|
|
21
|
+
}
|
|
22
|
+
|
|
11
23
|
export interface DelayedNodePromotionJob {
|
|
12
24
|
runId: string
|
|
13
25
|
nodeId: string
|
|
@@ -16,13 +28,13 @@ export interface DelayedNodePromotionJob {
|
|
|
16
28
|
|
|
17
29
|
export const DELAYED_NODE_PROMOTION_QUEUE = 'delayed-node-promotion'
|
|
18
30
|
|
|
19
|
-
interface
|
|
31
|
+
export interface DelayedNodePromotionWorkerDeps {
|
|
20
32
|
databaseService: Context.Service.Shape<typeof DatabaseServiceTag>
|
|
21
|
-
planExecutorService:
|
|
33
|
+
planExecutorService: DelayedNodePromotionPlanExecutorShape
|
|
22
34
|
}
|
|
23
35
|
|
|
24
36
|
function processDelayedNodePromotionJob(
|
|
25
|
-
deps:
|
|
37
|
+
deps: DelayedNodePromotionWorkerDeps,
|
|
26
38
|
job: Job<DelayedNodePromotionJob>,
|
|
27
39
|
): Promise<void> {
|
|
28
40
|
const { planExecutorService } = deps
|
|
@@ -35,36 +47,51 @@ function processDelayedNodePromotionJob(
|
|
|
35
47
|
).then(() => undefined)
|
|
36
48
|
}
|
|
37
49
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
jobName: 'promote-node',
|
|
42
|
-
concurrency: 1,
|
|
43
|
-
defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
|
|
44
|
-
prepare: ({ databaseService }) => databaseService.connect(),
|
|
45
|
-
processor: processDelayedNodePromotionJob,
|
|
46
|
-
})
|
|
47
|
-
|
|
48
|
-
export function enqueueDelayedNodePromotion(job: DelayedNodePromotionJob, delayMs: number) {
|
|
49
|
-
return delayedNodePromotion.enqueue(job, { delay: delayMs, jobId: `promote:${job.runId}:${job.nodeId}` })
|
|
50
|
+
export interface DelayedNodePromotionQueueRuntime {
|
|
51
|
+
enqueueDelayedNodePromotion(job: DelayedNodePromotionJob, delayMs: number): Promise<void>
|
|
52
|
+
startWorker(options: { registerSignals?: boolean; deps: DelayedNodePromotionWorkerDeps }): WorkerHandle
|
|
50
53
|
}
|
|
51
54
|
|
|
52
|
-
|
|
53
|
-
registerSignals?: boolean
|
|
55
|
+
interface MakeDelayedNodePromotionQueueRuntimeParams {
|
|
54
56
|
connectionProvider: () => IORedis
|
|
55
|
-
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
57
|
+
queueJobService: QueueJobService
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function makeDelayedNodePromotionQueueRuntime(
|
|
61
|
+
params: MakeDelayedNodePromotionQueueRuntimeParams,
|
|
62
|
+
): DelayedNodePromotionQueueRuntime {
|
|
63
|
+
const { connectionProvider, queueJobService } = params
|
|
64
|
+
const queue = createQueueFactoryWithDeps<DelayedNodePromotionJob, DelayedNodePromotionWorkerDeps>({
|
|
65
|
+
name: DELAYED_NODE_PROMOTION_QUEUE,
|
|
66
|
+
displayName: 'Delayed node promotion',
|
|
67
|
+
jobName: 'promote-node',
|
|
68
|
+
concurrency: 1,
|
|
69
|
+
defaultJobOptions: { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },
|
|
70
|
+
connectionProvider,
|
|
71
|
+
queueJobService,
|
|
72
|
+
prepare: ({ databaseService }) => Effect.runPromise(databaseService.connect()),
|
|
73
|
+
processor: processDelayedNodePromotionJob,
|
|
61
74
|
})
|
|
75
|
+
|
|
76
|
+
return {
|
|
77
|
+
enqueueDelayedNodePromotion: (job, delayMs) =>
|
|
78
|
+
queue.enqueue(job, { delay: delayMs, jobId: `promote:${job.runId}:${job.nodeId}` }),
|
|
79
|
+
startWorker: (options) =>
|
|
80
|
+
queue.startWorker({ deps: options.deps, registerSignals: options.registerSignals, connectionProvider }),
|
|
81
|
+
}
|
|
62
82
|
}
|
|
63
83
|
|
|
64
84
|
runStandaloneQueueWorker((runtime) => {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
85
|
+
void (async () => {
|
|
86
|
+
const { PlanExecutorServiceTag } = await import('../services/plan/plan-executor.service')
|
|
87
|
+
const resolve = <I, T>(tag: Context.Key<I, T>): T => runtime.runSync(Effect.service(tag))
|
|
88
|
+
const redis = resolve(RedisServiceTag)
|
|
89
|
+
const delayedNodePromotionQueue = makeDelayedNodePromotionQueueRuntime({
|
|
90
|
+
connectionProvider: () => redis.getConnectionForBullMQ(),
|
|
91
|
+
queueJobService: resolve(QueueJobServiceTag),
|
|
92
|
+
})
|
|
93
|
+
delayedNodePromotionQueue.startWorker({
|
|
94
|
+
deps: { databaseService: resolve(DatabaseServiceTag), planExecutorService: resolve(PlanExecutorServiceTag) },
|
|
95
|
+
})
|
|
96
|
+
})()
|
|
70
97
|
})
|