@lota-sdk/core 0.4.13 → 0.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/ai/embedding-cache.ts +17 -11
- package/src/ai-gateway/ai-gateway.ts +164 -94
- package/src/ai-gateway/index.ts +4 -1
- package/src/config/agent-defaults.ts +2 -2
- package/src/config/agent-types.ts +1 -1
- package/src/config/constants.ts +1 -1
- package/src/create-runtime.ts +259 -200
- package/src/db/cursor-pagination.ts +2 -9
- package/src/db/memory-store.ts +194 -175
- package/src/db/memory.ts +125 -71
- package/src/db/schema-fingerprint.ts +5 -4
- package/src/db/service-normalization.ts +4 -3
- package/src/db/service.ts +3 -2
- package/src/db/startup.ts +15 -16
- package/src/effect/errors.ts +161 -21
- package/src/effect/index.ts +0 -1
- package/src/embeddings/provider.ts +15 -7
- package/src/queues/autonomous-job.queue.ts +10 -22
- package/src/queues/delayed-node-promotion.queue.ts +8 -14
- package/src/queues/document-processor.queue.ts +13 -4
- package/src/queues/memory-consolidation.queue.ts +26 -14
- package/src/queues/plan-agent-heartbeat.queue.ts +48 -31
- package/src/queues/plan-scheduler.queue.ts +37 -15
- package/src/queues/queue-factory.ts +59 -35
- package/src/queues/standalone-worker.ts +3 -2
- package/src/redis/connection.ts +10 -3
- package/src/redis/org-memory-lock.ts +1 -1
- package/src/redis/redis-lease-lock.ts +5 -5
- package/src/redis/stream-context.ts +1 -1
- package/src/runtime/chat-message.ts +64 -1
- package/src/runtime/chat-run-orchestration.ts +33 -20
- package/src/runtime/context-compaction/context-compaction-runtime.ts +14 -7
- package/src/runtime/context-compaction/context-compaction.ts +78 -66
- package/src/runtime/domain-layer.ts +19 -13
- package/src/runtime/execution-plan.ts +7 -3
- package/src/runtime/memory/memory-block.ts +3 -9
- package/src/runtime/memory/memory-scope.ts +3 -1
- package/src/runtime/plugin-resolution.ts +2 -1
- package/src/runtime/post-turn-side-effects.ts +6 -5
- package/src/runtime/retrieval-adapters.ts +8 -20
- package/src/runtime/runtime-config.ts +3 -9
- package/src/runtime/runtime-extensions.ts +2 -4
- package/src/runtime/runtime-lifecycle.ts +56 -16
- package/src/runtime/runtime-services.ts +180 -102
- package/src/runtime/runtime-worker-registry.ts +3 -1
- package/src/runtime/social-chat/social-chat-agent-runner.ts +1 -1
- package/src/runtime/social-chat/social-chat-history.ts +21 -18
- package/src/runtime/social-chat/social-chat.ts +356 -223
- package/src/runtime/specialist-runner.ts +3 -1
- package/src/runtime/team-consultation/team-consultation-orchestrator.ts +3 -2
- package/src/runtime/thread-turn-context.ts +142 -102
- package/src/runtime/turn-lifecycle.ts +15 -46
- package/src/services/agent-activity.service.ts +1 -1
- package/src/services/agent-executor.service.ts +107 -77
- package/src/services/autonomous-job.service.ts +354 -293
- package/src/services/background-work.service.ts +3 -3
- package/src/services/context-compaction.service.ts +7 -2
- package/src/services/document-chunk.service.ts +50 -32
- package/src/services/execution-plan/execution-plan-schedule.ts +5 -3
- package/src/services/execution-plan/execution-plan.service.ts +162 -179
- package/src/services/feedback-loop.service.ts +5 -4
- package/src/services/graph-full-routing.ts +37 -36
- package/src/services/institutional-memory.service.ts +28 -30
- package/src/services/learned-skill.service.ts +107 -72
- package/src/services/memory/memory-errors.ts +4 -23
- package/src/services/memory/memory-org-memory.ts +10 -5
- package/src/services/memory/memory-rerank.ts +18 -6
- package/src/services/memory/memory.service.ts +170 -111
- package/src/services/memory/rerank.service.ts +29 -20
- package/src/services/organization-member.service.ts +1 -1
- package/src/services/organization.service.ts +69 -75
- package/src/services/ownership-dispatcher.service.ts +40 -39
- package/src/services/plan/plan-agent-heartbeat.service.ts +22 -24
- package/src/services/plan/plan-agent-query.service.ts +39 -31
- package/src/services/plan/plan-completion-side-effects.ts +13 -17
- package/src/services/plan/plan-coordination.service.ts +2 -1
- package/src/services/plan/plan-cycle.service.ts +6 -5
- package/src/services/plan/plan-deadline.service.ts +57 -54
- package/src/services/plan/plan-event-delivery.service.ts +5 -4
- package/src/services/plan/plan-executor-graph.ts +18 -15
- package/src/services/plan/plan-executor.service.ts +235 -262
- package/src/services/plan/plan-run.service.ts +169 -93
- package/src/services/plan/plan-scheduler.service.ts +192 -202
- package/src/services/plan/plan-template.service.ts +1 -1
- package/src/services/plan/plan-transaction-events.ts +1 -1
- package/src/services/plan/plan-workspace.service.ts +23 -14
- package/src/services/plugin-executor.service.ts +5 -9
- package/src/services/queue-job.service.ts +117 -59
- package/src/services/recent-activity-title.service.ts +13 -12
- package/src/services/recent-activity.service.ts +6 -1
- package/src/services/social-chat-history.service.ts +29 -25
- package/src/services/system-executor.service.ts +5 -9
- package/src/services/thread/thread-active-run.ts +2 -2
- package/src/services/thread/thread-listing.ts +61 -57
- package/src/services/thread/thread-memory-block.ts +73 -48
- package/src/services/thread/thread-message.service.ts +76 -65
- package/src/services/thread/thread-record-store.ts +8 -8
- package/src/services/thread/thread-title.service.ts +10 -4
- package/src/services/thread/thread-turn-execution.ts +43 -45
- package/src/services/thread/thread-turn-preparation.service.ts +257 -135
- package/src/services/thread/thread-turn-streaming.ts +83 -92
- package/src/services/thread/thread-turn.ts +18 -16
- package/src/services/thread/thread.service.ts +135 -100
- package/src/services/user.service.ts +45 -48
- package/src/storage/attachment-parser.ts +6 -2
- package/src/storage/attachment-storage.service.ts +5 -6
- package/src/storage/generated-document-storage.service.ts +1 -1
- package/src/system-agents/context-compaction.agent.ts +10 -9
- package/src/system-agents/delegated-agent-factory.ts +30 -6
- package/src/system-agents/memory-reranker.agent.ts +10 -9
- package/src/system-agents/memory.agent.ts +10 -9
- package/src/system-agents/recent-activity-title-refiner.agent.ts +13 -15
- package/src/system-agents/regular-chat-memory-digest.agent.ts +13 -12
- package/src/system-agents/skill-extractor.agent.ts +13 -12
- package/src/system-agents/skill-manager.agent.ts +13 -12
- package/src/system-agents/thread-router.agent.ts +11 -7
- package/src/system-agents/title-generator.agent.ts +13 -12
- package/src/tools/fetch-webpage.tool.ts +13 -13
- package/src/tools/memory-block.tool.ts +3 -1
- package/src/tools/plan-approval.tool.ts +4 -2
- package/src/tools/read-file-parts.tool.ts +10 -4
- package/src/tools/remember-memory.tool.ts +3 -1
- package/src/tools/research-topic.tool.ts +9 -5
- package/src/tools/search-web.tool.ts +16 -16
- package/src/tools/search.tool.ts +20 -5
- package/src/tools/team-think.tool.ts +61 -38
- package/src/utils/async.ts +5 -5
- package/src/utils/errors.ts +19 -18
- package/src/utils/sse-keepalive.ts +28 -25
- package/src/workers/bootstrap.ts +75 -11
- package/src/workers/memory-consolidation.worker.ts +82 -91
- package/src/workers/organization-learning.worker.ts +14 -4
- package/src/workers/regular-chat-memory-digest.runner.ts +105 -67
- package/src/workers/skill-extraction.runner.ts +97 -61
- package/src/workers/utils/repo-structure-extractor.ts +13 -8
- package/src/workers/utils/thread-message-query.ts +24 -24
- package/src/workers/worker-utils.ts +23 -4
- package/src/effect/helpers.ts +0 -123
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import { Cause, Effect } from 'effect'
|
|
2
|
+
import type { Context } from 'effect'
|
|
3
3
|
|
|
4
|
+
import type { AiGatewayModels } from '../ai-gateway/ai-gateway'
|
|
4
5
|
import type { ResolvedAgentConfig } from '../config/agent-defaults'
|
|
5
6
|
import { isAgentName } from '../config/agent-defaults'
|
|
6
7
|
import { serverLogger } from '../config/logger'
|
|
@@ -8,16 +9,15 @@ import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
|
8
9
|
import type { RecordIdRef } from '../db/record-id'
|
|
9
10
|
import type { SurrealDBService } from '../db/service'
|
|
10
11
|
import { TABLES } from '../db/tables'
|
|
11
|
-
import { effectTryPromise } from '../effect/helpers'
|
|
12
12
|
import { ProviderEmbeddings } from '../embeddings/provider'
|
|
13
13
|
import type { SkillExtractionJob } from '../queues/organization-learning.queue'
|
|
14
14
|
import { createHelperModelRuntime } from '../runtime/helper-model'
|
|
15
15
|
import type { LotaRuntimeAdapters } from '../runtime/runtime-extensions'
|
|
16
16
|
import type { LearnedSkillServiceTag } from '../services/learned-skill.service'
|
|
17
17
|
import type { SocialChatHistoryServiceTag } from '../services/social-chat-history.service'
|
|
18
|
-
import {
|
|
18
|
+
import { makeSkillExtractorAgentFactory, SkillExtractionOutputSchema } from '../system-agents/skill-extractor.agent'
|
|
19
19
|
import type { SkillCandidate } from '../system-agents/skill-extractor.agent'
|
|
20
|
-
import {
|
|
20
|
+
import { makeSkillManagerAgentFactory, SkillManagerOutputSchema } from '../system-agents/skill-manager.agent'
|
|
21
21
|
import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
|
|
22
22
|
import {
|
|
23
23
|
compareDigestMessageOrder,
|
|
@@ -30,12 +30,14 @@ const MIN_MESSAGE_THRESHOLD = 10
|
|
|
30
30
|
|
|
31
31
|
export interface SkillExtractionServices {
|
|
32
32
|
agentConfig: ResolvedAgentConfig
|
|
33
|
+
aiGatewayModels: AiGatewayModels
|
|
33
34
|
databaseService: SurrealDBService
|
|
34
35
|
learnedSkillService: Context.Service.Shape<typeof LearnedSkillServiceTag>
|
|
35
36
|
socialChatHistoryService: Context.Service.Shape<typeof SocialChatHistoryServiceTag>
|
|
36
37
|
runtimeAdapters: LotaRuntimeAdapters
|
|
37
38
|
embeddingModel: string
|
|
38
39
|
openRouterApiKey?: string
|
|
40
|
+
runPromise: <A, E>(effect: Effect.Effect<A, E>) => Promise<A>
|
|
39
41
|
}
|
|
40
42
|
|
|
41
43
|
interface SkillExtractionRunResult {
|
|
@@ -110,21 +112,31 @@ function runSkillExtractionEffect(
|
|
|
110
112
|
return Effect.gen(function* () {
|
|
111
113
|
const getBackgroundCursor = cursorAwareWorkspaceProvider.getBackgroundCursor.bind(cursorAwareWorkspaceProvider)
|
|
112
114
|
const setBackgroundCursor = cursorAwareWorkspaceProvider.setBackgroundCursor.bind(cursorAwareWorkspaceProvider)
|
|
113
|
-
const workspace = yield*
|
|
115
|
+
const workspace = yield* Effect.tryPromise({
|
|
116
|
+
try: () => cursorAwareWorkspaceProvider.getWorkspace(orgRef),
|
|
117
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
118
|
+
})
|
|
114
119
|
const lifecycleState = cursorAwareWorkspaceProvider.getLifecycleState
|
|
115
|
-
? yield*
|
|
120
|
+
? yield* Effect.tryPromise({
|
|
121
|
+
try: () => Promise.resolve(cursorAwareWorkspaceProvider.getLifecycleState?.(workspace)),
|
|
122
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
123
|
+
})
|
|
116
124
|
: undefined
|
|
117
125
|
if (lifecycleState?.bootstrapActive ?? false) {
|
|
118
126
|
serverLogger.info`Skipping skill extraction for ${orgId}: onboarding is not completed`
|
|
119
127
|
return { skipped: true, processedMessages: 0, extractedSkills: 0 }
|
|
120
128
|
}
|
|
121
129
|
const projectionState = cursorAwareWorkspaceProvider.readProfileProjectionState
|
|
122
|
-
? yield*
|
|
123
|
-
Promise.resolve(cursorAwareWorkspaceProvider.readProfileProjectionState?.(workspace)),
|
|
124
|
-
|
|
130
|
+
? yield* Effect.tryPromise({
|
|
131
|
+
try: () => Promise.resolve(cursorAwareWorkspaceProvider.readProfileProjectionState?.(workspace)),
|
|
132
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
133
|
+
})
|
|
125
134
|
: undefined
|
|
126
135
|
|
|
127
|
-
const existingCursor = yield*
|
|
136
|
+
const existingCursor = yield* Effect.tryPromise({
|
|
137
|
+
try: () => getBackgroundCursor('skill-extraction', orgRef),
|
|
138
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
139
|
+
})
|
|
128
140
|
const onboardingCutoff = resolveWorkspaceBootstrapCutoff({
|
|
129
141
|
hasExistingCursor: existingCursor !== null,
|
|
130
142
|
bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
|
|
@@ -135,10 +147,20 @@ function runSkillExtractionEffect(
|
|
|
135
147
|
bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
|
|
136
148
|
})
|
|
137
149
|
|
|
138
|
-
const threadIds = yield*
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
)
|
|
150
|
+
const threadIds = yield* Effect.tryPromise({
|
|
151
|
+
try: () => listThreadIdsForOrg(services.databaseService, orgRef),
|
|
152
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
153
|
+
})
|
|
154
|
+
const threadMessages = yield* Effect.tryPromise({
|
|
155
|
+
try: () =>
|
|
156
|
+
listEligibleThreadMessages({
|
|
157
|
+
db: services.databaseService,
|
|
158
|
+
threadIds,
|
|
159
|
+
cursor: existingCursor,
|
|
160
|
+
onboardingCutoff,
|
|
161
|
+
}),
|
|
162
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
163
|
+
})
|
|
142
164
|
const socialMessages = yield* services.socialChatHistoryService.listWorkspaceMessages({
|
|
143
165
|
workspaceId: orgId,
|
|
144
166
|
cursor: existingSocialCursor,
|
|
@@ -163,24 +185,26 @@ function runSkillExtractionEffect(
|
|
|
163
185
|
? existingSkills.map((skill, i) => `${i + 1}. ${skill.name}: ${skill.description}`).join('\n')
|
|
164
186
|
: 'None'
|
|
165
187
|
|
|
166
|
-
const extraction = yield*
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
188
|
+
const extraction = yield* Effect.tryPromise({
|
|
189
|
+
try: () =>
|
|
190
|
+
helperModelRuntime.generateHelperStructured({
|
|
191
|
+
tag: 'skill-extraction',
|
|
192
|
+
createAgent: makeSkillExtractorAgentFactory(services.aiGatewayModels),
|
|
193
|
+
timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
|
|
194
|
+
messages: [
|
|
195
|
+
{
|
|
196
|
+
role: 'user',
|
|
197
|
+
content: buildExtractionPrompt({
|
|
198
|
+
workspaceName: projectionState?.workspaceName || 'Workspace',
|
|
199
|
+
transcript,
|
|
200
|
+
existingSkills: existingSkillsSummary,
|
|
201
|
+
}),
|
|
202
|
+
},
|
|
203
|
+
],
|
|
204
|
+
schema: SkillExtractionOutputSchema,
|
|
205
|
+
}),
|
|
206
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
207
|
+
})
|
|
184
208
|
|
|
185
209
|
const skillCandidates = extraction.candidates.filter((c) => c.classification === 'skill')
|
|
186
210
|
|
|
@@ -195,37 +219,42 @@ function runSkillExtractionEffect(
|
|
|
195
219
|
|
|
196
220
|
const mostSimilar = yield* services.learnedSkillService.findMostSimilar(orgId, candidate.description)
|
|
197
221
|
|
|
198
|
-
const managerResult = yield*
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
+
const managerResult = yield* Effect.tryPromise({
|
|
223
|
+
try: () =>
|
|
224
|
+
helperModelRuntime.generateHelperStructured({
|
|
225
|
+
tag: 'skill-manager',
|
|
226
|
+
createAgent: makeSkillManagerAgentFactory(services.aiGatewayModels),
|
|
227
|
+
timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
|
|
228
|
+
messages: [
|
|
229
|
+
{
|
|
230
|
+
role: 'user',
|
|
231
|
+
content: buildManagerPrompt({
|
|
232
|
+
candidate,
|
|
233
|
+
existingSkill: mostSimilar
|
|
234
|
+
? {
|
|
235
|
+
name: mostSimilar.name,
|
|
236
|
+
description: mostSimilar.description,
|
|
237
|
+
instructions: mostSimilar.instructions,
|
|
238
|
+
version: mostSimilar.version,
|
|
239
|
+
}
|
|
240
|
+
: null,
|
|
241
|
+
}),
|
|
242
|
+
},
|
|
243
|
+
],
|
|
244
|
+
schema: SkillManagerOutputSchema,
|
|
245
|
+
}),
|
|
246
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
247
|
+
})
|
|
222
248
|
|
|
223
249
|
if (managerResult.decision === 'discard') {
|
|
224
250
|
serverLogger.info`Discarding skill candidate ${candidate.name}: ${managerResult.reason}`
|
|
225
251
|
return 0
|
|
226
252
|
}
|
|
227
253
|
|
|
228
|
-
const embedding = yield*
|
|
254
|
+
const embedding = yield* Effect.tryPromise({
|
|
255
|
+
try: () => embeddings.embedQuery(candidate.description),
|
|
256
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
257
|
+
})
|
|
229
258
|
if (embedding.length === 0) {
|
|
230
259
|
serverLogger.warn`Skipping skill candidate ${candidate.name}: empty embedding`
|
|
231
260
|
return 0
|
|
@@ -253,7 +282,10 @@ function runSkillExtractionEffect(
|
|
|
253
282
|
if (mostSimilar && managerResult.mergedSkill) {
|
|
254
283
|
const merged = managerResult.mergedSkill
|
|
255
284
|
const mergedHash = services.learnedSkillService.generateHash(merged.description, merged.instructions)
|
|
256
|
-
const mergedEmbedding = yield*
|
|
285
|
+
const mergedEmbedding = yield* Effect.tryPromise({
|
|
286
|
+
try: () => embeddings.embedQuery(merged.description),
|
|
287
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
288
|
+
})
|
|
257
289
|
|
|
258
290
|
yield* services.learnedSkillService.update(mostSimilar.id, {
|
|
259
291
|
name: merged.name,
|
|
@@ -291,7 +323,10 @@ function runSkillExtractionEffect(
|
|
|
291
323
|
const lastThreadMessage = threadMessages.at(-1)
|
|
292
324
|
const lastSocialMessage = socialMessages.at(-1)
|
|
293
325
|
if (lastThreadMessage) {
|
|
294
|
-
yield*
|
|
326
|
+
yield* Effect.tryPromise({
|
|
327
|
+
try: () => setBackgroundCursor('skill-extraction', orgRef, lastThreadMessage.cursor),
|
|
328
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
329
|
+
})
|
|
295
330
|
}
|
|
296
331
|
if (lastSocialMessage) {
|
|
297
332
|
yield* services.socialChatHistoryService.setBackgroundCursor('skill-extraction', orgId, lastSocialMessage.cursor)
|
|
@@ -319,10 +354,11 @@ export function runSkillExtraction(
|
|
|
319
354
|
const embeddings = new ProviderEmbeddings({
|
|
320
355
|
modelId: services.embeddingModel,
|
|
321
356
|
openRouterApiKey: services.openRouterApiKey,
|
|
357
|
+
runPromise: services.runPromise,
|
|
322
358
|
})
|
|
323
359
|
const withMemoryLock = runtimeAdapters.withWorkspaceMemoryLock
|
|
324
360
|
const runExtraction = () =>
|
|
325
|
-
|
|
361
|
+
services.runPromise(
|
|
326
362
|
runSkillExtractionEffect(services, orgRef, orgId, workspaceProvider, embeddings) as Effect.Effect<
|
|
327
363
|
SkillExtractionRunResult,
|
|
328
364
|
never,
|
|
@@ -6,10 +6,8 @@ import type {
|
|
|
6
6
|
RepositoryStructureSignal,
|
|
7
7
|
RepositoryStructureSummary,
|
|
8
8
|
} from '@lota-sdk/shared'
|
|
9
|
-
import
|
|
10
|
-
import { Effect } from 'effect'
|
|
9
|
+
import { Cause, Effect } from 'effect'
|
|
11
10
|
|
|
12
|
-
import { effectTryPromise } from '../../effect/helpers'
|
|
13
11
|
import { nowIsoDateTimeString } from '../../utils/date-time'
|
|
14
12
|
|
|
15
13
|
const EXTRACTOR_VERSION = 'repository-structure-extractor.v1'
|
|
@@ -152,14 +150,18 @@ function collectRelativeFilePathsEffect(
|
|
|
152
150
|
): Effect.Effect<string[], Cause.UnknownError, never> {
|
|
153
151
|
return Effect.gen(function* () {
|
|
154
152
|
const absoluteDir = currentDir ? joinPath(rootDir, currentDir) : rootDir
|
|
155
|
-
const entries = yield*
|
|
156
|
-
Array.fromAsync(new Bun.Glob('*').scan({ cwd: absoluteDir, onlyFiles: false, absolute: false })),
|
|
157
|
-
|
|
153
|
+
const entries = yield* Effect.tryPromise({
|
|
154
|
+
try: () => Array.fromAsync(new Bun.Glob('*').scan({ cwd: absoluteDir, onlyFiles: false, absolute: false })),
|
|
155
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
156
|
+
})
|
|
158
157
|
|
|
159
158
|
const filePaths: string[] = []
|
|
160
159
|
for (const entry of entries) {
|
|
161
160
|
const relativePath = currentDir ? joinPath(currentDir, entry) : normalizePath(entry)
|
|
162
|
-
const stats = yield*
|
|
161
|
+
const stats = yield* Effect.tryPromise({
|
|
162
|
+
try: () => Bun.file(joinPath(rootDir, relativePath)).stat(),
|
|
163
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
164
|
+
})
|
|
163
165
|
|
|
164
166
|
if (stats.isDirectory()) {
|
|
165
167
|
if (IGNORED_DIR_NAMES.has(entry)) continue
|
|
@@ -179,7 +181,10 @@ function readPackageJsonEffect(
|
|
|
179
181
|
rootDir: string,
|
|
180
182
|
relativePath: string,
|
|
181
183
|
): Effect.Effect<PackageJson | null, Cause.UnknownError, never> {
|
|
182
|
-
return
|
|
184
|
+
return Effect.tryPromise({
|
|
185
|
+
try: () => Bun.file(joinPath(rootDir, relativePath)).text(),
|
|
186
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
187
|
+
}).pipe(
|
|
183
188
|
Effect.map((raw) => {
|
|
184
189
|
try {
|
|
185
190
|
const parsed: unknown = JSON.parse(raw)
|
|
@@ -11,7 +11,7 @@ import { TABLES } from '../../db/tables'
|
|
|
11
11
|
import { ThreadMessageRowSchema } from '../../db/thread-message-row'
|
|
12
12
|
import type { ThreadMessageRow } from '../../db/thread-message-row'
|
|
13
13
|
import { normalizeTextBody } from '../../document/parsing'
|
|
14
|
-
import {
|
|
14
|
+
import { ERROR_TAGS } from '../../effect/errors'
|
|
15
15
|
import type { LotaRuntimeBackgroundCursor } from '../../runtime/runtime-extensions'
|
|
16
16
|
import type { SocialChatHistoryMessage } from '../../services/social-chat-history.service'
|
|
17
17
|
import { unsafeDateFrom } from '../../utils/date-time'
|
|
@@ -53,10 +53,10 @@ function readPersistedMessageParts(parts: ThreadMessageRow['parts']): ChatMessag
|
|
|
53
53
|
return (Array.isArray(parts) ? normalizeMessageValue(parts) : []) as ChatMessage['parts']
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
class ThreadMessageQueryError extends Schema.TaggedErrorClass<ThreadMessageQueryError>()(
|
|
57
|
-
|
|
58
|
-
cause: Schema.Defect,
|
|
59
|
-
|
|
56
|
+
class ThreadMessageQueryError extends Schema.TaggedErrorClass<ThreadMessageQueryError>()(
|
|
57
|
+
ERROR_TAGS.ThreadMessageQueryError,
|
|
58
|
+
{ message: Schema.String, cause: Schema.Defect },
|
|
59
|
+
) {}
|
|
60
60
|
|
|
61
61
|
function mapThreadRow(row: ThreadMessageRow): ThreadDigestMessage {
|
|
62
62
|
return {
|
|
@@ -72,10 +72,6 @@ function mapThreadRow(row: ThreadMessageRow): ThreadDigestMessage {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
const effectTryPromise = makeEffectTryPromiseWithMessage(
|
|
76
|
-
(message, cause) => new ThreadMessageQueryError({ message, cause }),
|
|
77
|
-
)
|
|
78
|
-
|
|
79
75
|
export function compareDigestMessageOrder(left: DigestMessage, right: DigestMessage): number {
|
|
80
76
|
const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
|
|
81
77
|
if (timeDiff !== 0) return timeDiff
|
|
@@ -85,19 +81,20 @@ export function compareDigestMessageOrder(left: DigestMessage, right: DigestMess
|
|
|
85
81
|
export function listThreadIdsForOrg(db: SurrealDBService, orgRef: RecordIdRef): Promise<RecordIdRef[]> {
|
|
86
82
|
const EntityIdRowSchema = z.string().trim().min(1)
|
|
87
83
|
return Effect.runPromise(
|
|
88
|
-
|
|
89
|
-
(
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
`SELECT VALUE type::string(id) FROM ${TABLES.THREAD}
|
|
84
|
+
db
|
|
85
|
+
.query<unknown>(
|
|
86
|
+
new BoundQuery(
|
|
87
|
+
`SELECT VALUE type::string(id) FROM ${TABLES.THREAD}
|
|
93
88
|
WHERE organizationId = $organizationId`,
|
|
94
|
-
|
|
95
|
-
|
|
89
|
+
{ organizationId: orgRef },
|
|
90
|
+
),
|
|
91
|
+
)
|
|
92
|
+
.pipe(
|
|
93
|
+
Effect.mapError(
|
|
94
|
+
(cause) => new ThreadMessageQueryError({ message: 'Failed to list thread ids for org digest.', cause }),
|
|
96
95
|
),
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
Effect.map((ids) => ids.map((value: unknown) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.THREAD))),
|
|
100
|
-
),
|
|
96
|
+
Effect.map((ids) => ids.map((value: unknown) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.THREAD))),
|
|
97
|
+
),
|
|
101
98
|
)
|
|
102
99
|
}
|
|
103
100
|
|
|
@@ -140,10 +137,13 @@ export function listEligibleThreadMessages(params: {
|
|
|
140
137
|
return []
|
|
141
138
|
}
|
|
142
139
|
|
|
143
|
-
const rows = yield*
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
140
|
+
const rows = yield* params.db
|
|
141
|
+
.query<unknown>(query)
|
|
142
|
+
.pipe(
|
|
143
|
+
Effect.mapError(
|
|
144
|
+
(cause) => new ThreadMessageQueryError({ message: 'Failed to list eligible thread messages.', cause }),
|
|
145
|
+
),
|
|
146
|
+
)
|
|
147
147
|
return rows.map((row: unknown) => mapThreadRow(ThreadMessageRowSchema.parse(row)))
|
|
148
148
|
}),
|
|
149
149
|
)
|
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
import { fileURLToPath } from 'node:url'
|
|
2
2
|
|
|
3
3
|
import type { Job, Worker } from 'bullmq'
|
|
4
|
-
import { Effect } from 'effect'
|
|
4
|
+
import { Effect, Schema } from 'effect'
|
|
5
5
|
import type { Context } from 'effect'
|
|
6
6
|
|
|
7
7
|
import { chatLogger } from '../config/logger'
|
|
8
|
+
import { ERROR_TAGS } from '../effect/errors'
|
|
8
9
|
import type { QueueJobServiceTag } from '../services/queue-job.service'
|
|
9
10
|
|
|
10
11
|
export type QueueJobService = Context.Service.Shape<typeof QueueJobServiceTag>
|
|
11
12
|
|
|
13
|
+
class QueueWorkerError extends Schema.TaggedErrorClass<QueueWorkerError>()(ERROR_TAGS.QueueWorkerError, {
|
|
14
|
+
phase: Schema.Literals(['close', 'process', 'shutdown-timeout']),
|
|
15
|
+
cause: Schema.Defect,
|
|
16
|
+
}) {}
|
|
17
|
+
|
|
12
18
|
export const DEFAULT_JOB_RETENTION = { removeOnComplete: true, removeOnFail: { age: 24 * 60 * 60, count: 200 } }
|
|
13
19
|
export const LOW_JOB_RETENTION = { removeOnComplete: true, removeOnFail: { age: 6 * 60 * 60, count: 50 } }
|
|
14
20
|
export const LONG_JOB_LOCK_DURATION_MS = 600_000
|
|
@@ -78,7 +84,14 @@ export const attachWorkerEvents = (worker: Worker, name: string, logger: typeof
|
|
|
78
84
|
export const createWorkerShutdown = (worker: Worker, name: string, logger: typeof chatLogger = chatLogger) => {
|
|
79
85
|
return () => {
|
|
80
86
|
logger.info`Shutting down ${name} worker`
|
|
81
|
-
return Effect.runPromise(
|
|
87
|
+
return Effect.runPromise(
|
|
88
|
+
Effect.asVoid(
|
|
89
|
+
Effect.tryPromise({
|
|
90
|
+
try: () => worker.close(),
|
|
91
|
+
catch: (cause) => new QueueWorkerError({ phase: 'close', cause }),
|
|
92
|
+
}),
|
|
93
|
+
),
|
|
94
|
+
)
|
|
82
95
|
}
|
|
83
96
|
}
|
|
84
97
|
|
|
@@ -111,7 +124,10 @@ export function createTracedWorkerProcessor<TJob extends TracedWorkerJobLike, TR
|
|
|
111
124
|
}),
|
|
112
125
|
)
|
|
113
126
|
|
|
114
|
-
const result = yield* Effect.tryPromise(
|
|
127
|
+
const result = yield* Effect.tryPromise({
|
|
128
|
+
try: () => processor(job),
|
|
129
|
+
catch: (cause) => new QueueWorkerError({ phase: 'process', cause }),
|
|
130
|
+
})
|
|
115
131
|
|
|
116
132
|
yield* Effect.catch(queueJobService.markAttemptCompleted(trackedJob, result), (error) =>
|
|
117
133
|
Effect.sync(() => {
|
|
@@ -163,7 +179,10 @@ export const registerShutdownSignals = ({
|
|
|
163
179
|
shuttingDown = true
|
|
164
180
|
void Effect.runFork(
|
|
165
181
|
Effect.gen(function* () {
|
|
166
|
-
yield* Effect.tryPromise(
|
|
182
|
+
yield* Effect.tryPromise({
|
|
183
|
+
try: () => Bun.sleep(timeoutMs),
|
|
184
|
+
catch: (cause) => new QueueWorkerError({ phase: 'shutdown-timeout', cause }),
|
|
185
|
+
})
|
|
167
186
|
if (!forcedExitArmed) return
|
|
168
187
|
logger.warn`Forced shutdown after ${timeoutMs}ms`
|
|
169
188
|
process.exit(0)
|
package/src/effect/helpers.ts
DELETED
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
import { Cause, Effect } from 'effect'
|
|
2
|
-
|
|
3
|
-
import { ServiceError } from './errors'
|
|
4
|
-
|
|
5
|
-
export function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
|
|
6
|
-
if (typeof value !== 'object' && typeof value !== 'function') {
|
|
7
|
-
return false
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
if (value === null) {
|
|
11
|
-
return false
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
return 'then' in value && typeof value.then === 'function'
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
type PromiseEffectEvaluator<A, E = never, R = never> =
|
|
18
|
-
| (() => PromiseLike<A> | Effect.Effect<A, E, R>)
|
|
19
|
-
| ((signal: AbortSignal) => PromiseLike<A> | Effect.Effect<A, E, R>)
|
|
20
|
-
|
|
21
|
-
function invokePromiseEffectEvaluator<A, E, R>(
|
|
22
|
-
evaluate: PromiseEffectEvaluator<A, E, R>,
|
|
23
|
-
): PromiseLike<A> | Effect.Effect<A, E, R> {
|
|
24
|
-
return (evaluate as () => PromiseLike<A> | Effect.Effect<A, E, R>)()
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export function effectTryPromise<A, E = never, R = never>(
|
|
28
|
-
evaluate: PromiseEffectEvaluator<A, E, R>,
|
|
29
|
-
): Effect.Effect<A, Cause.UnknownError, R>
|
|
30
|
-
export function effectTryPromise<A, E1, E2, R>(
|
|
31
|
-
evaluate: PromiseEffectEvaluator<A, E1, R>,
|
|
32
|
-
onError: (cause: unknown) => E2,
|
|
33
|
-
): Effect.Effect<A, E2, R>
|
|
34
|
-
export function effectTryPromise<A, E1, E2, R>(
|
|
35
|
-
evaluate: PromiseEffectEvaluator<A, E1, R>,
|
|
36
|
-
onError?: (cause: unknown) => E2,
|
|
37
|
-
): Effect.Effect<A, E2 | Cause.UnknownError, R> {
|
|
38
|
-
return Effect.suspend(() => {
|
|
39
|
-
try {
|
|
40
|
-
const value = invokePromiseEffectEvaluator(evaluate)
|
|
41
|
-
if (Effect.isEffect(value)) {
|
|
42
|
-
if (onError) {
|
|
43
|
-
return value.pipe(Effect.mapError((cause) => onError(cause)))
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
return value.pipe(Effect.mapError((cause) => new Cause.UnknownError(cause)))
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
return onError
|
|
50
|
-
? Effect.tryPromise({ try: () => value, catch: onError })
|
|
51
|
-
: Effect.tryPromise({ try: () => value, catch: (cause) => new Cause.UnknownError(cause) })
|
|
52
|
-
} catch (cause) {
|
|
53
|
-
return Effect.fail(onError ? onError(cause) : new Cause.UnknownError(cause))
|
|
54
|
-
}
|
|
55
|
-
})
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
export function effectTryServicePromise<A, E = never, R = never>(
|
|
59
|
-
evaluate: PromiseEffectEvaluator<A, E, R>,
|
|
60
|
-
message: string,
|
|
61
|
-
): Effect.Effect<A, ServiceError, R> {
|
|
62
|
-
return Effect.suspend(() => {
|
|
63
|
-
try {
|
|
64
|
-
const value = invokePromiseEffectEvaluator(evaluate)
|
|
65
|
-
if (Effect.isEffect(value)) {
|
|
66
|
-
return value.pipe(Effect.mapError((cause) => new ServiceError({ message, cause })))
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
return Effect.tryPromise({ try: () => value, catch: (cause) => new ServiceError({ message, cause }) })
|
|
70
|
-
} catch (cause) {
|
|
71
|
-
return Effect.fail(new ServiceError({ message, cause }))
|
|
72
|
-
}
|
|
73
|
-
})
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export function makeEffectTryPromiseWithMessage<E>(onError: (message: string, cause: unknown) => E) {
|
|
77
|
-
return <A, E1 = never, R = never>(
|
|
78
|
-
evaluate: PromiseEffectEvaluator<A, E1, R>,
|
|
79
|
-
message: string,
|
|
80
|
-
): Effect.Effect<A, E, R> => effectTryPromise(evaluate, (cause) => onError(message, cause))
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export function makeEffectTryPromiseWithOperation<E>(
|
|
84
|
-
onError: (operation: string, message: string, cause: unknown) => E,
|
|
85
|
-
) {
|
|
86
|
-
return <A, E1 = never, R = never>(
|
|
87
|
-
operation: string,
|
|
88
|
-
message: string,
|
|
89
|
-
evaluate: PromiseEffectEvaluator<A, E1, R>,
|
|
90
|
-
): Effect.Effect<A, E, R> => effectTryPromise(evaluate, (cause) => onError(operation, message, cause))
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
export function effectTryMaybeAsync<A>(evaluate: () => A | PromiseLike<A>): Effect.Effect<A, Cause.UnknownError, never>
|
|
94
|
-
export function effectTryMaybeAsync<A, E>(
|
|
95
|
-
evaluate: () => A | PromiseLike<A>,
|
|
96
|
-
onError: (cause: unknown) => E,
|
|
97
|
-
): Effect.Effect<A, E, never>
|
|
98
|
-
export function effectTryMaybeAsync<A, E>(
|
|
99
|
-
evaluate: () => A | PromiseLike<A>,
|
|
100
|
-
onError?: (cause: unknown) => E,
|
|
101
|
-
): Effect.Effect<A, E | Cause.UnknownError, never> {
|
|
102
|
-
return Effect.suspend(() => {
|
|
103
|
-
try {
|
|
104
|
-
const value = evaluate()
|
|
105
|
-
return isPromiseLike(value)
|
|
106
|
-
? onError
|
|
107
|
-
? effectTryPromise(() => value, onError)
|
|
108
|
-
: effectTryPromise(() => value)
|
|
109
|
-
: Effect.succeed(value)
|
|
110
|
-
} catch (cause) {
|
|
111
|
-
return Effect.fail(onError ? onError(cause) : new Cause.UnknownError(cause))
|
|
112
|
-
}
|
|
113
|
-
})
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
export function iterateEffect<A, E, R>(
|
|
117
|
-
initial: A,
|
|
118
|
-
options: { while: (state: A) => boolean; body: (state: A) => Effect.Effect<A, E, R> },
|
|
119
|
-
): Effect.Effect<A, E, R> {
|
|
120
|
-
const step = (state: A): Effect.Effect<A, E, R> =>
|
|
121
|
-
options.while(state) ? options.body(state).pipe(Effect.flatMap(step)) : Effect.succeed(state)
|
|
122
|
-
return Effect.suspend(() => step(initial))
|
|
123
|
-
}
|