@lota-sdk/core 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/ai/embedding-cache.ts +17 -11
- package/src/ai-gateway/ai-gateway.ts +164 -94
- package/src/ai-gateway/index.ts +4 -1
- package/src/config/agent-defaults.ts +2 -2
- package/src/config/agent-types.ts +1 -1
- package/src/create-runtime.ts +259 -200
- package/src/db/cursor-pagination.ts +2 -9
- package/src/db/memory-store.ts +194 -175
- package/src/db/memory.ts +125 -71
- package/src/db/schema-fingerprint.ts +5 -4
- package/src/db/service-normalization.ts +4 -3
- package/src/db/service.ts +3 -2
- package/src/db/startup.ts +15 -16
- package/src/effect/errors.ts +161 -21
- package/src/effect/index.ts +0 -1
- package/src/embeddings/provider.ts +15 -7
- package/src/queues/autonomous-job.queue.ts +10 -22
- package/src/queues/delayed-node-promotion.queue.ts +8 -14
- package/src/queues/document-processor.queue.ts +13 -4
- package/src/queues/memory-consolidation.queue.ts +26 -14
- package/src/queues/plan-agent-heartbeat.queue.ts +10 -9
- package/src/queues/plan-scheduler.queue.ts +37 -15
- package/src/queues/queue-factory.ts +59 -35
- package/src/queues/standalone-worker.ts +3 -2
- package/src/redis/connection.ts +10 -3
- package/src/redis/org-memory-lock.ts +1 -1
- package/src/redis/redis-lease-lock.ts +5 -5
- package/src/redis/stream-context.ts +1 -1
- package/src/runtime/chat-message.ts +64 -1
- package/src/runtime/chat-run-orchestration.ts +33 -20
- package/src/runtime/context-compaction/context-compaction-runtime.ts +14 -7
- package/src/runtime/context-compaction/context-compaction.ts +78 -66
- package/src/runtime/domain-layer.ts +13 -7
- package/src/runtime/execution-plan.ts +7 -3
- package/src/runtime/live-turn-trace.ts +6 -49
- package/src/runtime/memory/memory-block.ts +3 -9
- package/src/runtime/memory/memory-scope.ts +3 -1
- package/src/runtime/plugin-resolution.ts +2 -1
- package/src/runtime/post-turn-side-effects.ts +6 -5
- package/src/runtime/retrieval-adapters.ts +8 -20
- package/src/runtime/runtime-config.ts +3 -9
- package/src/runtime/runtime-extensions.ts +2 -4
- package/src/runtime/runtime-lifecycle.ts +56 -16
- package/src/runtime/runtime-services.ts +180 -102
- package/src/runtime/runtime-worker-registry.ts +3 -1
- package/src/runtime/social-chat/social-chat-agent-runner.ts +1 -1
- package/src/runtime/social-chat/social-chat-history.ts +21 -18
- package/src/runtime/social-chat/social-chat.ts +356 -223
- package/src/runtime/specialist-runner.ts +3 -1
- package/src/runtime/team-consultation/team-consultation-orchestrator.ts +3 -2
- package/src/runtime/thread-turn-context.ts +142 -102
- package/src/runtime/turn-lifecycle.ts +15 -46
- package/src/services/agent-activity.service.ts +1 -1
- package/src/services/agent-executor.service.ts +107 -77
- package/src/services/autonomous-job.service.ts +354 -293
- package/src/services/background-work.service.ts +3 -3
- package/src/services/context-compaction.service.ts +7 -2
- package/src/services/document-chunk.service.ts +50 -32
- package/src/services/execution-plan/execution-plan-schedule.ts +5 -3
- package/src/services/execution-plan/execution-plan.service.ts +162 -179
- package/src/services/feedback-loop.service.ts +5 -4
- package/src/services/graph-full-routing.ts +37 -36
- package/src/services/institutional-memory.service.ts +28 -30
- package/src/services/learned-skill.service.ts +107 -72
- package/src/services/memory/memory-errors.ts +4 -23
- package/src/services/memory/memory-org-memory.ts +10 -5
- package/src/services/memory/memory-rerank.ts +18 -6
- package/src/services/memory/memory.service.ts +170 -111
- package/src/services/memory/rerank.service.ts +29 -20
- package/src/services/organization-member.service.ts +1 -1
- package/src/services/organization.service.ts +69 -75
- package/src/services/ownership-dispatcher.service.ts +40 -39
- package/src/services/plan/plan-agent-heartbeat.service.ts +26 -23
- package/src/services/plan/plan-agent-query.service.ts +39 -31
- package/src/services/plan/plan-completion-side-effects.ts +13 -17
- package/src/services/plan/plan-coordination.service.ts +2 -1
- package/src/services/plan/plan-cycle.service.ts +6 -5
- package/src/services/plan/plan-deadline.service.ts +57 -54
- package/src/services/plan/plan-event-delivery.service.ts +5 -4
- package/src/services/plan/plan-executor-graph.ts +18 -15
- package/src/services/plan/plan-executor.service.ts +235 -262
- package/src/services/plan/plan-run.service.ts +169 -93
- package/src/services/plan/plan-scheduler.service.ts +192 -202
- package/src/services/plan/plan-template.service.ts +1 -1
- package/src/services/plan/plan-transaction-events.ts +1 -1
- package/src/services/plan/plan-workspace.service.ts +23 -14
- package/src/services/plugin-executor.service.ts +5 -9
- package/src/services/queue-job.service.ts +117 -59
- package/src/services/recent-activity-title.service.ts +13 -12
- package/src/services/recent-activity.service.ts +6 -1
- package/src/services/social-chat-history.service.ts +29 -25
- package/src/services/system-executor.service.ts +5 -9
- package/src/services/thread/thread-active-run.ts +2 -2
- package/src/services/thread/thread-listing.ts +61 -57
- package/src/services/thread/thread-memory-block.ts +73 -48
- package/src/services/thread/thread-message.service.ts +76 -65
- package/src/services/thread/thread-record-store.ts +8 -8
- package/src/services/thread/thread-title.service.ts +10 -4
- package/src/services/thread/thread-turn-execution.ts +43 -45
- package/src/services/thread/thread-turn-preparation.service.ts +257 -135
- package/src/services/thread/thread-turn-streaming.ts +82 -85
- package/src/services/thread/thread-turn.ts +8 -8
- package/src/services/thread/thread.service.ts +135 -100
- package/src/services/user.service.ts +45 -48
- package/src/storage/attachment-parser.ts +6 -2
- package/src/storage/attachment-storage.service.ts +5 -6
- package/src/storage/generated-document-storage.service.ts +1 -1
- package/src/system-agents/context-compaction.agent.ts +10 -9
- package/src/system-agents/delegated-agent-factory.ts +30 -6
- package/src/system-agents/memory-reranker.agent.ts +10 -9
- package/src/system-agents/memory.agent.ts +10 -9
- package/src/system-agents/recent-activity-title-refiner.agent.ts +13 -15
- package/src/system-agents/regular-chat-memory-digest.agent.ts +13 -12
- package/src/system-agents/skill-extractor.agent.ts +13 -12
- package/src/system-agents/skill-manager.agent.ts +13 -12
- package/src/system-agents/thread-router.agent.ts +10 -5
- package/src/system-agents/title-generator.agent.ts +13 -12
- package/src/tools/fetch-webpage.tool.ts +13 -13
- package/src/tools/memory-block.tool.ts +3 -1
- package/src/tools/plan-approval.tool.ts +4 -2
- package/src/tools/read-file-parts.tool.ts +10 -4
- package/src/tools/remember-memory.tool.ts +3 -1
- package/src/tools/research-topic.tool.ts +9 -5
- package/src/tools/search-web.tool.ts +16 -16
- package/src/tools/search.tool.ts +20 -5
- package/src/tools/team-think.tool.ts +61 -38
- package/src/utils/async.ts +5 -5
- package/src/utils/errors.ts +19 -18
- package/src/utils/sse-keepalive.ts +28 -25
- package/src/workers/bootstrap.ts +75 -11
- package/src/workers/memory-consolidation.worker.ts +82 -91
- package/src/workers/organization-learning.worker.ts +14 -4
- package/src/workers/regular-chat-memory-digest.runner.ts +105 -67
- package/src/workers/skill-extraction.runner.ts +97 -61
- package/src/workers/utils/repo-structure-extractor.ts +13 -8
- package/src/workers/utils/thread-message-query.ts +24 -24
- package/src/workers/worker-utils.ts +23 -4
- package/src/effect/helpers.ts +0 -123
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { Effect, Schema } from 'effect'
|
|
1
|
+
import { Cause, Effect, Schema } from 'effect'
|
|
2
2
|
import type { Context } from 'effect'
|
|
3
3
|
import { BoundQuery } from 'surrealdb'
|
|
4
4
|
import { z } from 'zod'
|
|
5
5
|
|
|
6
|
+
import type { AiGatewayModels } from '../ai-gateway/ai-gateway'
|
|
6
7
|
import type { ResolvedAgentConfig } from '../config/agent-defaults'
|
|
7
8
|
import { isAgentName } from '../config/agent-defaults'
|
|
8
9
|
import { serverLogger } from '../config/logger'
|
|
@@ -10,7 +11,7 @@ import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
|
10
11
|
import type { RecordIdRef } from '../db/record-id'
|
|
11
12
|
import type { SurrealDBService } from '../db/service'
|
|
12
13
|
import { TABLES } from '../db/tables'
|
|
13
|
-
import {
|
|
14
|
+
import { ERROR_TAGS } from '../effect/errors'
|
|
14
15
|
import type {
|
|
15
16
|
OrganizationLearningQueueRuntime,
|
|
16
17
|
RegularChatMemoryDigestJob,
|
|
@@ -19,7 +20,7 @@ import { createHelperModelRuntime } from '../runtime/helper-model'
|
|
|
19
20
|
import type { LotaRuntimeAdapters, LotaRuntimeBackgroundCursor } from '../runtime/runtime-extensions'
|
|
20
21
|
import type { MemoryServiceTag } from '../services/memory/memory.service'
|
|
21
22
|
import type { SocialChatHistoryServiceTag } from '../services/social-chat-history.service'
|
|
22
|
-
import {
|
|
23
|
+
import { makeRegularChatMemoryDigestAgentFactory } from '../system-agents/regular-chat-memory-digest.agent'
|
|
23
24
|
import { nowIsoDateTimeString } from '../utils/date-time'
|
|
24
25
|
import { compactWhitespace } from '../utils/string'
|
|
25
26
|
import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
|
|
@@ -35,8 +36,9 @@ import type { DigestMessage } from './utils/thread-message-query'
|
|
|
35
36
|
// runner handles the regular-chat path after onboarding so longer transcripts
|
|
36
37
|
// can be digested into durable memory and profile projections in the background.
|
|
37
38
|
const StructuredProfilePatchSchema = z.record(z.string(), z.unknown()).default({})
|
|
39
|
+
const encodeJsonString = Schema.encodeSync(Schema.fromJsonString(Schema.Json))
|
|
38
40
|
|
|
39
|
-
class MemoryDigestError extends Schema.TaggedErrorClass<MemoryDigestError>()(
|
|
41
|
+
class MemoryDigestError extends Schema.TaggedErrorClass<MemoryDigestError>()(ERROR_TAGS.MemoryDigestError, {
|
|
40
42
|
message: Schema.String,
|
|
41
43
|
cause: Schema.Defect,
|
|
42
44
|
}) {}
|
|
@@ -45,6 +47,7 @@ const REGULAR_CHAT_MEMORY_DIGEST_TIMEOUT_MS = 10 * 60 * 1000
|
|
|
45
47
|
const WorkspaceMemoryRowSchema = z.object({ content: z.string() })
|
|
46
48
|
export interface RegularChatDigestServices {
|
|
47
49
|
agentConfig: ResolvedAgentConfig
|
|
50
|
+
aiGatewayModels: AiGatewayModels
|
|
48
51
|
databaseService: SurrealDBService
|
|
49
52
|
memoryService: Context.Service.Shape<typeof MemoryServiceTag>
|
|
50
53
|
socialChatHistoryService: Context.Service.Shape<typeof SocialChatHistoryServiceTag>
|
|
@@ -156,7 +159,9 @@ function hasNewEligibleThreadMessages(params: {
|
|
|
156
159
|
: null
|
|
157
160
|
|
|
158
161
|
if (!query) return false
|
|
159
|
-
const rows = yield*
|
|
162
|
+
const rows = yield* params.db
|
|
163
|
+
.query<unknown>(query)
|
|
164
|
+
.pipe(Effect.mapError((cause) => new Cause.UnknownError(cause)))
|
|
160
165
|
return rows.length > 0
|
|
161
166
|
}),
|
|
162
167
|
)
|
|
@@ -164,20 +169,20 @@ function hasNewEligibleThreadMessages(params: {
|
|
|
164
169
|
|
|
165
170
|
function loadExistingOrganizationMemories(db: SurrealDBService, orgId: string): Promise<Array<{ content: string }>> {
|
|
166
171
|
return Effect.runPromise(
|
|
167
|
-
|
|
168
|
-
|
|
172
|
+
db
|
|
173
|
+
.queryMany(
|
|
169
174
|
new BoundQuery(
|
|
170
175
|
`SELECT content, createdAt, id FROM ${TABLES.MEMORY}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
+
WHERE metadata.orgId = $orgId
|
|
177
|
+
AND archivedAt IS NONE
|
|
178
|
+
AND (validUntil IS NONE OR validUntil > time::now())
|
|
179
|
+
ORDER BY createdAt DESC, id DESC
|
|
180
|
+
LIMIT 250`,
|
|
176
181
|
{ orgId },
|
|
177
182
|
),
|
|
178
183
|
WorkspaceMemoryRowSchema,
|
|
179
|
-
)
|
|
180
|
-
|
|
184
|
+
)
|
|
185
|
+
.pipe(Effect.mapError((cause) => new Cause.UnknownError(cause))),
|
|
181
186
|
)
|
|
182
187
|
}
|
|
183
188
|
|
|
@@ -196,33 +201,50 @@ function runRegularChatMemoryDigestEffect(
|
|
|
196
201
|
return { skipped: true, processedThreadMessages: 0, processedSocialMessages: 0, followUpScheduled: false }
|
|
197
202
|
}
|
|
198
203
|
|
|
199
|
-
const workspace = yield*
|
|
204
|
+
const workspace = yield* Effect.tryPromise({
|
|
205
|
+
try: () => workspaceProvider.getWorkspace(orgRef),
|
|
206
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
207
|
+
})
|
|
200
208
|
const lifecycleState = workspaceProvider.getLifecycleState
|
|
201
|
-
? yield*
|
|
209
|
+
? yield* Effect.tryPromise({
|
|
210
|
+
try: () => Promise.resolve(workspaceProvider.getLifecycleState?.(workspace)),
|
|
211
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
212
|
+
})
|
|
202
213
|
: undefined
|
|
203
214
|
if (lifecycleState?.bootstrapActive ?? false) {
|
|
204
215
|
serverLogger.info`Skipping regular chat memory digest for ${orgId}: onboarding is not completed`
|
|
205
216
|
return { skipped: true, processedThreadMessages: 0, processedSocialMessages: 0, followUpScheduled: false }
|
|
206
217
|
}
|
|
207
218
|
const projectionState = workspaceProvider.readProfileProjectionState
|
|
208
|
-
? yield*
|
|
219
|
+
? yield* Effect.tryPromise({
|
|
220
|
+
try: () => Promise.resolve(workspaceProvider.readProfileProjectionState?.(workspace)),
|
|
221
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
222
|
+
})
|
|
209
223
|
: undefined
|
|
210
224
|
|
|
211
|
-
const existingThreadCursor = yield*
|
|
225
|
+
const existingThreadCursor = yield* Effect.tryPromise({
|
|
226
|
+
try: () => getBackgroundCursor('regular-chat-digest', orgRef),
|
|
227
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
228
|
+
})
|
|
212
229
|
const threadOnboardingCutoff = resolveWorkspaceBootstrapCutoff({
|
|
213
230
|
hasExistingCursor: existingThreadCursor !== null,
|
|
214
231
|
bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
|
|
215
232
|
})
|
|
216
233
|
|
|
217
|
-
const threadIds = yield*
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
234
|
+
const threadIds = yield* Effect.tryPromise({
|
|
235
|
+
try: () => listThreadIdsForOrg(services.databaseService, orgRef),
|
|
236
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
237
|
+
})
|
|
238
|
+
const threadMessages = yield* Effect.tryPromise({
|
|
239
|
+
try: () =>
|
|
240
|
+
listEligibleThreadMessages({
|
|
241
|
+
db: services.databaseService,
|
|
242
|
+
threadIds,
|
|
243
|
+
cursor: existingThreadCursor,
|
|
244
|
+
onboardingCutoff: threadOnboardingCutoff,
|
|
245
|
+
}),
|
|
246
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
247
|
+
})
|
|
226
248
|
const existingSocialCursor = yield* services.socialChatHistoryService.getBackgroundCursor(
|
|
227
249
|
'regular-chat-digest',
|
|
228
250
|
orgId,
|
|
@@ -247,30 +269,36 @@ function runRegularChatMemoryDigestEffect(
|
|
|
247
269
|
messages: combinedMessages,
|
|
248
270
|
isKnownAgentName: (value: string) => isAgentName(services.agentConfig, value),
|
|
249
271
|
})
|
|
250
|
-
const existingMemories = yield*
|
|
251
|
-
loadExistingOrganizationMemories(services.databaseService, orgId),
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
272
|
+
const existingMemories = yield* Effect.tryPromise({
|
|
273
|
+
try: () => loadExistingOrganizationMemories(services.databaseService, orgId),
|
|
274
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
275
|
+
})
|
|
276
|
+
const currentStructuredProfile = yield* Schema.decodeUnknownEffect(Schema.Json)(
|
|
277
|
+
projectionState?.structuredProfile ?? {},
|
|
278
|
+
).pipe(Effect.mapError((cause) => new Cause.UnknownError(cause)))
|
|
279
|
+
|
|
280
|
+
const synthesis = yield* Effect.tryPromise({
|
|
281
|
+
try: () =>
|
|
282
|
+
helperModelRuntime.generateHelperStructured({
|
|
283
|
+
tag: 'regular-chat-memory-digest',
|
|
284
|
+
createAgent: makeRegularChatMemoryDigestAgentFactory(services.aiGatewayModels),
|
|
285
|
+
timeoutMs: REGULAR_CHAT_MEMORY_DIGEST_TIMEOUT_MS,
|
|
286
|
+
messages: [
|
|
287
|
+
{
|
|
288
|
+
role: 'user',
|
|
289
|
+
content: buildPrompt({
|
|
290
|
+
workspaceName: projectionState?.workspaceName || 'Workspace',
|
|
291
|
+
currentSummaryBlock: projectionState?.summaryBlock ?? '',
|
|
292
|
+
currentStructuredProfile: encodeJsonString(currentStructuredProfile),
|
|
293
|
+
existingMemories: buildMemoryContext(existingMemories),
|
|
294
|
+
transcript,
|
|
295
|
+
}),
|
|
296
|
+
},
|
|
297
|
+
],
|
|
298
|
+
schema: RegularChatMemoryDigestOutputSchema,
|
|
299
|
+
}),
|
|
300
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
301
|
+
})
|
|
274
302
|
|
|
275
303
|
const summaryBlock = normalizeBlock(synthesis.summaryBlock)
|
|
276
304
|
if (!summaryBlock) {
|
|
@@ -309,25 +337,31 @@ function runRegularChatMemoryDigestEffect(
|
|
|
309
337
|
})
|
|
310
338
|
}
|
|
311
339
|
|
|
312
|
-
yield*
|
|
313
|
-
applyProfileProjection(orgRef, { summaryBlock, structuredPatch: synthesis.structuredProfilePatch }),
|
|
314
|
-
|
|
340
|
+
yield* Effect.tryPromise({
|
|
341
|
+
try: () => applyProfileProjection(orgRef, { summaryBlock, structuredPatch: synthesis.structuredProfilePatch }),
|
|
342
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
343
|
+
})
|
|
315
344
|
if (processedThreadCursor) {
|
|
316
|
-
yield*
|
|
345
|
+
yield* Effect.tryPromise({
|
|
346
|
+
try: () => setBackgroundCursor('regular-chat-digest', orgRef, processedThreadCursor),
|
|
347
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
348
|
+
})
|
|
317
349
|
}
|
|
318
350
|
if (processedSocialCursor) {
|
|
319
351
|
yield* services.socialChatHistoryService.setBackgroundCursor('regular-chat-digest', orgId, processedSocialCursor)
|
|
320
352
|
}
|
|
321
353
|
|
|
322
354
|
const threadBoundaryCursor = processedThreadCursor ?? existingThreadCursor
|
|
323
|
-
const hasMoreThreadMessages = yield*
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
355
|
+
const hasMoreThreadMessages = yield* Effect.tryPromise({
|
|
356
|
+
try: () =>
|
|
357
|
+
hasNewEligibleThreadMessages({
|
|
358
|
+
db: services.databaseService,
|
|
359
|
+
threadIds,
|
|
360
|
+
cursor: threadBoundaryCursor,
|
|
361
|
+
onboardingCutoff: threadBoundaryCursor ? null : threadOnboardingCutoff,
|
|
362
|
+
}),
|
|
363
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
364
|
+
})
|
|
331
365
|
const socialBoundaryCursor = processedSocialCursor ?? existingSocialCursor
|
|
332
366
|
const hasMoreSocialMessages = yield* services.socialChatHistoryService.hasWorkspaceMessages({
|
|
333
367
|
workspaceId: orgId,
|
|
@@ -337,10 +371,14 @@ function runRegularChatMemoryDigestEffect(
|
|
|
337
371
|
|
|
338
372
|
const followUpScheduled = hasMoreThreadMessages || hasMoreSocialMessages
|
|
339
373
|
if (followUpScheduled) {
|
|
340
|
-
yield*
|
|
341
|
-
services.organizationLearningQueue.clearRegularChatMemoryDigestDeduplicationKey(orgId),
|
|
342
|
-
|
|
343
|
-
|
|
374
|
+
yield* Effect.tryPromise({
|
|
375
|
+
try: () => services.organizationLearningQueue.clearRegularChatMemoryDigestDeduplicationKey(orgId),
|
|
376
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
377
|
+
})
|
|
378
|
+
yield* Effect.tryPromise({
|
|
379
|
+
try: () => services.organizationLearningQueue.enqueueRegularChatMemoryDigest({ orgId }),
|
|
380
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
381
|
+
})
|
|
344
382
|
}
|
|
345
383
|
|
|
346
384
|
serverLogger.info`Regular chat memory digest completed for ${orgId}: threadMessages=${threadMessages.length}, socialMessages=${socialMessages.length}, facts=${synthesis.facts.length}, followUpScheduled=${followUpScheduled}`
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import { Cause, Effect } from 'effect'
|
|
2
|
+
import type { Context } from 'effect'
|
|
3
3
|
|
|
4
|
+
import type { AiGatewayModels } from '../ai-gateway/ai-gateway'
|
|
4
5
|
import type { ResolvedAgentConfig } from '../config/agent-defaults'
|
|
5
6
|
import { isAgentName } from '../config/agent-defaults'
|
|
6
7
|
import { serverLogger } from '../config/logger'
|
|
@@ -8,16 +9,15 @@ import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
|
8
9
|
import type { RecordIdRef } from '../db/record-id'
|
|
9
10
|
import type { SurrealDBService } from '../db/service'
|
|
10
11
|
import { TABLES } from '../db/tables'
|
|
11
|
-
import { effectTryPromise } from '../effect/helpers'
|
|
12
12
|
import { ProviderEmbeddings } from '../embeddings/provider'
|
|
13
13
|
import type { SkillExtractionJob } from '../queues/organization-learning.queue'
|
|
14
14
|
import { createHelperModelRuntime } from '../runtime/helper-model'
|
|
15
15
|
import type { LotaRuntimeAdapters } from '../runtime/runtime-extensions'
|
|
16
16
|
import type { LearnedSkillServiceTag } from '../services/learned-skill.service'
|
|
17
17
|
import type { SocialChatHistoryServiceTag } from '../services/social-chat-history.service'
|
|
18
|
-
import {
|
|
18
|
+
import { makeSkillExtractorAgentFactory, SkillExtractionOutputSchema } from '../system-agents/skill-extractor.agent'
|
|
19
19
|
import type { SkillCandidate } from '../system-agents/skill-extractor.agent'
|
|
20
|
-
import {
|
|
20
|
+
import { makeSkillManagerAgentFactory, SkillManagerOutputSchema } from '../system-agents/skill-manager.agent'
|
|
21
21
|
import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
|
|
22
22
|
import {
|
|
23
23
|
compareDigestMessageOrder,
|
|
@@ -30,12 +30,14 @@ const MIN_MESSAGE_THRESHOLD = 10
|
|
|
30
30
|
|
|
31
31
|
export interface SkillExtractionServices {
|
|
32
32
|
agentConfig: ResolvedAgentConfig
|
|
33
|
+
aiGatewayModels: AiGatewayModels
|
|
33
34
|
databaseService: SurrealDBService
|
|
34
35
|
learnedSkillService: Context.Service.Shape<typeof LearnedSkillServiceTag>
|
|
35
36
|
socialChatHistoryService: Context.Service.Shape<typeof SocialChatHistoryServiceTag>
|
|
36
37
|
runtimeAdapters: LotaRuntimeAdapters
|
|
37
38
|
embeddingModel: string
|
|
38
39
|
openRouterApiKey?: string
|
|
40
|
+
runPromise: <A, E>(effect: Effect.Effect<A, E>) => Promise<A>
|
|
39
41
|
}
|
|
40
42
|
|
|
41
43
|
interface SkillExtractionRunResult {
|
|
@@ -110,21 +112,31 @@ function runSkillExtractionEffect(
|
|
|
110
112
|
return Effect.gen(function* () {
|
|
111
113
|
const getBackgroundCursor = cursorAwareWorkspaceProvider.getBackgroundCursor.bind(cursorAwareWorkspaceProvider)
|
|
112
114
|
const setBackgroundCursor = cursorAwareWorkspaceProvider.setBackgroundCursor.bind(cursorAwareWorkspaceProvider)
|
|
113
|
-
const workspace = yield*
|
|
115
|
+
const workspace = yield* Effect.tryPromise({
|
|
116
|
+
try: () => cursorAwareWorkspaceProvider.getWorkspace(orgRef),
|
|
117
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
118
|
+
})
|
|
114
119
|
const lifecycleState = cursorAwareWorkspaceProvider.getLifecycleState
|
|
115
|
-
? yield*
|
|
120
|
+
? yield* Effect.tryPromise({
|
|
121
|
+
try: () => Promise.resolve(cursorAwareWorkspaceProvider.getLifecycleState?.(workspace)),
|
|
122
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
123
|
+
})
|
|
116
124
|
: undefined
|
|
117
125
|
if (lifecycleState?.bootstrapActive ?? false) {
|
|
118
126
|
serverLogger.info`Skipping skill extraction for ${orgId}: onboarding is not completed`
|
|
119
127
|
return { skipped: true, processedMessages: 0, extractedSkills: 0 }
|
|
120
128
|
}
|
|
121
129
|
const projectionState = cursorAwareWorkspaceProvider.readProfileProjectionState
|
|
122
|
-
? yield*
|
|
123
|
-
Promise.resolve(cursorAwareWorkspaceProvider.readProfileProjectionState?.(workspace)),
|
|
124
|
-
|
|
130
|
+
? yield* Effect.tryPromise({
|
|
131
|
+
try: () => Promise.resolve(cursorAwareWorkspaceProvider.readProfileProjectionState?.(workspace)),
|
|
132
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
133
|
+
})
|
|
125
134
|
: undefined
|
|
126
135
|
|
|
127
|
-
const existingCursor = yield*
|
|
136
|
+
const existingCursor = yield* Effect.tryPromise({
|
|
137
|
+
try: () => getBackgroundCursor('skill-extraction', orgRef),
|
|
138
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
139
|
+
})
|
|
128
140
|
const onboardingCutoff = resolveWorkspaceBootstrapCutoff({
|
|
129
141
|
hasExistingCursor: existingCursor !== null,
|
|
130
142
|
bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
|
|
@@ -135,10 +147,20 @@ function runSkillExtractionEffect(
|
|
|
135
147
|
bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
|
|
136
148
|
})
|
|
137
149
|
|
|
138
|
-
const threadIds = yield*
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
)
|
|
150
|
+
const threadIds = yield* Effect.tryPromise({
|
|
151
|
+
try: () => listThreadIdsForOrg(services.databaseService, orgRef),
|
|
152
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
153
|
+
})
|
|
154
|
+
const threadMessages = yield* Effect.tryPromise({
|
|
155
|
+
try: () =>
|
|
156
|
+
listEligibleThreadMessages({
|
|
157
|
+
db: services.databaseService,
|
|
158
|
+
threadIds,
|
|
159
|
+
cursor: existingCursor,
|
|
160
|
+
onboardingCutoff,
|
|
161
|
+
}),
|
|
162
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
163
|
+
})
|
|
142
164
|
const socialMessages = yield* services.socialChatHistoryService.listWorkspaceMessages({
|
|
143
165
|
workspaceId: orgId,
|
|
144
166
|
cursor: existingSocialCursor,
|
|
@@ -163,24 +185,26 @@ function runSkillExtractionEffect(
|
|
|
163
185
|
? existingSkills.map((skill, i) => `${i + 1}. ${skill.name}: ${skill.description}`).join('\n')
|
|
164
186
|
: 'None'
|
|
165
187
|
|
|
166
|
-
const extraction = yield*
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
188
|
+
const extraction = yield* Effect.tryPromise({
|
|
189
|
+
try: () =>
|
|
190
|
+
helperModelRuntime.generateHelperStructured({
|
|
191
|
+
tag: 'skill-extraction',
|
|
192
|
+
createAgent: makeSkillExtractorAgentFactory(services.aiGatewayModels),
|
|
193
|
+
timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
|
|
194
|
+
messages: [
|
|
195
|
+
{
|
|
196
|
+
role: 'user',
|
|
197
|
+
content: buildExtractionPrompt({
|
|
198
|
+
workspaceName: projectionState?.workspaceName || 'Workspace',
|
|
199
|
+
transcript,
|
|
200
|
+
existingSkills: existingSkillsSummary,
|
|
201
|
+
}),
|
|
202
|
+
},
|
|
203
|
+
],
|
|
204
|
+
schema: SkillExtractionOutputSchema,
|
|
205
|
+
}),
|
|
206
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
207
|
+
})
|
|
184
208
|
|
|
185
209
|
const skillCandidates = extraction.candidates.filter((c) => c.classification === 'skill')
|
|
186
210
|
|
|
@@ -195,37 +219,42 @@ function runSkillExtractionEffect(
|
|
|
195
219
|
|
|
196
220
|
const mostSimilar = yield* services.learnedSkillService.findMostSimilar(orgId, candidate.description)
|
|
197
221
|
|
|
198
|
-
const managerResult = yield*
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
+
const managerResult = yield* Effect.tryPromise({
|
|
223
|
+
try: () =>
|
|
224
|
+
helperModelRuntime.generateHelperStructured({
|
|
225
|
+
tag: 'skill-manager',
|
|
226
|
+
createAgent: makeSkillManagerAgentFactory(services.aiGatewayModels),
|
|
227
|
+
timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
|
|
228
|
+
messages: [
|
|
229
|
+
{
|
|
230
|
+
role: 'user',
|
|
231
|
+
content: buildManagerPrompt({
|
|
232
|
+
candidate,
|
|
233
|
+
existingSkill: mostSimilar
|
|
234
|
+
? {
|
|
235
|
+
name: mostSimilar.name,
|
|
236
|
+
description: mostSimilar.description,
|
|
237
|
+
instructions: mostSimilar.instructions,
|
|
238
|
+
version: mostSimilar.version,
|
|
239
|
+
}
|
|
240
|
+
: null,
|
|
241
|
+
}),
|
|
242
|
+
},
|
|
243
|
+
],
|
|
244
|
+
schema: SkillManagerOutputSchema,
|
|
245
|
+
}),
|
|
246
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
247
|
+
})
|
|
222
248
|
|
|
223
249
|
if (managerResult.decision === 'discard') {
|
|
224
250
|
serverLogger.info`Discarding skill candidate ${candidate.name}: ${managerResult.reason}`
|
|
225
251
|
return 0
|
|
226
252
|
}
|
|
227
253
|
|
|
228
|
-
const embedding = yield*
|
|
254
|
+
const embedding = yield* Effect.tryPromise({
|
|
255
|
+
try: () => embeddings.embedQuery(candidate.description),
|
|
256
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
257
|
+
})
|
|
229
258
|
if (embedding.length === 0) {
|
|
230
259
|
serverLogger.warn`Skipping skill candidate ${candidate.name}: empty embedding`
|
|
231
260
|
return 0
|
|
@@ -253,7 +282,10 @@ function runSkillExtractionEffect(
|
|
|
253
282
|
if (mostSimilar && managerResult.mergedSkill) {
|
|
254
283
|
const merged = managerResult.mergedSkill
|
|
255
284
|
const mergedHash = services.learnedSkillService.generateHash(merged.description, merged.instructions)
|
|
256
|
-
const mergedEmbedding = yield*
|
|
285
|
+
const mergedEmbedding = yield* Effect.tryPromise({
|
|
286
|
+
try: () => embeddings.embedQuery(merged.description),
|
|
287
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
288
|
+
})
|
|
257
289
|
|
|
258
290
|
yield* services.learnedSkillService.update(mostSimilar.id, {
|
|
259
291
|
name: merged.name,
|
|
@@ -291,7 +323,10 @@ function runSkillExtractionEffect(
|
|
|
291
323
|
const lastThreadMessage = threadMessages.at(-1)
|
|
292
324
|
const lastSocialMessage = socialMessages.at(-1)
|
|
293
325
|
if (lastThreadMessage) {
|
|
294
|
-
yield*
|
|
326
|
+
yield* Effect.tryPromise({
|
|
327
|
+
try: () => setBackgroundCursor('skill-extraction', orgRef, lastThreadMessage.cursor),
|
|
328
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
329
|
+
})
|
|
295
330
|
}
|
|
296
331
|
if (lastSocialMessage) {
|
|
297
332
|
yield* services.socialChatHistoryService.setBackgroundCursor('skill-extraction', orgId, lastSocialMessage.cursor)
|
|
@@ -319,10 +354,11 @@ export function runSkillExtraction(
|
|
|
319
354
|
const embeddings = new ProviderEmbeddings({
|
|
320
355
|
modelId: services.embeddingModel,
|
|
321
356
|
openRouterApiKey: services.openRouterApiKey,
|
|
357
|
+
runPromise: services.runPromise,
|
|
322
358
|
})
|
|
323
359
|
const withMemoryLock = runtimeAdapters.withWorkspaceMemoryLock
|
|
324
360
|
const runExtraction = () =>
|
|
325
|
-
|
|
361
|
+
services.runPromise(
|
|
326
362
|
runSkillExtractionEffect(services, orgRef, orgId, workspaceProvider, embeddings) as Effect.Effect<
|
|
327
363
|
SkillExtractionRunResult,
|
|
328
364
|
never,
|
|
@@ -6,10 +6,8 @@ import type {
|
|
|
6
6
|
RepositoryStructureSignal,
|
|
7
7
|
RepositoryStructureSummary,
|
|
8
8
|
} from '@lota-sdk/shared'
|
|
9
|
-
import
|
|
10
|
-
import { Effect } from 'effect'
|
|
9
|
+
import { Cause, Effect } from 'effect'
|
|
11
10
|
|
|
12
|
-
import { effectTryPromise } from '../../effect/helpers'
|
|
13
11
|
import { nowIsoDateTimeString } from '../../utils/date-time'
|
|
14
12
|
|
|
15
13
|
const EXTRACTOR_VERSION = 'repository-structure-extractor.v1'
|
|
@@ -152,14 +150,18 @@ function collectRelativeFilePathsEffect(
|
|
|
152
150
|
): Effect.Effect<string[], Cause.UnknownError, never> {
|
|
153
151
|
return Effect.gen(function* () {
|
|
154
152
|
const absoluteDir = currentDir ? joinPath(rootDir, currentDir) : rootDir
|
|
155
|
-
const entries = yield*
|
|
156
|
-
Array.fromAsync(new Bun.Glob('*').scan({ cwd: absoluteDir, onlyFiles: false, absolute: false })),
|
|
157
|
-
|
|
153
|
+
const entries = yield* Effect.tryPromise({
|
|
154
|
+
try: () => Array.fromAsync(new Bun.Glob('*').scan({ cwd: absoluteDir, onlyFiles: false, absolute: false })),
|
|
155
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
156
|
+
})
|
|
158
157
|
|
|
159
158
|
const filePaths: string[] = []
|
|
160
159
|
for (const entry of entries) {
|
|
161
160
|
const relativePath = currentDir ? joinPath(currentDir, entry) : normalizePath(entry)
|
|
162
|
-
const stats = yield*
|
|
161
|
+
const stats = yield* Effect.tryPromise({
|
|
162
|
+
try: () => Bun.file(joinPath(rootDir, relativePath)).stat(),
|
|
163
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
164
|
+
})
|
|
163
165
|
|
|
164
166
|
if (stats.isDirectory()) {
|
|
165
167
|
if (IGNORED_DIR_NAMES.has(entry)) continue
|
|
@@ -179,7 +181,10 @@ function readPackageJsonEffect(
|
|
|
179
181
|
rootDir: string,
|
|
180
182
|
relativePath: string,
|
|
181
183
|
): Effect.Effect<PackageJson | null, Cause.UnknownError, never> {
|
|
182
|
-
return
|
|
184
|
+
return Effect.tryPromise({
|
|
185
|
+
try: () => Bun.file(joinPath(rootDir, relativePath)).text(),
|
|
186
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
187
|
+
}).pipe(
|
|
183
188
|
Effect.map((raw) => {
|
|
184
189
|
try {
|
|
185
190
|
const parsed: unknown = JSON.parse(raw)
|