@lota-sdk/core 0.4.8 → 0.4.10

package/src/workers/skill-extraction.runner.ts

@@ -1,12 +1,18 @@
+import type { Context, Cause } from 'effect'
+import { Effect } from 'effect'
+
 import { serverLogger } from '../config/logger'
 import { ensureRecordId, recordIdToString } from '../db/record-id'
+import type { RecordIdRef } from '../db/record-id'
+import type { SurrealDBService } from '../db/service'
 import { TABLES } from '../db/tables'
-import { getDefaultEmbeddings } from '../embeddings/provider'
+import { effectTryPromise } from '../effect/helpers'
+import { ProviderEmbeddings } from '../embeddings/provider'
 import type { SkillExtractionJob } from '../queues/organization-learning.queue'
 import { createHelperModelRuntime } from '../runtime/helper-model'
-import { getRuntimeAdapters, withConfiguredWorkspaceMemoryLock } from '../runtime/runtime-extensions'
-import { learnedSkillService } from '../services/learned-skill.service'
-import { socialChatHistoryService } from '../services/social-chat-history.service'
+import type { LotaRuntimeAdapters } from '../runtime/runtime-extensions'
+import type { LearnedSkillServiceTag } from '../services/learned-skill.service'
+import type { SocialChatHistoryServiceTag } from '../services/social-chat-history.service'
 import { createSkillExtractorAgent, SkillExtractionOutputSchema } from '../system-agents/skill-extractor.agent'
 import type { SkillCandidate } from '../system-agents/skill-extractor.agent'
 import { createSkillManagerAgent, SkillManagerOutputSchema } from '../system-agents/skill-manager.agent'
@@ -20,16 +26,36 @@ import {
 const SKILL_EXTRACTION_TIMEOUT_MS = 10 * 60 * 1000
 const MIN_MESSAGE_THRESHOLD = 10
 
+export interface SkillExtractionServices {
+  databaseService: SurrealDBService
+  learnedSkillService: Context.Service.Shape<typeof LearnedSkillServiceTag>
+  socialChatHistoryService: Context.Service.Shape<typeof SocialChatHistoryServiceTag>
+  runtimeAdapters: LotaRuntimeAdapters
+  embeddingModel: string
+  openRouterApiKey?: string
+}
+
 interface SkillExtractionRunResult {
   skipped: boolean
   processedMessages: number
   extractedSkills: number
 }
 
-const embeddings = getDefaultEmbeddings()
-
 const helperModelRuntime = createHelperModelRuntime()
 
+type CursorAwareWorkspaceProvider = NonNullable<LotaRuntimeAdapters['workspaceProvider']> & {
+  getBackgroundCursor: NonNullable<NonNullable<LotaRuntimeAdapters['workspaceProvider']>['getBackgroundCursor']>
+  setBackgroundCursor: NonNullable<NonNullable<LotaRuntimeAdapters['workspaceProvider']>['setBackgroundCursor']>
+}
+
+function hasCursorAwareWorkspaceProvider(
+  workspaceProvider: LotaRuntimeAdapters['workspaceProvider'],
+): workspaceProvider is CursorAwareWorkspaceProvider {
+  const getBackgroundCursor = workspaceProvider?.getBackgroundCursor?.bind(workspaceProvider)
+  const setBackgroundCursor = workspaceProvider?.setBackgroundCursor?.bind(workspaceProvider)
+  return Boolean(getBackgroundCursor && setBackgroundCursor)
+}
+
 function buildExtractionPrompt(params: { workspaceName: string; transcript: string; existingSkills: string }): string {
   return [
     `Workspace name: ${params.workspaceName}`,
@@ -71,45 +97,46 @@ function buildManagerPrompt(params: {
   return parts.join('\n')
 }
 
-export async function runSkillExtraction(data: SkillExtractionJob): Promise<SkillExtractionRunResult> {
-  const orgRef = ensureRecordId(data.orgId, TABLES.ORGANIZATION)
-  const orgId = recordIdToString(orgRef, TABLES.ORGANIZATION)
-  const workspaceProvider = getRuntimeAdapters().workspaceProvider
-  const cursorAwareWorkspaceProvider =
-    workspaceProvider?.getBackgroundCursor && workspaceProvider.setBackgroundCursor
-      ? (workspaceProvider as typeof workspaceProvider & {
-          getBackgroundCursor: NonNullable<typeof workspaceProvider.getBackgroundCursor>
-          setBackgroundCursor: NonNullable<typeof workspaceProvider.setBackgroundCursor>
-        })
+function runSkillExtractionEffect(
+  services: SkillExtractionServices,
+  orgRef: RecordIdRef,
+  orgId: string,
+  cursorAwareWorkspaceProvider: CursorAwareWorkspaceProvider,
+  embeddings: ProviderEmbeddings,
+) {
+  return Effect.gen(function* () {
+    const getBackgroundCursor = cursorAwareWorkspaceProvider.getBackgroundCursor.bind(cursorAwareWorkspaceProvider)
+    const setBackgroundCursor = cursorAwareWorkspaceProvider.setBackgroundCursor.bind(cursorAwareWorkspaceProvider)
+    const workspace = yield* effectTryPromise(() => cursorAwareWorkspaceProvider.getWorkspace(orgRef))
+    const lifecycleState = cursorAwareWorkspaceProvider.getLifecycleState
+      ? yield* effectTryPromise(() => Promise.resolve(cursorAwareWorkspaceProvider.getLifecycleState?.(workspace)))
       : undefined
-  if (!cursorAwareWorkspaceProvider) {
-    serverLogger.info`Skipping skill extraction for ${orgId}: workspaceProvider background cursor methods are not configured`
-    return { skipped: true, processedMessages: 0, extractedSkills: 0 }
-  }
-
-  return withConfiguredWorkspaceMemoryLock(orgId, async () => {
-    const workspace = await cursorAwareWorkspaceProvider.getWorkspace(orgRef)
-    const lifecycleState = await cursorAwareWorkspaceProvider.getLifecycleState?.(workspace)
     if (lifecycleState?.bootstrapActive ?? false) {
       serverLogger.info`Skipping skill extraction for ${orgId}: onboarding is not completed`
       return { skipped: true, processedMessages: 0, extractedSkills: 0 }
     }
-    const projectionState = await cursorAwareWorkspaceProvider.readProfileProjectionState?.(workspace)
+    const projectionState = cursorAwareWorkspaceProvider.readProfileProjectionState
+      ? yield* effectTryPromise(() =>
+          Promise.resolve(cursorAwareWorkspaceProvider.readProfileProjectionState?.(workspace)),
+        )
+      : undefined
 
-    const existingCursor = await cursorAwareWorkspaceProvider.getBackgroundCursor('skill-extraction', orgRef)
+    const existingCursor = yield* effectTryPromise(() => getBackgroundCursor('skill-extraction', orgRef))
     const onboardingCutoff = resolveWorkspaceBootstrapCutoff({
       hasExistingCursor: existingCursor !== null,
       bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
     })
-    const existingSocialCursor = await socialChatHistoryService.getBackgroundCursor('skill-extraction', orgId)
+    const existingSocialCursor = yield* services.socialChatHistoryService.getBackgroundCursor('skill-extraction', orgId)
     const socialOnboardingCutoff = resolveWorkspaceBootstrapCutoff({
       hasExistingCursor: existingSocialCursor !== null,
       bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
     })
 
-    const threadIds = await listThreadIdsForOrg(orgRef)
-    const threadMessages = await listEligibleThreadMessages({ threadIds, cursor: existingCursor, onboardingCutoff })
-    const socialMessages = await socialChatHistoryService.listWorkspaceMessages({
+    const threadIds = yield* effectTryPromise(() => listThreadIdsForOrg(services.databaseService, orgRef))
+    const threadMessages = yield* effectTryPromise(() =>
+      listEligibleThreadMessages({ db: services.databaseService, threadIds, cursor: existingCursor, onboardingCutoff }),
+    )
+    const socialMessages = yield* services.socialChatHistoryService.listWorkspaceMessages({
       workspaceId: orgId,
       cursor: existingSocialCursor,
       onboardingCutoff: socialOnboardingCutoff,
@@ -124,79 +151,82 @@ export async function runSkillExtraction(data: SkillExtractionJob): Promise<Skil
     const sortedMessages = [...messages].sort(compareDigestMessageOrder)
     const { transcript } = buildDigestTranscript({ messages: sortedMessages })
 
-    const existingSkills = await learnedSkillService.listForOrg(orgId)
+    const existingSkills = yield* services.learnedSkillService.listForOrg(orgId)
     const existingSkillsSummary =
       existingSkills.length > 0
         ? existingSkills.map((skill, i) => `${i + 1}. ${skill.name}: ${skill.description}`).join('\n')
         : 'None'
 
-    const extraction = await helperModelRuntime.generateHelperStructured({
-      tag: 'skill-extraction',
-      createAgent: createSkillExtractorAgent,
-      timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
-      messages: [
-        {
-          role: 'user',
-          content: buildExtractionPrompt({
-            workspaceName: projectionState?.workspaceName || 'Workspace',
-            transcript,
-            existingSkills: existingSkillsSummary,
-          }),
-        },
-      ],
-      schema: SkillExtractionOutputSchema,
-    })
+    const extraction = yield* effectTryPromise(() =>
+      helperModelRuntime.generateHelperStructured({
+        tag: 'skill-extraction',
+        createAgent: createSkillExtractorAgent,
+        timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
+        messages: [
+          {
+            role: 'user',
+            content: buildExtractionPrompt({
+              workspaceName: projectionState?.workspaceName || 'Workspace',
+              transcript,
+              existingSkills: existingSkillsSummary,
+            }),
+          },
+        ],
+        schema: SkillExtractionOutputSchema,
+      }),
+    )
 
     const skillCandidates = extraction.candidates.filter((c) => c.classification === 'skill')
-    let extractedSkills = 0
 
-    for (const candidate of skillCandidates) {
-      try {
-        const hash = learnedSkillService.generateHash(candidate.description, candidate.instructions)
-        const existingByHash = await learnedSkillService.findByHash(orgId, hash)
-        if (existingByHash) {
+    const processCandidate = (candidate: SkillCandidate): Effect.Effect<number, Cause.UnknownError, never> => {
+      const processCandidateEffect = Effect.gen(function* () {
+        const hash = services.learnedSkillService.generateHash(candidate.description, candidate.instructions)
+        const existingByHash = yield* services.learnedSkillService.findByHash(orgId, hash)
+        if (existingByHash !== null) {
           serverLogger.info`Skipping duplicate skill candidate ${candidate.name} (hash match)`
-          continue
+          return 0
         }
 
-        const mostSimilar = await learnedSkillService.findMostSimilar(orgId, candidate.description)
-
-        const managerResult = await helperModelRuntime.generateHelperStructured({
-          tag: 'skill-manager',
-          createAgent: createSkillManagerAgent,
-          timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
-          messages: [
-            {
-              role: 'user',
-              content: buildManagerPrompt({
-                candidate,
-                existingSkill: mostSimilar
-                  ? {
-                      name: mostSimilar.name,
-                      description: mostSimilar.description,
-                      instructions: mostSimilar.instructions,
-                      version: mostSimilar.version,
-                    }
-                  : null,
-              }),
-            },
-          ],
-          schema: SkillManagerOutputSchema,
-        })
+        const mostSimilar = yield* services.learnedSkillService.findMostSimilar(orgId, candidate.description)
+
+        const managerResult = yield* effectTryPromise(() =>
+          helperModelRuntime.generateHelperStructured({
+            tag: 'skill-manager',
+            createAgent: createSkillManagerAgent,
+            timeoutMs: SKILL_EXTRACTION_TIMEOUT_MS,
+            messages: [
+              {
+                role: 'user',
+                content: buildManagerPrompt({
+                  candidate,
+                  existingSkill: mostSimilar
+                    ? {
+                        name: mostSimilar.name,
+                        description: mostSimilar.description,
+                        instructions: mostSimilar.instructions,
+                        version: mostSimilar.version,
+                      }
+                    : null,
+                }),
+              },
+            ],
+            schema: SkillManagerOutputSchema,
+          }),
+        )
 
         if (managerResult.decision === 'discard') {
           serverLogger.info`Discarding skill candidate ${candidate.name}: ${managerResult.reason}`
-          continue
+          return 0
         }
 
-        const embedding = await embeddings.embedQuery(candidate.description)
+        const embedding = yield* effectTryPromise(() => embeddings.embedQuery(candidate.description))
         if (embedding.length === 0) {
           serverLogger.warn`Skipping skill candidate ${candidate.name}: empty embedding`
-          continue
+          return 0
         }
 
         if (managerResult.decision === 'add') {
-          await learnedSkillService.create({
+          yield* services.learnedSkillService.create({
             name: candidate.name,
             description: candidate.description,
             instructions: candidate.instructions,
@@ -210,14 +240,16 @@ export async function runSkillExtraction(data: SkillExtractionJob): Promise<Skil
             embedding,
             hash,
           })
-          extractedSkills++
           serverLogger.info`Added new learned skill: ${candidate.name}`
-        } else if (mostSimilar && managerResult.mergedSkill) {
+          return 1
+        }
+
+        if (mostSimilar && managerResult.mergedSkill) {
           const merged = managerResult.mergedSkill
-          const mergedHash = learnedSkillService.generateHash(merged.description, merged.instructions)
-          const mergedEmbedding = await embeddings.embedQuery(merged.description)
+          const mergedHash = services.learnedSkillService.generateHash(merged.description, merged.instructions)
+          const mergedEmbedding = yield* effectTryPromise(() => embeddings.embedQuery(merged.description))
 
-          await learnedSkillService.update(mostSimilar.id, {
+          yield* services.learnedSkillService.update(mostSimilar.id, {
             name: merged.name,
             description: merged.description,
             instructions: merged.instructions,
@@ -229,21 +261,34 @@ export async function runSkillExtraction(data: SkillExtractionJob): Promise<Skil
             embedding: mergedEmbedding,
             hash: mergedHash,
           })
-          extractedSkills++
           serverLogger.info`Merged skill candidate into ${mostSimilar.name} (v${mostSimilar.version + 1})`
+          return 1
         }
-      } catch (candidateError) {
-        serverLogger.warn`Failed to process skill candidate ${candidate.name}: ${candidateError}`
-      }
+
+        return 0
+      })
+
+      return Effect.matchCauseEffect(processCandidateEffect, {
+        onFailure: (cause) =>
+          Effect.sync(() => {
+            serverLogger.warn`Failed to process skill candidate ${candidate.name}: ${cause}`
+            return 0
+          }),
+        onSuccess: (value) => Effect.succeed(value),
+      })
     }
+    const extractedSkills = (yield* Effect.forEach(skillCandidates, processCandidate, { concurrency: 2 })).reduce(
+      (sum, n) => sum + n,
+      0,
+    )
 
     const lastThreadMessage = threadMessages.at(-1)
     const lastSocialMessage = socialMessages.at(-1)
     if (lastThreadMessage) {
-      await cursorAwareWorkspaceProvider.setBackgroundCursor('skill-extraction', orgRef, lastThreadMessage.cursor)
+      yield* effectTryPromise(() => setBackgroundCursor('skill-extraction', orgRef, lastThreadMessage.cursor))
     }
     if (lastSocialMessage) {
-      await socialChatHistoryService.setBackgroundCursor('skill-extraction', orgId, lastSocialMessage.cursor)
+      yield* services.socialChatHistoryService.setBackgroundCursor('skill-extraction', orgId, lastSocialMessage.cursor)
     }
 
     serverLogger.info`Skill extraction completed for ${orgId}: messages=${messages.length}, socialMessages=${socialMessages.length}, extracted=${extractedSkills}`
@@ -251,3 +296,41 @@ export async function runSkillExtraction(data: SkillExtractionJob): Promise<Skil
     return { skipped: false, processedMessages: messages.length, extractedSkills }
   })
 }
+
+export function runSkillExtraction(
+  data: SkillExtractionJob,
+  services: SkillExtractionServices,
+): Promise<SkillExtractionRunResult> {
+  const { databaseService, learnedSkillService, socialChatHistoryService, runtimeAdapters } = services
+  const orgRef = ensureRecordId(data.orgId, TABLES.ORGANIZATION)
+  const orgId = recordIdToString(orgRef, TABLES.ORGANIZATION)
+  const workspaceProvider = runtimeAdapters.workspaceProvider
+  if (!hasCursorAwareWorkspaceProvider(workspaceProvider)) {
+    serverLogger.info`Skipping skill extraction for ${orgId}: workspaceProvider background cursor methods are not configured`
+    return Promise.resolve({ skipped: true, processedMessages: 0, extractedSkills: 0 })
+  }
+
+  const embeddings = new ProviderEmbeddings({
+    modelId: services.embeddingModel,
+    openRouterApiKey: services.openRouterApiKey,
+  })
+  const withMemoryLock = runtimeAdapters.withWorkspaceMemoryLock
+  const runExtraction = () =>
+    Effect.runPromise(
+      runSkillExtractionEffect(
+        {
+          databaseService,
+          learnedSkillService,
+          socialChatHistoryService,
+          runtimeAdapters,
+          embeddingModel: services.embeddingModel,
+        },
+        orgRef,
+        orgId,
+        workspaceProvider,
+        embeddings,
+      ),
+    )
+
+  return withMemoryLock ? withMemoryLock(orgId, runExtraction) : runExtraction()
+}

package/src/workers/utils/file-section-chunker.ts

@@ -50,7 +50,7 @@ function normalizeMinChunkChars(value: number | undefined, maxChars: number): nu
   return Math.min(normalized, Math.floor(maxChars * 0.6))
 }
 
-async function splitTextByCharBudget(text: string, maxChars: number): Promise<string[]> {
+function splitTextByCharBudget(text: string, maxChars: number): string[] {
   const source = text.trim()
   if (!source) return []
   if (source.length <= maxChars) return [source]
@@ -157,16 +157,16 @@ function mergeTinyTailParts(parts: string[], options: { minChunkChars: number; m
   return merged
 }
 
-async function splitOversizedSection(
+function splitOversizedSection(
   section: FileSection,
   options: { maxChars: number; minChunkChars: number; preserveCodeFenceIntegrity: boolean },
-): Promise<FileSection[]> {
+): FileSection[] {
   if (section.content.length <= options.maxChars) {
     return [section]
   }
 
   if (section.kind !== 'file' || !section.filePath) {
-    const chunks = await splitTextByCharBudget(section.content, options.maxChars)
+    const chunks = splitTextByCharBudget(section.content, options.maxChars)
     return chunks.map((content) => ({ kind: section.kind, content }))
   }
 
@@ -223,10 +223,10 @@ function mergeTinyChunks(
   return merged
 }
 
-export async function chunkFileSections(
+export function chunkFileSections(
   fileSections: readonly FileSection[],
   options: FileSectionChunkOptions = {},
-): Promise<FileSectionChunk[]> {
+): FileSectionChunk[] {
   const maxChars = normalizeMaxChars(options.maxChars)
   const minChunkChars = normalizeMinChunkChars(options.minChunkChars, maxChars)
   const preserveCodeFenceIntegrity = options.preserveCodeFenceIntegrity ?? true
@@ -235,10 +235,8 @@ export async function chunkFileSections(
     .map((section) => ({ kind: section.kind, content: section.content.trim(), filePath: section.filePath }))
     .filter((section) => section.content.length > 0)
 
-  const splitSections = await Promise.all(
-    rawSections.map(
-      async (section) => await splitOversizedSection(section, { maxChars, minChunkChars, preserveCodeFenceIntegrity }),
-    ),
+  const splitSections = rawSections.map((section) =>
+    splitOversizedSection(section, { maxChars, minChunkChars, preserveCodeFenceIntegrity }),
   )
   const sections = splitSections.flat()