@lota-sdk/core 0.4.8 → 0.4.10

package/src/workers/utils/repomix-file-sections.ts

@@ -34,9 +34,9 @@ export function parseRepomixFileSections(repomixOutput: string): FileSection[] {
   return sections
 }
 
-export async function chunkRepomixFileSections(
+export function chunkRepomixFileSections(
   repomixOutput: string,
   options: FileSectionChunkOptions = {},
-): Promise<FileSectionChunk[]> {
+): FileSectionChunk[] {
   return chunkFileSections(parseRepomixFileSections(repomixOutput), options)
 }

package/src/workers/utils/thread-message-query.ts

@@ -1,93 +1,152 @@
-import { requireTimestamp } from '@lota-sdk/shared'
+import { parseRowMetadata, requireTimestamp } from '@lota-sdk/shared'
+import type { ChatMessage } from '@lota-sdk/shared'
+import { Schema, Effect } from 'effect'
 import { BoundQuery } from 'surrealdb'
 import { z } from 'zod'
 
 import { ensureRecordId, recordIdToString } from '../../db/record-id'
 import type { RecordIdRef } from '../../db/record-id'
-import { databaseService } from '../../db/service'
+import type { SurrealDBService } from '../../db/service'
 import { TABLES } from '../../db/tables'
 import { ThreadMessageRowSchema } from '../../db/thread-message-row'
 import type { ThreadMessageRow } from '../../db/thread-message-row'
 import { normalizeTextBody } from '../../document/parsing'
+import { makeEffectTryPromiseWithMessage } from '../../effect/helpers'
 import type { LotaRuntimeBackgroundCursor } from '../../runtime/runtime-extensions'
+import type { SocialChatHistoryMessage } from '../../services/social-chat-history.service'
+import { unsafeDateFrom } from '../../utils/date-time'
 
-export type DigestCursor = LotaRuntimeBackgroundCursor
-
-export interface DigestMessage {
-  source: 'thread' | 'social'
+export interface ThreadDigestMessage {
+  source: 'thread'
   sourceId: string
   role: 'system' | 'user' | 'assistant'
-  parts: Array<Record<string, unknown>>
-  metadata?: Record<string, unknown>
-  cursor: DigestCursor
+  parts: ChatMessage['parts']
+  metadata?: ChatMessage['metadata']
+  cursor: LotaRuntimeBackgroundCursor
 }
 
-function mapThreadRow(row: ThreadMessageRow): DigestMessage {
+export type SocialDigestMessage = Pick<
+  SocialChatHistoryMessage,
+  'source' | 'sourceId' | 'role' | 'parts' | 'metadata' | 'cursor'
+>
+export type DigestMessage = ThreadDigestMessage | SocialDigestMessage
+
+function normalizeMessageValue(value: unknown): unknown {
+  if (value instanceof Date) {
+    return value.toISOString()
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((item) => normalizeMessageValue(item))
+  }
+
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>).map(([key, entry]) => [key, normalizeMessageValue(entry)]),
+    )
+  }
+
+  return value
+}
+
+function readPersistedMessageParts(parts: ThreadMessageRow['parts']): ChatMessage['parts'] {
+  return (Array.isArray(parts) ? normalizeMessageValue(parts) : []) as ChatMessage['parts']
+}
+
+class ThreadMessageQueryError extends Schema.TaggedErrorClass<ThreadMessageQueryError>()('ThreadMessageQueryError', {
+  message: Schema.String,
+  cause: Schema.Defect,
+}) {}
+
+function mapThreadRow(row: ThreadMessageRow): ThreadDigestMessage {
   return {
     source: 'thread',
     sourceId: recordIdToString(row.threadId, TABLES.THREAD),
     role: row.role,
-    parts: row.parts as Array<Record<string, unknown>>,
-    metadata: row.metadata ?? undefined,
+    parts: readPersistedMessageParts(row.parts),
+    metadata: parseRowMetadata(row.metadata),
     cursor: {
-      createdAt: new Date(requireTimestamp(row.createdAt)),
+      createdAt: unsafeDateFrom(requireTimestamp(row.createdAt)),
       id: recordIdToString(row.id, TABLES.THREAD_MESSAGE),
     },
   }
 }
 
+const effectTryPromise = makeEffectTryPromiseWithMessage(
+  (message, cause) => new ThreadMessageQueryError({ message, cause }),
+)
+
 export function compareDigestMessageOrder(left: DigestMessage, right: DigestMessage): number {
   const timeDiff = left.cursor.createdAt.getTime() - right.cursor.createdAt.getTime()
   if (timeDiff !== 0) return timeDiff
   return left.cursor.id.localeCompare(right.cursor.id)
 }
 
-export async function listThreadIdsForOrg(orgRef: RecordIdRef): Promise<RecordIdRef[]> {
+export function listThreadIdsForOrg(db: SurrealDBService, orgRef: RecordIdRef): Promise<RecordIdRef[]> {
   const EntityIdRowSchema = z.string().trim().min(1)
-  const ids = await databaseService.query<unknown>(
-    new BoundQuery(
-      `SELECT VALUE type::string(id) FROM ${TABLES.THREAD}
+  return Effect.runPromise(
+    effectTryPromise(
+      () =>
+        db.query<unknown>(
+          new BoundQuery(
+            `SELECT VALUE type::string(id) FROM ${TABLES.THREAD}
        WHERE organizationId = $organizationId`,
-      { organizationId: orgRef },
+            { organizationId: orgRef },
+          ),
+        ),
+      'Failed to list thread ids for org digest.',
+    ).pipe(
+      Effect.map((ids) => ids.map((value: unknown) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.THREAD))),
     ),
   )
-  return ids.map((value) => ensureRecordId(EntityIdRowSchema.parse(value), TABLES.THREAD))
 }
 
-export async function listEligibleThreadMessages(params: {
+export function listEligibleThreadMessages(params: {
+  db: SurrealDBService
   threadIds: RecordIdRef[]
-  cursor: DigestCursor | null
+  cursor: LotaRuntimeBackgroundCursor | null
   onboardingCutoff: Date | null
 }): Promise<DigestMessage[]> {
-  if (params.threadIds.length === 0) return []
+  if (params.threadIds.length === 0) return Promise.resolve([])
 
-  let query: BoundQuery | null = null
-  if (params.cursor) {
-    const cursorRowId = ensureRecordId(params.cursor.id, TABLES.THREAD_MESSAGE)
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(threadId) AS threadId, role, parts, metadata, createdAt FROM ${TABLES.THREAD_MESSAGE}
+  return Effect.runPromise(
+    Effect.gen(function* () {
+      const query = params.cursor
+        ? new BoundQuery(
+            `SELECT type::string(id) AS id, type::string(threadId) AS threadId, role, parts, metadata, createdAt FROM ${TABLES.THREAD_MESSAGE}
        WHERE threadId IN $threadIds
          AND (
            createdAt > $cursorCreatedAt
            OR (createdAt = $cursorCreatedAt AND id > $cursorRowId)
          )
        ORDER BY createdAt ASC, id ASC`,
-      { threadIds: params.threadIds, cursorCreatedAt: params.cursor.createdAt, cursorRowId },
-    )
-  } else if (params.onboardingCutoff) {
-    query = new BoundQuery(
-      `SELECT type::string(id) AS id, type::string(threadId) AS threadId, role, parts, metadata, createdAt FROM ${TABLES.THREAD_MESSAGE}
+            {
+              threadIds: params.threadIds,
+              cursorCreatedAt: params.cursor.createdAt,
+              cursorRowId: ensureRecordId(params.cursor.id, TABLES.THREAD_MESSAGE),
+            },
+          )
+        : params.onboardingCutoff
+          ? new BoundQuery(
+              `SELECT type::string(id) AS id, type::string(threadId) AS threadId, role, parts, metadata, createdAt FROM ${TABLES.THREAD_MESSAGE}
        WHERE threadId IN $threadIds
          AND createdAt > $onboardingCutoff
        ORDER BY createdAt ASC, id ASC`,
-      { threadIds: params.threadIds, onboardingCutoff: params.onboardingCutoff },
-    )
-  }
+              { threadIds: params.threadIds, onboardingCutoff: params.onboardingCutoff },
+            )
+          : null
 
-  if (!query) return []
+      if (!query) {
+        return []
+      }
 
-  const rows = await databaseService.query<unknown>(query)
-  return rows.map((row) => mapThreadRow(ThreadMessageRowSchema.parse(row)))
+      const rows = yield* effectTryPromise(
+        () => params.db.query<unknown>(query),
+        'Failed to list eligible thread messages.',
+      )
+      return rows.map((row: unknown) => mapThreadRow(ThreadMessageRowSchema.parse(row)))
+    }),
+  )
 }
 
 export function normalizeBlock(value: string): string {

package/src/workers/worker-utils.ts

@@ -1,10 +1,27 @@
-import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 
 import type { Job, Worker } from 'bullmq'
+import { Effect } from 'effect'
+import type { Context } from 'effect'
 
 import { chatLogger } from '../config/logger'
-import { queueJobService } from '../services/queue-job.service'
+import { resolveLotaService } from '../effect/runtime'
+import { QueueJobServiceTag } from '../services/queue-job.service'
+
+let currentQueueJobService: Context.Service.Shape<typeof QueueJobServiceTag> | null = null
+
+export function configureQueueJobService(service: Context.Service.Shape<typeof QueueJobServiceTag>): void {
+  currentQueueJobService = service
+}
+
+export function clearQueueJobService(): void {
+  currentQueueJobService = null
+}
+
+export function getQueueJobService(): Context.Service.Shape<typeof QueueJobServiceTag> {
+  return currentQueueJobService ?? resolveLotaService(QueueJobServiceTag)
+}
+
 export const DEFAULT_JOB_RETENTION = { removeOnComplete: true, removeOnFail: { age: 24 * 60 * 60, count: 200 } }
 export const LOW_JOB_RETENTION = { removeOnComplete: true, removeOnFail: { age: 6 * 60 * 60, count: 50 } }
 export const LONG_JOB_LOCK_DURATION_MS = 600_000
@@ -12,7 +29,7 @@ export const LONG_JOB_LOCK_DURATION_MS = 600_000
 const DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000
 
 export function getWorkerPath(workerName: string): string {
-  return fileURLToPath(new URL(path.join('.', workerName), import.meta.url))
+  return fileURLToPath(new URL(workerName, import.meta.url))
 }
 
 export interface WorkerHandle {
@@ -72,9 +89,9 @@ export const attachWorkerEvents = (worker: Worker, name: string, logger: typeof
 }
 
 export const createWorkerShutdown = (worker: Worker, name: string, logger: typeof chatLogger = chatLogger) => {
-  return async () => {
+  return () => {
     logger.info`Shutting down ${name} worker`
-    await worker.close()
+    return Effect.runPromise(Effect.asVoid(Effect.tryPromise(() => worker.close())))
   }
 }
 
@@ -82,7 +99,7 @@ export function createTracedWorkerProcessor<TJob extends TracedWorkerJobLike, TR
   queueName: string,
   processor: (job: TJob) => Promise<TResult>,
 ): (job: TJob) => Promise<TResult> {
-  return async (job: TJob) => {
+  return (job: TJob) => {
     const trackedJob = {
       queueName,
       id: typeof job.id === 'string' || typeof job.id === 'number' ? job.id : undefined,
@@ -93,31 +110,52 @@ export function createTracedWorkerProcessor<TJob extends TracedWorkerJobLike, TR
       timestamp: job.timestamp,
     }
 
-    try {
-      await queueJobService.markAttemptStarted(trackedJob)
-    } catch (error) {
-      chatLogger.error`Failed to persist queue job start (queue=${queueName}, job=${job.id}): ${error}`
+    const logPersistenceError = (phase: string, error: unknown) => {
+      chatLogger.error`Failed to persist queue job ${phase} (queue=${queueName}, job=${job.id}): ${error}`
     }
 
-    try {
-      const result = await processor(job)
-      try {
-        await queueJobService.markAttemptCompleted(trackedJob, result)
-      } catch (error) {
-        chatLogger.error`Failed to persist queue job completion (queue=${queueName}, job=${job.id}): ${error}`
-      }
-      return result
-    } catch (error) {
-      try {
-        await queueJobService.markAttemptFailed(trackedJob, error)
-      } catch (persistenceError) {
-        chatLogger.error`Failed to persist queue job failure (queue=${queueName}, job=${job.id}): ${persistenceError}`
-      }
-      throw error
-    }
+    return Effect.runPromise(
+      Effect.catch(
+        Effect.gen(function* () {
+          yield* Effect.catch(getQueueJobService().markAttemptStarted(trackedJob), (error) =>
+            Effect.sync(() => {
+              logPersistenceError('start', error)
+            }),
+          )
+
+          const result = yield* Effect.tryPromise(() => processor(job))
+
+          yield* Effect.catch(getQueueJobService().markAttemptCompleted(trackedJob, result), (error) =>
+            Effect.sync(() => {
+              logPersistenceError('completion', error)
+            }),
+          )
+
+          return result
+        }),
+        (error) =>
+          Effect.gen(function* () {
+            yield* Effect.catch(getQueueJobService().markAttemptFailed(trackedJob, error), (persistenceError) =>
+              Effect.sync(() => {
+                logPersistenceError('failure', persistenceError)
+              }),
+            )
+            return yield* error
+          }),
+      ).pipe(
+        Effect.withSpan(`worker.${queueName}.${job.name}`),
+        Effect.annotateSpans({
+          'job.id': trackedJob.id !== undefined ? String(trackedJob.id) : 'unknown',
+          'job.name': job.name,
+          'job.attemptsMade': typeof job.attemptsMade === 'number' ? job.attemptsMade : -1,
+          'queue.name': queueName,
+        }),
+      ),
+    )
   }
 }
 
+/** Registers process signal handlers for sandboxed workers that run outside an Effect scope. */
 export const registerShutdownSignals = ({
   name,
   shutdown,
@@ -130,21 +168,26 @@ export const registerShutdownSignals = ({
   timeoutMs?: number
 }) => {
   let shuttingDown = false
+  let forcedExitArmed = true
 
   const handleSignal = () => {
     if (shuttingDown) return
     shuttingDown = true
-    const timeout = setTimeout(() => {
-      logger.warn`Forced shutdown after ${timeoutMs}ms`
-      process.exit(0)
-    }, timeoutMs)
+    void Effect.runFork(
+      Effect.gen(function* () {
+        yield* Effect.tryPromise(() => Bun.sleep(timeoutMs))
+        if (!forcedExitArmed) return
+        logger.warn`Forced shutdown after ${timeoutMs}ms`
+        process.exit(0)
+      }),
+    )
 
     shutdown()
       .catch((err: unknown) => {
         logger.error`Failed to shutdown ${name} worker: ${err}`
       })
       .finally(() => {
-        clearTimeout(timeout)
+        forcedExitArmed = false
         process.exit(0)
       })
   }

package/src/config/debug-logger.ts

@@ -1,47 +0,0 @@
-import { chatLogger } from './logger'
-
-const isDebug = (): boolean => {
-  const level = process.env.LOG_LEVEL
-  const flag = process.env.LOTA_DEBUG
-  return level === 'debug' || flag === '1'
-}
-
-interface DebugTimer {
-  step(name: string): void
-  elapsed(): number
-}
-
-const NOOP_TIMER: DebugTimer = { step() {}, elapsed: () => 0 }
-
-function createTimer(label: string): DebugTimer {
-  const start = performance.now()
-  let lastStep = start
-
-  return {
-    step(name: string) {
-      const now = performance.now()
-      const stepMs = now - lastStep
-      const totalMs = now - start
-      chatLogger.debug`[ttft:${label}] ${name}: ${stepMs.toFixed(1)}ms (elapsed: ${totalMs.toFixed(1)}ms)`
-      lastStep = now
-    },
-    elapsed() {
-      return performance.now() - start
-    },
-  }
-}
-
-export const lotaDebugLogger = {
-  get enabled() {
-    return isDebug()
-  },
-
-  timer(label: string): DebugTimer {
-    return isDebug() ? createTimer(label) : NOOP_TIMER
-  },
-
-  step(name: string) {
-    if (!isDebug()) return
-    chatLogger.debug`[ttft] ${name}`
-  },
-}

package/src/config/search.ts

@@ -1,3 +0,0 @@
-export { VECTOR_SEARCH_OVERFETCH_MULTIPLIER } from '@lota-sdk/shared'
-
-export const DEFAULT_MEMORY_SEARCH_LIMIT = 10

package/src/redis/connection-accessor.ts

@@ -1,26 +0,0 @@
-import type IORedis from 'ioredis'
-
-export interface RedisConnectionAccessor {
-  getConnection(): IORedis
-  getConnectionForBullMQ(): IORedis
-}
-
-let redisManager: RedisConnectionAccessor | undefined
-
-export function setRedisConnectionManager(manager: RedisConnectionAccessor): void {
-  redisManager = manager
-}
-
-export function getRedisConnection(): IORedis {
-  if (!redisManager) {
-    throw new Error('Redis connection manager not configured. Call setRedisConnectionManager() first.')
-  }
-  return redisManager.getConnection()
-}
-
-export function getRedisConnectionForBullMQ(): IORedis {
-  if (!redisManager) {
-    throw new Error('Redis connection manager not configured. Call setRedisConnectionManager() first.')
-  }
-  return redisManager.getConnectionForBullMQ()
-}

package/src/runtime/agent-types.ts

@@ -1 +0,0 @@
-export type { ChatMode, CreateHelperToolLoopAgentOptions, CreateRoutedAgentOptions } from '@lota-sdk/shared'

package/src/runtime/context-compaction-runtime.ts

@@ -1,87 +0,0 @@
-import { createContextCompactionAgent } from '../system-agents/context-compaction.agent'
-import {
-  buildContextCompactionPrompt,
-  buildMemoryBlockCompactionPrompt,
-  ContextCompactionOutputSchema,
-  createContextCompactionRuntime,
-  parseCompactionOutput,
-} from './context-compaction'
-import type { ContextCompactionRunnerParams } from './context-compaction'
-import {
-  COMPACTION_CHUNK_MAX_CHARS,
-  CONTEXT_COMPACTION_INCLUDED_TOOL_NAMES,
-  CONTEXT_COMPACTION_INCLUDED_TOOL_PREFIXES,
-  CONTEXT_COMPACTION_THRESHOLD_RATIO,
-  CONTEXT_OUTPUT_RESERVE_TOKENS,
-  CONTEXT_SAFETY_MARGIN_TOKENS,
-  SUMMARY_ROLLUP_MAX_TOKENS,
-} from './context-compaction-constants'
-import type { GenerateHelperStructuredParams, GenerateHelperTextParams } from './helper-model'
-
-const CONTEXT_COMPACTION_MAX_OUTPUT_TOKENS = 512
-
-interface HelperModelRuntime {
-  generateHelperStructured<T>(params: GenerateHelperStructuredParams<T>): Promise<T>
-  generateHelperText(params: GenerateHelperTextParams): Promise<string>
-}
-
-interface CreateContextCompactionRuntimeDeps {
-  helperModelRuntime: HelperModelRuntime
-  now?: () => number
-  randomId?: () => string
-}
-
-async function runContextCompacter(helperModelRuntime: HelperModelRuntime, params: ContextCompactionRunnerParams) {
-  const output = await helperModelRuntime.generateHelperStructured({
-    tag: 'context-compaction',
-    createAgent: createContextCompactionAgent,
-    messages: [
-      {
-        role: 'user',
-        content: buildContextCompactionPrompt({
-          previousSummary: params.previousSummary,
-          transcript: params.transcript,
-        }),
-      },
-    ],
-    schema: ContextCompactionOutputSchema,
-    maxOutputTokens: 8_000,
-  })
-
-  return parseCompactionOutput(output)
-}
-
-export function createWiredContextCompactionRuntime(deps: CreateContextCompactionRuntimeDeps) {
-  const { helperModelRuntime } = deps
-
-  const runtime = createContextCompactionRuntime({
-    runCompacter: (params) => runContextCompacter(helperModelRuntime, params),
-    now: deps.now,
-    randomId: deps.randomId,
-    thresholdRatio: CONTEXT_COMPACTION_THRESHOLD_RATIO,
-    outputReserveTokens: CONTEXT_OUTPUT_RESERVE_TOKENS,
-    safetyMarginTokens: CONTEXT_SAFETY_MARGIN_TOKENS,
-    compactionChunkMaxChars: COMPACTION_CHUNK_MAX_CHARS,
-    summaryRollupMaxTokens: SUMMARY_ROLLUP_MAX_TOKENS,
-    includedToolNames: CONTEXT_COMPACTION_INCLUDED_TOOL_NAMES,
-    includedToolPrefixes: CONTEXT_COMPACTION_INCLUDED_TOOL_PREFIXES,
-  })
-
-  async function compactMemoryBlockSummary(params: {
-    previousSummary: string
-    newEntriesText: string
-  }): Promise<string> {
-    const previousSummary = params.previousSummary.trim()
-    const newEntriesText = params.newEntriesText.trim()
-    if (!previousSummary && !newEntriesText) return ''
-
-    return helperModelRuntime.generateHelperText({
-      tag: 'memory-block-compaction',
-      createAgent: createContextCompactionAgent,
-      messages: [{ role: 'user', content: buildMemoryBlockCompactionPrompt({ previousSummary, newEntriesText }) }],
-      maxOutputTokens: CONTEXT_COMPACTION_MAX_OUTPUT_TOKENS,
-    })
-  }
-
-  return { ...runtime, compactMemoryBlockSummary }
-}

package/src/runtime/memory-scope.ts

@@ -1,43 +0,0 @@
-export const ORG_SCOPE_PREFIX = 'org'
-
-const SCOPE_ID_MAX_LENGTH = 255
-const SCOPE_PREFIX_REGEX = /^[a-z][a-z0-9_]*$/
-
-function stripRecordPrefix(id: string): string {
-  if (typeof id !== 'string' || id.length === 0) {
-    throw new TypeError('id must be a non-empty string')
-  }
-  const [, ...rest] = id.split(':')
-  return rest.length > 0 ? rest.join(':') : id
-}
-
-export function scopeId(prefix: string, id: string): string {
-  if (typeof prefix !== 'string' || prefix.length === 0) {
-    throw new TypeError('prefix must be a non-empty string')
-  }
-  if (!SCOPE_PREFIX_REGEX.test(prefix)) {
-    throw new Error(`Invalid scope prefix: ${prefix}`)
-  }
-
-  const stripped = stripRecordPrefix(id)
-  const scoped = `${prefix}:${stripped}`
-  if (scoped.length > SCOPE_ID_MAX_LENGTH) {
-    throw new Error('scopeId exceeds maximum length')
-  }
-
-  return scoped
-}
-
-export function agentScopeId(orgId: string, agentName: string): string {
-  if (typeof agentName !== 'string' || agentName.trim().length === 0) {
-    throw new TypeError('agentName must be a non-empty string')
-  }
-  const strippedOrgId = stripRecordPrefix(orgId)
-  const scoped = `agent:${strippedOrgId}:${agentName.trim()}`
-
-  if (scoped.length > SCOPE_ID_MAX_LENGTH) {
-    throw new Error('scopeId exceeds maximum length')
-  }
-
-  return scoped
-}