@lota-sdk/core 0.4.8 → 0.4.10

package/src/services/thread/thread-active-run.ts

@@ -0,0 +1,203 @@
+import { Effect } from 'effect'
+import { surql } from 'surrealdb'
+
+import { serverLogger } from '../../config/logger'
+import type { RecordIdRef } from '../../db/record-id'
+import { ensureRecordId, recordIdToString } from '../../db/record-id'
+import type { SurrealDBService } from '../../db/service'
+import { TABLES } from '../../db/tables'
+import { ActiveThreadRunConflictError, DatabaseError, RedisError } from '../../effect/errors'
+import type { RedisConnectionManager } from '../../redis/connection'
+import { withLeaseLock } from '../../redis/redis-lease-lock'
+import type { ThreadRecordStore } from './thread-record-store'
+
+// Background turns can spend several seconds inside model/tool orchestration even
+// when the process is healthy. A 5s lease was short enough to lose legitimate
+// runs under normal jitter, so keep a wider safety margin while still expiring
+// crashed runs quickly enough for recovery.
+const THREAD_ACTIVE_RUN_LOCK_TTL_MS = 30_000
+const THREAD_ACTIVE_RUN_LOCK_MAX_WAIT_MS = 750
+const THREAD_ACTIVE_RUN_LOCK_RETRY_DELAY_MS = 75
+const THREAD_ACTIVE_RUN_LOCK_REFRESH_INTERVAL_MS = 5_000
+
+type ChatRunRegistryLike = {
+  has(runId: string): boolean
+  stopEffect(runId: string, reason: DOMException): Effect.Effect<boolean>
+}
+
+function buildActiveRunLockKey(threadId: RecordIdRef): string {
+  return `thread-active-run:${recordIdToString(ensureRecordId(threadId, TABLES.THREAD), TABLES.THREAD)}`
+}
+
+function normalizeActiveTurnValue(value: unknown): string | null {
+  if (typeof value !== 'string') {
+    return null
+  }
+
+  const normalized = value.trim()
+  return normalized.length > 0 ? normalized : null
+}
+
+export function createThreadActiveRunHelpers(deps: {
+  db: SurrealDBService
+  threadStore: ThreadRecordStore
+  redis: RedisConnectionManager
+  chatRunRegistry: ChatRunRegistryLike
+}) {
+  const setActiveTurnEffect = (threadId: RecordIdRef, runId: string, streamId?: string | null) =>
+    Effect.gen(function* () {
+      const threadRef = ensureRecordId(threadId, TABLES.THREAD)
+      if (streamId === null || streamId === undefined) {
+        yield* deps.db
+          .query(
+            surql`
+              UPDATE ONLY ${threadRef}
+              SET activeRunId = ${runId},
+                  activeStreamId = NONE
+            `,
+          )
+          .pipe(Effect.mapError((cause) => new DatabaseError({ message: 'Failed to set active run state.', cause })))
+        return
+      }
+
+      yield* deps.db
+        .query(
+          surql`
+            UPDATE ONLY ${threadRef}
+            SET activeRunId = ${runId},
+                activeStreamId = ${streamId}
+          `,
+        )
+        .pipe(Effect.mapError((cause) => new DatabaseError({ message: 'Failed to set active run state.', cause })))
+    })
+
+  const getActiveTurnEffect = (threadId: RecordIdRef) =>
+    Effect.gen(function* () {
+      const thread = yield* deps.threadStore
+        .getById(threadId)
+        .pipe(Effect.mapError((cause) => new DatabaseError({ message: 'Failed to load active thread state.', cause })))
+      return {
+        runId: normalizeActiveTurnValue(thread.activeRunId),
+        streamId: normalizeActiveTurnValue(thread.activeStreamId),
+      }
+    })
+
+  const getActiveRunIdEffect = (threadId: RecordIdRef) =>
+    Effect.gen(function* () {
+      const { runId } = yield* getActiveTurnEffect(threadId)
+      return runId
+    })
+
+  const hasActiveRunLeaseEffect = (threadId: RecordIdRef) =>
+    Effect.gen(function* () {
+      const count = yield* Effect.tryPromise({
+        try: () => deps.redis.getConnection().exists(buildActiveRunLockKey(threadId)),
+        catch: (error) => new RedisError({ message: 'Failed to inspect active run lease.', cause: error }),
+      })
+      return count > 0
+    })
+
+  const getActiveStreamIdEffect = (threadId: RecordIdRef) =>
+    Effect.gen(function* () {
+      const { streamId } = yield* getActiveTurnEffect(threadId)
+      return streamId
+    })
+
+  const clearActiveTurnEffect = (threadId: RecordIdRef, params: { runId: string; streamId?: string | null }) =>
+    Effect.gen(function* () {
+      const threadRef = ensureRecordId(threadId, TABLES.THREAD)
+      const currentStreamId = params.streamId ?? null
+      if (currentStreamId === null) {
+        yield* deps.db
+          .query(
+            surql`UPDATE ONLY ${threadRef} SET activeRunId = NONE, activeStreamId = NONE WHERE activeRunId = ${params.runId}`,
+          )
+          .pipe(Effect.mapError((cause) => new DatabaseError({ message: 'Failed to clear active run state.', cause })))
+        return
+      }
+
+      yield* deps.db
+        .query(surql`
+          UPDATE ONLY ${threadRef}
+          SET activeRunId = NONE,
+              activeStreamId = NONE
+          WHERE activeRunId = ${params.runId} AND activeStreamId = ${currentStreamId}
+        `)
+        .pipe(Effect.mapError((cause) => new DatabaseError({ message: 'Failed to clear active run state.', cause })))
+    })
+
+  const clearStaleActiveRunIfMissingFromRegistryEffect = (threadId: RecordIdRef) =>
+    Effect.gen(function* () {
+      const { runId: activeRunId, streamId: activeStreamId } = yield* getActiveTurnEffect(threadId)
+      if (!activeRunId) {
+        return false
+      }
+
+      const hasLease = yield* hasActiveRunLeaseEffect(threadId)
+      if (hasLease) {
+        return false
+      }
+
+      yield* clearActiveTurnEffect(threadId, { runId: activeRunId, streamId: activeStreamId })
+      serverLogger.warn`Cleared stale thread run after lease expired: thread=${recordIdToString(ensureRecordId(threadId, TABLES.THREAD), TABLES.THREAD)} run=${activeRunId}`
+      return true
+    })
+
+  const stopActiveRunEffect = (threadId: RecordIdRef) =>
+    Effect.gen(function* () {
+      const { runId: activeRunId } = yield* getActiveTurnEffect(threadId)
+      if (!activeRunId) return false
+
+      const stopped = yield* deps.chatRunRegistry.stopEffect(
+        activeRunId,
+        new DOMException('Run stopped by user.', 'AbortError'),
+      )
+      if (stopped) {
+        return true
+      }
+
+      return yield* clearStaleActiveRunIfMissingFromRegistryEffect(threadId)
+    })
+
+  function withActiveRunLease<T, E, R>(
+    threadId: RecordIdRef,
+    fn: (signal: AbortSignal) => Effect.Effect<T, E, R>,
+  ): Effect.Effect<T, E | ActiveThreadRunConflictError | RedisError, R> {
+    return withLeaseLock(
+      {
+        redis: deps.redis.getConnection(),
+        lockKey: buildActiveRunLockKey(threadId),
+        lockTtlMs: THREAD_ACTIVE_RUN_LOCK_TTL_MS,
+        refreshIntervalMs: THREAD_ACTIVE_RUN_LOCK_REFRESH_INTERVAL_MS,
+        retryDelayMs: THREAD_ACTIVE_RUN_LOCK_RETRY_DELAY_MS,
+        maxWaitMs: THREAD_ACTIVE_RUN_LOCK_MAX_WAIT_MS,
+        label: 'thread active run',
+        logger: serverLogger,
+      },
+      fn,
+    ).pipe(
+      Effect.catchTag('LockAcquisitionError', () =>
+        Effect.gen(function* () {
+          const activeRunId = (yield* Effect.catch(getActiveRunIdEffect(threadId), () => Effect.succeed(null))) ?? ''
+          return yield* new ActiveThreadRunConflictError({
+            threadId: recordIdToString(ensureRecordId(threadId, TABLES.THREAD), TABLES.THREAD),
+            activeRunId,
+            message: 'A chat run is already active.',
+          })
+        }),
+      ),
+    )
+  }
+
+  return {
+    setActiveTurn: setActiveTurnEffect,
+    getActiveTurn: getActiveTurnEffect,
+    getActiveRunId: getActiveRunIdEffect,
+    hasActiveRunLease: hasActiveRunLeaseEffect,
+    withActiveRunLease,
+    getActiveStreamId: getActiveStreamIdEffect,
+    clearActiveTurn: clearActiveTurnEffect,
+    clearStaleActiveRunIfMissingFromRegistry: clearStaleActiveRunIfMissingFromRegistryEffect,
+    stopActiveRun: stopActiveRunEffect,
+  }
+}

package/src/services/thread/thread-bootstrap.ts

@@ -0,0 +1,385 @@
+import { THREAD } from '@lota-sdk/shared'
+import { Effect } from 'effect'
+
+import { getAgentDisplayNames, getAgentRoster, getCoreThreadProfile } from '../../config/agent-defaults'
+import { serverLogger } from '../../config/logger'
+import { getThreadBootstrapConfig } from '../../config/thread-defaults'
+import type { RecordIdRef } from '../../db/record-id'
+import { ensureRecordId, recordIdToString } from '../../db/record-id'
+import { TABLES } from '../../db/tables'
+import { BadRequestError, DatabaseError } from '../../effect/errors'
+import type { LockAcquisitionError, LockLostError, NotFoundError, RedisError, ServiceError } from '../../effect/errors'
+import type { RedisConnectionManager } from '../../redis/connection'
+import { withLeaseLock } from '../../redis/redis-lease-lock'
+import type { makeThreadMessageService } from './thread-message.service'
+import type { ThreadRecordStore } from './thread-record-store'
+import type { NormalizedThread, ThreadRecord } from './thread.types'
+
+const THREAD_BOOTSTRAP_LOCK_TTL_MS = 15_000
+const THREAD_BOOTSTRAP_LOCK_REFRESH_INTERVAL_MS = 5_000
+const THREAD_BOOTSTRAP_LOCK_MAX_WAIT_MS = 5_000
+const THREAD_BOOTSTRAP_LOCK_RETRY_DELAY_MS = 100
+
+function getAgentDisplayName(agentId: string): string {
+  return getAgentDisplayNames()[agentId] ?? agentId
+}
+
+function buildBootstrapThreadsLockKey(userId: RecordIdRef, orgId: RecordIdRef): string {
+  return `thread-bootstrap:${recordIdToString(ensureRecordId(userId, TABLES.USER), TABLES.USER)}:${recordIdToString(ensureRecordId(orgId, TABLES.ORGANIZATION), TABLES.ORGANIZATION)}`
+}
+
+function haveSameMembers(left: string[], right: string[]): boolean {
+  return left.length === right.length && left.every((value, index) => value === right[index])
+}
+
+type ThreadNormalizationError = BadRequestError | ServiceError
+type ThreadBootstrapError =
+  | BadRequestError
+  | DatabaseError
+  | LockAcquisitionError
+  | LockLostError
+  | NotFoundError
+  | RedisError
+  | ServiceError
+
+type NormalizedThreadFactory = (thread: ThreadRecord) => Effect.Effect<NormalizedThread, ThreadNormalizationError>
+type DuplicateCreateErrorLike = { cause?: unknown; message?: unknown }
+
+function isDuplicateCreateErrorLike(value: unknown): value is DuplicateCreateErrorLike {
+  return (typeof value === 'object' || typeof value === 'function') && value !== null
+}
+
+export function createThreadBootstrapHelpers(deps: {
+  threadStore: ThreadRecordStore
+  threadMessageService: Pick<ReturnType<typeof makeThreadMessageService>, 'ensureBootstrapWelcomeMessageEffect'>
+  redis: RedisConnectionManager
+  normalizeThread: NormalizedThreadFactory
+}) {
+  const syncThreadConfigEffect = (
+    record: ThreadRecord,
+    config?: { title?: string; nameGenerated?: boolean; members?: string[] },
+  ) => {
+    if (!config) {
+      return Effect.succeed(record)
+    }
+
+    const updates: Partial<ThreadRecord> = {}
+    if (config.title !== undefined && record.title !== config.title) {
+      updates.title = config.title
+    }
+    if (config.nameGenerated !== undefined && record.nameGenerated !== config.nameGenerated) {
+      updates.nameGenerated = config.nameGenerated
+    }
+    if (config.members !== undefined && !haveSameMembers(record.members, config.members)) {
+      updates.members = config.members
+    }
+
+    if (Object.keys(updates).length === 0) {
+      return Effect.succeed(record)
+    }
+
+    return deps.threadStore.update(record.id, updates)
+  }
+
+  const findThreadByUniqueLookupEffect = (lookup: {
+    type: 'default' | 'thread'
+    organizationId: RecordIdRef
+    userId: RecordIdRef
+    agentId?: string
+    threadType?: string
+  }) => deps.threadStore.findThreadByUniqueLookup(lookup)
+
+  const waitForExistingThreadEffect = (lookup: {
+    type: 'default' | 'thread'
+    organizationId: RecordIdRef
+    userId: RecordIdRef
+    agentId?: string
+    threadType?: string
+  }) => deps.threadStore.waitForExistingThread(lookup)
+
+  const isDuplicateCreateError = (error: unknown): boolean => {
+    const seen = new Set<unknown>()
+    let current = error
+
+    while (isDuplicateCreateErrorLike(current) && !seen.has(current)) {
+      seen.add(current)
+      if (typeof current.message === 'string' && current.message.includes('already contains')) {
+        return true
+      }
+      current = current.cause
+    }
+
+    return false
+  }
+
+  const normalizeThreadEffect = (thread: ThreadRecord): Effect.Effect<NormalizedThread, ThreadNormalizationError> =>
+    deps.normalizeThread(thread)
+
+  const getOrCreateDefaultEffect = (
+    orgId: RecordIdRef,
+    userId: RecordIdRef,
+    agentId: string,
+    config?: { title?: string; nameGenerated?: boolean },
+  ): Effect.Effect<{ created: boolean; record: ThreadRecord }, DatabaseError | NotFoundError> => {
+    const lookup = { type: 'default' as const, organizationId: orgId, userId, agentId }
+
+    return Effect.gen(function* () {
+      const existing = yield* findThreadByUniqueLookupEffect(lookup)
+      if (existing) {
+        const record = yield* syncThreadConfigEffect(existing, config)
+        return { record, created: false as const }
+      }
+
+      return yield* Effect.matchEffect(
+        deps.threadStore.create({
+          type: 'default',
+          organizationId: orgId,
+          userId,
+          agentId,
+          members: [agentId],
+          title: config?.title ?? getAgentDisplayName(agentId),
+          status: 'active',
+          nameGenerated: config?.nameGenerated ?? false,
+          isCompacting: false,
+          turnCount: 0,
+        }),
+        {
+          onFailure: (error) => {
+            if (!isDuplicateCreateError(error)) {
+              return Effect.fail(error)
+            }
+
+            return Effect.gen(function* () {
+              const existingThread = yield* waitForExistingThreadEffect(lookup)
+              const record = yield* syncThreadConfigEffect(existingThread, config)
+              return { record, created: false as const }
+            })
+          },
+          onSuccess: (record) => Effect.succeed({ record, created: true as const }),
+        },
+      )
+    })
+  }
+
+  const getOrCreateThreadEffect = (
+    orgId: RecordIdRef,
+    userId: RecordIdRef,
+    threadType: string,
+    config: { members: string[]; title: string; nameGenerated?: boolean },
+  ): Effect.Effect<{ created: boolean; record: ThreadRecord }, DatabaseError | NotFoundError> => {
+    const lookup = { type: 'thread' as const, organizationId: orgId, userId, threadType }
+
+    return Effect.gen(function* () {
+      const existing = yield* findThreadByUniqueLookupEffect(lookup)
+      if (existing) {
+        const record = yield* syncThreadConfigEffect(existing, config)
+        return { record, created: false as const }
+      }
+
+      return yield* Effect.matchEffect(
+        deps.threadStore.create({
+          type: 'thread',
+          organizationId: orgId,
+          userId,
+          threadType,
+          members: config.members,
+          title: config.title,
+          status: 'active',
+          nameGenerated: config.nameGenerated ?? false,
+          isCompacting: false,
+          turnCount: 0,
+        }),
+        {
+          onFailure: (error) => {
+            if (!isDuplicateCreateError(error)) {
+              return Effect.fail(error)
+            }
+
+            return Effect.gen(function* () {
+              const existingThread = yield* waitForExistingThreadEffect(lookup)
+              const record = yield* syncThreadConfigEffect(existingThread, config)
+              return { record, created: false as const }
+            })
+          },
+          onSuccess: (record) => Effect.succeed({ record, created: true as const }),
+        },
+      )
+    })
+  }
+
+  const createThreadEffect = (input: {
+    userId: RecordIdRef
+    organizationId: RecordIdRef
+    type: string
+    agentId?: string
+    threadType?: string
+    members?: string[]
+    title?: string
+  }): Effect.Effect<NormalizedThread, ThreadBootstrapError> =>
+    Effect.gen(function* () {
+      switch (input.type) {
+        case 'default':
+          if (!input.agentId) return yield* new BadRequestError({ message: 'Default threads require agentId' })
+          if (input.threadType) return yield* new BadRequestError({ message: 'Default threads cannot have threadType' })
+          break
+        case 'topic':
+          if (!input.agentId) return yield* new BadRequestError({ message: 'Topic threads require agentId' })
+          if (input.threadType) return yield* new BadRequestError({ message: 'Topic threads cannot have threadType' })
+          break
+        case 'thread':
+          if (!input.threadType) return yield* new BadRequestError({ message: 'Thread threads require threadType' })
+          if (input.agentId) return yield* new BadRequestError({ message: 'Thread threads cannot have agentId' })
+          break
+        case 'group':
+          if (input.agentId) return yield* new BadRequestError({ message: 'Group threads cannot have agentId' })
+          if (input.threadType) return yield* new BadRequestError({ message: 'Group threads cannot have threadType' })
+          break
+      }
+
+      const title = input.title ?? THREAD.DEFAULT_TITLE
+      const nameGenerated = input.title !== undefined && input.title !== THREAD.DEFAULT_TITLE
+
+      if (input.type === 'default') {
+        const agentId = input.agentId
+        if (!agentId) return yield* new BadRequestError({ message: 'Default threads require agentId' })
+        const { record } = yield* getOrCreateDefaultEffect(input.organizationId, input.userId, agentId, undefined)
+        return yield* normalizeThreadEffect(record)
+      }
+
+      if (input.type === 'thread') {
+        const threadType = input.threadType
+        if (!threadType) return yield* new BadRequestError({ message: 'Thread threads require threadType' })
+        const { record } = yield* getOrCreateThreadEffect(input.organizationId, input.userId, threadType, {
+          members: input.members ?? [...getAgentRoster()],
+          title,
+          nameGenerated,
+        })
+        return yield* normalizeThreadEffect(record)
+      }
+
+      const thread = yield* deps.threadStore.create({
+        userId: input.userId,
+        organizationId: input.organizationId,
+        type: input.type,
+        agentId: input.agentId,
+        threadType: input.threadType,
+        members: input.members ?? [...getAgentRoster()],
+        title,
+        status: 'active',
+        nameGenerated,
+        isCompacting: false,
+        turnCount: 0,
+      })
+
+      return yield* normalizeThreadEffect(thread)
+    })
+
+  const ensureBootstrapThreadsEffect = (
+    userId: RecordIdRef,
+    orgId: RecordIdRef,
+    options?: { onboardStatus?: string; userName?: string | null },
+  ): Effect.Effect<void, ThreadBootstrapError> => {
+    const bootstrapConfig = getThreadBootstrapConfig()
+
+    return withLeaseLock(
+      {
+        redis: deps.redis.getConnection(),
+        lockKey: buildBootstrapThreadsLockKey(userId, orgId),
+        lockTtlMs: THREAD_BOOTSTRAP_LOCK_TTL_MS,
+        refreshIntervalMs: THREAD_BOOTSTRAP_LOCK_REFRESH_INTERVAL_MS,
+        retryDelayMs: THREAD_BOOTSTRAP_LOCK_RETRY_DELAY_MS,
+        maxWaitMs: THREAD_BOOTSTRAP_LOCK_MAX_WAIT_MS,
+        label: 'thread bootstrap',
+        logger: serverLogger,
+      },
+      (signal) =>
+        Effect.gen(function* () {
+          const failIfAborted = () =>
+            signal.aborted
+              ? Effect.fail(new DatabaseError({ message: 'Thread bootstrap lease was aborted.', cause: signal.reason }))
+              : Effect.void
+
+          yield* failIfAborted()
+          const onboardStatus = options?.onboardStatus ?? 'completed'
+          const onboardingCompleted = onboardStatus === 'completed'
+
+          const existingThreads = yield* deps.threadStore.findAll({ userId, organizationId: orgId })
+          yield* failIfAborted()
+
+          const hasGroupThread = existingThreads.some((t) => t.type === 'group')
+          const defaultThreadsByAgent = new Map<string, ThreadRecord>()
+          const threadThreadsByType = new Map<string, ThreadRecord>()
+
+          for (const thread of existingThreads) {
+            if (thread.type === 'default' && thread.agentId) {
+              defaultThreadsByAgent.set(thread.agentId, thread)
+            }
+            if (thread.type === 'thread' && typeof thread.threadType === 'string') {
+              threadThreadsByType.set(thread.threadType, thread)
+            }
+          }
+
+          const requiredDefaultAgents = onboardingCompleted
+            ? bootstrapConfig.completedDefaultAgents
+            : bootstrapConfig.onboardingDefaultAgents
+
+          for (const agentId of requiredDefaultAgents) {
+            if (defaultThreadsByAgent.has(agentId)) continue
+            yield* getOrCreateDefaultEffect(orgId, userId, agentId)
+            yield* failIfAborted()
+          }
+
+          if (onboardingCompleted && bootstrapConfig.ensureDefaultGroupOnCompleted && !hasGroupThread) {
+            const normalized = yield* createThreadEffect({
+              userId,
+              organizationId: orgId,
+              type: 'group',
+              title: THREAD.DEFAULT_TITLE,
+            })
+            yield* deps.threadStore.getById(normalized.id)
+          }
+
+          if (onboardingCompleted) {
+            for (const wsType of bootstrapConfig.threadTypesAfterOnboarding) {
+              if (threadThreadsByType.has(wsType)) continue
+              const profile = getCoreThreadProfile(wsType)
+              yield* getOrCreateThreadEffect(orgId, userId, wsType, {
+                members: [...profile.members],
+                title: profile.config.title,
+              })
+              yield* failIfAborted()
+            }
+          }
+
+          if (!onboardingCompleted && bootstrapConfig.onboardingWelcome) {
+            const onboardingWelcome = bootstrapConfig.onboardingWelcome
+            const ownerThread = defaultThreadsByAgent.get(onboardingWelcome.defaultAgentId)
+            if (ownerThread?.id) {
+              yield* failIfAborted()
+              const ownerThreadRef = ensureRecordId(ownerThread.id, TABLES.THREAD)
+              yield* deps.threadMessageService
+                .ensureBootstrapWelcomeMessageEffect({
+                  threadId: ownerThreadRef,
+                  agentId: onboardingWelcome.defaultAgentId,
+                  text: onboardingWelcome.buildMessageText({ userName: options?.userName }),
+                })
+                .pipe(
+                  Effect.mapError(
+                    (error) =>
+                      new DatabaseError({ message: 'Failed to write onboarding welcome message.', cause: error }),
+                  ),
+                )
+              yield* failIfAborted()
+            }
+          }
+        }),
+    )
+  }
+
+  return {
+    getOrCreateDefault: getOrCreateDefaultEffect,
+    getOrCreateThread: getOrCreateThreadEffect,
+    createThread: createThreadEffect,
+    ensureBootstrapThreads: ensureBootstrapThreadsEffect,
+  }
+}