@lota-sdk/core 0.1.5

package/src/index.ts

@@ -0,0 +1,302 @@
+import type { CoreWorkstreamProfile } from './config/agent-defaults'
+import type { LotaWorkstreamConfig } from './config/workstream-defaults'
+import type { SurrealDBService } from './db/service'
+import type { startContextCompactionWorker } from './queues/context-compaction.queue'
+import type {
+  scheduleRecurringConsolidation,
+  startMemoryConsolidationWorker,
+} from './queues/memory-consolidation.queue'
+import type { startPostChatMemoryWorker } from './queues/post-chat-memory.queue'
+import type { startRecentActivityTitleRefinementWorker } from './queues/recent-activity-title-refinement.queue'
+import type { startRegularChatMemoryDigestWorker } from './queues/regular-chat-memory-digest.queue'
+import type { startSkillExtractionWorker } from './queues/skill-extraction.queue'
+import type { RedisConnectionManager } from './redis/connection'
+import type { isApprovalContinuationRequest } from './runtime/approval-continuation'
+import type { LotaPlugin } from './runtime/plugin-types'
+import type { LotaRuntimeAdapters, LotaRuntimeTurnHooks } from './runtime/runtime-extensions'
+import type { attachmentService } from './services/attachment.service'
+import type { executionPlanService } from './services/execution-plan.service'
+import type { memoryService } from './services/memory.service'
+import type { verifyMutatingApproval } from './services/mutating-approval.service'
+import type { recentActivityTitleService } from './services/recent-activity-title.service'
+import type { recentActivityService } from './services/recent-activity.service'
+import type { workstreamMessageService } from './services/workstream-message.service'
+import type { workstreamTitleService } from './services/workstream-title.service'
+import type {
+  createWorkstreamApprovalContinuationStream,
+  createWorkstreamTurnStream,
+  runWorkstreamTurnInBackground,
+} from './services/workstream-turn'
+import type { workstreamService } from './services/workstream.service'
+import type { generatedDocumentStorageService } from './storage/generated-document-storage.service'
+
+export type LotaAgentFactoryRegistry = Record<string, (...args: unknown[]) => unknown>
+
+interface LotaRuntimeBuiltInWorkers {
+  startContextCompactionWorker: typeof startContextCompactionWorker
+  startMemoryConsolidationWorker: typeof startMemoryConsolidationWorker
+  startPostChatMemoryWorker: typeof startPostChatMemoryWorker
+  startRecentActivityTitleRefinementWorker: typeof startRecentActivityTitleRefinementWorker
+  startRegularChatMemoryDigestWorker: typeof startRegularChatMemoryDigestWorker
+  startSkillExtractionWorker: typeof startSkillExtractionWorker
+  scheduleRecurringConsolidation: typeof scheduleRecurringConsolidation
+}
+
+export interface LotaRuntimeConfig {
+  database: { url: string; namespace: string; database: string; username: string; password: string }
+  redis: { url: string }
+  aiGateway: { url: string; key: string; admin?: string; pass?: string; embeddingModel?: string }
+  s3: {
+    endpoint: string
+    bucket: string
+    region?: string
+    accessKeyId: string
+    secretAccessKey: string
+    attachmentUrlExpiresIn?: number
+  }
+  firecrawl: { apiKey: string; apiBaseUrl?: string }
+  logging?: { level?: 'trace' | 'debug' | 'info' | 'warning' | 'error' | 'fatal' }
+  memory?: { searchK?: number }
+  workstreams?: LotaWorkstreamConfig
+
+  agents: {
+    roster: readonly string[]
+    displayNames: Record<string, string>
+    shortDisplayNames?: Record<string, string>
+    teamConsultParticipants: readonly string[]
+    getCoreWorkstreamProfile?: (coreType: string) => CoreWorkstreamProfile
+    createAgent?: LotaAgentFactoryRegistry
+    buildAgentTools?: (...args: unknown[]) => unknown
+    getAgentRuntimeConfig?: (...args: unknown[]) => unknown
+  }
+
+  toolProviders?: Record<string, unknown>
+  extraSchemaFiles?: Array<string | URL>
+  extraWorkers?: Record<string, unknown>
+  pluginRuntime?: Record<string, LotaPlugin>
+  runtimeAdapters?: LotaRuntimeAdapters
+  turnHooks?: LotaRuntimeTurnHooks
+}
+
+export interface LotaRuntime {
+  services: {
+    database: SurrealDBService
+    databaseService: SurrealDBService
+    redis: RedisConnectionManager
+    closeRedisConnection: () => Promise<void>
+    attachmentService: typeof attachmentService
+    generatedDocumentStorageService: typeof generatedDocumentStorageService
+    memoryService: typeof memoryService
+    verifyMutatingApproval: typeof verifyMutatingApproval
+    recentActivityService: typeof recentActivityService
+    recentActivityTitleService: typeof recentActivityTitleService
+    executionPlanService: typeof executionPlanService
+    workstreamMessageService: typeof workstreamMessageService
+    workstreamService: typeof workstreamService
+    workstreamTitleService: typeof workstreamTitleService
+    createWorkstreamApprovalContinuationStream: typeof createWorkstreamApprovalContinuationStream
+    createWorkstreamTurnStream: typeof createWorkstreamTurnStream
+    isApprovalContinuationRequest: typeof isApprovalContinuationRequest
+    runWorkstreamTurnInBackground: typeof runWorkstreamTurnInBackground
+  }
+  redis: {
+    manager: RedisConnectionManager
+    getConnection: () => ReturnType<RedisConnectionManager['getConnection']>
+    getConnectionForBullMQ: () => ReturnType<RedisConnectionManager['getConnectionForBullMQ']>
+    closeConnection: () => Promise<void>
+  }
+  workers: LotaRuntimeBuiltInWorkers & Record<string, unknown>
+  schemaFiles: Array<string | URL>
+  contributions: { envKeys: readonly string[]; schemaFiles: Array<string | URL> }
+  config: LotaRuntimeConfig
+  plugins: Record<string, LotaPlugin>
+  connectPluginDatabases(): Promise<void>
+  connect(): Promise<void>
+  disconnect(): Promise<void>
+}
+
+export async function createLotaRuntime(config: LotaRuntimeConfig): Promise<LotaRuntime> {
+  const { lotaSdkEnvKeys, setEnv } = await import('./config/env-shapes')
+  const { configureLogger } = await import('./config/logger')
+  const { SurrealDBService: SurrealDBServiceClass, setDatabaseService } = await import('./db/service')
+  const { createRedisConnectionManager } = await import('./redis/connection')
+  const { setRedisConnectionManager } = await import('./redis/index')
+  const { configureAgents, configureAgentFactory } = await import('./config/agent-defaults')
+  const { configureWorkstreams } = await import('./config/workstream-defaults')
+  const { configureRuntimeExtensions } = await import('./runtime/runtime-extensions')
+
+  setEnv({
+    AI_GATEWAY_URL: config.aiGateway.url,
+    AI_GATEWAY_KEY: config.aiGateway.key,
+    AI_GATEWAY_ADMIN: config.aiGateway.admin,
+    AI_GATEWAY_PASS: config.aiGateway.pass,
+    AI_EMBEDDING_MODEL: config.aiGateway.embeddingModel ?? 'openai/text-embedding-3-small',
+    REDIS_URL: config.redis.url,
+    LOG_LEVEL: config.logging?.level ?? 'info',
+    S3_ENDPOINT: config.s3.endpoint,
+    S3_BUCKET: config.s3.bucket,
+    S3_REGION: config.s3.region ?? 'garage',
+    S3_ACCESS_KEY_ID: config.s3.accessKeyId,
+    S3_SECRET_ACCESS_KEY: config.s3.secretAccessKey,
+    ATTACHMENT_URL_EXPIRES_IN: config.s3.attachmentUrlExpiresIn ?? 1800,
+    FIRECRAWL_API_KEY: config.firecrawl.apiKey,
+    FIRECRAWL_API_BASE_URL: config.firecrawl.apiBaseUrl,
+    MEMORY_SEARCH_K: config.memory?.searchK ?? 6,
+  })
+
+  await configureLogger(config.logging?.level ?? 'info')
+
+  const db = new SurrealDBServiceClass({
+    url: config.database.url,
+    namespace: config.database.namespace,
+    database: config.database.database,
+    username: config.database.username,
+    password: config.database.password,
+  })
+  setDatabaseService(db)
+
+  const redisManager = createRedisConnectionManager({ url: config.redis.url })
+  setRedisConnectionManager(redisManager)
+
+  configureAgents({
+    roster: config.agents.roster,
+    displayNames: config.agents.displayNames,
+    shortDisplayNames: config.agents.shortDisplayNames,
+    teamConsultParticipants: config.agents.teamConsultParticipants,
+    getCoreWorkstreamProfile: config.agents.getCoreWorkstreamProfile,
+  })
+  configureWorkstreams({ agentRoster: config.agents.roster, config: config.workstreams })
+
+  if (config.agents.createAgent || config.agents.buildAgentTools || config.agents.getAgentRuntimeConfig) {
+    configureAgentFactory({
+      createAgent: config.agents.createAgent ?? {},
+      buildAgentTools: config.agents.buildAgentTools,
+      getAgentRuntimeConfig: config.agents.getAgentRuntimeConfig,
+      pluginRuntime: config.pluginRuntime,
+    })
+  }
+
+  const { attachmentService } = await import('./services/attachment.service')
+  const { recentActivityService } = await import('./services/recent-activity.service')
+  const { recentActivityTitleService } = await import('./services/recent-activity-title.service')
+  const { executionPlanService } = await import('./services/execution-plan.service')
+  const { memoryService } = await import('./services/memory.service')
+  const { verifyMutatingApproval } = await import('./services/mutating-approval.service')
+  const { workstreamMessageService } = await import('./services/workstream-message.service')
+  const { workstreamService } = await import('./services/workstream.service')
+  const { workstreamTitleService } = await import('./services/workstream-title.service')
+  const {
+    createWorkstreamApprovalContinuationStream,
+    createWorkstreamTurnStream,
+    isApprovalContinuationRequest,
+    runWorkstreamTurnInBackground,
+  } = await import('./services/workstream-turn')
+  const { generatedDocumentStorageService } = await import('./storage/generated-document-storage.service')
+  const { startContextCompactionWorker } = await import('./queues/context-compaction.queue')
+  const { scheduleRecurringConsolidation, startMemoryConsolidationWorker } =
+    await import('./queues/memory-consolidation.queue')
+  const { startPostChatMemoryWorker } = await import('./queues/post-chat-memory.queue')
+  const { startRecentActivityTitleRefinementWorker } = await import('./queues/recent-activity-title-refinement.queue')
+  const { startRegularChatMemoryDigestWorker } = await import('./queues/regular-chat-memory-digest.queue')
+  const { startSkillExtractionWorker } = await import('./queues/skill-extraction.queue')
+
+  configureRuntimeExtensions({
+    adapters: config.runtimeAdapters,
+    turnHooks: config.turnHooks,
+    toolProviders: (config.toolProviders ?? {}) as never,
+    extraWorkers: config.extraWorkers,
+  })
+
+  const pluginRuntime = config.pluginRuntime ?? {}
+  const pluginContributions = Object.values(pluginRuntime).map((plugin) => plugin.contributions)
+  const schemaFiles = [
+    ...getBuiltInSchemaFiles(),
+    ...pluginContributions.flatMap((plugin) => plugin.schemaFiles),
+    ...(config.extraSchemaFiles ?? []),
+  ]
+  const contributionEnvKeys = [...lotaSdkEnvKeys, ...pluginContributions.flatMap((plugin) => plugin.envKeys)]
+  const connectPluginDatabases = createPluginDatabaseConnector(pluginRuntime)
+  const builtInWorkers = {
+    startContextCompactionWorker,
+    startMemoryConsolidationWorker,
+    startPostChatMemoryWorker,
+    startRecentActivityTitleRefinementWorker,
+    startRegularChatMemoryDigestWorker,
+    startSkillExtractionWorker,
+    scheduleRecurringConsolidation,
+  } satisfies LotaRuntimeBuiltInWorkers
+
+  return {
+    services: {
+      database: db,
+      databaseService: db,
+      redis: redisManager,
+      closeRedisConnection: async () => await redisManager.closeConnection(),
+      attachmentService,
+      generatedDocumentStorageService,
+      memoryService,
+      verifyMutatingApproval,
+      recentActivityService,
+      recentActivityTitleService,
+      executionPlanService,
+      workstreamMessageService,
+      workstreamService,
+      workstreamTitleService,
+      createWorkstreamApprovalContinuationStream,
+      createWorkstreamTurnStream,
+      isApprovalContinuationRequest,
+      runWorkstreamTurnInBackground,
+    },
+    redis: {
+      manager: redisManager,
+      getConnection: () => redisManager.getConnection(),
+      getConnectionForBullMQ: () => redisManager.getConnectionForBullMQ(),
+      closeConnection: async () => await redisManager.closeConnection(),
+    },
+    workers: { ...builtInWorkers, ...config.extraWorkers },
+    schemaFiles,
+    contributions: { envKeys: [...new Set(contributionEnvKeys)], schemaFiles },
+    config,
+    plugins: pluginRuntime,
+    async connectPluginDatabases() {
+      await connectPluginDatabases()
+    },
+    async connect() {
+      await db.connect()
+    },
+    async disconnect() {
+      await db.disconnect()
+      await redisManager.closeConnection()
+    },
+  }
+}
+
+function getBuiltInSchemaFiles(): URL[] {
+  return [
+    new URL('../infrastructure/schema/00_workstream.surql', import.meta.url),
+    new URL('../infrastructure/schema/01_memory.surql', import.meta.url),
+    new URL('../infrastructure/schema/02_execution_plan.surql', import.meta.url),
+    new URL('../infrastructure/schema/03_learned_skill.surql', import.meta.url),
+    new URL('../infrastructure/schema/04_runtime_bootstrap.surql', import.meta.url),
+  ]
+}
+
+function createPluginDatabaseConnector(pluginRuntime: Record<string, LotaPlugin>): () => Promise<void> {
+  return async () => {
+    for (const plugin of Object.values(pluginRuntime)) {
+      const services = plugin.services as Record<string, unknown>
+      const connectDatabase = services.connectDatabase
+      if (typeof connectDatabase !== 'function') {
+        continue
+      }
+
+      await Reflect.apply(connectDatabase, services, [])
+    }
+  }
+}
+
+export type { CoreWorkstreamProfile } from './config/agent-defaults'
+export type { SurrealDBService } from './db/service'
+export type { RedisConnectionManager } from './redis/connection'
+export type { LotaPlugin, LotaPluginContributions } from './runtime/plugin-types'
+export type { LotaRuntimeAdapters, LotaRuntimeTurnHooks } from './runtime/runtime-extensions'

package/src/queues/context-compaction.queue.ts

@@ -0,0 +1,82 @@
+import { Queue, Worker } from 'bullmq'
+import type { Job } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import { ensureRecordId } from '../db/record-id'
+import { databaseService } from '../db/service'
+import { TABLES } from '../db/tables'
+import { getRedisConnectionForBullMQ } from '../redis'
+import { contextCompactionService } from '../services/context-compaction.service'
+import { workstreamService } from '../services/workstream.service'
+import {
+  attachWorkerEvents,
+  createTracedWorkerProcessor,
+  createWorkerShutdown,
+  registerShutdownSignals,
+} from '../workers/worker-utils'
+import type { WorkerHandle } from '../workers/worker-utils'
+
+interface ContextCompactionJob {
+  domain: 'workstream'
+  entityId: string
+  contextSize?: number
+}
+
+const CONTEXT_COMPACTION_QUEUE = 'context-compaction'
+
+let _contextCompactionQueue: Queue<ContextCompactionJob> | null = null
+function getContextCompactionQueue(): Queue<ContextCompactionJob> {
+  if (!_contextCompactionQueue) {
+    _contextCompactionQueue = new Queue<ContextCompactionJob>(CONTEXT_COMPACTION_QUEUE, {
+      connection: getRedisConnectionForBullMQ(),
+      defaultJobOptions: {
+        removeOnComplete: 200,
+        removeOnFail: 200,
+        attempts: 2,
+        backoff: { type: 'exponential', delay: 3_000 },
+      },
+    })
+  }
+  return _contextCompactionQueue
+}
+
+export async function enqueueContextCompaction(job: ContextCompactionJob) {
+  return await getContextCompactionQueue().add('compact', job, {
+    deduplication: { id: `compact:${job.domain}:${job.entityId}` },
+  })
+}
+
+async function processContextCompactionJob(job: Job<ContextCompactionJob>): Promise<void> {
+  await databaseService.connect()
+
+  const { entityId, contextSize } = job.data
+  const workstreamRef = ensureRecordId(entityId, TABLES.WORKSTREAM)
+  await workstreamService.setCompacting(workstreamRef, true)
+  try {
+    await contextCompactionService.compactWorkstreamHistory({ workstreamId: workstreamRef, contextSize })
+  } finally {
+    await workstreamService.setCompacting(workstreamRef, false)
+  }
+}
+
+export function startContextCompactionWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
+  const { registerSignals = import.meta.main } = options
+  const worker = new Worker(
+    CONTEXT_COMPACTION_QUEUE,
+    createTracedWorkerProcessor(CONTEXT_COMPACTION_QUEUE, processContextCompactionJob),
+    { connection: getRedisConnectionForBullMQ(), concurrency: 2, lockDuration: 300_000 },
+  )
+
+  attachWorkerEvents(worker, 'Context compaction', serverLogger)
+  const shutdown = createWorkerShutdown(worker, 'Context compaction', serverLogger)
+
+  if (registerSignals) {
+    registerShutdownSignals({ name: 'Context compaction', shutdown, logger: serverLogger })
+  }
+
+  return { worker, shutdown }
+}
+
+if (import.meta.main) {
+  startContextCompactionWorker()
+}

package/src/queues/document-processor.queue.ts

@@ -0,0 +1,118 @@
+import { createHash } from 'node:crypto'
+
+import { Queue, Worker } from 'bullmq'
+import type IORedis from 'ioredis'
+
+import type { chatLogger } from '../config/logger'
+import { attachWorkerEvents, createWorkerShutdown, registerShutdownSignals } from '../workers/worker-utils'
+import type { WorkerHandle } from '../workers/worker-utils'
+
+export type DocumentSourceChannel = string
+
+export interface DocumentProcessorAttachmentRef {
+  storageKey: string
+  name: string
+  contentType: string
+  sizeBytes?: number
+}
+
+export interface DocumentProcessorJob {
+  orgId: string
+  source: DocumentSourceChannel
+  sourceId: string
+  title: string
+  contentType?: string
+  sourceCanonicalKey: string
+  sourceVersionKey: string
+  text?: string
+  attachment?: DocumentProcessorAttachmentRef
+  metadata?: Record<string, unknown>
+}
+
+const DEFAULT_DOCUMENT_PROCESSOR_QUEUE = 'document-processor'
+const DEFAULT_WORKER_NAME = 'Document processor'
+
+export function buildDocumentProcessorJobId(
+  job: Pick<
+    DocumentProcessorJob,
+    'orgId' | 'source' | 'sourceId' | 'sourceCanonicalKey' | 'sourceVersionKey' | 'title'
+  >,
+): string {
+  const digest = createHash('sha256')
+    .update(
+      JSON.stringify({
+        orgId: job.orgId,
+        source: job.source,
+        sourceId: job.sourceId,
+        sourceCanonicalKey: job.sourceCanonicalKey,
+        sourceVersionKey: job.sourceVersionKey,
+        title: job.title,
+      }),
+    )
+    .digest('hex')
+
+  return `doc__${digest}`
+}
+
+export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcessorJob>(params: {
+  getConnectionForBullMQ: () => IORedis
+  getWorkerPath: () => string
+  logger: typeof chatLogger
+  queueName?: string
+  workerName?: string
+  concurrency?: number
+  lockDuration?: number
+}): {
+  enqueue: (job: TJob) => Promise<unknown>
+  startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle
+} {
+  const queueName = params.queueName ?? DEFAULT_DOCUMENT_PROCESSOR_QUEUE
+  const workerName = params.workerName ?? DEFAULT_WORKER_NAME
+  const concurrency = params.concurrency ?? 4
+  const lockDuration = params.lockDuration ?? 300_000
+  const jobName = 'process-document' as Parameters<Queue<TJob, unknown, string>['add']>[0]
+  const toQueueData = (job: TJob): Parameters<Queue<TJob, unknown, string>['add']>[1] =>
+    job as Parameters<Queue<TJob, unknown, string>['add']>[1]
+  let queue: Queue<TJob, unknown, string> | null = null
+
+  const getQueue = (): Queue<TJob, unknown, string> => {
+    if (queue) {
+      return queue
+    }
+
+    queue = new Queue<TJob, unknown, string>(queueName, {
+      connection: params.getConnectionForBullMQ(),
+      defaultJobOptions: {
+        removeOnComplete: 200,
+        removeOnFail: 200,
+        attempts: 3,
+        backoff: { type: 'exponential', delay: 1000 },
+      },
+    })
+
+    return queue
+  }
+
+  return {
+    enqueue: async (job) =>
+      await getQueue().add(jobName, toQueueData(job), { jobId: buildDocumentProcessorJobId(job) }),
+    startWorker: (options = {}) => {
+      const { registerSignals = import.meta.main } = options
+      const worker = new Worker(queueName, params.getWorkerPath(), {
+        connection: params.getConnectionForBullMQ(),
+        concurrency,
+        lockDuration,
+      })
+
+      attachWorkerEvents(worker, workerName, params.logger)
+
+      const shutdown = createWorkerShutdown(worker, workerName, params.logger)
+
+      if (registerSignals) {
+        registerShutdownSignals({ name: workerName, shutdown, logger: params.logger })
+      }
+
+      return { worker, shutdown }
+    },
+  }
+}

package/src/queues/memory-consolidation.queue.ts

@@ -0,0 +1,65 @@
+import { Queue, Worker } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import { getRedisConnectionForBullMQ } from '../redis'
+import {
+  attachWorkerEvents,
+  getWorkerPath,
+  createWorkerShutdown,
+  registerShutdownSignals,
+} from '../workers/worker-utils'
+import type { WorkerHandle } from '../workers/worker-utils'
+
+export interface MemoryConsolidationJob {
+  scopeId?: string
+}
+
+const MEMORY_CONSOLIDATION_QUEUE = 'memory-consolidation'
+
+let _memoryConsolidationQueue: Queue<MemoryConsolidationJob> | null = null
+function getMemoryConsolidationQueue(): Queue<MemoryConsolidationJob> {
+  if (!_memoryConsolidationQueue) {
+    _memoryConsolidationQueue = new Queue<MemoryConsolidationJob>(MEMORY_CONSOLIDATION_QUEUE, {
+      connection: getRedisConnectionForBullMQ(),
+      defaultJobOptions: {
+        removeOnComplete: 50,
+        removeOnFail: 50,
+        attempts: 2,
+        backoff: { type: 'exponential', delay: 5000 },
+      },
+    })
+  }
+  return _memoryConsolidationQueue
+}
+
+export async function scheduleRecurringConsolidation() {
+  await getMemoryConsolidationQueue().add(
+    'consolidate',
+    {},
+    { repeat: { every: 24 * 60 * 60 * 1000 }, jobId: 'memory-consolidation-recurring' },
+  )
+}
+
+export function startMemoryConsolidationWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
+  const { registerSignals = import.meta.main } = options
+  const processorPath = getWorkerPath('memory-consolidation.worker.ts')
+  const worker = new Worker(MEMORY_CONSOLIDATION_QUEUE, processorPath, {
+    connection: getRedisConnectionForBullMQ(),
+    lockDuration: 600_000,
+    concurrency: 1,
+  })
+
+  attachWorkerEvents(worker, 'Memory consolidation', serverLogger)
+
+  const shutdown = createWorkerShutdown(worker, 'Memory consolidation', serverLogger)
+
+  if (registerSignals) {
+    registerShutdownSignals({ name: 'Memory consolidation', shutdown, logger: serverLogger })
+  }
+
+  return { worker, shutdown }
+}
+
+if (import.meta.main) {
+  startMemoryConsolidationWorker()
+}

package/src/queues/post-chat-memory.queue.ts

@@ -0,0 +1,128 @@
+type OrganizationOnboardStatus = string
+import { Queue, Worker } from 'bullmq'
+import type { Job } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import { databaseService } from '../db/service'
+import { getRedisConnectionForBullMQ } from '../redis'
+import { memoryService } from '../services/memory.service'
+import {
+  attachWorkerEvents,
+  createTracedWorkerProcessor,
+  createWorkerShutdown,
+  registerShutdownSignals,
+} from '../workers/worker-utils'
+import type { WorkerHandle } from '../workers/worker-utils'
+
+interface PostChatMemoryMessage {
+  role: 'user' | 'agent'
+  content: string
+  agentName?: string
+}
+
+interface PostChatMemoryExtractionJob {
+  orgId: string
+  workstreamId: string
+  sourceId: string
+  onboardStatus?: OrganizationOnboardStatus
+  userMessage: string
+  historyMessages: PostChatMemoryMessage[]
+  agentMessages: Array<{ content: string; agentName?: string }>
+  memoryBlock?: string
+  attachmentContext?: string
+}
+
+const POST_CHAT_MEMORY_QUEUE = 'post-chat-memory'
+const POST_CHAT_MEMORY_CONCURRENCY = 3
+const POST_CHAT_MEMORY_LOCK_DURATION_MS = 900_000
+const POST_CHAT_MEMORY_MAX_STALLED_COUNT = 10
+const POST_CHAT_MEMORY_STALLED_INTERVAL_MS = 120_000
+
+let _postChatMemoryQueue: Queue<PostChatMemoryExtractionJob> | null = null
+function getPostChatMemoryQueue(): Queue<PostChatMemoryExtractionJob> {
+  if (!_postChatMemoryQueue) {
+    _postChatMemoryQueue = new Queue<PostChatMemoryExtractionJob>(POST_CHAT_MEMORY_QUEUE, {
+      connection: getRedisConnectionForBullMQ(),
+      defaultJobOptions: {
+        removeOnComplete: 200,
+        removeOnFail: 200,
+        attempts: 3,
+        backoff: { type: 'exponential', delay: 2_000 },
+      },
+    })
+  }
+  return _postChatMemoryQueue
+}
+
+export async function enqueuePostChatMemory(job: PostChatMemoryExtractionJob) {
+  return await getPostChatMemoryQueue().add('extract-memory', job)
+}
+
+async function processPostChatMemoryJob(job: Job<PostChatMemoryExtractionJob>): Promise<void> {
+  await databaseService.connect()
+
+  const data = job.data
+  const userMessage = data.userMessage.trim()
+  const agentMessages = data.agentMessages
+    .map((item) => ({
+      content: item.content.trim(),
+      ...(typeof item.agentName === 'string' && item.agentName.trim() ? { agentName: item.agentName.trim() } : {}),
+    }))
+    .filter((item) => item.content.length > 0)
+
+  if (!userMessage || agentMessages.length === 0) return
+
+  const joinedOutput = agentMessages
+    .map((item) => (item.agentName ? `[${item.agentName}] ${item.content}` : item.content))
+    .join('\n\n')
+
+  const uniqueAgentNames = [
+    ...new Set(
+      agentMessages
+        .map((item) => item.agentName)
+        .filter((value): value is string => typeof value === 'string' && value.length > 0),
+    ),
+  ]
+
+  await memoryService.addConversationMemories({
+    orgId: data.orgId,
+    input: userMessage,
+    output: joinedOutput,
+    sourceId: data.sourceId,
+    onboardStatus: data.onboardStatus,
+    ...(uniqueAgentNames.length > 0 ? { agentName: uniqueAgentNames[0] } : {}),
+    historyMessages: data.historyMessages,
+    memoryBlock: data.memoryBlock,
+    attachmentContext: data.attachmentContext,
+    agentNames: uniqueAgentNames,
+  })
+}
+
+export function startPostChatMemoryWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
+  const { registerSignals = import.meta.main } = options
+  const worker = new Worker(
+    POST_CHAT_MEMORY_QUEUE,
+    createTracedWorkerProcessor(POST_CHAT_MEMORY_QUEUE, processPostChatMemoryJob),
+    {
+      connection: getRedisConnectionForBullMQ(),
+      concurrency: POST_CHAT_MEMORY_CONCURRENCY,
+      lockDuration: POST_CHAT_MEMORY_LOCK_DURATION_MS,
+      maxStalledCount: POST_CHAT_MEMORY_MAX_STALLED_COUNT,
+      stalledInterval: POST_CHAT_MEMORY_STALLED_INTERVAL_MS,
+    },
+  )
+
+  attachWorkerEvents(worker, 'Post-chat memory', serverLogger)
+
+  const shutdown = createWorkerShutdown(worker, 'Post-chat memory', serverLogger)
+
+  if (registerSignals) {
+    registerShutdownSignals({ name: 'Post-chat memory', shutdown, logger: serverLogger })
+  }
+
+  return { worker, shutdown }
+}
+
+if (import.meta.main) {
+  startPostChatMemoryWorker()
+}