@lota-sdk/core 0.1.5

package/src/queues/recent-activity-title-refinement.queue.ts

@@ -0,0 +1,69 @@
+import { Queue, Worker } from 'bullmq'
+import type { Job } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import { databaseService } from '../db/service'
+import { getRedisConnectionForBullMQ } from '../redis'
+import { recentActivityTitleService } from '../services/recent-activity-title.service'
+import {
+  attachWorkerEvents,
+  createTracedWorkerProcessor,
+  createWorkerShutdown,
+  registerShutdownSignals,
+} from '../workers/worker-utils'
+import type { WorkerHandle } from '../workers/worker-utils'
+
+interface RecentActivityTitleRefinementJob {
+  activityId: string
+}
+
+const RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE = 'recent-activity-title-refinement'
+
+let _recentActivityTitleRefinementQueue: Queue<RecentActivityTitleRefinementJob> | null = null
+function getRecentActivityTitleRefinementQueue(): Queue<RecentActivityTitleRefinementJob> {
+  if (!_recentActivityTitleRefinementQueue) {
+    _recentActivityTitleRefinementQueue = new Queue<RecentActivityTitleRefinementJob>(
+      RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE,
+      {
+        connection: getRedisConnectionForBullMQ(),
+        defaultJobOptions: {
+          removeOnComplete: 200,
+          removeOnFail: 200,
+          attempts: 3,
+          backoff: { type: 'exponential', delay: 2_000 },
+        },
+      },
+    )
+  }
+  return _recentActivityTitleRefinementQueue
+}
+
+export async function enqueueRecentActivityTitleRefinement(job: RecentActivityTitleRefinementJob) {
+  return await getRecentActivityTitleRefinementQueue().add('refine-recent-activity-title', job, {
+    jobId: `recent-activity-title:${job.activityId}`,
+  })
+}
+
+async function processRecentActivityTitleRefinementJob(job: Job<RecentActivityTitleRefinementJob>): Promise<void> {
+  await databaseService.connect()
+  await recentActivityTitleService.refineRecentActivityTitle(job.data.activityId)
+}
+
+export function startRecentActivityTitleRefinementWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
+  const { registerSignals = import.meta.main } = options
+  const worker = new Worker(
+    RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE,
+    createTracedWorkerProcessor(RECENT_ACTIVITY_TITLE_REFINEMENT_QUEUE, processRecentActivityTitleRefinementJob),
+    { connection: getRedisConnectionForBullMQ(), concurrency: 2, lockDuration: 300_000 },
+  )
+
+  attachWorkerEvents(worker, 'Recent activity title refinement', serverLogger)
+
+  const shutdown = createWorkerShutdown(worker, 'Recent activity title refinement', serverLogger)
+
+  if (registerSignals) {
+    registerShutdownSignals({ name: 'Recent activity title refinement', shutdown, logger: serverLogger })
+  }
+
+  return { worker, shutdown }
+}

package/src/queues/regular-chat-memory-digest.config.ts

@@ -0,0 +1,12 @@
+const REGULAR_CHAT_MEMORY_DIGEST_DELAY_MS = 15 * 60 * 1000
+
+export function buildRegularChatMemoryDigestDeduplicationId(orgId: string): string {
+  return `regular-chat-digest:${orgId}`
+}
+
+export function buildRegularChatMemoryDigestJobOptions(orgId: string) {
+  return {
+    delay: REGULAR_CHAT_MEMORY_DIGEST_DELAY_MS,
+    deduplication: { id: buildRegularChatMemoryDigestDeduplicationId(orgId) },
+  }
+}

package/src/queues/regular-chat-memory-digest.queue.ts

@@ -0,0 +1,73 @@
+import { Queue, Worker } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import { getRedisConnectionForBullMQ } from '../redis'
+import {
+  attachWorkerEvents,
+  getWorkerPath,
+  createWorkerShutdown,
+  registerShutdownSignals,
+} from '../workers/worker-utils'
+import type { WorkerHandle } from '../workers/worker-utils'
+import {
+  buildRegularChatMemoryDigestDeduplicationId,
+  buildRegularChatMemoryDigestJobOptions,
+} from './regular-chat-memory-digest.config'
+
+export interface RegularChatMemoryDigestJob {
+  orgId: string
+}
+
+const REGULAR_CHAT_MEMORY_DIGEST_QUEUE = 'regular-chat-memory-digest'
+
+let _regularChatMemoryDigestQueue: Queue<RegularChatMemoryDigestJob> | null = null
+function getRegularChatMemoryDigestQueue(): Queue<RegularChatMemoryDigestJob> {
+  if (!_regularChatMemoryDigestQueue) {
+    _regularChatMemoryDigestQueue = new Queue<RegularChatMemoryDigestJob>(REGULAR_CHAT_MEMORY_DIGEST_QUEUE, {
+      connection: getRedisConnectionForBullMQ(),
+      defaultJobOptions: {
+        removeOnComplete: 200,
+        removeOnFail: 200,
+        attempts: 2,
+        backoff: { type: 'exponential', delay: 5000 },
+      },
+    })
+  }
+  return _regularChatMemoryDigestQueue
+}
+
+export async function enqueueRegularChatMemoryDigest(job: RegularChatMemoryDigestJob) {
+  return await getRegularChatMemoryDigestQueue().add(
+    'run-digest',
+    job,
+    buildRegularChatMemoryDigestJobOptions(job.orgId),
+  )
+}
+
+export async function clearRegularChatMemoryDigestDeduplicationKey(orgId: string): Promise<void> {
+  await getRegularChatMemoryDigestQueue().removeDeduplicationKey(buildRegularChatMemoryDigestDeduplicationId(orgId))
+}
+
+export function startRegularChatMemoryDigestWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
+  const { registerSignals = import.meta.main } = options
+  const processorPath = getWorkerPath('regular-chat-memory-digest.worker.ts')
+  const worker = new Worker(REGULAR_CHAT_MEMORY_DIGEST_QUEUE, processorPath, {
+    connection: getRedisConnectionForBullMQ(),
+    concurrency: 1,
+    lockDuration: 600_000,
+  })
+
+  attachWorkerEvents(worker, 'Regular chat memory digest', serverLogger)
+
+  const shutdown = createWorkerShutdown(worker, 'Regular chat memory digest', serverLogger)
+
+  if (registerSignals) {
+    registerShutdownSignals({ name: 'Regular chat memory digest', shutdown, logger: serverLogger })
+  }
+
+  return { worker, shutdown }
+}
+
+if (import.meta.main) {
+  startRegularChatMemoryDigestWorker()
+}

package/src/queues/skill-extraction.config.ts

@@ -0,0 +1,9 @@
+const SKILL_EXTRACTION_DELAY_MS = 15 * 60 * 1000
+
+export function buildSkillExtractionDeduplicationId(orgId: string): string {
+  return `skill-extraction:${orgId}`
+}
+
+export function buildSkillExtractionJobOptions(orgId: string) {
+  return { delay: SKILL_EXTRACTION_DELAY_MS, deduplication: { id: buildSkillExtractionDeduplicationId(orgId) } }
+}

package/src/queues/skill-extraction.queue.ts

@@ -0,0 +1,62 @@
+import { Queue, Worker } from 'bullmq'
+
+import { serverLogger } from '../config/logger'
+import { getRedisConnectionForBullMQ } from '../redis'
+import {
+  attachWorkerEvents,
+  getWorkerPath,
+  createWorkerShutdown,
+  registerShutdownSignals,
+} from '../workers/worker-utils'
+import type { WorkerHandle } from '../workers/worker-utils'
+import { buildSkillExtractionJobOptions } from './skill-extraction.config'
+
+export interface SkillExtractionJob {
+  orgId: string
+}
+
+const SKILL_EXTRACTION_QUEUE = 'skill-extraction'
+
+let _skillExtractionQueue: Queue<SkillExtractionJob> | null = null
+function getSkillExtractionQueue(): Queue<SkillExtractionJob> {
+  if (!_skillExtractionQueue) {
+    _skillExtractionQueue = new Queue<SkillExtractionJob>(SKILL_EXTRACTION_QUEUE, {
+      connection: getRedisConnectionForBullMQ(),
+      defaultJobOptions: {
+        removeOnComplete: 200,
+        removeOnFail: 200,
+        attempts: 2,
+        backoff: { type: 'exponential', delay: 5000 },
+      },
+    })
+  }
+  return _skillExtractionQueue
+}
+
+export async function enqueueSkillExtraction(job: SkillExtractionJob) {
+  return await getSkillExtractionQueue().add('run-extraction', job, buildSkillExtractionJobOptions(job.orgId))
+}
+
+export function startSkillExtractionWorker(options: { registerSignals?: boolean } = {}): WorkerHandle {
+  const { registerSignals = import.meta.main } = options
+  const processorPath = getWorkerPath('skill-extraction.worker.ts')
+  const worker = new Worker(SKILL_EXTRACTION_QUEUE, processorPath, {
+    connection: getRedisConnectionForBullMQ(),
+    concurrency: 1,
+    lockDuration: 600_000,
+  })
+
+  attachWorkerEvents(worker, 'Skill extraction', serverLogger)
+
+  const shutdown = createWorkerShutdown(worker, 'Skill extraction', serverLogger)
+
+  if (registerSignals) {
+    registerShutdownSignals({ name: 'Skill extraction', shutdown, logger: serverLogger })
+  }
+
+  return { worker, shutdown }
+}
+
+if (import.meta.main) {
+  startSkillExtractionWorker()
+}

package/src/redis/connection.ts

@@ -0,0 +1,176 @@
+import IORedis from 'ioredis'
+import type { RedisOptions } from 'ioredis'
+
+import { getErrorMessage } from '../utils/error'
+
+export interface RedisConnectionLogger {
+  debug?: (message: string) => void
+  info?: (message: string) => void
+  warn?: (message: string) => void
+  error?: (message: string) => void
+}
+
+export interface CreateRedisConnectionManagerOptions {
+  url: string
+  redisOptions?: RedisOptions
+  healthCheckIntervalMs?: number
+  logger?: RedisConnectionLogger
+}
+
+export interface RedisConnectionManager {
+  getConnection(): IORedis
+  getConnectionForBullMQ(): IORedis
+  isConnectionHealthy(): boolean
+  closeConnection(): Promise<void>
+}
+
+const DEFAULT_HEALTH_CHECK_INTERVAL_MS = 30_000
+
+export const DEFAULT_REDIS_OPTIONS: RedisOptions = {
+  maxRetriesPerRequest: null,
+  enableReadyCheck: true,
+  enableOfflineQueue: true,
+  retryStrategy: (times: number) => {
+    const delay = Math.min(times * 50, 2000)
+    return delay
+  },
+  reconnectOnError: (err: Error) => {
+    const targetErrors = ['READONLY', 'ETIMEDOUT', 'ECONNREFUSED', 'EAI_AGAIN']
+    return targetErrors.some((candidate) => err.message.includes(candidate))
+  },
+  connectTimeout: 10000,
+  lazyConnect: false,
+}
+
+function log(logger: RedisConnectionLogger | undefined, level: keyof RedisConnectionLogger, message: string): void {
+  const sink = logger?.[level]
+  if (typeof sink === 'function') {
+    sink(message)
+  }
+}
+
+class RedisConnectionManagerImpl implements RedisConnectionManager {
+  private redis: IORedis | null = null
+  private isHealthy = false
+  private healthCheckInterval: ReturnType<typeof setInterval> | null = null
+  private isInitialized = false
+  private isClosing = false
+
+  constructor(private readonly options: CreateRedisConnectionManagerOptions) {
+    this.initializeConnection()
+  }
+
+  private initializeConnection(): void {
+    if (this.isInitialized) {
+      return
+    }
+
+    this.isInitialized = true
+
+    try {
+      const redisOptions = this.options.redisOptions ?? DEFAULT_REDIS_OPTIONS
+      this.redis = new IORedis(this.options.url, redisOptions)
+      this.setupEventHandlers()
+      this.startHealthChecks()
+    } catch (error) {
+      log(this.options.logger, 'error', `Failed to initialize Redis connection: ${getErrorMessage(error)}`)
+    }
+  }
+
+  private setupEventHandlers(): void {
+    if (!this.redis) return
+
+    this.redis.on('connect', () => {
+      log(this.options.logger, 'info', 'Redis connected')
+      this.isHealthy = true
+    })
+
+    this.redis.on('ready', () => {
+      log(this.options.logger, 'info', 'Redis ready')
+      this.isHealthy = true
+    })
+
+    this.redis.on('error', (error: Error) => {
+      log(this.options.logger, 'error', `Redis error: ${error.message}`)
+      this.isHealthy = false
+    })
+
+    this.redis.on('close', () => {
+      log(
+        this.options.logger,
+        this.isClosing ? 'info' : 'warn',
+        this.isClosing ? 'Redis connection closed during shutdown' : 'Redis connection closed',
+      )
+      this.isHealthy = false
+    })
+
+    this.redis.on('reconnecting', () => {
+      log(this.options.logger, 'info', 'Redis reconnecting...')
+    })
+
+    this.redis.on('end', () => {
+      log(
+        this.options.logger,
+        this.isClosing ? 'info' : 'warn',
+        this.isClosing ? 'Redis connection ended during shutdown' : 'Redis connection ended',
+      )
+      this.isHealthy = false
+    })
+  }
+
+  private startHealthChecks(): void {
+    if (this.healthCheckInterval) {
+      return
+    }
+
+    const intervalMs = this.options.healthCheckIntervalMs ?? DEFAULT_HEALTH_CHECK_INTERVAL_MS
+    this.healthCheckInterval = setInterval(async () => {
+      try {
+        if (this.redis && this.redis.status === 'ready') {
+          await this.redis.ping()
+          this.isHealthy = true
+        }
+      } catch (error) {
+        log(this.options.logger, 'warn', `Redis health check failed: ${getErrorMessage(error)}`)
+        this.isHealthy = false
+      }
+    }, intervalMs)
+  }
+
+  getConnection(): IORedis {
+    if (!this.redis) {
+      throw new Error('Redis connection not initialized')
+    }
+    return this.redis
+  }
+
+  getConnectionForBullMQ(): IORedis {
+    return this.getConnection()
+  }
+
+  isConnectionHealthy(): boolean {
+    return this.isHealthy && this.redis?.status === 'ready'
+  }
+
+  async closeConnection(): Promise<void> {
+    if (this.healthCheckInterval) {
+      clearInterval(this.healthCheckInterval)
+      this.healthCheckInterval = null
+    }
+
+    this.isClosing = true
+
+    try {
+      if (this.redis && this.redis.status !== 'end') {
+        await this.redis.quit()
+      }
+    } catch (error) {
+      log(this.options.logger, 'warn', `Redis quit failed, forcing disconnect: ${getErrorMessage(error)}`)
+      this.redis?.disconnect()
+    }
+  }
+}
+
+export function createRedisConnectionManager(options: CreateRedisConnectionManagerOptions): RedisConnectionManager {
+  return new RedisConnectionManagerImpl(options)
+}

package/src/redis/index.ts

@@ -0,0 +1,30 @@
+import type IORedis from 'ioredis'
+
+import { createRedisConnectionManager } from './connection'
+import type { RedisConnectionManager } from './connection'
+
+export { createRedisConnectionManager }
+export type { RedisConnectionManager }
+
+let _redisManager: { getConnection(): IORedis; getConnectionForBullMQ(): IORedis } | undefined
+
+export function setRedisConnectionManager(manager: {
+  getConnection(): IORedis
+  getConnectionForBullMQ(): IORedis
+}): void {
+  _redisManager = manager
+}
+
+export function getRedisConnection(): IORedis {
+  if (!_redisManager) {
+    throw new Error('Redis connection manager not configured. Call setRedisConnectionManager() first.')
+  }
+  return _redisManager.getConnection()
+}
+
+export function getRedisConnectionForBullMQ(): IORedis {
+  if (!_redisManager) {
+    throw new Error('Redis connection manager not configured. Call setRedisConnectionManager() first.')
+  }
+  return _redisManager.getConnectionForBullMQ()
+}

package/src/redis/org-memory-lock.ts

@@ -0,0 +1,43 @@
+import { getRedisConnection } from '.'
+import { serverLogger } from '../config/logger'
+import { withRedisLeaseLock } from './redis-lease-lock'
+
+const ORG_MEMORY_LOCK_PREFIX = 'lock:org-memory:org:'
+const ORG_MEMORY_LOCK_TTL_MS = 120_000
+const ORG_MEMORY_LOCK_RETRY_DELAY_MS = 500
+const ORG_MEMORY_LOCK_REFRESH_INTERVAL_MS = 30_000
+const ORG_MEMORY_LOCK_WAIT_LOG_INTERVAL_MS = 30_000
+const ORG_MEMORY_LOCK_MAX_WAIT_MS = 15 * 60 * 1000
+
+export async function withOrgMemoryLock<T>(orgId: string, fn: () => Promise<T>): Promise<T> {
+  const normalizedOrgId = orgId.trim()
+
+  if (!normalizedOrgId) {
+    throw new Error('Organization id is required for memory lock')
+  }
+
+  return withRedisLeaseLock(
+    {
+      redis: getRedisConnection(),
+      lockKey: `${ORG_MEMORY_LOCK_PREFIX}${normalizedOrgId}`,
+      lockTtlMs: ORG_MEMORY_LOCK_TTL_MS,
+      retryDelayMs: ORG_MEMORY_LOCK_RETRY_DELAY_MS,
+      refreshIntervalMs: ORG_MEMORY_LOCK_REFRESH_INTERVAL_MS,
+      waitLogIntervalMs: ORG_MEMORY_LOCK_WAIT_LOG_INTERVAL_MS,
+      maxWaitMs: ORG_MEMORY_LOCK_MAX_WAIT_MS,
+      label: 'org memory lock',
+      logger: {
+        debug: (message) => {
+          serverLogger.debug`${message}`
+        },
+        info: (message) => {
+          serverLogger.info`${message}`
+        },
+        warn: (message) => {
+          serverLogger.warn`${message}`
+        },
+      },
+    },
+    fn,
+  )
+}

package/src/redis/redis-lease-lock.ts

@@ -0,0 +1,158 @@
+import { randomUUID } from 'node:crypto'
+import { setTimeout as delay } from 'node:timers/promises'
+
+import type IORedis from 'ioredis'
+
+import { getErrorMessage } from '../utils/error'
+
+interface RedisLeaseLockLogger {
+  debug?: (message: string) => void
+  info?: (message: string) => void
+  warn?: (message: string) => void
+}
+
+export interface RedisLeaseLockOptions {
+  redis: IORedis
+  lockKey: string
+  lockTtlMs: number
+  retryDelayMs?: number
+  refreshIntervalMs?: number
+  waitLogIntervalMs?: number
+  maxWaitMs?: number
+  acquiredWaitInfoThresholdMs?: number
+  heldInfoThresholdMs?: number
+  logger?: RedisLeaseLockLogger
+  label?: string
+}
+
+const RELEASE_LOCK_SCRIPT = `
+if redis.call("get", KEYS[1]) == ARGV[1] then
+  return redis.call("del", KEYS[1])
+else
+  return 0
+end
+`
+
+const REFRESH_LOCK_SCRIPT = `
+if redis.call("get", KEYS[1]) == ARGV[1] then
+  return redis.call("pexpire", KEYS[1], ARGV[2])
+else
+  return 0
+end
+`
+
+function log(logger: RedisLeaseLockLogger | undefined, level: keyof RedisLeaseLockLogger, message: string): void {
+  const sink = logger?.[level]
+  if (typeof sink === 'function') {
+    sink(message)
+  }
+}
+
+async function acquireLeaseLock(
+  options: Required<Pick<RedisLeaseLockOptions, 'retryDelayMs' | 'waitLogIntervalMs' | 'maxWaitMs'>> &
+    RedisLeaseLockOptions & { lockValue: string; label: string },
+): Promise<void> {
+  const waitStart = Date.now()
+  let lastWaitLogAt = waitStart
+
+  for (;;) {
+    const result = await options.redis.set(options.lockKey, options.lockValue, 'PX', options.lockTtlMs, 'NX')
+
+    if (result === 'OK') {
+      return
+    }
+
+    const now = Date.now()
+    const waitedMs = now - waitStart
+    if (waitedMs >= options.maxWaitMs) {
+      throw new Error(`Timed out waiting for ${options.label} (${options.lockKey}) waitedMs=${waitedMs}`)
+    }
+
+    if (now - lastWaitLogAt >= options.waitLogIntervalMs) {
+      log(options.logger, 'info', `Waiting for ${options.label} (${options.lockKey}) waitedMs=${waitedMs}`)
+      lastWaitLogAt = now
+    }
+
+    await delay(options.retryDelayMs)
+  }
+}
+
+async function refreshLeaseLock(options: RedisLeaseLockOptions & { lockValue: string; label: string }): Promise<void> {
+  const refreshed = await options.redis.eval(
+    REFRESH_LOCK_SCRIPT,
+    1,
+    options.lockKey,
+    options.lockValue,
+    options.lockTtlMs.toString(),
+  )
+
+  const refreshedCount = typeof refreshed === 'number' ? refreshed : Number(refreshed)
+  if (refreshedCount === 1) return
+  throw new Error(`${options.label} refresh was rejected for key ${options.lockKey}`)
+}
+
+async function releaseLeaseLock(options: RedisLeaseLockOptions & { lockValue: string }): Promise<void> {
+  await options.redis.eval(RELEASE_LOCK_SCRIPT, 1, options.lockKey, options.lockValue)
+}
+
+function startLeaseLockRefreshLoop(
+  options: Required<Pick<RedisLeaseLockOptions, 'refreshIntervalMs'>> &
+    RedisLeaseLockOptions & { lockValue: string; label: string },
+): () => void {
+  let stopped = false
+
+  const timer = setInterval(() => {
+    if (stopped) return
+    void refreshLeaseLock(options).catch((error: unknown) => {
+      log(options.logger, 'warn', `Failed to refresh ${options.label} (${options.lockKey}): ${getErrorMessage(error)}`)
+    })
+  }, options.refreshIntervalMs)
+
+  timer.unref()
+
+  return () => {
+    stopped = true
+    clearInterval(timer)
+  }
+}
+
+export async function withRedisLeaseLock<T>(options: RedisLeaseLockOptions, fn: () => Promise<T>): Promise<T> {
+  const lockKey = options.lockKey.trim()
+  if (!lockKey) {
+    throw new Error('Redis lease lock requires a non-empty lock key')
+  }
+
+  const retryDelayMs = options.retryDelayMs ?? 500
+  const refreshIntervalMs = options.refreshIntervalMs ?? 30_000
+  const waitLogIntervalMs = options.waitLogIntervalMs ?? 30_000
+  const maxWaitMs = options.maxWaitMs ?? 45_000
+  const acquiredWaitInfoThresholdMs = options.acquiredWaitInfoThresholdMs ?? 1_000
+  const heldInfoThresholdMs = options.heldInfoThresholdMs ?? 5_000
+  const label = options.label ?? 'redis lease lock'
+
+  const lockValue = randomUUID()
+  const waitStart = Date.now()
+  await acquireLeaseLock({ ...options, lockKey, lockValue, label, retryDelayMs, waitLogIntervalMs, maxWaitMs })
+  const waitedMs = Date.now() - waitStart
+  if (waitedMs >= acquiredWaitInfoThresholdMs) {
+    log(options.logger, 'info', `Acquired ${label} (${lockKey}) after waiting waitedMs=${waitedMs}`)
+  }
+
+  const stopRefreshLoop = startLeaseLockRefreshLoop({ ...options, lockKey, lockValue, label, refreshIntervalMs })
+  const holdStart = Date.now()
+
+  try {
+    return await fn()
+  } finally {
+    stopRefreshLoop()
+    const heldMs = Date.now() - holdStart
+    await releaseLeaseLock({ ...options, lockKey, lockValue }).catch((error: unknown) => {
+      log(options.logger, 'warn', `Failed to release ${label} (${lockKey}): ${getErrorMessage(error)}`)
+    })
+    if (heldMs >= heldInfoThresholdMs) {
+      log(options.logger, 'info', `Released ${label} (${lockKey}) heldMs=${heldMs}`)
+    } else {
+      log(options.logger, 'debug', `Released ${label} (${lockKey}) heldMs=${heldMs}`)
+    }
+  }
+}

package/src/runtime/agent-contract.ts

@@ -0,0 +1 @@
+export type AgentSkill = string