@lota-sdk/core 0.4.27 → 0.4.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lota-sdk/core",
-  "version": "0.4.27",
+  "version": "0.4.29",
   "files": [
     "src",
     "infrastructure/schema"
@@ -32,7 +32,7 @@
     "@ai-sdk/provider": "^3.0.9",
     "@chat-adapter/slack": "^4.26.0",
     "@chat-adapter/state-ioredis": "^4.26.0",
-    "@lota-sdk/shared": "0.4.27",
+    "@lota-sdk/shared": "0.4.29",
     "@mendable/firecrawl-js": "^4.20.0",
     "@surrealdb/node": "^3.0.3",
     "ai": "^6.0.170",

package/src/ai-gateway/ai-gateway.ts

@@ -38,8 +38,8 @@ class AiGatewayStreamAttemptTag extends Context.Service<
 
 const EXPECTED_GATEWAY_KEY_PREFIX = 'sk-bf-'
 const AI_GATEWAY_VIRTUAL_KEY_HEADER = 'x-bf-vk'
-const AI_GATEWAY_TIMEOUT_MS = 180_000
-const AI_GATEWAY_STREAM_IDLE_TIMEOUT_MS = 30_000
+const AI_GATEWAY_TIMEOUT_MS = 360_000
+const AI_GATEWAY_STREAM_IDLE_TIMEOUT_MS = 180_000
 const AI_GATEWAY_MAX_RETRIES = 4
 const AI_GATEWAY_MAX_RETRY_DELAY_MS = 15_000
 const OPENAI_RESPONSES_PROVIDER_ID = 'openai.responses'
@@ -748,13 +748,60 @@ function isOpenRouterModel(modelId: string): boolean {
   return modelId.trim().toLowerCase().startsWith('openrouter/')
 }
 
+function mergeAbortSignals(signals: Array<AbortSignal | undefined>): { signal?: AbortSignal; cleanup: () => void } {
+  const activeSignals = signals.filter((signal): signal is AbortSignal => Boolean(signal))
+  if (activeSignals.length === 0) return { cleanup: () => undefined }
+  if (activeSignals.length === 1) return { signal: activeSignals[0], cleanup: () => undefined }
+
+  const controller = new AbortController()
+  const listeners: Array<() => void> = []
+  const abortFrom = (signal: AbortSignal) => {
+    if (!controller.signal.aborted) controller.abort(signal.reason)
+  }
+
+  for (const signal of activeSignals) {
+    if (signal.aborted) {
+      abortFrom(signal)
+      continue
+    }
+
+    const listener = () => abortFrom(signal)
+    signal.addEventListener('abort', listener, { once: true })
+    listeners.push(() => signal.removeEventListener('abort', listener))
+  }
+
+  return {
+    signal: controller.signal,
+    cleanup: () => {
+      for (const cleanup of listeners) cleanup()
+    },
+  }
+}
+
+async function callAbortableProvider<T>(
+  params: AiGatewayCallOptions,
+  effectSignal: AbortSignal,
+  evaluate: (params: AiGatewayCallOptions) => PromiseLike<T>,
+): Promise<T> {
+  const { signal, cleanup } = mergeAbortSignals([effectSignal, params.abortSignal])
+  try {
+    return await evaluate(signal ? { ...params, abortSignal: signal } : params)
+  } finally {
+    cleanup()
+  }
+}
+
 function attemptAiGatewayGenerate(
   source: string,
-  evaluate: () => PromiseLike<AiGatewayGenerateResult>,
+  params: AiGatewayCallOptions,
+  evaluate: (params: AiGatewayCallOptions) => PromiseLike<AiGatewayGenerateResult>,
 ): Effect.Effect<AiGatewayAttemptResult<AiGatewayGenerateResult>, AiGenerationError> {
   return withAiGatewayResilience(
     source,
-    Effect.tryPromise({ try: evaluate, catch: (cause) => classifyAiGatewayError(source, cause) }),
+    Effect.tryPromise({
+      try: (signal) => callAbortableProvider(params, signal, evaluate),
+      catch: (cause) => classifyAiGatewayError(source, cause),
+    }),
   ).pipe(
     Effect.map((result) => ({ source, result })),
     Effect.withSpan('AiGateway.generateAttempt'),
@@ -764,11 +811,15 @@ function attemptAiGatewayGenerate(
 
 function attemptAiGatewayStream(
   source: string,
-  evaluate: () => PromiseLike<AiGatewayStreamResult>,
+  params: AiGatewayCallOptions,
+  evaluate: (params: AiGatewayCallOptions) => PromiseLike<AiGatewayStreamResult>,
 ): Effect.Effect<AiGatewayAttemptResult<AiGatewayStreamResult>, AiGenerationError> {
   return withAiGatewayResilience(
     source,
-    Effect.tryPromise({ try: evaluate, catch: (cause) => classifyAiGatewayError(source, cause) }),
+    Effect.tryPromise({
+      try: (signal) => callAbortableProvider(params, signal, evaluate),
+      catch: (cause) => classifyAiGatewayError(source, cause),
+    }),
   ).pipe(
     Effect.map((result) => ({ source, result })),
     Effect.withSpan('AiGateway.streamAttempt'),
@@ -778,10 +829,11 @@ function attemptAiGatewayStream(
 
 function executeGenerateAttemptPlan(
   modelId: string,
-  doGenerate: () => PromiseLike<AiGatewayGenerateResult>,
+  params: AiGatewayCallOptions,
+  doGenerate: (params: AiGatewayCallOptions) => PromiseLike<AiGatewayGenerateResult>,
 ): Effect.Effect<AiGatewayAttemptResult<AiGatewayGenerateResult>, AiGenerationError> {
   const primary = Layer.succeed(AiGatewayGenerateAttemptTag, {
-    execute: attemptAiGatewayGenerate('ai-gateway.generate', doGenerate),
+    execute: attemptAiGatewayGenerate('ai-gateway.generate', params, doGenerate),
   })
   const effect = Effect.gen(function* () {
     const attempt = yield* AiGatewayGenerateAttemptTag
@@ -797,10 +849,11 @@ function executeGenerateAttemptPlan(
 
 function executeStreamAttemptPlan(
   modelId: string,
-  doStream: () => PromiseLike<AiGatewayStreamResult>,
+  params: AiGatewayCallOptions,
+  doStream: (params: AiGatewayCallOptions) => PromiseLike<AiGatewayStreamResult>,
 ): Effect.Effect<AiGatewayAttemptResult<AiGatewayStreamResult>, AiGenerationError> {
   const primary = Layer.succeed(AiGatewayStreamAttemptTag, {
-    execute: attemptAiGatewayStream('ai-gateway.stream', doStream),
+    execute: attemptAiGatewayStream('ai-gateway.stream', params, doStream),
   })
   const effect = Effect.gen(function* () {
     const attempt = yield* AiGatewayStreamAttemptTag
@@ -1046,7 +1099,7 @@ function createAiGatewayLanguageModelMiddleware(
       const model = resolveProviderModel(resolvedDeps.gateway.provider, modelId, providerId)
       return resolvedDeps.runPromise(
         withAiGatewayConcurrency(
-          executeGenerateAttemptPlan(modelId, () => model.doGenerate(params)).pipe(
+          executeGenerateAttemptPlan(modelId, params, (attemptParams) => model.doGenerate(attemptParams)).pipe(
             Effect.map(({ result }) => ({
               ...result,
               content: injectAiGatewayChatReasoningContent(
@@ -1063,7 +1116,7 @@ function createAiGatewayLanguageModelMiddleware(
       const model = resolveProviderModel(resolvedDeps.gateway.provider, modelId, providerId)
       return resolvedDeps.runPromise(
         withAiGatewayStreamConcurrency(
-          executeStreamAttemptPlan(modelId, () => model.doStream(params)).pipe(
+          executeStreamAttemptPlan(modelId, params, (attemptParams) => model.doStream(attemptParams)).pipe(
             Effect.map((attempt) => ({
               ...attempt,
               result: isReasoningEnabled(params)

package/src/queues/document-processor.queue.ts

@@ -66,11 +66,13 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
   queueName?: string
   workerName?: string
   concurrency?: number
+  attempts?: number
   lockDuration?: number
 }): { enqueue: (job: TJob) => Promise<void>; startWorker: (options?: { registerSignals?: boolean }) => WorkerHandle } {
   const queueName = params.queueName ?? DEFAULT_DOCUMENT_PROCESSOR_QUEUE
   const workerName = params.workerName ?? DEFAULT_WORKER_NAME
   const concurrency = params.concurrency ?? 10
+  const attempts = params.attempts ?? 3
   const lockDuration = params.lockDuration ?? 300_000
   const queueRuntime = createQueueFactory<TJob>({
     name: queueName,
@@ -80,7 +82,7 @@ export function createDocumentProcessorQueueRuntime<TJob extends DocumentProcess
     lockDuration,
     logger: params.logger,
     connectionProvider: params.getConnectionForBullMQ,
-    defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts: 3, backoff: { type: 'exponential', delay: 1000 } },
+    defaultJobOptions: { ...DEFAULT_JOB_RETENTION, attempts, backoff: { type: 'exponential', delay: 1000 } },
     processorPath: params.getWorkerPath(),
     queueJobService: params.queueJobService,
   })

package/src/queues/queue-factory.ts

@@ -9,6 +9,7 @@ import { ERROR_TAGS } from '../effect/errors'
 import type { TrackedBullJobLike } from '../services/queue-job.service'
 import {
   attachWorkerEvents,
+  attachSandboxChildRecycling,
   createTracedWorkerProcessor,
   createWorkerShutdown,
   DEFAULT_JOB_RETENTION,
@@ -42,6 +43,7 @@ interface QueueFactoryConfigBase {
   stalledInterval?: number
   maxStalledCount?: number
   defaultJobOptions?: JobsOptions
+  recycleSandboxChildren?: boolean
   connectionProvider: () => IORedis
   queueJobService: QueueJobService
 }
@@ -280,6 +282,9 @@ function createQueueFactoryRuntime<TJob>(config: QueueFactoryConfigBase): {
     // leaks.
     try {
       attachWorkerEvents(worker, config.displayName, logger)
+      if ((config.recycleSandboxChildren ?? true) && workerConfig.processorPath) {
+        attachSandboxChildRecycling(worker, config.displayName, logger)
+      }
       const shutdown = createWorkerShutdown(worker, config.displayName, logger)
 
       if (registerSignals) {

package/src/services/recent-activity-title.service.ts

@@ -4,7 +4,7 @@ import { AiGatewayModelsTag, isAiGenerationContentFilterError } from '../ai-gate
 import type { AiGatewayModels } from '../ai-gateway/ai-gateway'
 import type { ResolvedAgentConfig } from '../config/agent-defaults'
 import { chatLogger } from '../config/logger'
-import { ERROR_TAGS, ServiceError } from '../effect/errors'
+import { ServiceError } from '../effect/errors'
 import { AgentConfigServiceTag } from '../effect/services'
 import type { HelperModelRuntime } from '../runtime/helper-model'
 import { HelperModelTag } from '../runtime/helper-model'
@@ -13,10 +13,19 @@ import {
   makeRecentActivityTitleRefinerAgentFactory,
   RECENT_ACTIVITY_TITLE_REFINER_PROMPT,
 } from '../system-agents/recent-activity-title-refiner.agent'
+import { compactWhitespace, truncateText } from '../utils/string'
 import type { makeRecentActivityService } from './recent-activity.service'
 import { RecentActivityServiceTag } from './recent-activity.service'
 
 const RECENT_ACTIVITY_TITLE_TIMEOUT_MS = 60_000
+const RECENT_ACTIVITY_TITLE_FIELD_MAX_CHARS = 800
+
+function formatPromptField(label: string, value: string | undefined): string | null {
+  if (!value) return null
+  const normalized = compactWhitespace(value)
+  if (!normalized) return null
+  return `${label}=${truncateText(normalized, RECENT_ACTIVITY_TITLE_FIELD_MAX_CHARS)}`
+}
 
 function buildRefinementPromptInput(
   candidate: {
@@ -32,12 +41,12 @@ function buildRefinementPromptInput(
 
   const metadata = candidate.metadata
   const lines = [
-    `sourceLabel=${candidate.sourceLabel}`,
-    `systemTitle=${candidate.systemTitle}`,
-    metadata.agentName ? `agentName=${metadata.agentName}` : null,
-    metadata.threadTitle ? `threadTitle=${metadata.threadTitle}` : null,
-    metadata.userMessageText ? `userMessage=${metadata.userMessageText}` : null,
-    metadata.assistantSummary ? `assistantSummary=${metadata.assistantSummary}` : null,
+    formatPromptField('sourceLabel', candidate.sourceLabel),
+    formatPromptField('systemTitle', candidate.systemTitle),
+    formatPromptField('agentName', metadata.agentName),
+    formatPromptField('threadTitle', metadata.threadTitle),
+    formatPromptField('userMessage', metadata.userMessageText),
+    formatPromptField('assistantSummary', metadata.assistantSummary),
   ].filter((line): line is string => Boolean(line))
 
   if (lines.length === 0) return null
@@ -77,13 +86,12 @@ export function makeRecentActivityTitleService(
             ? cause
             : new ServiceError({ message: 'Failed to generate recent activity title refinement.', cause }),
       }).pipe(
-        Effect.catchTag(ERROR_TAGS.AiGenerationError, (error) =>
-          isAiGenerationContentFilterError(error)
-            ? Effect.sync(() => {
-                chatLogger.warn`Skipping recent activity title refinement after provider content filter (activityId=${activityId})`
-                return null
-              })
-            : Effect.fail(error),
+        Effect.catch((error) =>
+          Effect.sync(() => {
+            const reason = isAiGenerationContentFilterError(error) ? 'provider content filter' : 'non-fatal error'
+            chatLogger.warn`Skipping recent activity title refinement after ${reason} (activityId=${activityId}): ${error}`
+            return null
+          }),
         ),
       )
       if (maybeRefinedTitle === null) {

package/src/workers/worker-utils.ts

@@ -1,3 +1,4 @@
+import { setTimeout as delay } from 'node:timers/promises'
 import { fileURLToPath } from 'node:url'
 
 import type { Job, Worker } from 'bullmq'
@@ -30,6 +31,32 @@ export interface WorkerHandle {
   shutdown: () => Promise<void>
 }
 
+async function closeWorkerWithTimeout(
+  worker: Worker,
+  name: string,
+  logger: typeof chatLogger,
+  timeoutMs = DEFAULT_SHUTDOWN_TIMEOUT_MS,
+): Promise<void> {
+  const gracefulClose = worker.close(false)
+  const gracefulOutcome = gracefulClose.then(
+    () => ({ status: 'closed' as const }),
+    (error: unknown) => ({ status: 'failed' as const, error }),
+  )
+
+  const outcome = await Promise.race([gracefulOutcome, delay(timeoutMs).then(() => ({ status: 'timed_out' as const }))])
+
+  if (outcome.status === 'closed') return
+
+  if (outcome.status === 'timed_out') {
+    logger.warn`${name} worker did not close within ${timeoutMs}ms; force-closing`
+  } else {
+    logger.warn`${name} worker graceful close failed; force-closing: ${outcome.error}`
+  }
+
+  await worker.close(true)
+  if (outcome.status === 'failed') throw outcome.error
+}
+
 interface TracedWorkerJobLike {
   id?: unknown
   name: string
@@ -87,7 +114,7 @@ export const createWorkerShutdown = (worker: Worker, name: string, logger: typeo
     return Effect.runPromise(
       Effect.asVoid(
         Effect.tryPromise({
-          try: () => worker.close(true),
+          try: () => closeWorkerWithTimeout(worker, name, logger),
           catch: (cause) => new QueueWorkerError({ phase: 'close', cause }),
         }),
       ),
@@ -95,6 +122,60 @@ export const createWorkerShutdown = (worker: Worker, name: string, logger: typeo
   }
 }
 
+interface SandboxChildLike {
+  pid?: number
+}
+
+interface SandboxChildPoolLike {
+  getAllFree?: () => SandboxChildLike[]
+  kill?: (child: SandboxChildLike, signal?: NodeJS.Signals) => Promise<void>
+}
+
+interface SandboxedWorkerLike {
+  childPool?: SandboxChildPoolLike
+}
+
+function getSandboxChildPool(worker: Worker): SandboxChildPoolLike | null {
+  const pool = (worker as unknown as SandboxedWorkerLike).childPool
+  return pool && typeof pool.getAllFree === 'function' && typeof pool.kill === 'function' ? pool : null
+}
+
+export function recycleIdleSandboxChildren(
+  worker: Worker,
+  name: string,
+  logger: typeof chatLogger = chatLogger,
+): Promise<void> {
+  const pool = getSandboxChildPool(worker)
+  if (!pool) return Promise.resolve()
+
+  const idleChildren = pool.getAllFree?.() ?? []
+  if (idleChildren.length === 0) return Promise.resolve()
+
+  return Promise.all(
+    idleChildren.map((child) =>
+      (pool.kill?.(child, 'SIGTERM') ?? Promise.resolve()).catch((error: unknown) => {
+        logger.warn`Failed to recycle idle ${name} sandbox child (${child.pid ?? 'unknown'}): ${error}`
+      }),
+    ),
+  ).then(() => undefined)
+}
+
+export function attachSandboxChildRecycling(
+  worker: Worker,
+  name: string,
+  logger: typeof chatLogger = chatLogger,
+): void {
+  const recycle = () => {
+    // @effect-diagnostics-next-line globalTimers:off -- BullMQ worker event callback; defer until BullMQ releases the child.
+    setTimeout(() => {
+      void recycleIdleSandboxChildren(worker, name, logger)
+    }, 0)
+  }
+
+  worker.on('completed', recycle)
+  worker.on('failed', recycle)
+}
+
 export function createTracedWorkerProcessor<TJob extends TracedWorkerJobLike, TResult = void>(
   queueName: string,
   processor: (job: TJob) => Promise<TResult>,