@lota-sdk/core 0.4.12 → 0.4.14

package/src/utils/sse-keepalive.ts

@@ -1,12 +1,25 @@
-import { Duration, Effect, Fiber } from 'effect'
+import type { Fiber } from 'effect'
+import { Duration, Effect, Exit, Schema, Scope } from 'effect'
+
+import { ERROR_TAGS } from '../effect/errors'
 
 const KEEPALIVE_COMMENT = new TextEncoder().encode(': keepalive\n\n')
 const DEFAULT_KEEPALIVE_INTERVAL_MS = 20_000
 
+class SseKeepaliveError extends Schema.TaggedErrorClass<SseKeepaliveError>()(ERROR_TAGS.SseKeepaliveError, {
+  phase: Schema.Literals(['read']),
+  cause: Schema.Defect,
+}) {}
+
 /**
  * Wraps an SSE Response body with periodic keepalive comments.
  * SSE comments (`: keepalive\n\n`) are ignored by standard SSE parsers,
  * so no client changes are needed.
+ *
+ * Lifecycle: a function-local `Scope` owns both the keepalive and body-pump
+ * fibers via `Effect.forkScoped`. Stream `cancel` closes the scope, which
+ * auto-interrupts both fibers and runs their `ensuring` finalizers. No
+ * manual `Fiber.interrupt` bookkeeping.
  */
 export function wrapResponseWithKeepalive(response: Response, intervalMs = DEFAULT_KEEPALIVE_INTERVAL_MS): Response {
   const body = response.body
@@ -14,24 +27,10 @@ export function wrapResponseWithKeepalive(response: Response, intervalMs = DEFAU
 
   let closed = false
   let reader: ReadableStreamDefaultReader<Uint8Array> | null = null
-  let keepaliveFiber: ReturnType<typeof Effect.runFork> | null = null
-  let bodyPumpFiber: ReturnType<typeof Effect.runFork> | null = null
-
-  const interruptFiber = (fiber: ReturnType<typeof Effect.runFork> | null) => {
-    if (!fiber) return
-    void Effect.runFork(Fiber.interrupt(fiber))
-  }
+  const scope = Scope.makeUnsafe()
 
-  const stopKeepalive = () => {
-    const fiber = keepaliveFiber
-    keepaliveFiber = null
-    interruptFiber(fiber)
-  }
-
-  const stopBodyPump = () => {
-    const fiber = bodyPumpFiber
-    bodyPumpFiber = null
-    interruptFiber(fiber)
+  const closeScope = () => {
+    void Effect.runPromise(Scope.close(scope, Exit.void))
   }
 
   const releaseReader = (bodyReader: ReadableStreamDefaultReader<Uint8Array>) => {
@@ -90,7 +89,10 @@ export function wrapResponseWithKeepalive(response: Response, intervalMs = DEFAU
       for (;;) {
         if (closed) return
 
-        const { done, value } = yield* Effect.tryPromise(() => bodyReader.read())
+        const { done, value } = yield* Effect.tryPromise({
+          try: () => bodyReader.read(),
+          catch: (cause) => new SseKeepaliveError({ phase: 'read', cause }),
+        })
         if (done) {
           yield* Effect.sync(() => closeStream(controller))
           return
@@ -105,25 +107,26 @@ export function wrapResponseWithKeepalive(response: Response, intervalMs = DEFAU
       Effect.ensuring(
         Effect.sync(() => {
           closed = true
-          stopKeepalive()
-          bodyPumpFiber = null
           reader = null
           releaseReader(bodyReader)
+          closeScope()
         }),
       ),
     )
 
+  const startFiber = <A, E>(effect: Effect.Effect<A, E>): Fiber.Fiber<Fiber.Fiber<A, E>, never> =>
+    Effect.runFork(Scope.provide(Effect.forkScoped(effect), scope))
+
   const transformed = new ReadableStream<Uint8Array>({
     start(controller) {
       const bodyReader = body.getReader()
       reader = bodyReader
-      keepaliveFiber = Effect.runFork(keepaliveEffect(controller))
-      bodyPumpFiber = Effect.runFork(pumpBodyEffect(bodyReader, controller))
+      startFiber(keepaliveEffect(controller))
+      startFiber(pumpBodyEffect(bodyReader, controller))
     },
     cancel(reason) {
       closed = true
-      stopKeepalive()
-      stopBodyPump()
+      closeScope()
 
       const bodyReader = reader
       reader = null

package/src/workers/bootstrap.ts

@@ -1,17 +1,25 @@
-import { ConfigProvider, Option, Schema, Effect, Layer, ManagedRuntime, Redacted } from 'effect'
-
-import { AiGatewayLive } from '../ai-gateway/ai-gateway'
+import { ConfigProvider, Deferred, Option, Schema, Effect, Layer, ManagedRuntime, Redacted } from 'effect'
+
+import {
+  AiGatewayModelsTag,
+  AiGatewayTag,
+  RuntimeBridgeTag,
+  createAiGatewayModels,
+  makeAiGatewayService,
+} from '../ai-gateway/ai-gateway'
+import type { AiGatewayModels, RuntimeBridge } from '../ai-gateway/ai-gateway'
 import { EmbeddingCacheLive } from '../ai/embedding-cache'
 import { serverLogger } from '../config/logger'
 import { connectWithStartupRetry, waitForDatabaseBootstrap } from '../db/startup'
 import { buildWorkerInfrastructureLayer } from '../effect'
+import { ERROR_TAGS } from '../effect/errors'
 import { DatabaseServiceTag, RuntimeAdaptersServiceTag } from '../effect/services'
 import { lotaRuntimeEnvConfig, parseLotaRuntimeConfig, parseWorkerBootstrapEnv } from '../runtime/runtime-config'
 import { FirecrawlLive } from '../tools/firecrawl-client'
 import { getErrorMessage } from '../utils/errors'
 
 class SandboxedWorkerBootstrapError extends Schema.TaggedErrorClass<SandboxedWorkerBootstrapError>()(
-  'SandboxedWorkerBootstrapError',
+  ERROR_TAGS.SandboxedWorkerBootstrapError,
   {
     stage: Schema.Literals(['setup', 'initialize', 'connect-db', 'connect-plugin-db', 'bootstrap-wait']),
     message: Schema.String,
@@ -86,12 +94,35 @@ function ensureSandboxedWorkerRuntimeConfigured(): Promise<WorkerManagedRuntime>
       const runtimeConfig = yield* buildSandboxedWorkerRuntimeConfigEffect()
 
       const infrastructureLayer = buildWorkerInfrastructureLayer(runtimeConfig)
-
-      const layer = Layer.mergeAll(infrastructureLayer, Layer.provide(AiGatewayLive, infrastructureLayer))
-
-      const fullLayer = Layer.mergeAll(layer, Layer.provide(Layer.mergeAll(EmbeddingCacheLive, FirecrawlLive), layer))
+      const aiGateway = yield* makeAiGatewayService(runtimeConfig).pipe(
+        Effect.mapError((error) => toSandboxedWorkerBootstrapError('setup', error)),
+      )
+      const aiGatewayModelsDeferred = yield* Deferred.make<AiGatewayModels>()
+      const runtimeBridgeDeferred = yield* Deferred.make<RuntimeBridge>()
+      const bridgeLayer = Layer.mergeAll(
+        Layer.succeed(AiGatewayTag, aiGateway),
+        Layer.effect(AiGatewayModelsTag, Deferred.await(aiGatewayModelsDeferred)),
+        Layer.effect(RuntimeBridgeTag, Deferred.await(runtimeBridgeDeferred)),
+      )
+      const layerWithBridge = Layer.mergeAll(infrastructureLayer, bridgeLayer)
+      const fullLayer = Layer.mergeAll(
+        layerWithBridge,
+        Layer.provide(Layer.mergeAll(EmbeddingCacheLive, FirecrawlLive), layerWithBridge),
+      )
 
       const managedRuntime = ManagedRuntime.make(fullLayer)
+      const runtimeBridge: RuntimeBridge = {
+        runPromise: (effect, options) => managedRuntime.runPromise(effect, options),
+        runFork: (effect) => managedRuntime.runFork(effect),
+      }
+      const aiGatewayModels = createAiGatewayModels({
+        gateway: aiGateway,
+        runtimeConfig,
+        runPromise: runtimeBridge.runPromise,
+        runFork: runtimeBridge.runFork,
+      })
+      yield* Deferred.succeed(runtimeBridgeDeferred, runtimeBridge)
+      yield* Deferred.succeed(aiGatewayModelsDeferred, aiGatewayModels)
 
       return managedRuntime
     }),
@@ -111,7 +142,10 @@ export function initializeSandboxedWorkerRuntime(): Promise<WorkerManagedRuntime
   // Assign before the async kicks off so concurrent callers observe the in-flight promise.
   sandboxedWorkerInitPromise = Effect.runPromise(
     Effect.gen(function* () {
-      const env = parseWorkerBootstrapEnv(Bun.env)
+      const env = yield* Effect.try({
+        try: () => parseWorkerBootstrapEnv(Bun.env),
+        catch: (error) => toSandboxedWorkerBootstrapError('setup', error),
+      })
       const runtime = yield* Effect.tryPromise({
         try: () => ensureSandboxedWorkerRuntimeConfigured(),
         catch: (error) => toSandboxedWorkerBootstrapError('setup', error),
@@ -124,7 +158,7 @@ export function initializeSandboxedWorkerRuntime(): Promise<WorkerManagedRuntime
       yield* Effect.tryPromise({
         try: () =>
           connectWithStartupRetry({
-            connect: () => db.connect(),
+            connect: () => runtime.runPromise(db.connect()),
             label: 'sandboxed worker AI database runtime',
             logger: serverLogger,
           }),
@@ -156,7 +190,7 @@ export function initializeSandboxedWorkerRuntime(): Promise<WorkerManagedRuntime
             expectedFingerprint: env.DB_SCHEMA_FINGERPRINT,
             label: 'sandboxed worker runtime',
             logger: serverLogger,
-            connect: () => db.connect(),
+            connect: () => runtime.runPromise(db.connect()),
           }),
         catch: (error) => toSandboxedWorkerBootstrapError('bootstrap-wait', error),
       })
@@ -170,3 +204,33 @@ export function initializeSandboxedWorkerRuntime(): Promise<WorkerManagedRuntime
 
   return sandboxedWorkerInitPromise
 }
+
+/**
+ * Dispose the sandboxed worker runtime and clear both module-level caches so
+ * a subsequent initialize call rebuilds a fresh runtime. Registered as a
+ * SIGTERM/SIGINT handler when the worker process runs as the main entrypoint.
+ */
+// @effect-diagnostics-next-line asyncFunction:off -- host-boundary dispose handler returns Promise by design.
+export async function disposeSandboxedWorkerRuntime(): Promise<void> {
+  const setupPromise = sandboxedWorkerSetupPromise
+  sandboxedWorkerSetupPromise = null
+  sandboxedWorkerInitPromise = null
+  if (!setupPromise) return
+  try {
+    const runtime = await setupPromise
+    await runtime.dispose()
+  } catch (error) {
+    serverLogger.warn`Failed to dispose sandboxed worker runtime: ${error}`
+  }
+}
+
+let sandboxedWorkerShutdownHandlersRegistered = false
+export function registerSandboxedWorkerShutdownHandlers(): void {
+  if (sandboxedWorkerShutdownHandlersRegistered) return
+  sandboxedWorkerShutdownHandlersRegistered = true
+  const handle = () => {
+    void disposeSandboxedWorkerRuntime().catch(() => undefined)
+  }
+  process.once('SIGTERM', handle)
+  process.once('SIGINT', handle)
+}

package/src/workers/memory-consolidation.worker.ts

@@ -8,7 +8,7 @@ import { ensureRecordId, recordIdToString } from '../db/record-id'
 import type { RecordIdInput } from '../db/record-id'
 import type { SurrealDBService } from '../db/service'
 import { TABLES } from '../db/tables'
-import { effectTryPromise } from '../effect/helpers'
+import { ERROR_TAGS } from '../effect/errors'
 import { DatabaseServiceTag } from '../effect/services'
 import type { MemoryConsolidationJob } from '../queues/memory-consolidation.queue'
 import { QueueJobServiceTag } from '../services/queue-job.service'
@@ -18,27 +18,30 @@ import { initializeSandboxedWorkerRuntime } from './bootstrap'
 import { toSandboxedWorkerError } from './utils/sandbox-error'
 import { createTracedWorkerProcessor } from './worker-utils'
 
-class MemoryConsolidationError extends Schema.TaggedErrorClass<MemoryConsolidationError>()('MemoryConsolidationError', {
-  stage: Schema.Literals([
-    'read-memories',
-    'read-neighbors',
-    'archive-memory',
-    'relate-memories',
-    'write-history',
-    'load-stale',
-    'update-stale',
-    'load-middle-nodes',
-    'read-existing-relation',
-    'write-relation',
-    'update-middle-node',
-    'decay-standard',
-    'decay-ephemeral',
-    'cleanup-relations',
-    'load-scope-ids',
-  ]),
-  message: Schema.String,
-  cause: Schema.Defect,
-}) {}
+class MemoryConsolidationError extends Schema.TaggedErrorClass<MemoryConsolidationError>()(
+  ERROR_TAGS.MemoryConsolidationError,
+  {
+    stage: Schema.Literals([
+      'read-memories',
+      'read-neighbors',
+      'archive-memory',
+      'relate-memories',
+      'write-history',
+      'load-stale',
+      'update-stale',
+      'load-middle-nodes',
+      'read-existing-relation',
+      'write-relation',
+      'update-middle-node',
+      'decay-standard',
+      'decay-ephemeral',
+      'cleanup-relations',
+      'load-scope-ids',
+    ]),
+    message: Schema.String,
+    cause: Schema.Defect,
+  },
+) {}
 
 function toMemoryConsolidationError(
   stage: MemoryConsolidationError['stage'],
@@ -47,13 +50,6 @@ function toMemoryConsolidationError(
   return new MemoryConsolidationError({ stage, message: getErrorMessage(cause), cause })
 }
 
-function tryMemoryConsolidationEffect<A, R = never>(
-  stage: MemoryConsolidationError['stage'],
-  thunk: () => PromiseLike<A> | Effect.Effect<A, unknown, R>,
-): Effect.Effect<A, MemoryConsolidationError, R> {
-  return effectTryPromise(thunk, (cause) => toMemoryConsolidationError(stage, cause))
-}
-
 const runtime = await initializeSandboxedWorkerRuntime()
 
 const memoryConsolidationDatabaseService: SurrealDBService = await runtime.runPromise(
@@ -93,8 +89,8 @@ function isContentSubsumed(shorter: string, longer: string): boolean {
 
 function deduplicateScopeEffect(scopeId: string) {
   return Effect.gen(function* () {
-    const memoryRows = yield* tryMemoryConsolidationEffect('read-memories', () =>
-      db().query<{
+    const memoryRows = yield* db()
+      .query<{
         id: RecordIdInput
         content: string
         importance: number
@@ -110,8 +106,8 @@ function deduplicateScopeEffect(scopeId: string) {
            LIMIT $limit`,
           { scopeId, limit: MAX_MEMORIES_PER_SCOPE },
         ),
-      ),
-    )
+      )
+      .pipe(Effect.mapError((cause) => toMemoryConsolidationError('read-memories', cause)))
 
     const memories = memoryRows.map((row) => ({ ...row, id: toMemoryId(row.id) }))
     if (memories.length < 2) return 0
@@ -123,8 +119,8 @@ function deduplicateScopeEffect(scopeId: string) {
       if (archived.has(memory.id)) continue
 
       const candidateLimit = 20
-      const neighborStatements = yield* tryMemoryConsolidationEffect('read-neighbors', () =>
-        db().queryAll<{
+      const neighborStatements = yield* db()
+        .queryAll<{
           id: RecordIdInput
           content: string
           importance: number
@@ -159,8 +155,8 @@ function deduplicateScopeEffect(scopeId: string) {
              LIMIT ${candidateLimit}`,
             { scopeId, memoryId: ensureRecordId(memory.id, TABLES.MEMORY), embedding: memory.embedding },
           ),
-        ),
-      )
+        )
+        .pipe(Effect.mapError((cause) => toMemoryConsolidationError('read-neighbors', cause)))
 
       const neighbors = (neighborStatements.at(-1) ?? []).map((row) => ({ ...row, id: toMemoryId(row.id) }))
 
@@ -185,31 +181,31 @@ function deduplicateScopeEffect(scopeId: string) {
         const winner = keepMemory ? memory : neighbor
         const loser = keepMemory ? neighbor : memory
 
-        yield* tryMemoryConsolidationEffect('archive-memory', () =>
-          db().query(
+        yield* db()
+          .query(
             new BoundQuery(
               `UPDATE ${MEMORY_TABLE}
                SET archivedAt = time::now(), validUntil = time::now()
                WHERE id = $loserId AND archivedAt IS NONE`,
               { loserId: ensureRecordId(loser.id, TABLES.MEMORY) },
             ),
-          ),
-        )
-        yield* tryMemoryConsolidationEffect('relate-memories', () =>
-          db().relate(
+          )
+          .pipe(Effect.mapError((cause) => toMemoryConsolidationError('archive-memory', cause)))
+        yield* db()
+          .relate(
             ensureRecordId(winner.id, TABLES.MEMORY),
             MEMORY_RELATION_TABLE,
             ensureRecordId(loser.id, TABLES.MEMORY),
             { relationType: RELATION_SUPERSEDES, confidence: 1.0 },
-          ),
-        )
-        yield* tryMemoryConsolidationEffect('write-history', () =>
-          db().insert<Record<string, unknown>>(MEMORY_HISTORY_TABLE, {
+          )
+          .pipe(Effect.mapError((cause) => toMemoryConsolidationError('relate-memories', cause)))
+        yield* db()
+          .insert<Record<string, unknown>>(MEMORY_HISTORY_TABLE, {
             memoryId: ensureRecordId(loser.id, TABLES.MEMORY),
             prevValue: loser.content,
             event: 'DELETE',
-          }),
-        )
+          })
+          .pipe(Effect.mapError((cause) => toMemoryConsolidationError('write-history', cause)))
 
         archived.add(loser.id)
         mergeCount++
@@ -224,8 +220,8 @@ function deduplicateScopeEffect(scopeId: string) {
 
 function pruneStaleMemoriesEffect() {
   return Effect.gen(function* () {
-    const stale = yield* tryMemoryConsolidationEffect('load-stale', () =>
-      db().query<{ id: RecordIdInput }>(
+    const stale = yield* db()
+      .query<{ id: RecordIdInput }>(
         new BoundQuery(
           `SELECT id FROM ${MEMORY_TABLE}
            WHERE accessCount = 0
@@ -234,15 +230,15 @@ function pruneStaleMemoriesEffect() {
              AND importance < 0.5
              LIMIT 200`,
         ),
-      ),
-    )
+      )
+      .pipe(Effect.mapError((cause) => toMemoryConsolidationError('load-stale', cause)))
 
     if (stale.length === 0) return 0
 
     const staleIds = stale.map((row) => ensureRecordId(row.id, TABLES.MEMORY))
-    yield* tryMemoryConsolidationEffect('update-stale', () =>
-      db().updateWhere(MEMORY_TABLE, inside('id', staleIds), { archivedAt: nowDate() }),
-    )
+    yield* db()
+      .updateWhere(MEMORY_TABLE, inside('id', staleIds), { archivedAt: nowDate() })
+      .pipe(Effect.mapError((cause) => toMemoryConsolidationError('update-stale', cause)))
 
     return stale.length
   })
@@ -250,8 +246,8 @@ function pruneStaleMemoriesEffect() {
 
 function collapseSupersededChainEffect() {
   return Effect.gen(function* () {
-    const middleNodes = yield* tryMemoryConsolidationEffect('load-middle-nodes', () =>
-      db().query<{ middleId: RecordIdInput; predecessors: RecordIdInput[]; successors: RecordIdInput[] }>(
+    const middleNodes = yield* db()
+      .query<{ middleId: RecordIdInput; predecessors: RecordIdInput[]; successors: RecordIdInput[] }>(
         new BoundQuery(
           `SELECT
              id AS middleId,
@@ -263,8 +259,8 @@ function collapseSupersededChainEffect() {
              AND count(<-${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']) > 0
              LIMIT ${MEMORY.MAX_KNN_LIMIT}`,
         ),
-      ),
-    )
+      )
+      .pipe(Effect.mapError((cause) => toMemoryConsolidationError('load-middle-nodes', cause)))
 
     let collapsed = 0
 
@@ -273,33 +269,28 @@ function collapseSupersededChainEffect() {
         for (const succId of node.successors) {
           const predRef = ensureRecordId(predId, TABLES.MEMORY)
           const succRef = ensureRecordId(succId, TABLES.MEMORY)
-          const existing = yield* tryMemoryConsolidationEffect('read-existing-relation', () =>
-            db().query<{ id: RecordIdInput }>(
+          const existing = yield* db()
+            .query<{ id: RecordIdInput }>(
               new BoundQuery(
                 `SELECT id FROM ${MEMORY_RELATION_TABLE}
                  WHERE in = $predId AND out = $succId AND relationType = '${RELATION_SUPERSEDES}'
                  LIMIT 1`,
                 { predId: predRef, succId: succRef },
               ),
-            ),
-          )
+            )
+            .pipe(Effect.mapError((cause) => toMemoryConsolidationError('read-existing-relation', cause)))
 
           if (existing.length === 0) {
-            yield* tryMemoryConsolidationEffect('write-relation', () =>
-              db().relate(predRef, MEMORY_RELATION_TABLE, succRef, {
-                relationType: RELATION_SUPERSEDES,
-                confidence: 1.0,
-              }),
-            )
+            yield* db()
+              .relate(predRef, MEMORY_RELATION_TABLE, succRef, { relationType: RELATION_SUPERSEDES, confidence: 1.0 })
+              .pipe(Effect.mapError((cause) => toMemoryConsolidationError('write-relation', cause)))
           }
         }
       }
 
-      yield* tryMemoryConsolidationEffect('update-middle-node', () =>
-        db().updateWhere(MEMORY_TABLE, eq('id', ensureRecordId(node.middleId, TABLES.MEMORY)), {
-          archivedAt: nowDate(),
-        }),
-      )
+      yield* db()
+        .updateWhere(MEMORY_TABLE, eq('id', ensureRecordId(node.middleId, TABLES.MEMORY)), { archivedAt: nowDate() })
+        .pipe(Effect.mapError((cause) => toMemoryConsolidationError('update-middle-node', cause)))
 
       collapsed++
     }
@@ -311,8 +302,8 @@ function collapseSupersededChainEffect() {
 function decayImportanceEffect() {
   return Effect.gen(function* () {
     const [standardResult, ephemeralResult] = yield* Effect.all([
-      tryMemoryConsolidationEffect('decay-standard', () =>
-        db().query<{ count: number }>(
+      db()
+        .query<{ count: number }>(
           new BoundQuery(
             `UPDATE ${MEMORY_TABLE}
              SET importance = math::max([0.1, importance * 0.95])
@@ -323,10 +314,10 @@ function decayImportanceEffect() {
                AND (durability = 'standard' OR durability IS NONE)
                RETURN count() AS count`,
           ),
-        ),
-      ),
-      tryMemoryConsolidationEffect('decay-ephemeral', () =>
-        db().query<{ count: number }>(
+        )
+        .pipe(Effect.mapError((cause) => toMemoryConsolidationError('decay-standard', cause))),
+      db()
+        .query<{ count: number }>(
           new BoundQuery(
             `UPDATE ${MEMORY_TABLE}
              SET importance = math::max([0.1, importance * 0.85])
@@ -337,8 +328,8 @@ function decayImportanceEffect() {
                AND durability = 'ephemeral'
                RETURN count() AS count`,
           ),
-        ),
-      ),
+        )
+        .pipe(Effect.mapError((cause) => toMemoryConsolidationError('decay-ephemeral', cause))),
     ])
 
     return (standardResult[0]?.count ?? 0) + (ephemeralResult[0]?.count ?? 0)
@@ -347,15 +338,15 @@ function decayImportanceEffect() {
 
 function cleanupOrphanedRelationsEffect() {
   return Effect.gen(function* () {
-    const result = yield* tryMemoryConsolidationEffect('cleanup-relations', () =>
-      db().query<{ count: number }>(
+    const result = yield* db()
+      .query<{ count: number }>(
         new BoundQuery(
           `DELETE ${MEMORY_RELATION_TABLE}
            WHERE in.archivedAt IS NOT NONE OR out.archivedAt IS NOT NONE
            RETURN count() AS count`,
         ),
-      ),
-    )
+      )
+      .pipe(Effect.mapError((cause) => toMemoryConsolidationError('cleanup-relations', cause)))
     return result[0]?.count ?? 0
   })
 }
@@ -369,11 +360,11 @@ const handler = (job: SandboxedJob<MemoryConsolidationJob>) =>
       if (targetScope) {
         totalMerged = yield* deduplicateScopeEffect(targetScope)
       } else {
-        const scopeIds = yield* tryMemoryConsolidationEffect('load-scope-ids', () =>
-          db().query<string>(
+        const scopeIds = yield* db()
+          .query<string>(
             new BoundQuery(`SELECT VALUE scopeId FROM ${MEMORY_TABLE} WHERE archivedAt IS NONE GROUP BY scopeId`),
-          ),
-        )
+          )
+          .pipe(Effect.mapError((cause) => toMemoryConsolidationError('load-scope-ids', cause)))
 
         const results = yield* Effect.forEach(scopeIds, deduplicateScopeEffect, { concurrency: 2 })
         totalMerged = results.reduce((a, b) => a + b, 0)

package/src/workers/organization-learning.worker.ts

@@ -1,9 +1,9 @@
 import type { SandboxedJob } from 'bullmq'
-import { Effect } from 'effect'
+import { Cause, Effect } from 'effect'
 import type { Context } from 'effect'
 
+import { AiGatewayModelsTag } from '../ai-gateway/ai-gateway'
 import { serverLogger } from '../config/logger'
-import { effectTryPromise } from '../effect/helpers'
 import {
   AgentConfigServiceTag,
   DatabaseServiceTag,
@@ -27,9 +27,11 @@ import { createTracedWorkerProcessor } from './worker-utils'
 const runtime = await initializeSandboxedWorkerRuntime()
 const resolve = <I, T>(tag: Context.Key<I, T>): Promise<T> => runtime.runPromise(Effect.service(tag))
 const agentConfig = await resolve(AgentConfigServiceTag)
+const aiGatewayModels = await resolve(AiGatewayModelsTag)
 const queues = await resolve(LotaQueuesServiceTag)
 const regularChatDigestServices: RegularChatDigestServices = {
   agentConfig,
+  aiGatewayModels,
   databaseService: await resolve(DatabaseServiceTag),
   memoryService: await resolve(MemoryServiceTag),
   socialChatHistoryService: await resolve(SocialChatHistoryServiceTag),
@@ -39,12 +41,14 @@ const regularChatDigestServices: RegularChatDigestServices = {
 const workerRuntimeConfig = await resolve(RuntimeConfigServiceTag)
 const skillExtractionServices: SkillExtractionServices = {
   agentConfig,
+  aiGatewayModels,
   databaseService: await resolve(DatabaseServiceTag),
   learnedSkillService: await resolve(LearnedSkillServiceTag),
   socialChatHistoryService: await resolve(SocialChatHistoryServiceTag),
   runtimeAdapters: await resolve(RuntimeAdaptersServiceTag),
   embeddingModel: workerRuntimeConfig.aiGateway.embeddingModel,
   openRouterApiKey: workerRuntimeConfig.aiGateway.openRouterApiKey,
+  runPromise: (effect) => runtime.runPromise(effect),
 }
 const organizationLearningQueueJobService = await resolve(QueueJobServiceTag)
 
@@ -55,11 +59,17 @@ const handler = (job: SandboxedJob<OrganizationLearningJob>) =>
     Effect.gen(function* () {
       if (job.data.kind === 'regular-chat-memory-digest') {
         const digestJob = job.data
-        return yield* effectTryPromise(() => runRegularChatMemoryDigest(digestJob, regularChatDigestServices))
+        return yield* Effect.tryPromise({
+          try: () => runRegularChatMemoryDigest(digestJob, regularChatDigestServices),
+          catch: (cause) => new Cause.UnknownError(cause),
+        })
       }
 
       const skillJob = job.data
-      return yield* effectTryPromise(() => runSkillExtraction(skillJob, skillExtractionServices))
+      return yield* Effect.tryPromise({
+        try: () => runSkillExtraction(skillJob, skillExtractionServices),
+        catch: (cause) => new Cause.UnknownError(cause),
+      })
     }).pipe(
       Effect.catch((error) =>
         Effect.gen(function* () {