@lota-sdk/core 0.1.6 → 0.1.7

package/infrastructure/schema/00_workstream.surql

@@ -18,6 +18,7 @@ DEFINE FIELD IF NOT EXISTS lastCompactedMessageId ON TABLE workstream TYPE optio
 DEFINE FIELD IF NOT EXISTS nameGenerated ON TABLE workstream TYPE bool DEFAULT false;
 DEFINE FIELD IF NOT EXISTS isCompacting ON TABLE workstream TYPE bool DEFAULT false;
 DEFINE FIELD IF NOT EXISTS state ON TABLE workstream TYPE option<object> FLEXIBLE;
+DEFINE FIELD IF NOT EXISTS turnCount ON TABLE workstream TYPE int DEFAULT 0;
 
 DEFINE INDEX IF NOT EXISTS workstreamOrgIdx ON TABLE workstream COLUMNS organizationId;
 DEFINE INDEX IF NOT EXISTS workstreamUserIdx ON TABLE workstream COLUMNS userId;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lota-sdk/core",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/ai/embedding-cache.ts

@@ -0,0 +1,48 @@
+import { createHash } from 'node:crypto'
+
+import type IORedis from 'ioredis'
+
+import { aiLogger } from '../config/logger'
+
+const DEFAULT_TTL_SECONDS = 3600
+
+export class EmbeddingCache {
+  constructor(
+    private redis: IORedis,
+    private ttlSeconds: number = DEFAULT_TTL_SECONDS,
+  ) {}
+
+  private buildKey(model: string, text: string): string {
+    const hash = createHash('sha256').update(text).digest('hex')
+    return `emb:${model}:${hash}`
+  }
+
+  async get(model: string, text: string): Promise<number[] | null> {
+    try {
+      const cached = await this.redis.getBuffer(this.buildKey(model, text))
+      if (!cached) return null
+      return JSON.parse(cached.toString()) as number[]
+    } catch (error) {
+      aiLogger.debug`Embedding cache get failed: ${error}`
+      return null
+    }
+  }
+
+  async set(model: string, text: string, embedding: number[]): Promise<void> {
+    try {
+      await this.redis.set(this.buildKey(model, text), JSON.stringify(embedding), 'EX', this.ttlSeconds)
+    } catch (error) {
+      aiLogger.debug`Embedding cache set failed: ${error}`
+    }
+  }
+}
+
+let embeddingCacheInstance: EmbeddingCache | null = null
+
+export function configureEmbeddingCache(redis: IORedis, ttlSeconds?: number): void {
+  embeddingCacheInstance = new EmbeddingCache(redis, ttlSeconds ?? DEFAULT_TTL_SECONDS)
+}
+
+export function getEmbeddingCache(): EmbeddingCache | null {
+  return embeddingCacheInstance
+}

package/src/config/background-processing.ts

@@ -0,0 +1,33 @@
+export interface BackgroundProcessingConfig {
+  memoryExtractionFrequency: number
+  skillExtractionFrequency: number
+  memoryDigestFrequency: number
+  memoryConsolidationFrequency: number
+}
+
+const DEFAULT_CONFIG: BackgroundProcessingConfig = {
+  memoryExtractionFrequency: 3,
+  skillExtractionFrequency: 5,
+  memoryDigestFrequency: 1,
+  memoryConsolidationFrequency: 10,
+}
+
+let resolvedConfig: BackgroundProcessingConfig = { ...DEFAULT_CONFIG }
+
+export function configureBackgroundProcessing(config?: Partial<BackgroundProcessingConfig>): void {
+  resolvedConfig = {
+    memoryExtractionFrequency: config?.memoryExtractionFrequency ?? DEFAULT_CONFIG.memoryExtractionFrequency,
+    skillExtractionFrequency: config?.skillExtractionFrequency ?? DEFAULT_CONFIG.skillExtractionFrequency,
+    memoryDigestFrequency: config?.memoryDigestFrequency ?? DEFAULT_CONFIG.memoryDigestFrequency,
+    memoryConsolidationFrequency: config?.memoryConsolidationFrequency ?? DEFAULT_CONFIG.memoryConsolidationFrequency,
+  }
+}
+
+export function getBackgroundProcessingConfig(): BackgroundProcessingConfig {
+  return resolvedConfig
+}
+
+export function shouldRunAtFrequency(turnCount: number, frequency: number): boolean {
+  if (frequency <= 1) return true
+  return turnCount > 0 && turnCount % frequency === 0
+}

package/src/config/model-constants.ts

@@ -22,6 +22,10 @@ export const OPENROUTER_XHIGH_REASONING_PROVIDER_OPTIONS = {
   openai: { forceReasoning: true, reasoningEffort: 'xhigh', reasoningSummary: 'auto' },
 } as const
 
+export const OPENROUTER_MEDIUM_REASONING_PROVIDER_OPTIONS = {
+  openai: { forceReasoning: true, reasoningEffort: 'medium', reasoningSummary: 'auto' },
+} as const
+
 export const OPENROUTER_LOW_REASONING_PROVIDER_OPTIONS = {
   openai: { forceReasoning: true, reasoningEffort: 'low', reasoningSummary: 'auto' },
 } as const

package/src/db/memory-store.ts

@@ -1,5 +1,7 @@
 import { BoundQuery, eq, inside } from 'surrealdb'
 
+import { getEmbeddingCache } from '../ai/embedding-cache'
+import { env } from '../config/env-shapes'
 import { aiLogger } from '../config/logger'
 import { DEFAULT_MEMORY_SEARCH_LIMIT } from '../config/search'
 import { createDefaultEmbeddings } from '../embeddings/provider'
@@ -7,7 +9,14 @@ import { memoryQueryBuilder } from './memory-query-builder'
 import type { RelationCounts } from './memory-store.helpers'
 import { hashContent, mapRowToMemoryRecord, processGraphAwareRows } from './memory-store.helpers'
 import type { BasicSearchRow, SurrealMemoryRow } from './memory-store.rows'
-import type { LinearNormalization, MemoryEvent, MemoryRecord, MemorySearchResult, RelationType } from './memory-types'
+import type {
+  LinearNormalization,
+  MemoryEvent,
+  MemoryListOptions,
+  MemoryRecord,
+  MemorySearchResult,
+  RelationType,
+} from './memory-types'
 import { ensureRecordId, recordIdToString } from './record-id'
 import type { RecordIdInput, RecordIdRef } from './record-id'
 import { databaseService } from './service'
@@ -33,6 +42,18 @@ export class SurrealMemoryStore {
 
   constructor(private embeddings: EmbeddingClient) {}
 
+  private toMetadataFieldPath(key: string): string {
+    const segments = key.split('.').map((segment) => segment.trim())
+    if (
+      segments.length === 0 ||
+      segments.some((segment) => segment.length === 0 || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(segment))
+    ) {
+      throw new Error(`Invalid memory metadata filter key: ${key}`)
+    }
+
+    return `metadata.${segments.join('.')}`
+  }
+
   private tokenizeQuery(query: string): string[] {
     return query
       .toLowerCase()
@@ -188,6 +209,7 @@ export class SurrealMemoryStore {
     const now = Date.now()
     this.pruneEmbeddingCache(now)
 
+    // L1: in-memory cache
     const cached = this.embeddingCache.get(cacheKey)
     if (cached && now - cached.ts <= EMBEDDING_CACHE_TTL_MS) {
       return cached.embedding
@@ -198,16 +220,32 @@ export class SurrealMemoryStore {
       return await inFlight
     }
 
-    const request = this.embeddings
-      .embedQuery(normalized)
-      .then((embedding) => {
-        this.embeddingCache.set(cacheKey, { embedding, ts: Date.now() })
-        this.pruneEmbeddingCache(Date.now())
-        return embedding
-      })
-      .finally(() => {
-        this.embeddingInFlight.delete(cacheKey)
-      })
+    const request = (async () => {
+      // L2: Redis cache
+      const redisCache = getEmbeddingCache()
+      if (redisCache) {
+        const model = env.AI_EMBEDDING_MODEL
+        const redisCached = await redisCache.get(model, normalized)
+        if (redisCached) {
+          this.embeddingCache.set(cacheKey, { embedding: redisCached, ts: Date.now() })
+          return redisCached
+        }
+      }
+
+      // L3: API call
+      const embedding = await this.embeddings.embedQuery(normalized)
+      this.embeddingCache.set(cacheKey, { embedding, ts: Date.now() })
+      this.pruneEmbeddingCache(Date.now())
+
+      // Backfill Redis
+      if (redisCache) {
+        void redisCache.set(env.AI_EMBEDDING_MODEL, normalized, embedding)
+      }
+
+      return embedding
+    })().finally(() => {
+      this.embeddingInFlight.delete(cacheKey)
+    })
 
     this.embeddingInFlight.set(cacheKey, request)
     return await request
@@ -558,6 +596,8 @@ export class SurrealMemoryStore {
     return processed
   }
 
+  private static HYBRID_SEARCH_TIMEOUT_MS = 2000
+
   async hybridSearchWeighted(
     query: string,
     options: {
@@ -569,6 +609,7 @@ export class SurrealMemoryStore {
       fastMode?: boolean
     },
   ): Promise<MemorySearchResult[]> {
+    const searchStart = performance.now()
     const queryEmbedding = await this.generateEmbedding(query)
 
     const tokens = this.tokenizeQuery(query)
@@ -601,7 +642,25 @@ export class SurrealMemoryStore {
 
     type LinearRow = BasicSearchRow & { linearScore: number }
 
-    const results = await this.queryFinalStatement<LinearRow>(sql, bindVars)
+    let results: LinearRow[]
+    try {
+      results = await Promise.race([
+        this.queryFinalStatement<LinearRow>(sql, bindVars),
+        new Promise<never>((_, reject) =>
+          setTimeout(() => reject(new Error('Hybrid search timeout')), SurrealMemoryStore.HYBRID_SEARCH_TIMEOUT_MS),
+        ),
+      ])
+    } catch {
+      const elapsed = performance.now() - searchStart
+      aiLogger.warn`Hybrid search timed out after ${elapsed.toFixed(0)}ms (scopeId: ${options.scopeId}). Falling back to vector-only.`
+      return this.vectorSearchWithEmbedding({
+        embedding: queryEmbedding,
+        scopeId: options.scopeId,
+        limit: options.limit,
+        memoryType: options.memoryType,
+        fastMode: options.fastMode,
+      })
+    }
 
     if (results.length === 0) {
       aiLogger.debug`Weighted hybrid search returned 0 raw results (scopeId: ${options.scopeId})`
@@ -644,7 +703,8 @@ export class SurrealMemoryStore {
       })
     }
 
-    aiLogger.info`[SUCCESS_WEIGHTED_SEARCH] Weighted hybrid search succeeded (scopeId: ${options.scopeId}, rawResults: ${results.length}, returned: ${processed.length}, weights: ${weights.join(',')}, normalization: ${normalization})`
+    const elapsed = performance.now() - searchStart
+    aiLogger.info`[SUCCESS_WEIGHTED_SEARCH] Weighted hybrid search succeeded (scopeId: ${options.scopeId}, rawResults: ${results.length}, returned: ${processed.length}, weights: ${weights.join(',')}, normalization: ${normalization}, latencyMs: ${elapsed.toFixed(0)})`
     this.touchMemories(processed.map((row) => row.id))
     return processed
   }
@@ -709,16 +769,47 @@ export class SurrealMemoryStore {
     await this.recordHistory(id, existing.content, null, 'DELETE')
   }
 
-  async list(scopeId: string, memoryType?: MemoryRecord['memoryType']): Promise<MemoryRecord[]> {
-    const typeFilter = memoryType ? 'AND memoryType = $memoryType' : ''
+  async list(options: MemoryListOptions): Promise<MemoryRecord[]> {
+    const whereClauses = [
+      'scopeId = $scopeId',
+      'archivedAt IS NONE',
+      '(validUntil IS NONE OR validUntil > time::now())',
+    ]
+    const bindVars: Record<string, unknown> = { scopeId: options.scopeId }
+
+    if (options.memoryType) {
+      whereClauses.push('memoryType = $memoryType')
+      bindVars.memoryType = options.memoryType
+    }
+
+    for (const [index, [key, value]] of Object.entries(options.metadataEquals ?? {}).entries()) {
+      const fieldPath = this.toMetadataFieldPath(key)
+      const bindKey = `metadataEquals_${index}`
+      whereClauses.push(`${fieldPath} = $${bindKey}`)
+      bindVars[bindKey] = value
+    }
+
+    for (const [index, [key, value]] of Object.entries(options.metadataNotEquals ?? {}).entries()) {
+      const fieldPath = this.toMetadataFieldPath(key)
+      const bindKey = `metadataNotEquals_${index}`
+      whereClauses.push(`(${fieldPath} IS NONE OR ${fieldPath} != $${bindKey})`)
+      bindVars[bindKey] = value
+    }
+
+    const sortDirection = options.sort === 'createdAtAsc' ? 'ASC' : 'DESC'
+    const limitClause = typeof options.limit === 'number' ? 'LIMIT $limit' : ''
+    if (typeof options.limit === 'number') {
+      bindVars.limit = options.limit
+    }
+
     const sql = `
       SELECT * FROM ${MEMORY_TABLE}
-      WHERE scopeId = $scopeId ${typeFilter}
-        AND archivedAt IS NONE AND (validUntil IS NONE OR validUntil > time::now())
-      ORDER BY createdAt DESC
+      WHERE ${whereClauses.join('\n        AND ')}
+      ORDER BY createdAt ${sortDirection}
+      ${limitClause}
     `
 
-    const results = await databaseService.query<SurrealMemoryRow>(new BoundQuery(sql, { scopeId, memoryType }))
+    const results = await databaseService.query<SurrealMemoryRow>(new BoundQuery(sql, bindVars))
 
     return results.map((row) => mapRowToMemoryRecord(row))
   }

package/src/db/memory-types.ts

@@ -55,6 +55,17 @@ export interface SearchOptions {
   pointInTime?: string
 }
 
+export type MemoryListScalar = boolean | null | number | string
+
+export interface MemoryListOptions {
+  scopeId: string
+  limit?: number
+  memoryType?: MemoryType
+  metadataEquals?: Record<string, MemoryListScalar>
+  metadataNotEquals?: Record<string, MemoryListScalar>
+  sort?: 'createdAtAsc' | 'createdAtDesc'
+}
+
 export type LinearNormalization = 'minmax' | 'zscore'
 
 export interface WeightedSearchOptions extends SearchOptions {

package/src/db/memory.ts

@@ -21,11 +21,13 @@ import type {
   Durability,
   ExtractedFact,
   MemoryConfig,
+  MemoryListOptions,
   MemorySearchResult,
   MemoryType,
   MemoryUpdateOutput,
   Message,
   MemoryRecord,
+  RelationType,
   SearchOptions,
   WeightedSearchOptions,
 } from './memory-types'
@@ -152,10 +154,18 @@ export class Memory {
     return await this.store.listTopMemories(options)
   }
 
+  async list(options: MemoryListOptions): Promise<MemoryRecord[]> {
+    return await this.store.list(options)
+  }
+
   async updateMemory(id: string, newContent: string): Promise<void> {
     await this.store.update(id, newContent)
   }
 
+  async addRelation(fromId: string, toId: string, relationType: RelationType, confidence = 1): Promise<void> {
+    await this.store.addRelation(fromId, toId, relationType, confidence)
+  }
+
   async getStaleMemories(scopeId: string, limit?: number): Promise<MemorySearchResult[]> {
     return await this.store.getStaleMemories(scopeId, limit)
   }
@@ -237,7 +247,7 @@ export class Memory {
   private async prepareFactsForScope(facts: ExtractedFact[], options: AddOptions): Promise<PreparedScopeUpdate> {
     const factMaps = buildMemoryFactMaps(facts)
 
-    const existingMemories = await this.store.list(options.scopeId, options.memoryType)
+    const existingMemories = await this.store.list({ scopeId: options.scopeId, memoryType: options.memoryType })
     const oldMemoryFormat = existingMemories.map((m) => ({ id: m.id, text: m.content }))
 
     const factContents = facts.map((f) => f.content)

package/src/index.ts

@@ -19,6 +19,7 @@ import type { routeWorkstreamChatMessages } from './runtime/chat-request-routing
 import type { LotaPlugin } from './runtime/plugin-types'
 import type { LotaRuntimeAdapters, LotaRuntimeTurnHooks } from './runtime/runtime-extensions'
 import type { attachmentService } from './services/attachment.service'
+import type { documentChunkService } from './services/document-chunk.service'
 import type { executionPlanService } from './services/execution-plan.service'
 import type { memoryService } from './services/memory.service'
 import type { verifyMutatingApproval } from './services/mutating-approval.service'
@@ -74,8 +75,14 @@ export interface LotaRuntimeConfig {
   }
   firecrawl: { apiKey: string; apiBaseUrl?: string }
   logging?: { level?: 'trace' | 'debug' | 'info' | 'warning' | 'error' | 'fatal' }
-  memory?: { searchK?: number }
+  memory?: { searchK?: number; embeddingCacheTtlSeconds?: number }
   workstreams?: LotaWorkstreamConfig
+  backgroundProcessing?: {
+    memoryExtractionFrequency?: number
+    skillExtractionFrequency?: number
+    memoryDigestFrequency?: number
+    memoryConsolidationFrequency?: number
+  }
 
   agents: {
     roster: readonly string[]
@@ -103,6 +110,7 @@ export interface LotaRuntime {
     redis: RedisConnectionManager
     closeRedisConnection: () => Promise<void>
     attachmentService: typeof attachmentService
+    documentChunkService: typeof documentChunkService
     generatedDocumentStorageService: typeof generatedDocumentStorageService
     memoryService: typeof memoryService
     verifyMutatingApproval: typeof verifyMutatingApproval
@@ -200,6 +208,20 @@ export async function createLotaRuntime(config: LotaRuntimeConfig): Promise<Lota
   const { configureWorkstreams } = await import('./config/workstream-defaults')
   const { configureRuntimeExtensions } = await import('./runtime/runtime-extensions')
   const { routeWorkstreamChatMessages } = await import('./runtime/chat-request-routing')
+  const { configureBackgroundProcessing } = await import('./config/background-processing')
+  const { configureEmbeddingCache } = await import('./ai/embedding-cache')
+
+  // Resolve config defaults
+  const memory = {
+    searchK: config.memory?.searchK ?? 6,
+    embeddingCacheTtlSeconds: config.memory?.embeddingCacheTtlSeconds ?? 3600,
+  }
+  const backgroundProcessing = {
+    memoryExtractionFrequency: config.backgroundProcessing?.memoryExtractionFrequency ?? 3,
+    skillExtractionFrequency: config.backgroundProcessing?.skillExtractionFrequency ?? 5,
+    memoryDigestFrequency: config.backgroundProcessing?.memoryDigestFrequency ?? 1,
+    memoryConsolidationFrequency: config.backgroundProcessing?.memoryConsolidationFrequency ?? 10,
+  }
 
   setEnv({
     AI_GATEWAY_URL: config.aiGateway.url,
@@ -217,7 +239,7 @@ export async function createLotaRuntime(config: LotaRuntimeConfig): Promise<Lota
     ATTACHMENT_URL_EXPIRES_IN: config.s3.attachmentUrlExpiresIn ?? 1800,
     FIRECRAWL_API_KEY: config.firecrawl.apiKey,
     FIRECRAWL_API_BASE_URL: config.firecrawl.apiBaseUrl,
-    MEMORY_SEARCH_K: config.memory?.searchK ?? 6,
+    MEMORY_SEARCH_K: memory.searchK,
   })
 
   await configureLogger(config.logging?.level ?? 'info')
@@ -233,6 +255,8 @@ export async function createLotaRuntime(config: LotaRuntimeConfig): Promise<Lota
 
   const redisManager = createRedisConnectionManager({ url: config.redis.url })
   setRedisConnectionManager(redisManager)
+  configureEmbeddingCache(redisManager.getConnection(), memory.embeddingCacheTtlSeconds)
+  configureBackgroundProcessing(backgroundProcessing)
 
   configureAgents({
     roster: config.agents.roster,
@@ -253,6 +277,7 @@ export async function createLotaRuntime(config: LotaRuntimeConfig): Promise<Lota
   }
 
   const { attachmentService } = await import('./services/attachment.service')
+  const { documentChunkService } = await import('./services/document-chunk.service')
   const { recentActivityService } = await import('./services/recent-activity.service')
   const { recentActivityTitleService } = await import('./services/recent-activity-title.service')
   const { executionPlanService } = await import('./services/execution-plan.service')
@@ -392,6 +417,7 @@ export async function createLotaRuntime(config: LotaRuntimeConfig): Promise<Lota
       redis: redisManager,
       closeRedisConnection: async () => await redisManager.closeConnection(),
       attachmentService,
+      documentChunkService,
       generatedDocumentStorageService,
       memoryService,
       verifyMutatingApproval,

package/src/queues/memory-consolidation.queue.ts

@@ -32,6 +32,12 @@ function getMemoryConsolidationQueue(): Queue<MemoryConsolidationJob> {
   return _memoryConsolidationQueue
 }
 
+export async function enqueueMemoryConsolidation(job: MemoryConsolidationJob = {}) {
+  await getMemoryConsolidationQueue().add('consolidate-turn', job, {
+    jobId: job.scopeId ? `consolidate-turn:${job.scopeId}` : undefined,
+  })
+}
+
 export async function scheduleRecurringConsolidation() {
   await getMemoryConsolidationQueue().add(
     'consolidate',

package/src/runtime/agent-types.ts

@@ -1,22 +1,5 @@
-import type { Output, PrepareStepFunction, StopCondition, ToolLoopAgentOnFinishCallback, ToolSet } from 'ai'
-
-export type ChatMode = 'direct' | 'workstreamMode' | 'fixedWorkstreamMode'
-
-export interface CreateRoutedAgentOptions<TTools extends ToolSet = ToolSet> {
-  mode: ChatMode
-  tools: TTools
-  extraInstructions?: string
-  stopWhen?: StopCondition<TTools> | Array<StopCondition<TTools>>
-  prepareStep?: PrepareStepFunction<TTools>
-  maxRetries?: number
-  modelOverride?: { model: unknown; providerOptions?: Record<string, unknown> }
-  onFinish?: ToolLoopAgentOnFinishCallback<TTools>
-}
-
-export interface CreateHelperToolLoopAgentOptions {
-  instructions?: string
-  maxOutputTokens?: number
-  temperature?: number
-  output?: Output.Output
-  maxRetries?: number
-}
+export type {
+  ChatMode,
+  CreateHelperToolLoopAgentOptions,
+  CreateRoutedAgentOptions,
+} from '@lota-sdk/shared/runtime/agent-types'

package/src/runtime/memory-digest-policy.ts

@@ -1,3 +1,5 @@
+import { getBackgroundProcessingConfig, shouldRunAtFrequency } from '../config/background-processing'
+
 export function shouldEnqueueOnboardingPostChatMemory(params: {
   onboardingActive: boolean
   userMessageText: string
@@ -9,6 +11,32 @@ export function shouldEnqueueOnboardingPostChatMemory(params: {
   return params.userMessageText.trim().length > 0 || params.hasAttachmentContext
 }
 
-export function shouldEnqueueRegularDigestForWorkstream(params: { onboardingActive: boolean }): boolean {
-  return !params.onboardingActive
+export function shouldEnqueueRegularDigestForWorkstream(params: {
+  onboardingActive: boolean
+  turnCount?: number
+}): boolean {
+  if (params.onboardingActive) return false
+  const { memoryDigestFrequency } = getBackgroundProcessingConfig()
+  if (typeof params.turnCount === 'number') {
+    return shouldRunAtFrequency(params.turnCount, memoryDigestFrequency)
+  }
+  return true
+}
+
+export function shouldEnqueueMemoryExtraction(params: { onboardingActive: boolean; turnCount?: number }): boolean {
+  if (params.onboardingActive) return true
+  const { memoryExtractionFrequency } = getBackgroundProcessingConfig()
+  if (typeof params.turnCount === 'number') {
+    return shouldRunAtFrequency(params.turnCount, memoryExtractionFrequency)
+  }
+  return true
+}
+
+export function shouldEnqueueMemoryConsolidation(params: { onboardingActive: boolean; turnCount?: number }): boolean {
+  if (params.onboardingActive) return false
+  const { memoryConsolidationFrequency } = getBackgroundProcessingConfig()
+  if (typeof params.turnCount === 'number') {
+    return shouldRunAtFrequency(params.turnCount, memoryConsolidationFrequency)
+  }
+  return false
 }

package/src/runtime/skill-extraction-policy.ts

@@ -1,3 +1,10 @@
-export function shouldEnqueueSkillExtraction(params: { onboardingActive: boolean }): boolean {
-  return !params.onboardingActive
+import { getBackgroundProcessingConfig, shouldRunAtFrequency } from '../config/background-processing'
+
+export function shouldEnqueueSkillExtraction(params: { onboardingActive: boolean; turnCount?: number }): boolean {
+  if (params.onboardingActive) return false
+  const { skillExtractionFrequency } = getBackgroundProcessingConfig()
+  if (typeof params.turnCount === 'number') {
+    return shouldRunAtFrequency(params.turnCount, skillExtractionFrequency)
+  }
+  return true
 }

package/src/services/memory.service.ts

@@ -8,10 +8,12 @@ import { isUniqueIndexConflict } from '../db/memory-store.helpers'
 import type {
   AddOptions,
   ExtractedFact,
+  MemoryListScalar,
   MemoryRecord,
   MemorySearchResult,
   MemoryType,
   Message,
+  RelationType,
 } from '../db/memory-types'
 import { withOrgMemoryLock } from '../redis/org-memory-lock'
 import { createHelperModelRuntime } from '../runtime/helper-model'
@@ -413,6 +415,19 @@ class MemoryService {
     return `Agent memory (${agentName}):\n${agentResult}\n\nGlobal org memory:\n${orgResult}`
   }
 
+  async listOrganizationMemoryRecords(params: {
+    orgId: string
+    limit?: number
+    memoryType?: MemoryType
+    metadataEquals?: Record<string, MemoryListScalar>
+    metadataNotEquals?: Record<string, MemoryListScalar>
+    sort?: 'createdAtAsc' | 'createdAtDesc'
+  }): Promise<MemoryRecord[]> {
+    const { orgId, ...listOptions } = params
+    const orgMemory = this.getOrgMemory(orgId)
+    return await orgMemory.list({ scopeId: scopeId(ORG_SCOPE_PREFIX, orgId), ...listOptions })
+  }
+
   async getTopMemories(params: { orgId: string; agentName?: string; limit?: number }): Promise<string | undefined> {
     const orgMemory = this.getOrgMemory(params.orgId)
     const orgScopeId = scopeId(ORG_SCOPE_PREFIX, params.orgId)
@@ -568,12 +583,14 @@ class MemoryService {
     memoryType,
     metadata,
     importance,
+    durability,
   }: {
     orgId: string
     content: string
     memoryType: MemoryType
     metadata?: Record<string, unknown>
     importance?: number
+    durability?: MemoryRecord['durability']
   }): Promise<string> {
     const orgScopeId = scopeId(ORG_SCOPE_PREFIX, orgId)
     aiLogger.info`[MEMORY_DEBUG] createOrganizationMemory - orgId: "${orgId}", scopeId: "${orgScopeId}", content preview: "${content.slice(0, 50)}"`
@@ -583,6 +600,7 @@ class MemoryService {
         scopeId: orgScopeId,
         memoryType,
         importance: importance ?? 1,
+        durability,
         metadata: { orgId, ...metadata },
       })
     } catch (error) {
@@ -594,6 +612,23 @@ class MemoryService {
     }
   }
 
+  async addOrganizationMemoryRelation({
+    orgId,
+    fromMemoryId,
+    toMemoryId,
+    relationType,
+    confidence,
+  }: {
+    orgId: string
+    fromMemoryId: string
+    toMemoryId: string
+    relationType: RelationType
+    confidence?: number
+  }): Promise<void> {
+    const memory = this.getOrgMemory(orgId)
+    await memory.addRelation(fromMemoryId, toMemoryId, relationType, confidence)
+  }
+
   async createAgentMemory({
     orgId,
     agentName,

package/src/services/workstream-turn-preparation.ts

@@ -24,6 +24,7 @@ import type { RecordIdRef } from '../db/record-id'
 import { recordIdToString } from '../db/record-id'
 import { TABLES } from '../db/tables'
 import { enqueueContextCompaction } from '../queues/context-compaction.queue'
+import { enqueueMemoryConsolidation } from '../queues/memory-consolidation.queue'
 import { enqueuePostChatMemory } from '../queues/post-chat-memory.queue'
 import { enqueueRecentActivityTitleRefinement } from '../queues/recent-activity-title-refinement.queue'
 import { enqueueRegularChatMemoryDigest } from '../queues/regular-chat-memory-digest.queue'
@@ -44,6 +45,8 @@ import { CONTEXT_SIZE } from '../runtime/context-compaction-constants'
 import { createExecutionPlanInstructionSectionCache } from '../runtime/execution-plan'
 import { mergeInstructionSections } from '../runtime/instruction-sections'
 import {
+  shouldEnqueueMemoryConsolidation,
+  shouldEnqueueMemoryExtraction,
   shouldEnqueueOnboardingPostChatMemory,
   shouldEnqueueRegularDigestForWorkstream,
 } from '../runtime/memory-digest-policy'
@@ -1211,6 +1214,7 @@ export async function prepareWorkstreamRunCore(params: WorkstreamRunCoreParams):
             }
 
             if (allAssistantMessages.length > 0 && shouldProcessPostRunSideEffects) {
+              const turnCount = await workstreamService.incrementTurnCount(workstreamRef)
               const agentMessages = buildAgentHistoryMessages(allAssistantMessages)
               const historyMessagesForMemory = appendCompactionContextToHistoryMessages(
                 toHistoryMessages(recentHistory),
@@ -1221,14 +1225,16 @@ export async function prepareWorkstreamRunCore(params: WorkstreamRunCoreParams):
               const readableUploads = listReadableUploads()
               const attachmentMetadataContext = buildReadableUploadMetadataContext(readableUploads)
               const hasAttachmentContext = Boolean(attachmentMetadataContext)
-              if (
-                shouldEnqueueOnboardingPostChatMemory({
-                  onboardingActive,
-                  userMessageText,
-                  hasAttachmentContext,
-                  agentMessageCount: agentMessages.length,
-                })
-              ) {
+              const shouldExtractMemory = onboardingActive
+                ? shouldEnqueueOnboardingPostChatMemory({
+                    onboardingActive,
+                    userMessageText,
+                    hasAttachmentContext,
+                    agentMessageCount: agentMessages.length,
+                  })
+                : shouldEnqueueMemoryExtraction({ onboardingActive, turnCount }) && userMessageText.length > 0
+
+              if (shouldExtractMemory) {
                 const memoryUserMessage = userMessageText || 'User uploaded attachment(s).'
                 await safeEnqueue(
                   () =>
@@ -1315,17 +1321,23 @@ export async function prepareWorkstreamRunCore(params: WorkstreamRunCoreParams):
                 }
               }
 
-              if (shouldEnqueueRegularDigestForWorkstream({ onboardingActive })) {
+              if (shouldEnqueueRegularDigestForWorkstream({ onboardingActive, turnCount })) {
                 await safeEnqueue(() => enqueueRegularChatMemoryDigest({ orgId: orgIdString }), {
                   operationName: 'regular chat memory digest enqueue',
                 })
               }
 
-              if (shouldEnqueueSkillExtraction({ onboardingActive })) {
+              if (shouldEnqueueSkillExtraction({ onboardingActive, turnCount })) {
                 await safeEnqueue(() => enqueueSkillExtraction({ orgId: orgIdString }), {
                   operationName: 'skill extraction enqueue',
                 })
               }
+
+              if (shouldEnqueueMemoryConsolidation({ onboardingActive, turnCount })) {
+                await safeEnqueue(() => enqueueMemoryConsolidation({ scopeId: orgIdString }), {
+                  operationName: 'memory consolidation enqueue',
+                })
+              }
             }
 
             if (allAssistantMessages.length > 0) {

package/src/services/workstream-turn.ts

@@ -2,10 +2,12 @@ import type { ChatMessage } from '@lota-sdk/shared/schemas/chat-message'
 import { createUIMessageStream } from 'ai'
 
 import { hasApprovalRespondedParts, isApprovalContinuationRequest } from '../runtime/approval-continuation'
+import { wrapResponseWithKeepalive } from '../utils/sse-keepalive'
 import { prepareWorkstreamRunCore } from './workstream-turn-preparation'
 import type { WorkstreamTurnParams, WorkstreamApprovalContinuationParams } from './workstream-turn-preparation'
 
 export { hasApprovalRespondedParts, isApprovalContinuationRequest }
+export { wrapResponseWithKeepalive }
 
 export async function createWorkstreamApprovalContinuationStream(params: WorkstreamApprovalContinuationParams) {
   const prepared = await prepareWorkstreamRunCore({ ...params, kind: 'approvalContinuation' })

package/src/services/workstream.service.ts

@@ -568,6 +568,48 @@ class WorkstreamService extends BaseService<typeof WorkstreamSchema> {
     return { workstreams: sliced.map((workstream) => this.normalizeWorkstream(workstream)), hasMore }
   }
 
+  async listOrganizationWorkstreams(params: {
+    orgId: RecordIdRef
+    mode?: 'direct' | 'group'
+    agentId?: string
+    core?: boolean
+    includeArchived?: boolean
+  }): Promise<NormalizedWorkstream[]> {
+    const whereClauses = ['organizationId = $orgId']
+    const variables: Record<string, unknown> = { orgId: params.orgId }
+
+    if (params.mode) {
+      whereClauses.push('mode = $mode')
+      variables.mode = params.mode
+    }
+
+    if (typeof params.core === 'boolean') {
+      whereClauses.push('core = $core')
+      variables.core = params.core
+    }
+
+    if (params.agentId) {
+      whereClauses.push('agentId = $agentId')
+      variables.agentId = params.agentId
+    }
+
+    if (params.includeArchived !== true) {
+      whereClauses.push('status = "regular"')
+    }
+
+    const workstreams = await databaseService.queryMany<typeof WorkstreamSchema>(
+      new BoundQuery(
+        `SELECT * FROM ${TABLES.WORKSTREAM}
+         WHERE ${whereClauses.join('\n           AND ')}
+         ORDER BY createdAt ASC, id ASC`,
+        variables,
+      ),
+      WorkstreamSchema,
+    )
+
+    return workstreams.map((workstream) => this.normalizeWorkstream(workstream))
+  }
+
   async getWorkstream(workstreamId: RecordIdRef): Promise<NormalizedWorkstream> {
     const workstream = await this.getById(workstreamId)
     return this.normalizeWorkstream(workstream)
@@ -842,6 +884,16 @@ class WorkstreamService extends BaseService<typeof WorkstreamSchema> {
     return { ...publicWorkstream, workstreamState }
   }
 
+  async incrementTurnCount(workstreamId: RecordIdRef): Promise<number> {
+    const workstreamRef = ensureRecordId(workstreamId, TABLES.WORKSTREAM)
+    const result = await databaseService.query<{ turnCount: number }>(surql`
+      UPDATE ONLY ${workstreamRef}
+      SET turnCount += 1
+      RETURN turnCount
+    `)
+    return result[0]?.turnCount ?? 0
+  }
+
   async persistGeneratedTitle(workstreamId: RecordIdRef, title: string): Promise<void> {
     await this.update(workstreamId, { title, nameGenerated: true })
   }

package/src/services/workstream.types.ts

@@ -111,6 +111,7 @@ export const WorkstreamSchema = z.object({
   nameGenerated: z.boolean().optional().default(false),
   isCompacting: z.boolean().optional(),
   state: z.unknown().optional(),
+  turnCount: z.number().int().optional().default(0),
   createdAt: z.coerce.date(),
   updatedAt: z.coerce.date(),
   userId: z.any(), // RecordId

package/src/tools/research-topic.tool.ts

@@ -1,5 +1,8 @@
 import { bifrostChatModel } from '../bifrost/bifrost'
-import { OPENROUTER_WEB_RESEARCH_MODEL_ID } from '../config/model-constants'
+import {
+  OPENROUTER_MEDIUM_REASONING_PROVIDER_OPTIONS,
+  OPENROUTER_WEB_RESEARCH_MODEL_ID,
+} from '../config/model-constants'
 import { createDelegatedAgentTool } from '../system-agents/delegated-agent-factory'
 import { RESEARCHER_PROMPT } from '../system-agents/researcher.agent'
 import { fetchWebpageTool } from './fetch-webpage.tool'
@@ -10,6 +13,7 @@ export const researchTopicTool = createDelegatedAgentTool({
   description:
     'Delegate a research task to a dedicated research agent that searches the web, fetches pages, and returns a synthesized markdown report. Call multiple instances in parallel for broad research across different topics.',
   model: bifrostChatModel(OPENROUTER_WEB_RESEARCH_MODEL_ID),
+  providerOptions: OPENROUTER_MEDIUM_REASONING_PROVIDER_OPTIONS,
   instructions: RESEARCHER_PROMPT,
   tools: { searchWeb: searchWebTool.create(), fetchWebpage: fetchWebpageTool.create() },
 })

package/src/utils/sse-keepalive.ts

@@ -0,0 +1,40 @@
+const KEEPALIVE_COMMENT = new TextEncoder().encode(': keepalive\n\n')
+const DEFAULT_KEEPALIVE_INTERVAL_MS = 20_000
+
+/**
+ * Wraps an SSE Response body with periodic keepalive comments.
+ * SSE comments (`: keepalive\n\n`) are ignored by standard SSE parsers,
+ * so no client changes are needed.
+ */
+export function wrapResponseWithKeepalive(response: Response, intervalMs = DEFAULT_KEEPALIVE_INTERVAL_MS): Response {
+  const body = response.body
+  if (!body) return response
+
+  let intervalHandle: ReturnType<typeof setInterval> | null = null
+
+  const transformed = body.pipeThrough(
+    new TransformStream<Uint8Array, Uint8Array>({
+      start(controller) {
+        intervalHandle = setInterval(() => {
+          try {
+            controller.enqueue(KEEPALIVE_COMMENT)
+          } catch {
+            if (intervalHandle) clearInterval(intervalHandle)
+          }
+        }, intervalMs)
+      },
+      transform(chunk, controller) {
+        controller.enqueue(chunk)
+      },
+      flush() {
+        if (intervalHandle) clearInterval(intervalHandle)
+      },
+    }),
+  )
+
+  return new Response(transformed, {
+    headers: response.headers,
+    status: response.status,
+    statusText: response.statusText,
+  })
+}