@lota-sdk/core 0.1.5

package/src/utils/hono-error-handler.ts

@@ -0,0 +1,71 @@
+import type { ErrorHandler } from 'hono'
+import { HTTPException } from 'hono/http-exception'
+import { ZodError } from 'zod'
+
+import { getErrorMessage } from './error'
+import { AppError } from './errors'
+
+type AppErrorLike = Pick<AppError, 'code' | 'message' | 'statusCode' | 'toResponse'> & { name?: string }
+
+type HonoErrorLogger = Pick<Console, 'warn' | 'error'>
+
+function isAppErrorLike(error: unknown): error is AppErrorLike {
+  if (!error || typeof error !== 'object') {
+    return false
+  }
+
+  const candidate = error as Partial<AppErrorLike>
+  return (
+    typeof candidate.code === 'string' &&
+    typeof candidate.message === 'string' &&
+    typeof candidate.statusCode === 'number' &&
+    typeof candidate.toResponse === 'function'
+  )
+}
+
+function createValidationErrorResponse(issues: Array<{ path: string; message: string }>) {
+  return { error: { code: 'VALIDATION_ERROR', message: 'Validation failed', issues } }
+}
+
+function createHttpErrorResponse(message: string) {
+  return { error: { code: 'HTTP_ERROR', message } }
+}
+
+function createServerErrorResponse(message: string) {
+  return { error: { code: 'INTERNAL_SERVER_ERROR', message } }
+}
+
+export function createHonoErrorHandler(logger: HonoErrorLogger): ErrorHandler {
+  return (error, c) => {
+    const appError = error instanceof AppError || isAppErrorLike(error) ? error : null
+
+    if (appError) {
+      const log = appError.statusCode >= 500 ? logger.error : logger.warn
+      const errorName = typeof appError.name === 'string' && appError.name.length > 0 ? appError.name : 'AppError'
+      log(`Request failed: ${errorName} (${appError.code}) ${appError.message}`)
+      const { status, body } = appError.toResponse()
+      const typedStatus = status as Parameters<typeof c.json>[1]
+      return c.json(body, typedStatus)
+    }
+
+    if (error instanceof HTTPException) {
+      const log = error.status >= 500 ? logger.error : logger.warn
+      log(`Request failed: HTTPException ${error.status} ${error.message}`)
+      return error.res ?? c.json(createHttpErrorResponse(error.message), error.status)
+    }
+
+    if (error instanceof ZodError) {
+      logger.warn(`Request failed: ZodError ${error.message}`)
+      const issues = error.issues.map((issue) => ({ path: issue.path.join('.'), message: issue.message }))
+      return c.json(createValidationErrorResponse(issues), 400)
+    }
+
+    if (error instanceof Error) {
+      logger.error(`Server error: ${error.name} ${error.message}\n${error.stack ?? ''}`)
+      return c.json(createServerErrorResponse('Internal Server Error'), 500)
+    }
+
+    logger.error(`Server error: ${getErrorMessage(error)}`)
+    return c.json(createServerErrorResponse('Internal Server Error'), 500)
+  }
+}

package/src/utils/string.ts

@@ -0,0 +1,51 @@
+/**
+ * Returns true if the value is a non-null, non-array object (i.e. a plain record).
+ */
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
+}
+
+/**
+ * Reads a trimmed, non-empty string from an unknown value.
+ * Returns null for non-strings and empty/whitespace-only strings.
+ */
+export function readString(value: unknown): string | null {
+  if (typeof value !== 'string') return null
+  const trimmed = value.trim()
+  return trimmed.length > 0 ? trimmed : null
+}
+
+/**
+ * Reads a trimmed, non-empty string from a record field by key.
+ * Returns undefined when the field is missing or not a non-empty string.
+ */
+export function readStringField(record: Record<string, unknown>, key: string): string | undefined {
+  const value = record[key]
+  return typeof value === 'string' ? value : undefined
+}
+
+/**
+ * Truncates a string to the given character limit, appending "..." when truncated.
+ * Returns the original string when it fits within maxChars.
+ */
+export function truncateText(value: string, maxChars: number): string {
+  return value.length <= maxChars ? value : `${value.slice(0, maxChars).trimEnd()}...`
+}
+
+/**
+ * Truncates an optional string value with trimming. Returns undefined for
+ * falsy or whitespace-only input.
+ */
+export function truncateOptionalText(value: string | undefined, maxChars: number): string | undefined {
+  if (!value) return undefined
+  const trimmed = value.trim()
+  if (!trimmed) return undefined
+  return trimmed.length <= maxChars ? trimmed : `${trimmed.slice(0, maxChars)}...`
+}
+
+/**
+ * Collapses all runs of whitespace into a single space and trims.
+ */
+export function compactWhitespace(value: string): string {
+  return value.trim().replace(/\s+/g, ' ')
+}

package/src/workers/bootstrap.ts

@@ -0,0 +1,44 @@
+import { configureLogger, serverLogger } from '../config/logger'
+import { databaseService } from '../db/service'
+import { connectWithStartupRetry, waitForDatabaseBootstrap } from '../db/startup'
+import { getConfiguredPluginDatabaseConnector } from '../runtime/runtime-extensions'
+
+let sandboxedWorkerRuntimePromise: Promise<void> | null = null
+
+export async function initializeSandboxedWorkerRuntime(): Promise<void> {
+  if (sandboxedWorkerRuntimePromise) {
+    await sandboxedWorkerRuntimePromise
+    return
+  }
+
+  sandboxedWorkerRuntimePromise = (async () => {
+    await configureLogger()
+
+    await connectWithStartupRetry({
+      connect: () => databaseService.connect(),
+      label: 'sandboxed worker AI database runtime',
+      logger: serverLogger,
+    })
+
+    await connectWithStartupRetry({
+      connect: async () => {
+        const connectPluginRuntimeDatabases = getConfiguredPluginDatabaseConnector()
+        if (connectPluginRuntimeDatabases) {
+          await connectPluginRuntimeDatabases()
+        }
+      },
+      label: 'sandboxed worker plugin database runtime',
+      logger: serverLogger,
+    })
+
+    await waitForDatabaseBootstrap({
+      databaseService,
+      expectedFingerprint: process.env.DB_SCHEMA_FINGERPRINT,
+      label: 'sandboxed worker runtime',
+      logger: serverLogger,
+      connect: () => databaseService.connect(),
+    })
+  })()
+
+  await sandboxedWorkerRuntimePromise
+}

package/src/workers/memory-consolidation.worker.ts

@@ -0,0 +1,318 @@
+import type { SandboxedJob } from 'bullmq'
+import { BoundQuery, eq, inside } from 'surrealdb'
+
+import { serverLogger } from '../config/logger'
+import { ensureRecordId, recordIdToString } from '../db/record-id'
+import type { RecordIdInput } from '../db/record-id'
+import { databaseService } from '../db/service'
+import { TABLES } from '../db/tables'
+import type { MemoryConsolidationJob } from '../queues/memory-consolidation.queue'
+import { initializeSandboxedWorkerRuntime } from './bootstrap'
+import { toSandboxedWorkerError } from './utils/sandbox-error'
+import { createTracedWorkerProcessor } from './worker-utils'
+
+await initializeSandboxedWorkerRuntime()
+
+const MEMORY_TABLE = TABLES.MEMORY
+const MEMORY_RELATION_TABLE = TABLES.MEMORY_RELATION
+const MEMORY_HISTORY_TABLE = TABLES.MEMORY_HISTORY
+const HARD_SIMILARITY_THRESHOLD = 0.95
+const SOFT_SIMILARITY_THRESHOLD = 0.9
+const MAX_MEMORIES_PER_SCOPE = 500
+
+function toMemoryId(value: RecordIdInput): string {
+  return recordIdToString(value, TABLES.MEMORY)
+}
+
+function isContentSubsumed(shorter: string, longer: string): boolean {
+  const a = shorter.toLowerCase().trim()
+  const b = longer.toLowerCase().trim()
+  if (b.includes(a)) return true
+  const aWords = new Set(a.split(/\s+/))
+  const bWords = new Set(b.split(/\s+/))
+  let overlap = 0
+  for (const word of aWords) {
+    if (bWords.has(word)) overlap++
+  }
+  return aWords.size > 0 && overlap / aWords.size >= 0.8
+}
+
+async function deduplicateScope(scopeId: string): Promise<number> {
+  const memoryRows = await databaseService.query<{
+    id: RecordIdInput
+    content: string
+    importance: number
+    embedding: number[]
+    updatedAt?: string | Date | number | null
+    createdAt: string | Date | number
+  }>(
+    new BoundQuery(
+      `SELECT id, content, importance, embedding, updatedAt, createdAt
+     FROM ${MEMORY_TABLE}
+     WHERE scopeId = $scopeId AND archivedAt IS NONE
+     ORDER BY createdAt DESC
+     LIMIT $limit`,
+      { scopeId, limit: MAX_MEMORIES_PER_SCOPE },
+    ),
+  )
+
+  const memories = memoryRows.map((row) => ({ ...row, id: toMemoryId(row.id) }))
+  if (memories.length < 2) return 0
+
+  const archived = new Set<string>()
+  let mergeCount = 0
+
+  for (const memory of memories) {
+    if (archived.has(memory.id)) continue
+
+    const candidateLimit = 20
+    const neighborStatements = await databaseService.queryAll<{
+      id: RecordIdInput
+      content: string
+      importance: number
+      similarity: number
+      updatedAt?: string | Date | number | null
+      createdAt: string | Date | number
+    }>(
+      new BoundQuery(
+        `LET $candidateRows = (
+         SELECT
+           id,
+           vector::distance::knn() AS distance
+         FROM ${MEMORY_TABLE}
+         WHERE scopeId = $scopeId
+           AND archivedAt IS NONE
+           AND id != $memoryId
+           AND embedding <|${candidateLimit}|> $embedding
+         ORDER BY distance ASC
+         LIMIT ${candidateLimit}
+       );
+
+       SELECT
+         id,
+         content,
+         importance,
+         updatedAt,
+         createdAt,
+         vector::similarity::cosine(embedding, $embedding) AS similarity
+       FROM ${MEMORY_TABLE}
+       WHERE id IN $candidateRows.id
+       ORDER BY similarity DESC
+       LIMIT ${candidateLimit}`,
+        { scopeId, memoryId: ensureRecordId(memory.id, TABLES.MEMORY), embedding: memory.embedding },
+      ),
+    )
+
+    const neighbors = (neighborStatements.at(-1) ?? []).map((row) => ({ ...row, id: toMemoryId(row.id) }))
+
+    for (const neighbor of neighbors) {
+      if (neighbor.similarity < SOFT_SIMILARITY_THRESHOLD) break
+      if (archived.has(neighbor.id)) continue
+
+      const isHardMatch = neighbor.similarity >= HARD_SIMILARITY_THRESHOLD
+      if (!isHardMatch) {
+        const [shorter, longer] =
+          memory.content.length <= neighbor.content.length
+            ? [memory.content, neighbor.content]
+            : [neighbor.content, memory.content]
+        if (!isContentSubsumed(shorter, longer)) continue
+      }
+
+      const memoryCreatedAt = new Date(memory.createdAt).getTime()
+      const neighborCreatedAt = new Date(neighbor.createdAt).getTime()
+      const keepMemory =
+        memory.importance > neighbor.importance ||
+        (memory.importance === neighbor.importance && memoryCreatedAt >= neighborCreatedAt)
+      const winner = keepMemory ? memory : neighbor
+      const loser = keepMemory ? neighbor : memory
+
+      await databaseService.relate(
+        ensureRecordId(winner.id, TABLES.MEMORY),
+        MEMORY_RELATION_TABLE,
+        ensureRecordId(loser.id, TABLES.MEMORY),
+        { relationType: 'supersedes', confidence: 1.0 },
+      )
+
+      await databaseService.query(
+        new BoundQuery(
+          `UPDATE ${MEMORY_TABLE}
+         SET archivedAt = time::now(), validUntil = time::now()
+         WHERE id = $loserId AND archivedAt IS NONE`,
+          { loserId: ensureRecordId(loser.id, TABLES.MEMORY) },
+        ),
+      )
+
+      await databaseService.insert<Record<string, unknown>>(MEMORY_HISTORY_TABLE, {
+        memoryId: ensureRecordId(loser.id, TABLES.MEMORY),
+        prevValue: loser.content,
+        event: 'DELETE',
+      })
+
+      archived.add(loser.id)
+      mergeCount++
+
+      if (!keepMemory) break
+    }
+  }
+
+  return mergeCount
+}
+
+async function pruneStaleMemories(): Promise<number> {
+  const stale = await databaseService.query<{ id: RecordIdInput }>(
+    new BoundQuery(
+      `SELECT id FROM ${MEMORY_TABLE}
+     WHERE accessCount = 0
+       AND createdAt < time::now() - 90d
+       AND archivedAt IS NONE
+       AND importance < 0.5
+     LIMIT 200`,
+    ),
+  )
+
+  if (stale.length === 0) return 0
+
+  const staleIds = stale.map((row) => ensureRecordId(row.id, TABLES.MEMORY))
+  await databaseService.updateWhere(MEMORY_TABLE, inside('id', staleIds), { archivedAt: new Date() })
+
+  return stale.length
+}
+
+async function collapseSupersedeCh(): Promise<number> {
+  const middleNodes = await databaseService.query<{
+    middleId: RecordIdInput
+    predecessors: RecordIdInput[]
+    successors: RecordIdInput[]
+  }>(
+    new BoundQuery(
+      `SELECT
+       id AS middleId,
+       <-${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']<-${MEMORY_TABLE}.id AS predecessors,
+       ->${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']->${MEMORY_TABLE}.id AS successors
+     FROM ${MEMORY_TABLE}
+     WHERE archivedAt IS NONE
+       AND count(->${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']) > 0
+       AND count(<-${MEMORY_RELATION_TABLE}[WHERE relationType = 'supersedes']) > 0
+     LIMIT 100`,
+    ),
+  )
+
+  let collapsed = 0
+
+  for (const node of middleNodes) {
+    const predecessors = node.predecessors
+    const successors = node.successors
+
+    for (const predId of predecessors) {
+      for (const succId of successors) {
+        const predRef = ensureRecordId(predId, TABLES.MEMORY)
+        const succRef = ensureRecordId(succId, TABLES.MEMORY)
+        const existing = await databaseService.query<{ id: RecordIdInput }>(
+          new BoundQuery(
+            `SELECT id FROM ${MEMORY_RELATION_TABLE}
+           WHERE in = $predId AND out = $succId AND relationType = 'supersedes'
+           LIMIT 1`,
+            { predId: predRef, succId: succRef },
+          ),
+        )
+
+        if (existing.length === 0) {
+          await databaseService.relate(predRef, MEMORY_RELATION_TABLE, succRef, {
+            relationType: 'supersedes',
+            confidence: 1.0,
+          })
+        }
+      }
+    }
+
+    await databaseService.updateWhere(MEMORY_TABLE, eq('id', ensureRecordId(node.middleId, TABLES.MEMORY)), {
+      archivedAt: new Date(),
+    })
+
+    collapsed++
+  }
+
+  return collapsed
+}
+
+async function decayImportance(): Promise<number> {
+  const standardResult = await databaseService.query<{ count: number }>(
+    new BoundQuery(
+      `UPDATE ${MEMORY_TABLE}
+     SET importance = math::max([0.1, importance * 0.95])
+     WHERE lastAccessedAt IS NOT NONE
+       AND lastAccessedAt < time::now() - 30d
+       AND archivedAt IS NONE
+       AND importance > 0.1
+       AND (durability = 'standard' OR durability IS NONE)
+     RETURN count() AS count`,
+    ),
+  )
+
+  const ephemeralResult = await databaseService.query<{ count: number }>(
+    new BoundQuery(
+      `UPDATE ${MEMORY_TABLE}
+     SET importance = math::max([0.1, importance * 0.85])
+     WHERE lastAccessedAt IS NOT NONE
+       AND lastAccessedAt < time::now() - 30d
+       AND archivedAt IS NONE
+       AND importance > 0.1
+       AND durability = 'ephemeral'
+     RETURN count() AS count`,
+    ),
+  )
+
+  return (standardResult[0]?.count ?? 0) + (ephemeralResult[0]?.count ?? 0)
+}
+
+async function cleanupOrphanedRelations(): Promise<number> {
+  const result = await databaseService.query<{ count: number }>(
+    new BoundQuery(
+      `DELETE ${MEMORY_RELATION_TABLE}
+     WHERE in.archivedAt IS NOT NONE OR out.archivedAt IS NOT NONE
+     RETURN count() AS count`,
+    ),
+  )
+  return result[0]?.count ?? 0
+}
+
+async function pruneOldOrgActions(): Promise<number> {
+  const result = await databaseService.query<{ count: number }>(
+    new BoundQuery(`DELETE ${TABLES.ORG_ACTION} WHERE createdAt < time::now() - 90d RETURN count() AS count`),
+  )
+  return result[0]?.count ?? 0
+}
+
+const handler = async (job: SandboxedJob<MemoryConsolidationJob>) => {
+  const targetScope = job.data.scopeId
+
+  try {
+    let totalMerged = 0
+    if (targetScope) {
+      totalMerged = await deduplicateScope(targetScope)
+    } else {
+      const scopeIds = await databaseService.query<string>(
+        new BoundQuery(`SELECT VALUE scopeId FROM ${MEMORY_TABLE} WHERE archivedAt IS NONE GROUP BY scopeId`),
+      )
+
+      for (const scopeId of scopeIds) {
+        const merged = await deduplicateScope(scopeId)
+        totalMerged += merged
+      }
+    }
+
+    const pruned = await pruneStaleMemories()
+    const collapsed = await collapseSupersedeCh()
+    const decayed = await decayImportance()
+    const orphaned = await cleanupOrphanedRelations()
+    const prunedActions = await pruneOldOrgActions()
+
+    serverLogger.info`Memory consolidation complete (merged: ${totalMerged}, pruned: ${pruned}, collapsed: ${collapsed}, decayed: ${decayed}, orphaned relations: ${orphaned}, pruned actions: ${prunedActions})`
+  } catch (error) {
+    const serialized = toSandboxedWorkerError(error, 'Memory consolidation job failed')
+    serverLogger.error`${serialized.message}`
+    throw serialized
+  }
+}
+
+export default createTracedWorkerProcessor('memory-consolidation', handler)

package/src/workers/regular-chat-memory-digest.helpers.ts

@@ -0,0 +1,100 @@
+import { isAgentName } from '../config/agent-defaults'
+
+interface DigestMessageForTranscript {
+  source: 'workstream'
+  sourceId: string
+  role: 'system' | 'user' | 'assistant'
+  parts: Array<Record<string, unknown>>
+  metadata?: Record<string, unknown>
+}
+
+function normalizeWhitespace(value: string): string {
+  return value.replace(/\s+/g, ' ').trim()
+}
+
+function normalizeFilePartMetadata(part: Record<string, unknown>): string | null {
+  if (part.type !== 'file') return null
+
+  const filename = typeof part.filename === 'string' && part.filename.trim() ? part.filename.trim() : 'attachment'
+  const mediaType = typeof part.mediaType === 'string' && part.mediaType.trim() ? part.mediaType.trim() : 'unknown'
+  const storageKey = typeof part.storageKey === 'string' && part.storageKey.trim() ? part.storageKey.trim() : 'unknown'
+  const sizeBytes =
+    typeof part.sizeBytes === 'number' && Number.isFinite(part.sizeBytes) && part.sizeBytes >= 0
+      ? Math.trunc(part.sizeBytes)
+      : null
+
+  const sizeSegment = sizeBytes === null ? '' : `, sizeBytes=${sizeBytes}`
+  return `${filename} (${mediaType}${sizeSegment}, storageKey=${storageKey})`
+}
+
+function extractAssistantLabel(message: DigestMessageForTranscript): string {
+  const metadataAgentId =
+    message.metadata && typeof message.metadata.agentId === 'string'
+      ? message.metadata.agentId.trim().toLowerCase()
+      : ''
+  if (metadataAgentId && isAgentName(metadataAgentId)) {
+    return metadataAgentId
+  }
+
+  const metadataAgentName =
+    message.metadata && typeof message.metadata.agentName === 'string' ? message.metadata.agentName.trim() : ''
+  if (metadataAgentName) {
+    return metadataAgentName
+  }
+
+  return 'assistant'
+}
+
+export function buildDigestTranscript(params: { messages: DigestMessageForTranscript[] }): {
+  transcript: string
+  involvedAgentNames: string[]
+} {
+  const lines: string[] = []
+  const involvedAgentNames = new Set<string>()
+
+  for (const message of params.messages) {
+    if (message.role !== 'user' && message.role !== 'assistant') continue
+
+    const sourcePrefix = `[${message.source}:${message.sourceId}]`
+    const textParts = message.parts
+      .flatMap((part) =>
+        part.type === 'text' && typeof part.text === 'string' ? [normalizeWhitespace(part.text)] : [],
+      )
+      .filter((value) => value.length > 0)
+    const fileParts = message.parts
+      .map((part) => normalizeFilePartMetadata(part))
+      .filter((value): value is string => typeof value === 'string' && value.length > 0)
+
+    if (message.role === 'user') {
+      for (const textPart of textParts) {
+        lines.push(`${sourcePrefix} User: ${textPart}`)
+      }
+      if (fileParts.length > 0) {
+        lines.push(`${sourcePrefix} User files: ${fileParts.join('; ')}`)
+      }
+      continue
+    }
+
+    const assistantLabel = extractAssistantLabel(message)
+    if (isAgentName(assistantLabel)) {
+      involvedAgentNames.add(assistantLabel)
+    }
+
+    for (const textPart of textParts) {
+      lines.push(`${sourcePrefix} [${assistantLabel}] ${textPart}`)
+    }
+    if (fileParts.length > 0) {
+      lines.push(`${sourcePrefix} [${assistantLabel}] files: ${fileParts.join('; ')}`)
+    }
+  }
+
+  return { transcript: lines.join('\n'), involvedAgentNames: [...involvedAgentNames] }
+}
+
+export function resolveWorkspaceBootstrapCutoff(params: {
+  hasExistingCursor: boolean
+  bootstrapCompletedAt?: Date | null
+}): Date | null {
+  if (params.hasExistingCursor) return null
+  return params.bootstrapCompletedAt instanceof Date ? params.bootstrapCompletedAt : null
+}