@lota-sdk/core 0.1.5

package/src/db/startup.ts

@@ -0,0 +1,152 @@
+import { BoundQuery, RecordId } from 'surrealdb'
+import { z } from 'zod'
+
+import type { SurrealDBService, SurrealDatabaseLogger } from '../db/service'
+import { TABLES } from '../db/tables'
+import { getErrorMessage } from '../utils/error'
+
+const DATABASE_BOOTSTRAP_KEY = 'database-schema-ready'
+const DEFAULT_RETRY_DELAY_MS = 1_000
+const DEFAULT_MAX_WAIT_MS = 3 * 60 * 1_000
+const RETRY_LOG_INTERVAL = 5
+
+const RuntimeBootstrapRecordSchema = z.object({
+  id: z.unknown(),
+  key: z.string(),
+  schemaFingerprint: z.string(),
+  readyAt: z.union([z.date(), z.string(), z.number()]),
+  updatedAt: z.union([z.date(), z.string(), z.number()]),
+})
+
+type StartupLogger = Pick<SurrealDatabaseLogger, 'info' | 'warn' | 'error'>
+
+function shouldLogRetry(attempt: number): boolean {
+  return attempt === 1 || attempt % RETRY_LOG_INTERVAL === 0
+}
+
+export async function connectWithStartupRetry(params: {
+  connect: () => Promise<void>
+  label: string
+  logger?: StartupLogger
+  retryDelayMs?: number
+  maxWaitMs?: number
+}): Promise<void> {
+  const retryDelayMs = params.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS
+  const maxWaitMs = params.maxWaitMs ?? DEFAULT_MAX_WAIT_MS
+  const startedAt = Date.now()
+
+  let attempt = 0
+  let lastError: unknown = null
+
+  while (Date.now() - startedAt <= maxWaitMs) {
+    attempt += 1
+
+    try {
+      await params.connect()
+      return
+    } catch (error) {
+      lastError = error
+      if (shouldLogRetry(attempt)) {
+        params.logger?.warn?.(
+          `Waiting for ${params.label} (${attempt}, elapsed=${Date.now() - startedAt}ms): ${getErrorMessage(error)}`,
+        )
+      }
+      await Bun.sleep(retryDelayMs)
+    }
+  }
+
+  params.logger?.error?.(`Timed out waiting for ${params.label}: ${getErrorMessage(lastError)}`)
+  throw lastError instanceof Error ? lastError : new Error(`Timed out waiting for ${params.label}`)
+}
+
+async function readDatabaseBootstrapRecord(
+  databaseService: SurrealDBService,
+): Promise<z.infer<typeof RuntimeBootstrapRecordSchema> | null> {
+  return await databaseService.queryOne(
+    new BoundQuery(
+      `SELECT *
+       FROM ${TABLES.RUNTIME_BOOTSTRAP}
+       WHERE key = $key
+       LIMIT 1`,
+      { key: DATABASE_BOOTSTRAP_KEY },
+    ),
+    RuntimeBootstrapRecordSchema,
+  )
+}
+
+export async function waitForDatabaseBootstrap(params: {
+  databaseService: SurrealDBService
+  expectedFingerprint?: string | null
+  label: string
+  logger?: StartupLogger
+  connect?: () => Promise<void>
+  retryDelayMs?: number
+  maxWaitMs?: number
+}): Promise<void> {
+  const expectedFingerprint = params.expectedFingerprint?.trim()
+  if (!expectedFingerprint) {
+    return
+  }
+
+  const retryDelayMs = params.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS
+  const maxWaitMs = params.maxWaitMs ?? DEFAULT_MAX_WAIT_MS
+  const startedAt = Date.now()
+
+  let attempt = 0
+  let lastError: unknown = null
+
+  while (Date.now() - startedAt <= maxWaitMs) {
+    attempt += 1
+
+    try {
+      if (params.connect) {
+        await params.connect()
+      }
+
+      const record = await readDatabaseBootstrapRecord(params.databaseService)
+      if (record?.schemaFingerprint === expectedFingerprint) {
+        return
+      }
+
+      if (shouldLogRetry(attempt)) {
+        const currentFingerprint =
+          typeof record?.schemaFingerprint === 'string' && record.schemaFingerprint.length > 0
+            ? record.schemaFingerprint
+            : 'missing'
+        params.logger?.info?.(
+          `Waiting for ${params.label} schema readiness (${attempt}, expected=${expectedFingerprint}, current=${currentFingerprint})`,
+        )
+      }
+    } catch (error) {
+      lastError = error
+      if (shouldLogRetry(attempt)) {
+        params.logger?.warn?.(`Waiting for ${params.label} schema readiness (${attempt}): ${getErrorMessage(error)}`)
+      }
+    }
+
+    await Bun.sleep(retryDelayMs)
+  }
+
+  if (lastError instanceof Error) {
+    throw lastError
+  }
+
+  throw new Error(`Timed out waiting for ${params.label} schema readiness`)
+}
+
+export async function publishDatabaseBootstrap(params: {
+  databaseService: SurrealDBService
+  schemaFingerprint: string
+}): Promise<void> {
+  await params.databaseService.upsert(
+    TABLES.RUNTIME_BOOTSTRAP,
+    new RecordId(TABLES.RUNTIME_BOOTSTRAP, DATABASE_BOOTSTRAP_KEY),
+    {
+      key: DATABASE_BOOTSTRAP_KEY,
+      schemaFingerprint: params.schemaFingerprint,
+      readyAt: new Date(),
+      updatedAt: new Date(),
+    },
+    RuntimeBootstrapRecordSchema,
+  )
+}

package/src/db/tables.ts

@@ -0,0 +1,20 @@
+export const TABLES = {
+  WORKSTREAM_MESSAGE: 'workstreamMessage',
+  WORKSTREAM: 'workstream',
+  RUNTIME_BOOTSTRAP: 'runtimeBootstrap',
+  WORKSTREAM_ATTACHMENT: 'workstreamAttachment',
+  MEMORY: 'memory',
+  MEMORY_RELATION: 'memoryRelation',
+  MEMORY_HISTORY: 'memoryHistory',
+  LEARNED_SKILL: 'learnedSkill',
+  PLAN: 'plan',
+  PLAN_TASK: 'planTask',
+  PLAN_EVENT: 'planEvent',
+  ORGANIZATION: 'organization',
+  USER: 'user',
+  ORG_ACTION: 'orgAction',
+  RECENT_ACTIVITY_EVENT: 'recentActivityEvent',
+  RECENT_ACTIVITY: 'recentActivity',
+} as const
+
+export type DatabaseTable = (typeof TABLES)[keyof typeof TABLES] | (string & {})

package/src/document/org-document-chunking.ts

@@ -0,0 +1,224 @@
+import { normalizeKey, normalizeTextBody, normalizeWhitespace } from './parsing'
+
+export type ParsedDocumentChunk = {
+  chunkKey: string
+  chunkIndex: number
+  content: string
+  sectionPath?: string
+  pageStart?: number
+  pageEnd?: number
+}
+
+type ChunkBase = Omit<ParsedDocumentChunk, 'chunkIndex'>
+
+type TextPage = { pageNumber: number; text: string }
+
+const MARKDOWN_CHUNK_CHARS = 1_700
+const TEXT_CHUNK_CHARS = 1_600
+const PDF_CHUNK_CHARS = 1_400
+
+function joinSectionPath(parts: Array<string | undefined>): string | undefined {
+  const normalized = parts.map((part) => normalizeWhitespace(part ?? '')).filter((part) => part.length > 0)
+  return normalized.length > 0 ? normalized.join(' > ') : undefined
+}
+
+function splitParagraphUnits(text: string): string[] {
+  const normalized = normalizeTextBody(text)
+  if (!normalized) return []
+
+  return normalized
+    .split(/\n{2,}/)
+    .map((value) => value.trim())
+    .filter((value) => value.length > 0)
+}
+
+function splitOversizedUnit(unit: string, maxChars: number): string[] {
+  const normalized = normalizeTextBody(unit)
+  if (!normalized) return []
+  if (normalized.length <= maxChars) return [normalized]
+
+  const sentenceParts = normalized
+    .split(/(?<=[.!?])\s+/)
+    .map((value) => value.trim())
+    .filter((value) => value.length > 0)
+
+  if (sentenceParts.length > 1) {
+    return sentenceParts.flatMap((part) => splitOversizedUnit(part, maxChars))
+  }
+
+  const lineParts = normalized
+    .split(/\n+/)
+    .map((value) => value.trim())
+    .filter((value) => value.length > 0)
+
+  if (lineParts.length > 1) {
+    return lineParts.flatMap((part) => splitOversizedUnit(part, maxChars))
+  }
+
+  const chunks: string[] = []
+  let cursor = 0
+  while (cursor < normalized.length) {
+    let end = Math.min(cursor + maxChars, normalized.length)
+    if (end < normalized.length) {
+      const breakAt = normalized.lastIndexOf(' ', end)
+      if (breakAt > cursor + Math.floor(maxChars * 0.55)) {
+        end = breakAt
+      }
+    }
+
+    const slice = normalized.slice(cursor, end).trim()
+    if (slice) chunks.push(slice)
+    cursor = end
+    while (cursor < normalized.length && normalized[cursor] === ' ') {
+      cursor += 1
+    }
+  }
+
+  return chunks
+}
+
+function buildChunkBodies(units: string[], maxChars: number): string[] {
+  const chunks: string[] = []
+  let current = ''
+
+  const commit = () => {
+    const value = normalizeTextBody(current)
+    if (value) chunks.push(value)
+    current = ''
+  }
+
+  for (const unit of units) {
+    const normalized = normalizeTextBody(unit)
+    if (!normalized) continue
+
+    if (normalized.length > maxChars) {
+      if (current) commit()
+      for (const split of splitOversizedUnit(normalized, maxChars)) {
+        const value = normalizeTextBody(split)
+        if (value) chunks.push(value)
+      }
+      continue
+    }
+
+    const nextValue = current ? `${current}\n\n${normalized}` : normalized
+    if (nextValue.length > maxChars && current) {
+      commit()
+      current = normalized
+      continue
+    }
+
+    current = nextValue
+  }
+
+  commit()
+  return chunks
+}
+
+function toChunkEntries(params: {
+  prefix: string
+  contents: string[]
+  sectionPath?: string
+  pageStart?: number
+  pageEnd?: number
+}): ChunkBase[] {
+  return params.contents.map((content, index) => ({
+    chunkKey: `${params.prefix}:${String(index + 1).padStart(3, '0')}`,
+    content,
+    sectionPath: params.sectionPath,
+    pageStart: params.pageStart,
+    pageEnd: params.pageEnd,
+  }))
+}
+
+function withChunkIndexes(chunks: ChunkBase[]): ParsedDocumentChunk[] {
+  return chunks.map((chunk, index) => ({ ...chunk, chunkIndex: index }))
+}
+
+export function chunkPlainTextDocument(params: {
+  text: string
+  chunkChars?: number
+  chunkKeyPrefix?: string
+  sectionPath?: string
+}): ParsedDocumentChunk[] {
+  const chunkChars = params.chunkChars ?? TEXT_CHUNK_CHARS
+  const prefix = params.chunkKeyPrefix ?? 'text'
+  const units = splitParagraphUnits(params.text)
+  const contents = buildChunkBodies(units, chunkChars)
+  return withChunkIndexes(toChunkEntries({ prefix, contents, sectionPath: params.sectionPath }))
+}
+
+export function chunkMarkdownDocument(params: {
+  text: string
+  chunkChars?: number
+  chunkKeyPrefix?: string
+  baseSectionPath?: string
+}): ParsedDocumentChunk[] {
+  const chunkChars = params.chunkChars ?? MARKDOWN_CHUNK_CHARS
+  const prefix = params.chunkKeyPrefix ?? 'markdown'
+  const lines = normalizeTextBody(params.text).split('\n')
+  const chunks: ChunkBase[] = []
+  const headingStack: string[] = []
+  let currentLines: string[] = []
+  let sectionCounter = 0
+
+  const flushCurrent = () => {
+    const content = normalizeTextBody(currentLines.join('\n'))
+    if (!content) {
+      currentLines = []
+      return
+    }
+
+    sectionCounter += 1
+    const sectionPath = joinSectionPath([params.baseSectionPath, ...headingStack])
+    const sectionKeyBase = sectionPath ? normalizeKey(sectionPath) : 'section'
+    const sectionKey = `${sectionKeyBase}-${String(sectionCounter).padStart(3, '0')}`
+    const contents = buildChunkBodies(splitParagraphUnits(content), chunkChars)
+    chunks.push(...toChunkEntries({ prefix: `${prefix}:${sectionKey}`, contents, sectionPath }))
+    currentLines = []
+  }
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,6})\s+(.+?)\s*$/)
+    if (!headingMatch) {
+      currentLines.push(line)
+      continue
+    }
+
+    flushCurrent()
+    const depth = headingMatch[1].length
+    const headingText = normalizeWhitespace(headingMatch[2])
+    headingStack.splice(depth - 1)
+    headingStack[depth - 1] = headingText
+    currentLines = [line]
+  }
+
+  flushCurrent()
+  return withChunkIndexes(chunks)
+}
+
+export function chunkPagedDocument(params: {
+  pages: TextPage[]
+  chunkChars?: number
+  chunkKeyPrefix?: string
+}): ParsedDocumentChunk[] {
+  const chunkChars = params.chunkChars ?? PDF_CHUNK_CHARS
+  const prefix = params.chunkKeyPrefix ?? 'page'
+  const chunks: ChunkBase[] = []
+
+  for (const page of params.pages) {
+    const pageText = normalizeTextBody(page.text)
+    if (!pageText) continue
+
+    const contents = buildChunkBodies(splitParagraphUnits(pageText), chunkChars)
+    chunks.push(
+      ...toChunkEntries({
+        prefix: `${prefix}:${String(page.pageNumber).padStart(4, '0')}`,
+        contents,
+        pageStart: page.pageNumber,
+        pageEnd: page.pageNumber,
+      }),
+    )
+  }
+
+  return withChunkIndexes(chunks)
+}

package/src/document/parsing.ts

@@ -0,0 +1,40 @@
+export function normalizeWhitespace(value: string): string {
+  return value.replace(/\s+/g, ' ').trim()
+}
+
+export function normalizeTextBody(value: string): string {
+  return value.replaceAll(String.fromCharCode(0), '').replace(/\r/g, '').trim()
+}
+
+export function normalizeKey(value: string): string {
+  return normalizeWhitespace(value)
+    .toLowerCase()
+    .replace(/[^\w\s.-]/g, '')
+    .replace(/\s+/g, '-')
+    .slice(0, 120)
+}
+
+export function makeMemoryKey(kind: string, rawKey: string): string {
+  const normalized = normalizeKey(rawKey)
+  return normalized ? `${kind}:${normalized}` : `${kind}:item`
+}
+
+export function truncateForModel(value: string, maxChars: number): string {
+  if (value.length <= maxChars) return value
+  return `${value.slice(0, maxChars)}\n\n[...truncated due to size...]`
+}
+
+export function dedupeStrings(items: string[], limit: number): string[] {
+  const out: string[] = []
+  const seen = new Set<string>()
+  for (const raw of items) {
+    const value = normalizeWhitespace(raw)
+    if (!value) continue
+    const key = value.toLowerCase()
+    if (seen.has(key)) continue
+    seen.add(key)
+    out.push(value)
+    if (out.length >= limit) break
+  }
+  return out
+}

package/src/embeddings/provider.ts

@@ -0,0 +1,76 @@
+import { embed, embedMany } from 'ai'
+
+import { bifrostEmbeddingModel } from '../bifrost/bifrost'
+import { env } from '../config/env-shapes'
+
+const SUPPORTED_EMBEDDING_PREFIXES = ['openai/', 'openrouter/'] as const
+
+function resolveEmbeddingModel(modelId: string) {
+  const normalized = modelId.trim()
+  if (!normalized) {
+    throw new Error('[embeddings-provider] Model id is required.')
+  }
+
+  if (!SUPPORTED_EMBEDDING_PREFIXES.some((prefix) => normalized.startsWith(prefix))) {
+    throw new Error(
+      `[embeddings-provider] Unsupported model id "${modelId}". Use one of: ${SUPPORTED_EMBEDDING_PREFIXES.join(', ')}*.`,
+    )
+  }
+
+  return bifrostEmbeddingModel(normalized)
+}
+
+class ProviderEmbeddings {
+  private _model: ReturnType<typeof resolveEmbeddingModel> | null = null
+
+  private getModel() {
+    if (!this._model) {
+      this._model = resolveEmbeddingModel(env.AI_EMBEDDING_MODEL)
+    }
+    return this._model
+  }
+
+  async embedQuery(text: string): Promise<number[]> {
+    const input = text.trim()
+    if (!input) return []
+
+    const result = await embed({ model: this.getModel(), value: input, maxRetries: 2 })
+
+    return result.embedding.map((value) => Number(value))
+  }
+
+  async embedDocuments(values: string[]): Promise<number[][]> {
+    if (values.length === 0) return []
+
+    const normalized = values.map((value) => value.trim())
+    const nonEmptyEntries = normalized
+      .map((value, index) => ({ value, index }))
+      .filter((entry) => entry.value.length > 0)
+
+    if (nonEmptyEntries.length === 0) {
+      return normalized.map(() => [])
+    }
+
+    const result = await embedMany({
+      model: this.getModel(),
+      values: nonEmptyEntries.map((entry) => entry.value),
+      maxRetries: 2,
+    })
+
+    const embeddingsByIndex = new Map<number, number[]>()
+    result.embeddings.forEach((embedding, index) => {
+      const entry = nonEmptyEntries.at(index)
+      if (!entry) return
+      embeddingsByIndex.set(
+        entry.index,
+        embedding.map((value) => Number(value)),
+      )
+    })
+
+    return normalized.map((_, index) => embeddingsByIndex.get(index) ?? [])
+  }
+}
+
+export function createDefaultEmbeddings(): ProviderEmbeddings {
+  return new ProviderEmbeddings()
+}