@vox-ai-app/storage 1.0.2 → 1.0.3

package/src/repos/vectors.js

@@ -0,0 +1,105 @@
+export function vectorUpsert(db, collection, id, embedding, metadata = {}) {
+  const buffer = Buffer.from(new Float32Array(embedding).buffer)
+  const now = new Date().toISOString()
+
+  db.prepare(
+    `INSERT INTO vectors (id, collection, embedding, metadata, created_at, updated_at)
+     VALUES (?, ?, ?, ?, ?, ?)
+     ON CONFLICT (id, collection)
+     DO UPDATE SET embedding = excluded.embedding, metadata = excluded.metadata, updated_at = excluded.updated_at`
+  ).run(id, collection, buffer, JSON.stringify(metadata), now, now)
+}
+
+export function vectorSearch(db, collection, queryEmbedding, queryText, topK = 5) {
+  const rows = db
+    .prepare(`SELECT id, embedding, metadata FROM vectors WHERE collection = ?`)
+    .all(collection)
+
+  if (rows.length === 0) return []
+
+  const candidates = rows.map((row) => {
+    const stored = new Float32Array(
+      row.embedding.buffer,
+      row.embedding.byteOffset,
+      row.embedding.byteLength / 4
+    )
+    const meta = JSON.parse(row.metadata)
+    return {
+      id: row.id,
+      metadata: meta,
+      score: cosineSimilarity(queryEmbedding, stored)
+    }
+  })
+
+  const candidateLimit = Math.max(topK * 5, topK)
+  candidates.sort((a, b) => b.score - a.score)
+  const topCandidates = candidates.slice(0, candidateLimit)
+
+  if (!queryText) return topCandidates.slice(0, topK)
+  return rerankRows(topCandidates, queryText, topK)
+}
+
+export function vectorRemove(db, collection, id) {
+  db.prepare(`DELETE FROM vectors WHERE collection = ? AND id = ?`).run(collection, id)
+}
+
+export function vectorCount(db, collection) {
+  const row = db.prepare(`SELECT COUNT(*) as cnt FROM vectors WHERE collection = ?`).get(collection)
+  return row?.cnt || 0
+}
+
+function cosineSimilarity(a, b) {
+  let dot = 0
+  let normA = 0
+  let normB = 0
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i]
+    normA += a[i] * a[i]
+    normB += b[i] * b[i]
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB)
+  return denom === 0 ? 0 : dot / denom
+}
+
+function tokenize(value) {
+  if (typeof value !== 'string') return []
+  return value
+    .toLowerCase()
+    .split(/[^\p{L}\p{N}]+/gu)
+    .filter(Boolean)
+}
+
+function lexicalScore(queryText, candidateText) {
+  const queryTokens = new Set(tokenize(queryText))
+  if (queryTokens.size === 0) return 0
+  const candidateTokens = new Set(tokenize(candidateText))
+  if (candidateTokens.size === 0) return 0
+  let overlap = 0
+  for (const t of queryTokens) {
+    if (candidateTokens.has(t)) overlap++
+  }
+  return overlap / Math.sqrt(queryTokens.size * candidateTokens.size)
+}
+
+function rerankRows(rows, queryText, topK) {
+  const VECTOR_WEIGHT = 0.6
+  const LEXICAL_WEIGHT = 0.3
+  const PHRASE_BOOST = 0.1
+  const normalizedQuery = String(queryText || '')
+    .toLowerCase()
+    .trim()
+
+  return rows
+    .map((row) => {
+      const text = row.metadata?.text || row.metadata?.instructions || ''
+      const lexical = lexicalScore(queryText, text)
+      const vector = Math.max(0, row.score || 0)
+      const phraseMatch = normalizedQuery.length > 0 && text.toLowerCase().includes(normalizedQuery)
+      return {
+        ...row,
+        score: VECTOR_WEIGHT * vector + LEXICAL_WEIGHT * lexical + (phraseMatch ? PHRASE_BOOST : 0)
+      }
+    })
+    .sort((a, b) => b.score - a.score)
+    .slice(0, topK)
+}

package/src/config.js

@@ -1,62 +0,0 @@
-import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
-import path from 'node:path'
-
-function resolveConfigPath(configPath) {
-  const normalized = String(configPath || '').trim()
-  if (!normalized) {
-    throw new Error('A config path is required.')
-  }
-  return path.resolve(normalized)
-}
-
-function readConfigFile(configPath) {
-  const resolvedPath = resolveConfigPath(configPath)
-  if (!existsSync(resolvedPath)) {
-    return {}
-  }
-
-  try {
-    const raw = readFileSync(resolvedPath, 'utf8')
-    if (!raw.trim()) return {}
-    const parsed = JSON.parse(raw)
-    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
-      return {}
-    }
-    return parsed
-  } catch {
-    return {}
-  }
-}
-
-function writeConfigFile(configPath, value) {
-  const resolvedPath = resolveConfigPath(configPath)
-  mkdirSync(path.dirname(resolvedPath), { recursive: true })
-  const tempPath = `${resolvedPath}.tmp`
-  writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, 'utf8')
-  renameSync(tempPath, resolvedPath)
-}
-
-export function configGet(configPath, key) {
-  const config = readConfigFile(configPath)
-  return config[String(key)]
-}
-
-export function configSet(configPath, key, value) {
-  const config = readConfigFile(configPath)
-  config[String(key)] = value
-  writeConfigFile(configPath, config)
-  return value
-}
-
-export function configDelete(configPath, key) {
-  const config = readConfigFile(configPath)
-  const normalizedKey = String(key)
-  const existed = Object.prototype.hasOwnProperty.call(config, normalizedKey)
-  delete config[normalizedKey]
-  writeConfigFile(configPath, config)
-  return existed
-}
-
-export function configGetAll(configPath) {
-  return { ...readConfigFile(configPath) }
-}

package/src/messages.js

@@ -1,213 +0,0 @@
-export const DEFAULT_CONVERSATION_ID = 'main'
-
-function getConversationId(conversationId) {
-  const normalized = String(conversationId || '').trim()
-  return normalized || DEFAULT_CONVERSATION_ID
-}
-
-function normalizeLimit(limit) {
-  const parsed = Number.parseInt(limit, 10)
-  return Number.isFinite(parsed) && parsed > 0 ? parsed : null
-}
-
-function normalizeBeforeId(beforeId) {
-  const parsed = Number.parseInt(beforeId, 10)
-  return Number.isFinite(parsed) && parsed > 0 ? parsed : null
-}
-
-function mapRow(row) {
-  if (!row) return null
-  return {
-    id: row.id,
-    conversationId: row.conversation_id,
-    role: row.role,
-    content: row.content,
-    createdAt: row.created_at
-  }
-}
-
-export function ensureConversation(db, conversationId = DEFAULT_CONVERSATION_ID) {
-  const id = getConversationId(conversationId)
-  const now = new Date().toISOString()
-
-  db.prepare(
-    `
-    INSERT INTO conversations (id, created_at, updated_at)
-    VALUES (?, ?, ?)
-    ON CONFLICT(id) DO NOTHING
-  `
-  ).run(id, now, now)
-
-  return db
-    .prepare(
-      `
-    SELECT id, created_at, updated_at
-    FROM conversations
-    WHERE id = ?
-  `
-    )
-    .get(id)
-}
-
-export function touchConversation(db, conversationId = DEFAULT_CONVERSATION_ID) {
-  const id = getConversationId(conversationId)
-  const now = new Date().toISOString()
-
-  db.prepare(
-    `
-    INSERT INTO conversations (id, created_at, updated_at)
-    VALUES (?, ?, ?)
-    ON CONFLICT(id) DO UPDATE SET updated_at = excluded.updated_at
-  `
-  ).run(id, now, now)
-
-  return db
-    .prepare(
-      `
-    SELECT id, created_at, updated_at
-    FROM conversations
-    WHERE id = ?
-  `
-    )
-    .get(id)
-}
-
-export function appendMessage(db, role, content, conversationId = DEFAULT_CONVERSATION_ID) {
-  const id = getConversationId(conversationId)
-  const now = new Date().toISOString()
-  const normalizedRole = String(role || '').trim() || 'user'
-  const normalizedContent = String(content ?? '')
-
-  touchConversation(db, id)
-
-  const result = db
-    .prepare(
-      `
-    INSERT INTO messages (conversation_id, role, content, created_at)
-    VALUES (?, ?, ?, ?)
-  `
-    )
-    .run(id, normalizedRole, normalizedContent, now)
-
-  return mapRow(
-    db
-      .prepare(
-        `
-    SELECT id, conversation_id, role, content, created_at
-    FROM messages
-    WHERE id = ?
-  `
-      )
-      .get(result.lastInsertRowid)
-  )
-}
-
-export function getMessages(db, conversationId = DEFAULT_CONVERSATION_ID, limit) {
-  const id = getConversationId(conversationId)
-  const normalizedLimit = normalizeLimit(limit)
-
-  if (!normalizedLimit) {
-    return db
-      .prepare(
-        `
-      SELECT id, conversation_id, role, content, created_at
-      FROM messages
-      WHERE conversation_id = ?
-      ORDER BY id ASC
-    `
-      )
-      .all(id)
-      .map(mapRow)
-  }
-
-  return db
-    .prepare(
-      `
-    SELECT id, conversation_id, role, content, created_at
-    FROM (
-      SELECT id, conversation_id, role, content, created_at
-      FROM messages
-      WHERE conversation_id = ?
-      ORDER BY id DESC
-      LIMIT ?
-    )
-    ORDER BY id ASC
-  `
-    )
-    .all(id, normalizedLimit)
-    .map(mapRow)
-}
-
-export function getMessagesBeforeId(
-  db,
-  beforeId,
-  conversationId = DEFAULT_CONVERSATION_ID,
-  limit = 50
-) {
-  const id = getConversationId(conversationId)
-  const normalizedBeforeId = normalizeBeforeId(beforeId)
-  const normalizedLimit = normalizeLimit(limit) || 50
-
-  if (!normalizedBeforeId) {
-    return []
-  }
-
-  return db
-    .prepare(
-      `
-    SELECT id, conversation_id, role, content, created_at
-    FROM (
-      SELECT id, conversation_id, role, content, created_at
-      FROM messages
-      WHERE conversation_id = ? AND id < ?
-      ORDER BY id DESC
-      LIMIT ?
-    )
-    ORDER BY id ASC
-  `
-    )
-    .all(id, normalizedBeforeId, normalizedLimit)
-    .map(mapRow)
-}
-
-export function clearMessages(db, conversationId = DEFAULT_CONVERSATION_ID) {
-  const id = getConversationId(conversationId)
-  ensureConversation(db, id)
-  return db
-    .prepare(
-      `
-    DELETE FROM messages
-    WHERE conversation_id = ?
-  `
-    )
-    .run(id)
-}
-
-export function saveSummaryCheckpoint(
-  db,
-  summary,
-  checkpointId,
-  conversationId = DEFAULT_CONVERSATION_ID
-) {
-  const id = getConversationId(conversationId)
-  ensureConversation(db, id)
-  db.prepare(
-    `UPDATE conversations SET context_summary = ?, context_checkpoint_id = ?, updated_at = ? WHERE id = ?`
-  ).run(summary, checkpointId, new Date().toISOString(), id)
-}
-
-export function loadSummaryCheckpoint(db, conversationId = DEFAULT_CONVERSATION_ID) {
-  const id = getConversationId(conversationId)
-  const row = db
-    .prepare(`SELECT context_summary, context_checkpoint_id FROM conversations WHERE id = ?`)
-    .get(id)
-  if (!row || !row.context_summary) return null
-  return { summary: row.context_summary, checkpointId: row.context_checkpoint_id }
-}
-
-export function clearSummaryCheckpoint(db, conversationId = DEFAULT_CONVERSATION_ID) {
-  const id = getConversationId(conversationId)
-  db.prepare(
-    `UPDATE conversations SET context_summary = NULL, context_checkpoint_id = NULL, updated_at = ? WHERE id = ?`
-  ).run(new Date().toISOString(), id)
-}

package/src/tasks.js

@@ -1,219 +0,0 @@
-function parseJson(value, fallback) {
-  if (value === null || value === undefined || value === '') {
-    return fallback
-  }
-
-  try {
-    return JSON.parse(value)
-  } catch {
-    return fallback
-  }
-}
-
-function stringifyJson(value, fallback = null) {
-  if (value === null || value === undefined) {
-    return fallback
-  }
-
-  try {
-    return JSON.stringify(value)
-  } catch {
-    return fallback
-  }
-}
-
-function mapTask(row) {
-  if (!row) return null
-
-  return {
-    taskId: row.task_id,
-    instructions: row.instructions,
-    context: row.context,
-    status: row.status,
-    createdAt: row.created_at,
-    updatedAt: row.updated_at,
-    currentPlan: row.current_plan,
-    message: row.message,
-    result: row.result,
-    completedAt: row.completed_at,
-    failedAt: row.failed_at
-  }
-}
-
-function mapActivity(row) {
-  if (!row) return null
-
-  return {
-    id: row.id,
-    taskId: row.task_id,
-    type: row.type,
-    name: row.name || null,
-    rawResult: parseJson(row.raw_result, row.raw_result),
-    timestamp: row.timestamp,
-    data: parseJson(row.data, {})
-  }
-}
-
-export function upsertTask(db, task) {
-  const taskId = String(task?.taskId || '').trim()
-  if (!taskId) {
-    throw new Error('taskId is required.')
-  }
-
-  const createdAt = String(task?.createdAt || new Date().toISOString())
-  const updatedAt = String(task?.updatedAt || createdAt)
-
-  db.prepare(
-    `
-    INSERT INTO tasks (
-      task_id,
-      instructions,
-      context,
-      status,
-      created_at,
-      updated_at,
-      current_plan,
-      message,
-      result,
-      completed_at,
-      failed_at
-    )
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(task_id) DO UPDATE SET
-      instructions = excluded.instructions,
-      context = excluded.context,
-      status = excluded.status,
-      updated_at = excluded.updated_at,
-      current_plan = excluded.current_plan,
-      message = excluded.message,
-      result = excluded.result,
-      completed_at = excluded.completed_at,
-      failed_at = excluded.failed_at
-  `
-  ).run(
-    taskId,
-    String(task?.instructions || ''),
-    String(task?.context || ''),
-    String(task?.status || 'queued'),
-    createdAt,
-    updatedAt,
-    String(task?.currentPlan || ''),
-    String(task?.message || ''),
-    task?.result === null || task?.result === undefined ? null : String(task.result),
-    String(task?.completedAt || ''),
-    String(task?.failedAt || '')
-  )
-
-  return getTask(db, taskId)
-}
-
-export function getTask(db, taskId) {
-  return mapTask(
-    db
-      .prepare(
-        `
-    SELECT
-      task_id,
-      instructions,
-      context,
-      status,
-      created_at,
-      updated_at,
-      current_plan,
-      message,
-      result,
-      completed_at,
-      failed_at
-    FROM tasks
-    WHERE task_id = ?
-  `
-      )
-      .get(String(taskId || '').trim())
-  )
-}
-
-export function loadTasks(db) {
-  return db
-    .prepare(
-      `
-    SELECT
-      task_id,
-      instructions,
-      context,
-      status,
-      created_at,
-      updated_at,
-      current_plan,
-      message,
-      result,
-      completed_at,
-      failed_at
-    FROM tasks
-    ORDER BY created_at DESC, task_id DESC
-  `
-    )
-    .all()
-    .map(mapTask)
-}
-
-export function appendTaskActivity(db, activity) {
-  const id = String(activity?.id || '').trim()
-  const taskId = String(activity?.taskId || '').trim()
-  if (!id || !taskId) {
-    throw new Error('Task activity requires id and taskId.')
-  }
-
-  db.prepare(
-    `
-    INSERT INTO task_activity (id, task_id, type, name, raw_result, timestamp, data)
-    VALUES (?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(id) DO UPDATE SET
-      task_id = excluded.task_id,
-      type = excluded.type,
-      name = excluded.name,
-      raw_result = excluded.raw_result,
-      timestamp = excluded.timestamp,
-      data = excluded.data
-  `
-  ).run(
-    id,
-    taskId,
-    String(activity?.type || ''),
-    activity?.name ? String(activity.name) : null,
-    stringifyJson(
-      activity?.rawResult,
-      activity?.rawResult === undefined ? null : String(activity.rawResult)
-    ),
-    String(activity?.timestamp || new Date().toISOString()),
-    stringifyJson(activity?.data || {}, '{}')
-  )
-
-  return id
-}
-
-export function loadTaskActivity(db, taskId) {
-  return db
-    .prepare(
-      `
-    SELECT id, task_id, type, name, raw_result, timestamp, data
-    FROM task_activity
-    WHERE task_id = ?
-    ORDER BY timestamp ASC, id ASC
-  `
-    )
-    .all(String(taskId || '').trim())
-    .map(mapActivity)
-}
-
-export function loadAllTaskActivity(db) {
-  return db
-    .prepare(
-      `
-    SELECT id, task_id, type, name, raw_result, timestamp, data
-    FROM task_activity
-    ORDER BY timestamp ASC, id ASC
-  `
-    )
-    .all()
-    .map(mapActivity)
-}