@conversionpros/aiva 1.0.1 → 2.0.0

package/knowledge-base.js

@@ -1,472 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-
-const WORKSPACE = path.join(process.env.HOME || '/Users/brandonburgan', '.openclaw', 'workspace');
-const DB_PATH = path.join(__dirname, 'data', 'knowledge.db');
-const EMBEDDING_DIM = 1536;
-
-// Try to find an embedding endpoint - gateway chat completions doesn't support embeddings,
-// so we fall back to BM25 text search which works great for this use case
-let EMBEDDING_URL = null;
-let EMBEDDING_TOKEN = null;
-let EMBEDDING_MODEL = 'openai/text-embedding-3-small';
-
-const WATCHED_PATHS = [
-  path.join(WORKSPACE, 'memory'),
-  path.join(WORKSPACE, 'MEMORY.md'),
-  path.join(WORKSPACE, 'TOOLS.md'),
-  path.join(WORKSPACE, 'AGENTS.md'),
-  path.join(WORKSPACE, 'SOUL.md'),
-  path.join(WORKSPACE, 'USER.md'),
-];
-
-let db = null;
-let lastIndexedAt = null;
-let watchers = [];
-let debounceTimers = {};
-
-function initKnowledgeBase() {
-  const Database = require('better-sqlite3');
-  const dataDir = path.join(__dirname, 'data');
-  if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir, { recursive: true });
-
-  db = new Database(DB_PATH);
-  db.pragma('journal_mode = WAL');
-
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS chunks (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      file_path TEXT NOT NULL,
-      file_mtime REAL NOT NULL,
-      chunk_index INTEGER NOT NULL,
-      start_line INTEGER NOT NULL,
-      end_line INTEGER NOT NULL,
-      text TEXT NOT NULL,
-      embedding BLOB,
-      created_at TEXT DEFAULT CURRENT_TIMESTAMP
-    );
-    CREATE INDEX IF NOT EXISTS idx_file_path ON chunks(file_path);
-  `);
-
-  // Try to detect embedding API availability
-  tryDetectEmbeddingAPI();
-
-  console.log('[knowledge-base] Initialized SQLite at', DB_PATH);
-  return db;
-}
-
-async function tryDetectEmbeddingAPI() {
-  // TIER 1: Platform infrastructure — uses CMP's key from .env, NOT client token. This is intentional.
-  if (process.env.OPENAI_API_KEY) {
-    EMBEDDING_URL = 'https://api.openai.com/v1/embeddings';
-    EMBEDDING_TOKEN = process.env.OPENAI_API_KEY;
-    EMBEDDING_MODEL = 'text-embedding-3-small';
-    console.log('[knowledge-base] Using OpenAI embeddings API');
-    return;
-  }
-  // Check OpenClaw gateway (may support embeddings in future)
-  try {
-    const resp = await fetch('http://127.0.0.1:18789/v1/embeddings', {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${process.env.OPENCLAW_GATEWAY_TOKEN || 'Ttsrgr812!'}` },
-      body: JSON.stringify({ model: 'openai/text-embedding-3-small', input: 'test' })
-    });
-    if (resp.ok) {
-      EMBEDDING_URL = 'http://127.0.0.1:18789/v1/embeddings';
-      EMBEDDING_TOKEN = process.env.OPENCLAW_GATEWAY_TOKEN || 'Ttsrgr812!';
-      console.log('[knowledge-base] Using OpenClaw gateway embeddings');
-      return;
-    }
-  } catch (e) {}
-  console.log('[knowledge-base] No embedding API available — using BM25 text search');
-}
-
-// ========== BM25 SEARCH ==========
-// Simple but effective text search when no embedding API is available
-
-function tokenize(text) {
-  return text.toLowerCase()
-    .replace(/[^a-z0-9\s-]/g, ' ')
-    .split(/\s+/)
-    .filter(t => t.length > 1);
-}
-
-function bm25Search(query, chunks, limit = 5) {
-  const queryTokens = tokenize(query);
-  if (queryTokens.length === 0) return [];
-
-  const N = chunks.length;
-  const avgDl = chunks.reduce((s, c) => s + tokenize(c.text).length, 0) / N;
-  const k1 = 1.5, b = 0.75;
-
-  // Compute IDF for query terms
-  const df = {};
-  for (const token of queryTokens) df[token] = 0;
-  for (const chunk of chunks) {
-    const tokens = new Set(tokenize(chunk.text));
-    for (const qt of queryTokens) {
-      if (tokens.has(qt)) df[qt]++;
-    }
-  }
-
-  const idf = {};
-  for (const [term, freq] of Object.entries(df)) {
-    idf[term] = Math.log((N - freq + 0.5) / (freq + 0.5) + 1);
-  }
-
-  // Score each chunk
-  const scored = chunks.map(chunk => {
-    const tokens = tokenize(chunk.text);
-    const dl = tokens.length;
-    const tf = {};
-    for (const t of tokens) {
-      tf[t] = (tf[t] || 0) + 1;
-    }
-
-    let score = 0;
-    for (const qt of queryTokens) {
-      const termTf = tf[qt] || 0;
-      if (termTf === 0) continue;
-      const tfNorm = (termTf * (k1 + 1)) / (termTf + k1 * (1 - b + b * (dl / avgDl)));
-      score += (idf[qt] || 0) * tfNorm;
-    }
-
-    return {
-      filePath: chunk.file_path,
-      startLine: chunk.start_line,
-      endLine: chunk.end_line,
-      text: chunk.text,
-      score
-    };
-  });
-
-  scored.sort((a, b) => b.score - a.score);
-  return scored.slice(0, limit).filter(r => r.score > 0);
-}
-
-// ========== EMBEDDING SEARCH ==========
-
-async function getEmbedding(text) {
-  if (!EMBEDDING_URL) return null;
-  const resp = await fetch(EMBEDDING_URL, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${EMBEDDING_TOKEN}` },
-    body: JSON.stringify({ model: EMBEDDING_MODEL, input: text })
-  });
-  if (!resp.ok) throw new Error(`Embedding API error: ${resp.status}`);
-  const data = await resp.json();
-  return new Float32Array(data.data[0].embedding);
-}
-
-async function getEmbeddingsBatch(texts) {
-  if (!EMBEDDING_URL) return texts.map(() => null);
-  const results = [];
-  for (let i = 0; i < texts.length; i += 20) {
-    const batch = texts.slice(i, i + 20);
-    const resp = await fetch(EMBEDDING_URL, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${EMBEDDING_TOKEN}` },
-      body: JSON.stringify({ model: EMBEDDING_MODEL, input: batch })
-    });
-    if (!resp.ok) throw new Error(`Embedding API error: ${resp.status}`);
-    const data = await resp.json();
-    for (const item of data.data) results.push(new Float32Array(item.embedding));
-  }
-  return results;
-}
-
-function cosineSimilarity(a, b) {
-  let dot = 0;
-  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
-  return dot;
-}
-
-// ========== CHUNKING ==========
-
-function chunkMarkdown(text) {
-  const lines = text.split('\n');
-  const chunks = [];
-  const MAX_CHARS = 2000;
-
-  // Build paragraphs
-  let paragraphs = [];
-  let currentPara = [];
-  let paraStart = 0;
-
-  for (let i = 0; i < lines.length; i++) {
-    if (lines[i].trim() === '' && currentPara.length > 0) {
-      paragraphs.push({ text: currentPara.join('\n'), startLine: paraStart + 1, endLine: i });
-      currentPara = [];
-      paraStart = i + 1;
-    } else {
-      if (currentPara.length === 0) paraStart = i;
-      currentPara.push(lines[i]);
-    }
-  }
-  if (currentPara.length > 0) {
-    paragraphs.push({ text: currentPara.join('\n'), startLine: paraStart + 1, endLine: lines.length });
-  }
-
-  // Group into chunks
-  let chunk = { texts: [], startLine: 0, endLine: 0, charCount: 0 };
-
-  for (const para of paragraphs) {
-    if (para.text.length > MAX_CHARS) {
-      if (chunk.texts.length > 0) {
-        chunks.push({ text: chunk.texts.join('\n\n'), startLine: chunk.startLine, endLine: chunk.endLine });
-        chunk = { texts: [], startLine: 0, endLine: 0, charCount: 0 };
-      }
-      const sentences = para.text.match(/[^.!?\n]+[.!?\n]+|[^.!?\n]+$/g) || [para.text];
-      let sentChunk = [], sentCharCount = 0;
-      for (const sent of sentences) {
-        if (sentCharCount + sent.length > MAX_CHARS && sentChunk.length > 0) {
-          chunks.push({ text: sentChunk.join(' '), startLine: para.startLine, endLine: para.endLine });
-          sentChunk = sentChunk.slice(-2);
-          sentCharCount = sentChunk.join(' ').length;
-        }
-        sentChunk.push(sent.trim());
-        sentCharCount += sent.length;
-      }
-      if (sentChunk.length > 0) {
-        chunks.push({ text: sentChunk.join(' '), startLine: para.startLine, endLine: para.endLine });
-      }
-    } else {
-      if (chunk.charCount + para.text.length > MAX_CHARS && chunk.texts.length > 0) {
-        chunks.push({ text: chunk.texts.join('\n\n'), startLine: chunk.startLine, endLine: chunk.endLine });
-        const lastText = chunk.texts[chunk.texts.length - 1];
-        chunk = { texts: [lastText], startLine: chunk.startLine, endLine: chunk.endLine, charCount: lastText.length };
-      }
-      if (chunk.texts.length === 0) chunk.startLine = para.startLine;
-      chunk.texts.push(para.text);
-      chunk.endLine = para.endLine;
-      chunk.charCount += para.text.length;
-    }
-  }
-  if (chunk.texts.length > 0) {
-    chunks.push({ text: chunk.texts.join('\n\n'), startLine: chunk.startLine, endLine: chunk.endLine });
-  }
-
-  return chunks;
-}
-
-// ========== INDEXING ==========
-
-async function indexFile(filePath) {
-  if (!db) return 0;
-  try {
-    if (!fs.existsSync(filePath)) return 0;
-    const stat = fs.statSync(filePath);
-    const mtime = stat.mtimeMs;
-
-    const existing = db.prepare('SELECT file_mtime FROM chunks WHERE file_path = ? LIMIT 1').get(filePath);
-    if (existing && existing.file_mtime === mtime) return 0;
-
-    const text = fs.readFileSync(filePath, 'utf8');
-    if (!text.trim()) return 0;
-
-    const chunks = chunkMarkdown(text);
-    if (chunks.length === 0) return 0;
-
-    // Try embeddings, fall back to null
-    let embeddings;
-    try {
-      embeddings = await getEmbeddingsBatch(chunks.map(c => c.text));
-    } catch (e) {
-      embeddings = chunks.map(() => null);
-    }
-
-    db.prepare('DELETE FROM chunks WHERE file_path = ?').run(filePath);
-
-    const insert = db.prepare('INSERT INTO chunks (file_path, file_mtime, chunk_index, start_line, end_line, text, embedding) VALUES (?, ?, ?, ?, ?, ?, ?)');
-    const insertMany = db.transaction((items) => { for (const item of items) insert.run(...item); });
-
-    insertMany(chunks.map((c, i) => [
-      filePath, mtime, i, c.startLine, c.endLine, c.text,
-      embeddings[i] ? Buffer.from(embeddings[i].buffer) : null
-    ]));
-
-    return chunks.length;
-  } catch (e) {
-    console.error(`[knowledge-base] Error indexing ${filePath}:`, e.message);
-    return 0;
-  }
-}
-
-function findMdFiles(dir) {
-  const files = [];
-  if (!fs.existsSync(dir)) return files;
-  const stat = fs.statSync(dir);
-  if (stat.isFile() && dir.endsWith('.md')) return [dir];
-  if (!stat.isDirectory()) return files;
-  for (const entry of fs.readdirSync(dir)) {
-    if (entry.startsWith('.') || entry === 'node_modules') continue;
-    const full = path.join(dir, entry);
-    const s = fs.statSync(full);
-    if (s.isDirectory()) files.push(...findMdFiles(full));
-    else if (entry.endsWith('.md')) files.push(full);
-  }
-  return files;
-}
-
-async function indexAllFiles() {
-  if (!db) return { filesIndexed: 0, chunksCreated: 0 };
-  let filesIndexed = 0, chunksCreated = 0;
-  const allFiles = [];
-  for (const wp of WATCHED_PATHS) allFiles.push(...findMdFiles(wp));
-  for (const f of allFiles) {
-    const n = await indexFile(f);
-    if (n > 0) { filesIndexed++; chunksCreated += n; }
-  }
-  lastIndexedAt = new Date().toISOString();
-  console.log(`[knowledge-base] Indexed ${filesIndexed} files, ${chunksCreated} chunks`);
-  return { filesIndexed, chunksCreated };
-}
-
-// ========== SEARCH ==========
-
-async function search(query, limit = 5) {
-  if (!db) return [];
-  const allChunks = db.prepare('SELECT id, file_path, start_line, end_line, text, embedding FROM chunks').all();
-  if (allChunks.length === 0) return [];
-
-  // Check if we have embeddings and an embedding API
-  const hasEmbeddings = allChunks.some(c => c.embedding != null);
-
-  if (hasEmbeddings && EMBEDDING_URL) {
-    try {
-      const queryEmb = await getEmbedding(query);
-      if (queryEmb) {
-        const scored = allChunks.filter(r => r.embedding).map(row => {
-          const emb = new Float32Array(new Uint8Array(row.embedding).buffer);
-          return { filePath: row.file_path, startLine: row.start_line, endLine: row.end_line, text: row.text, score: cosineSimilarity(queryEmb, emb) };
-        });
-        scored.sort((a, b) => b.score - a.score);
-        return scored.slice(0, limit);
-      }
-    } catch (e) {
-      console.error('[knowledge-base] Embedding search failed, falling back to BM25:', e.message);
-    }
-  }
-
-  // BM25 fallback
-  return bm25Search(query, allChunks, limit);
-}
-
-// ========== STATUS ==========
-
-function getStatus() {
-  if (!db) return { totalFiles: 0, totalChunks: 0, lastIndexedAt: null, watchedPaths: WATCHED_PATHS, searchMode: 'offline' };
-  const totalChunks = db.prepare('SELECT COUNT(*) as c FROM chunks').get().c;
-  const totalFiles = db.prepare('SELECT COUNT(DISTINCT file_path) as c FROM chunks').get().c;
-  const hasEmbeddings = db.prepare('SELECT COUNT(*) as c FROM chunks WHERE embedding IS NOT NULL').get().c > 0;
-  return {
-    totalFiles, totalChunks, lastIndexedAt, watchedPaths: WATCHED_PATHS,
-    searchMode: hasEmbeddings && EMBEDDING_URL ? 'vector' : 'bm25'
-  };
-}
-
-// ========== FILE WATCHER ==========
-
-function startWatcher() {
-  for (const wp of WATCHED_PATHS) {
-    if (!fs.existsSync(wp)) continue;
-    try {
-      const isDir = fs.statSync(wp).isDirectory();
-      const watcher = fs.watch(wp, { recursive: isDir }, (eventType, filename) => {
-        if (!filename) return;
-        if (!filename.endsWith('.md')) return;
-        const fullPath = isDir ? path.join(wp, filename) : wp;
-
-        if (debounceTimers[fullPath]) clearTimeout(debounceTimers[fullPath]);
-        debounceTimers[fullPath] = setTimeout(async () => {
-          delete debounceTimers[fullPath];
-          if (fs.existsSync(fullPath)) {
-            const n = await indexFile(fullPath);
-            if (n > 0) console.log(`[knowledge-base] Re-indexed ${path.basename(fullPath)} (${n} chunks)`);
-          }
-        }, 2000);
-      });
-      watchers.push(watcher);
-    } catch (e) {
-      console.error(`[knowledge-base] Could not watch ${wp}:`, e.message);
-    }
-  }
-  console.log(`[knowledge-base] Watching ${watchers.length} paths`);
-}
-
-// ========== ROUTES ==========
-
-function setupRoutes(app) {
-  app.get('/api/knowledge/search', async (req, res) => {
-    try {
-      const q = req.query.q;
-      if (!q) return res.status(400).json({ error: 'Missing q parameter' });
-      const limit = parseInt(req.query.limit) || 5;
-      const results = await search(q, limit);
-      res.json({ results });
-    } catch (e) {
-      res.status(500).json({ error: e.message });
-    }
-  });
-
-  app.post('/api/knowledge/reindex', async (req, res) => {
-    try {
-      db.prepare('DELETE FROM chunks').run();
-      const result = await indexAllFiles();
-      res.json({ success: true, ...result });
-    } catch (e) {
-      res.status(500).json({ error: e.message });
-    }
-  });
-
-  app.get('/api/knowledge/status', (req, res) => {
-    res.json(getStatus());
-  });
-}
-
-// Index arbitrary text (e.g., voice note transcripts) into the knowledge base
-function indexText(id, text, metadata = {}) {
-  if (!db) initKnowledgeBase();
-  const filePath = id; // e.g. "voice-note:uuid"
-  const mtime = Date.now();
-
-  // Delete existing chunks for this id
-  db.prepare('DELETE FROM chunks WHERE file_path = ?').run(filePath);
-
-  // Split text into chunks
-  const lines = text.split('\n');
-  const chunkSize = 500; // chars per chunk
-  let chunks = [];
-  let current = '';
-  let startLine = 1;
-  let lineNum = 1;
-
-  for (const line of lines) {
-    if (current.length + line.length > chunkSize && current.length > 0) {
-      chunks.push({ text: current.trim(), startLine, endLine: lineNum - 1 });
-      current = '';
-      startLine = lineNum;
-    }
-    current += line + '\n';
-    lineNum++;
-  }
-  if (current.trim()) {
-    chunks.push({ text: current.trim(), startLine, endLine: lineNum - 1 });
-  }
-
-  // Prepend metadata to first chunk for context
-  if (chunks.length > 0 && metadata.title) {
-    chunks[0].text = `[Voice Note: ${metadata.title}] ${metadata.tags?.length ? 'Tags: ' + metadata.tags.join(', ') + '. ' : ''}${chunks[0].text}`;
-  }
-
-  const insert = db.prepare('INSERT INTO chunks (file_path, file_mtime, chunk_index, start_line, end_line, text, embedding) VALUES (?, ?, ?, ?, ?, ?, ?)');
-  const emptyEmbedding = Buffer.alloc(0);
-  chunks.forEach((chunk, i) => {
-    insert.run(filePath, mtime, i, chunk.startLine, chunk.endLine, chunk.text, emptyEmbedding);
-  });
-
-  console.log(`[knowledge-base] Indexed text "${id}": ${chunks.length} chunks`);
-  return { chunks: chunks.length };
-}
-
-module.exports = { initKnowledgeBase, indexAllFiles, search, getStatus, startWatcher, setupRoutes, indexText };

package/lib/cli.js

@@ -1,19 +0,0 @@
-'use strict';
-
-const { execSync } = require('child_process');
-const pc = require('picocolors');
-
-function update() {
-  console.log(pc.cyan('Checking for updates...'));
-  try {
-    execSync('npm update -g @conversionpros/aiva', { stdio: 'inherit' });
-    console.log(pc.green('\nUpdate complete. Restarting AIVA...'));
-    const { restartServer } = require('./process');
-    restartServer();
-  } catch (e) {
-    console.error(pc.red('Update failed:'), e.message);
-    process.exit(1);
-  }
-}
-
-module.exports = { update };

package/lib/server.js

@@ -1,42 +0,0 @@
-'use strict';
-
-// AIVA Server - CLI adapter
-// This wraps the main server.js, setting up environment variables
-// so that it resolves paths from ~/.aiva/ instead of hardcoded locations.
-
-const path = require('path');
-const fs = require('fs');
-
-// Determine AIVA home - either from env or default ~/.aiva/
-const AIVA_HOME = process.env.AIVA_HOME || path.join(require('os').homedir(), '.aiva');
-const configPath = process.env.AIVA_CONFIG_PATH || path.join(AIVA_HOME, 'config.json');
-
-// Load config
-let config = {};
-try {
-  config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
-} catch (e) {
-  console.warn('[aiva] No config found at', configPath, '- using defaults');
-}
-
-// Set environment variables that the server reads
-process.env.PORT = process.env.PORT || String(config.port || 3847);
-process.env.AIVA_HOME = AIVA_HOME;
-process.env.AIVA_DATA_DIR = process.env.AIVA_DATA_DIR || path.join(AIVA_HOME, 'data');
-process.env.AIVA_UPLOADS_DIR = process.env.AIVA_UPLOADS_DIR || path.join(AIVA_HOME, 'uploads');
-process.env.AIVA_LOGS_DIR = process.env.AIVA_LOGS_DIR || path.join(AIVA_HOME, 'logs');
-
-// Ensure data directories exist
-[process.env.AIVA_DATA_DIR, process.env.AIVA_UPLOADS_DIR, process.env.AIVA_LOGS_DIR].forEach(dir => {
-  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-});
-
-// Load .env from AIVA_HOME if it exists (user-specific environment)
-const envPath = path.join(AIVA_HOME, '.env');
-if (fs.existsSync(envPath)) {
-  require('dotenv').config({ path: envPath });
-}
-
-// Now load the actual server - it lives alongside this file in the package
-const serverPath = path.join(__dirname, '..', 'server.js');
-require(serverPath);