multis 0.1.0

package/src/indexer/store.js

@@ -0,0 +1,256 @@
+const Database = require('better-sqlite3');
+const path = require('path');
+const fs = require('fs');
+const { MULTIS_DIR } = require('../config');
+
+const DB_PATH = path.join(MULTIS_DIR, 'documents.db');
+
+/**
+ * DocumentStore - SQLite storage for document chunks with FTS5 search.
+ * Ported from aurora_core.store.sqlite (Python).
+ * Includes activation columns for future ACT-R (POC5).
+ */
+class DocumentStore {
+  constructor(dbPath = DB_PATH) {
+    // Ensure directory exists
+    const dir = path.dirname(dbPath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    this.db = new Database(dbPath);
+    this.db.pragma('journal_mode = WAL');
+    this.db.pragma('foreign_keys = ON');
+    this._initSchema();
+  }
+
+  _initSchema() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        chunk_id TEXT PRIMARY KEY,
+        file_path TEXT NOT NULL,
+        page_start INTEGER DEFAULT 0,
+        page_end INTEGER DEFAULT 0,
+        element_type TEXT DEFAULT 'paragraph',
+        name TEXT DEFAULT '',
+        content TEXT DEFAULT '',
+        parent_chunk_id TEXT,
+        section_path TEXT DEFAULT '[]',
+        section_level INTEGER DEFAULT 0,
+        document_type TEXT DEFAULT 'unknown',
+        metadata TEXT DEFAULT '{}',
+        -- Activation columns for ACT-R (POC5)
+        activation REAL DEFAULT 0.0,
+        access_count INTEGER DEFAULT 0,
+        last_accessed TEXT,
+        -- Timestamps
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path);
+      CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(element_type);
+      CREATE INDEX IF NOT EXISTS idx_chunks_doc_type ON chunks(document_type);
+      CREATE INDEX IF NOT EXISTS idx_chunks_activation ON chunks(activation DESC);
+
+      -- FTS5 virtual table for full-text search (BM25)
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        chunk_id UNINDEXED,
+        name,
+        content,
+        section_path,
+        content=chunks,
+        content_rowid=rowid,
+        tokenize='porter unicode61'
+      );
+
+      -- Triggers to keep FTS in sync
+      CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO chunks_fts(rowid, chunk_id, name, content, section_path)
+        VALUES (new.rowid, new.chunk_id, new.name, new.content, new.section_path);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, chunk_id, name, content, section_path)
+        VALUES ('delete', old.rowid, old.chunk_id, old.name, old.content, old.section_path);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, chunk_id, name, content, section_path)
+        VALUES ('delete', old.rowid, old.chunk_id, old.name, old.content, old.section_path);
+        INSERT INTO chunks_fts(rowid, chunk_id, name, content, section_path)
+        VALUES (new.rowid, new.chunk_id, new.name, new.content, new.section_path);
+      END;
+
+      -- Access history for ACT-R activation tracking (POC5)
+      CREATE TABLE IF NOT EXISTS access_history (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        chunk_id TEXT NOT NULL,
+        accessed_at TEXT NOT NULL,
+        query TEXT,
+        FOREIGN KEY (chunk_id) REFERENCES chunks(chunk_id) ON DELETE CASCADE
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_access_chunk ON access_history(chunk_id);
+    `);
+  }
+
+  /**
+   * Save a DocChunk to the store (insert or replace)
+   */
+  saveChunk(chunk) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO chunks
+        (chunk_id, file_path, page_start, page_end, element_type, name, content,
+         parent_chunk_id, section_path, section_level, document_type, metadata,
+         created_at, updated_at)
+      VALUES
+        (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    stmt.run(
+      chunk.chunkId,
+      chunk.filePath,
+      chunk.pageStart,
+      chunk.pageEnd,
+      chunk.elementType,
+      chunk.name,
+      chunk.content,
+      chunk.parentChunkId,
+      JSON.stringify(chunk.sectionPath),
+      chunk.sectionLevel,
+      chunk.documentType,
+      JSON.stringify(chunk.metadata),
+      chunk.createdAt,
+      chunk.updatedAt
+    );
+  }
+
+  /**
+   * Save multiple chunks in a transaction
+   */
+  saveChunks(chunks) {
+    const tx = this.db.transaction((items) => {
+      for (const chunk of items) {
+        this.saveChunk(chunk);
+      }
+    });
+    tx(chunks);
+  }
+
+  /**
+   * BM25 full-text search using FTS5
+   * @param {string} query - Search query
+   * @param {number} limit - Max results
+   * @returns {Array} - Chunks sorted by BM25 relevance
+   */
+  search(query, limit = 10) {
+    // Convert natural language to FTS5 OR query
+    // Filter out stopwords, join remaining with OR for broader matching
+    const stopwords = new Set(['a','an','the','is','are','was','were','be','been',
+      'being','have','has','had','do','does','did','will','would','could','should',
+      'may','might','can','i','me','my','we','our','you','your','he','she','it',
+      'they','them','this','that','what','which','who','how','when','where','why',
+      'not','no','so','if','or','and','but','in','on','at','to','for','of','with',
+      'by','from','as','into','about','than','after','before']);
+
+    const terms = query.toLowerCase()
+      .replace(/[^\w\s]/g, '')
+      .split(/\s+/)
+      .filter(t => t.length > 1 && !stopwords.has(t));
+
+    if (terms.length === 0) return [];
+
+    const ftsQuery = terms.join(' OR ');
+
+    const stmt = this.db.prepare(`
+      SELECT c.*, rank
+      FROM chunks_fts fts
+      JOIN chunks c ON fts.chunk_id = c.chunk_id
+      WHERE chunks_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?
+    `);
+
+    const rows = stmt.all(ftsQuery, limit);
+    return rows.map(row => ({
+      chunkId: row.chunk_id,
+      filePath: row.file_path,
+      pageStart: row.page_start,
+      pageEnd: row.page_end,
+      elementType: row.element_type,
+      name: row.name,
+      content: row.content,
+      parentChunkId: row.parent_chunk_id,
+      sectionPath: JSON.parse(row.section_path || '[]'),
+      sectionLevel: row.section_level,
+      documentType: row.document_type,
+      metadata: JSON.parse(row.metadata || '{}'),
+      activation: row.activation,
+      rank: row.rank,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at
+    }));
+  }
+
+  /**
+   * Get chunk by ID
+   */
+  getChunk(chunkId) {
+    const row = this.db.prepare('SELECT * FROM chunks WHERE chunk_id = ?').get(chunkId);
+    if (!row) return null;
+    return {
+      chunkId: row.chunk_id,
+      filePath: row.file_path,
+      name: row.name,
+      content: row.content,
+      sectionPath: JSON.parse(row.section_path || '[]'),
+      documentType: row.document_type,
+      activation: row.activation
+    };
+  }
+
+  /**
+   * Delete all chunks for a file (for re-indexing)
+   */
+  deleteByFile(filePath) {
+    this.db.prepare('DELETE FROM chunks WHERE file_path = ?').run(filePath);
+  }
+
+  /**
+   * Get stats about indexed documents
+   */
+  getStats() {
+    const total = this.db.prepare('SELECT COUNT(*) as count FROM chunks').get();
+    const byType = this.db.prepare(
+      'SELECT document_type, COUNT(*) as count FROM chunks GROUP BY document_type'
+    ).all();
+    const files = this.db.prepare(
+      'SELECT DISTINCT file_path FROM chunks'
+    ).all();
+    return {
+      totalChunks: total.count,
+      byType: Object.fromEntries(byType.map(r => [r.document_type, r.count])),
+      indexedFiles: files.length
+    };
+  }
+
+  /**
+   * Record a search access for ACT-R activation tracking (POC5)
+   */
+  recordAccess(chunkId, query) {
+    this.db.prepare(
+      'INSERT INTO access_history (chunk_id, accessed_at, query) VALUES (?, ?, ?)'
+    ).run(chunkId, new Date().toISOString(), query);
+
+    this.db.prepare(
+      'UPDATE chunks SET access_count = access_count + 1, last_accessed = ? WHERE chunk_id = ?'
+    ).run(new Date().toISOString(), chunkId);
+  }
+
+  close() {
+    this.db.close();
+  }
+}
+
+module.exports = { DocumentStore, DB_PATH };

package/src/llm/anthropic.js

@@ -0,0 +1,106 @@
+const https = require('https');
+const { LLMProvider } = require('./base');
+
+/**
+ * Anthropic Claude provider (vanilla Node.js https)
+ */
+class AnthropicProvider extends LLMProvider {
+  constructor(apiKey, model = 'claude-sonnet-4-5-20250929') {
+    super();
+    this.apiKey = apiKey;
+    this.model = model;
+    this.apiVersion = '2023-06-01';
+  }
+
+  async generate(prompt, options = {}) {
+    const body = {
+      model: this.model,
+      max_tokens: options.maxTokens || 2048,
+      temperature: options.temperature || 0.7,
+      messages: [{ role: 'user', content: prompt }]
+    };
+
+    if (options.system) {
+      body.system = options.system;
+    }
+
+    const response = await this._makeRequest(body);
+
+    return response.content[0].text;
+  }
+
+  async generateWithTools(prompt, tools, options = {}) {
+    const response = await this._makeRequest({
+      model: this.model,
+      max_tokens: options.maxTokens || 2048,
+      tools: tools.map(t => ({
+        name: t.name,
+        description: t.description,
+        input_schema: t.inputSchema
+      })),
+      messages: [{ role: 'user', content: prompt }]
+    });
+
+    return response;
+  }
+
+  async generateWithMessages(messages, options = {}) {
+    const body = {
+      model: this.model,
+      max_tokens: options.maxTokens || 2048,
+      temperature: options.temperature || 0.7,
+      messages
+    };
+
+    if (options.system) {
+      body.system = options.system;
+    }
+
+    const response = await this._makeRequest(body);
+    return response.content[0].text;
+  }
+
+  _makeRequest(body) {
+    return new Promise((resolve, reject) => {
+      const data = JSON.stringify(body);
+
+      const options = {
+        hostname: 'api.anthropic.com',
+        path: '/v1/messages',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(data),
+          'x-api-key': this.apiKey,
+          'anthropic-version': this.apiVersion
+        }
+      };
+
+      const req = https.request(options, (res) => {
+        let responseBody = '';
+
+        res.on('data', (chunk) => {
+          responseBody += chunk;
+        });
+
+        res.on('end', () => {
+          if (res.statusCode >= 200 && res.statusCode < 300) {
+            try {
+              resolve(JSON.parse(responseBody));
+            } catch (err) {
+              reject(new Error('Failed to parse response: ' + err.message));
+            }
+          } else {
+            reject(new Error(`API error (${res.statusCode}): ${responseBody}`));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.write(data);
+      req.end();
+    });
+  }
+}
+
+module.exports = { AnthropicProvider };

package/src/llm/base.js

@@ -0,0 +1,38 @@
+/**
+ * Base LLM Provider interface
+ * All LLM providers must extend this class
+ */
+class LLMProvider {
+  /**
+   * Generate a response from the LLM
+   * @param {string} prompt - The user prompt
+   * @param {Object} options - Additional options (context, temperature, etc.)
+   * @returns {Promise<string>} - The generated response
+   */
+  async generate(prompt, options = {}) {
+    throw new Error('Must implement generate() method');
+  }
+
+  /**
+   * Generate a response with tool calling support
+   * @param {string} prompt - The user prompt
+   * @param {Array} tools - Available tools
+   * @param {Object} options - Additional options
+   * @returns {Promise<Object>} - Response with possible tool calls
+   */
+  async generateWithTools(prompt, tools, options = {}) {
+    throw new Error('Must implement generateWithTools() method');
+  }
+
+  /**
+   * Generate a response from a messages array (for conversation memory)
+   * @param {Array<{role: string, content: string}>} messages - Conversation messages
+   * @param {Object} options - Additional options (system, temperature, etc.)
+   * @returns {Promise<string>} - The generated response
+   */
+  async generateWithMessages(messages, options = {}) {
+    throw new Error('Must implement generateWithMessages() method');
+  }
+}
+
+module.exports = { LLMProvider };

package/src/llm/client.js

@@ -0,0 +1,34 @@
+const { AnthropicProvider } = require('./anthropic');
+const { OpenAIProvider } = require('./openai');
+const { OllamaProvider } = require('./ollama');
+
+/**
+ * Factory function to create LLM client based on config
+ * @param {Object} config - LLM configuration
+ * @returns {LLMProvider} - Configured LLM provider
+ */
+function createLLMClient(config) {
+  const provider = config.provider || 'anthropic';
+
+  switch (provider.toLowerCase()) {
+    case 'anthropic':
+      if (!config.apiKey) {
+        throw new Error('Anthropic API key is required');
+      }
+      return new AnthropicProvider(config.apiKey, config.model);
+
+    case 'openai':
+      if (!config.apiKey) {
+        throw new Error('OpenAI API key is required');
+      }
+      return new OpenAIProvider(config.apiKey, config.model);
+
+    case 'ollama':
+      return new OllamaProvider(config.model, config.baseUrl);
+
+    default:
+      throw new Error(`Unknown LLM provider: ${provider}`);
+  }
+}
+
+module.exports = { createLLMClient };

package/src/llm/ollama.js

@@ -0,0 +1,148 @@
+const http = require('http');
+const { LLMProvider } = require('./base');
+
+/**
+ * Ollama local LLM provider (vanilla Node.js http)
+ */
+class OllamaProvider extends LLMProvider {
+  constructor(model = 'llama3.1:8b', baseUrl = 'http://localhost:11434') {
+    super();
+    this.model = model;
+    this.baseUrl = baseUrl;
+  }
+
+  async generate(prompt, options = {}) {
+    const body = {
+      model: this.model,
+      prompt,
+      stream: false,
+      options: {
+        temperature: options.temperature || 0.7,
+        num_predict: options.maxTokens || 2048
+      }
+    };
+
+    if (options.system) {
+      body.system = options.system;
+    }
+
+    const response = await this._makeRequest(body);
+
+    return response.response;
+  }
+
+  async generateWithTools(prompt, tools, options = {}) {
+    // Ollama doesn't natively support tool calling yet
+    // We can implement a simple version by prompting the model
+    const toolsDescription = tools.map(t =>
+      `Tool: ${t.name}\nDescription: ${t.description}\nParameters: ${JSON.stringify(t.inputSchema)}`
+    ).join('\n\n');
+
+    const systemPrompt = `You have access to these tools:\n\n${toolsDescription}\n\nTo use a tool, respond with JSON in this format: {"tool": "tool_name", "parameters": {...}}`;
+
+    const response = await this.generate(prompt, {
+      ...options,
+      system: systemPrompt
+    });
+
+    return { content: response };
+  }
+
+  async generateWithMessages(messages, options = {}) {
+    const body = {
+      model: this.model,
+      messages,
+      stream: false,
+      options: {
+        temperature: options.temperature || 0.7,
+        num_predict: options.maxTokens || 2048
+      }
+    };
+
+    if (options.system) {
+      body.messages = [{ role: 'system', content: options.system }, ...messages];
+    }
+
+    const response = await this._makeChatRequest(body);
+    return response.message.content;
+  }
+
+  _makeChatRequest(body) {
+    return new Promise((resolve, reject) => {
+      const data = JSON.stringify(body);
+      const url = new URL('/api/chat', this.baseUrl);
+
+      const options = {
+        hostname: url.hostname,
+        port: url.port || 80,
+        path: url.pathname,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(data)
+        }
+      };
+
+      const req = http.request(options, (res) => {
+        let responseBody = '';
+        res.on('data', (chunk) => { responseBody += chunk; });
+        res.on('end', () => {
+          if (res.statusCode >= 200 && res.statusCode < 300) {
+            try { resolve(JSON.parse(responseBody)); }
+            catch (err) { reject(new Error('Failed to parse response: ' + err.message)); }
+          } else {
+            reject(new Error(`Ollama error (${res.statusCode}): ${responseBody}`));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.write(data);
+      req.end();
+    });
+  }
+
+  _makeRequest(body) {
+    return new Promise((resolve, reject) => {
+      const data = JSON.stringify(body);
+      const url = new URL('/api/generate', this.baseUrl);
+
+      const options = {
+        hostname: url.hostname,
+        port: url.port || 80,
+        path: url.pathname,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(data)
+        }
+      };
+
+      const req = http.request(options, (res) => {
+        let responseBody = '';
+
+        res.on('data', (chunk) => {
+          responseBody += chunk;
+        });
+
+        res.on('end', () => {
+          if (res.statusCode >= 200 && res.statusCode < 300) {
+            try {
+              resolve(JSON.parse(responseBody));
+            } catch (err) {
+              reject(new Error('Failed to parse response: ' + err.message));
+            }
+          } else {
+            reject(new Error(`Ollama error (${res.statusCode}): ${responseBody}`));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.write(data);
+      req.end();
+    });
+  }
+}
+
+module.exports = { OllamaProvider };

package/src/llm/openai.js

@@ -0,0 +1,107 @@
+const https = require('https');
+const { LLMProvider } = require('./base');
+
+/**
+ * OpenAI GPT provider (vanilla Node.js https)
+ */
+class OpenAIProvider extends LLMProvider {
+  constructor(apiKey, model = 'gpt-4o') {
+    super();
+    this.apiKey = apiKey;
+    this.model = model;
+  }
+
+  async generate(prompt, options = {}) {
+    const messages = options.system
+      ? [
+          { role: 'system', content: options.system },
+          { role: 'user', content: prompt }
+        ]
+      : [{ role: 'user', content: prompt }];
+
+    const response = await this._makeRequest({
+      model: this.model,
+      max_tokens: options.maxTokens || 2048,
+      temperature: options.temperature || 0.7,
+      messages
+    });
+
+    return response.choices[0].message.content;
+  }
+
+  async generateWithTools(prompt, tools, options = {}) {
+    const response = await this._makeRequest({
+      model: this.model,
+      max_tokens: options.maxTokens || 2048,
+      tools: tools.map(t => ({
+        type: 'function',
+        function: {
+          name: t.name,
+          description: t.description,
+          parameters: t.inputSchema
+        }
+      })),
+      messages: [{ role: 'user', content: prompt }]
+    });
+
+    return response;
+  }
+
+  async generateWithMessages(messages, options = {}) {
+    const msgs = options.system
+      ? [{ role: 'system', content: options.system }, ...messages]
+      : [...messages];
+
+    const response = await this._makeRequest({
+      model: this.model,
+      max_tokens: options.maxTokens || 2048,
+      temperature: options.temperature || 0.7,
+      messages: msgs
+    });
+
+    return response.choices[0].message.content;
+  }
+
+  _makeRequest(body) {
+    return new Promise((resolve, reject) => {
+      const data = JSON.stringify(body);
+
+      const options = {
+        hostname: 'api.openai.com',
+        path: '/v1/chat/completions',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(data),
+          'Authorization': `Bearer ${this.apiKey}`
+        }
+      };
+
+      const req = https.request(options, (res) => {
+        let responseBody = '';
+
+        res.on('data', (chunk) => {
+          responseBody += chunk;
+        });
+
+        res.on('end', () => {
+          if (res.statusCode >= 200 && res.statusCode < 300) {
+            try {
+              resolve(JSON.parse(responseBody));
+            } catch (err) {
+              reject(new Error('Failed to parse response: ' + err.message));
+            }
+          } else {
+            reject(new Error(`API error (${res.statusCode}): ${responseBody}`));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.write(data);
+      req.end();
+    });
+  }
+}
+
+module.exports = { OpenAIProvider };