multis 0.1.0

package/src/index.js

@@ -0,0 +1,71 @@
+const { loadConfig, ensureMultisDir } = require('./config');
+const { logAudit } = require('./governance/audit');
+const { createMessageRouter } = require('./bot/handlers');
+const { TelegramPlatform } = require('./platforms/telegram');
+const { BeeperPlatform } = require('./platforms/beeper');
+
+async function main() {
+  ensureMultisDir();
+  const config = loadConfig();
+
+  console.log('multis v0.1.0');
+  console.log(`Pairing code: ${config.pairing_code}`);
+  console.log(`Paired users: ${config.allowed_users.length}`);
+  console.log(`LLM provider: ${config.llm.provider}`);
+
+  const handler = createMessageRouter(config);
+  const platforms = [];
+
+  // Telegram — enabled by default (backward compat)
+  if (config.platforms?.telegram?.enabled !== false) {
+    try {
+      const telegram = new TelegramPlatform(config);
+      telegram.onMessage(handler);
+      platforms.push(telegram);
+    } catch (err) {
+      console.error(`Telegram: ${err.message}`);
+    }
+  }
+
+  // Beeper — opt-in
+  if (config.platforms?.beeper?.enabled) {
+    try {
+      const beeper = new BeeperPlatform(config);
+      beeper.onMessage(handler);
+      platforms.push(beeper);
+    } catch (err) {
+      console.error(`Beeper: ${err.message}`);
+    }
+  }
+
+  if (platforms.length === 0) {
+    console.error('No platforms configured. Set up at least one platform.');
+    process.exit(1);
+  }
+
+  logAudit({ action: 'bot_start', platforms: platforms.map(p => p.name), paired_users: config.allowed_users.length });
+
+  for (const p of platforms) {
+    await p.start();
+  }
+
+  console.log(`Running on: ${platforms.map(p => p.name).join(', ')}`);
+
+  // Graceful shutdown
+  const shutdown = async (signal) => {
+    console.log(`\nShutting down (${signal})...`);
+    logAudit({ action: 'bot_stop', reason: signal });
+    for (const p of platforms) {
+      await p.stop();
+    }
+    process.exit(0);
+  };
+
+  process.once('SIGINT', () => shutdown('SIGINT'));
+  process.once('SIGTERM', () => shutdown('SIGTERM'));
+}
+
+main().catch(err => {
+  console.error('Fatal:', err.message);
+  process.exit(1);
+});

package/src/indexer/chunk.js

@@ -0,0 +1,68 @@
+const crypto = require('crypto');
+
+/**
+ * DocChunk - represents a document section/chunk.
+ * Ported from aurora_core.chunks.DocChunk (Python dataclass).
+ */
+class DocChunk {
+  constructor({
+    chunkId = null,
+    filePath,
+    pageStart = 0,
+    pageEnd = 0,
+    elementType = 'paragraph',  // toc_entry, section, paragraph, table
+    name = '',
+    content = '',
+    parentChunkId = null,
+    sectionPath = [],           // breadcrumb array: ["Chapter 1", "Section 1.2"]
+    sectionLevel = 0,           // heading depth 1-5, 0 = body
+    documentType = 'unknown',   // pdf, docx, md, txt
+    metadata = {},
+    createdAt = null,
+    updatedAt = null
+  }) {
+    this.chunkId = chunkId || DocChunk.generateId(filePath, name, content);
+    this.filePath = filePath;
+    this.pageStart = pageStart;
+    this.pageEnd = pageEnd;
+    this.elementType = elementType;
+    this.name = name;
+    this.content = content;
+    this.parentChunkId = parentChunkId;
+    this.sectionPath = sectionPath;
+    this.sectionLevel = sectionLevel;
+    this.documentType = documentType;
+    this.metadata = metadata;
+    this.createdAt = createdAt || new Date().toISOString();
+    this.updatedAt = updatedAt || new Date().toISOString();
+  }
+
+  static generateId(filePath, name, content) {
+    const hash = crypto.createHash('sha256')
+      .update(`${filePath}:${name}:${content.slice(0, 200)}`)
+      .digest('hex')
+      .slice(0, 16);
+    return `doc:${hash}`;
+  }
+
+  toJSON() {
+    return {
+      chunk_id: this.chunkId,
+      file_path: this.filePath,
+      page_start: this.pageStart,
+      page_end: this.pageEnd,
+      element_type: this.elementType,
+      name: this.name,
+      content: this.content,
+      parent_chunk_id: this.parentChunkId,
+      section_path: this.sectionPath,
+      section_level: this.sectionLevel,
+      document_type: this.documentType,
+      metadata: this.metadata,
+      created_at: this.createdAt,
+      updated_at: this.updatedAt
+    };
+  }
+}
+
+module.exports = { DocChunk };

package/src/indexer/chunker.js

@@ -0,0 +1,87 @@
+const { DocChunk } = require('./chunk');
+
+/**
+ * DocumentChunker - section-aware chunk splitting with overlap.
+ * Ported from aurora_context_doc.chunker.DocumentChunker (Python).
+ */
+class DocumentChunker {
+  constructor({ maxChunkSize = 2000, overlap = 200 } = {}) {
+    this.maxChunkSize = maxChunkSize;
+    this.overlap = overlap;
+  }
+
+  /**
+   * Split a large chunk into smaller overlapping pieces at sentence boundaries.
+   * @param {DocChunk} chunk
+   * @returns {DocChunk[]}
+   */
+  splitLarge(chunk) {
+    if (chunk.content.length <= this.maxChunkSize) {
+      return [chunk];
+    }
+
+    const chunks = [];
+    const content = chunk.content;
+    let start = 0;
+    let partNum = 0;
+
+    while (start < content.length) {
+      let end = Math.min(start + this.maxChunkSize, content.length);
+
+      // Try to break at sentence boundary
+      if (end < content.length) {
+        for (const marker of ['. ', '! ', '? ', '\n\n', '\n']) {
+          const lastBreak = content.lastIndexOf(marker, end);
+          if (lastBreak > start) {
+            end = lastBreak + marker.length;
+            break;
+          }
+        }
+      }
+
+      const sliceContent = content.slice(start, end).trim();
+      if (sliceContent) {
+        chunks.push(new DocChunk({
+          chunkId: `${chunk.chunkId}-p${partNum}`,
+          filePath: chunk.filePath,
+          pageStart: chunk.pageStart,
+          pageEnd: chunk.pageEnd,
+          elementType: chunk.elementType,
+          name: `${chunk.name} (part ${partNum + 1})`,
+          content: sliceContent,
+          parentChunkId: chunk.parentChunkId,
+          sectionPath: chunk.sectionPath,
+          sectionLevel: chunk.sectionLevel,
+          documentType: chunk.documentType,
+          metadata: chunk.metadata,
+          createdAt: chunk.createdAt,
+          updatedAt: chunk.updatedAt
+        }));
+      }
+
+      if (end >= content.length) break;
+
+      // Move forward with overlap, ensure progress
+      const nextStart = end - this.overlap;
+      start = nextStart <= start ? start + 1 : nextStart;
+      partNum++;
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Process an array of chunks: split large ones.
+   * @param {DocChunk[]} chunks
+   * @returns {DocChunk[]}
+   */
+  process(chunks) {
+    const result = [];
+    for (const chunk of chunks) {
+      result.push(...this.splitLarge(chunk));
+    }
+    return result;
+  }
+}
+
+module.exports = { DocumentChunker };

package/src/indexer/index.js

@@ -0,0 +1,150 @@
+const fs = require('fs');
+const path = require('path');
+const { getParser } = require('./parsers');
+const { DocumentChunker } = require('./chunker');
+const { DocumentStore } = require('./store');
+const { logAudit } = require('../governance/audit');
+
+/**
+ * DocumentIndexer - orchestrates parsing, chunking, and storage.
+ * Ported from aurora_context_doc.indexer.DocumentIndexer (Python).
+ */
+class DocumentIndexer {
+  constructor(store = null) {
+    this.store = store || new DocumentStore();
+    this.chunker = new DocumentChunker();
+  }
+
+  /**
+   * Index a single file: parse → chunk → store
+   * @param {string} filePath - Path to document
+   * @returns {Promise<number>} - Number of chunks created
+   */
+  async indexFile(filePath) {
+    const resolved = path.resolve(filePath);
+
+    if (!fs.existsSync(resolved)) {
+      throw new Error(`File not found: ${filePath}`);
+    }
+
+    const parser = getParser(resolved);
+    if (!parser) {
+      const ext = path.extname(resolved);
+      throw new Error(`Unsupported file type: ${ext}. Supported: .pdf, .docx, .md, .txt`);
+    }
+
+    // Delete existing chunks for this file (re-index)
+    this.store.deleteByFile(resolved);
+
+    // Parse
+    const rawChunks = await parser(resolved);
+    if (!rawChunks || rawChunks.length === 0) {
+      return 0;
+    }
+
+    // Chunk (split large sections)
+    const processed = this.chunker.process(rawChunks);
+
+    // Store
+    this.store.saveChunks(processed);
+
+    logAudit({
+      action: 'index_file',
+      file: resolved,
+      raw_chunks: rawChunks.length,
+      stored_chunks: processed.length
+    });
+
+    return processed.length;
+  }
+
+  /**
+   * Index a buffer (e.g. from Telegram file upload)
+   * @param {Buffer} buffer - File contents
+   * @param {string} filename - Original filename
+   * @returns {Promise<number>} - Number of chunks created
+   */
+  async indexBuffer(buffer, filename) {
+    // Write to temp file, index it, then clean up
+    const tmpDir = path.join(require('../config').MULTIS_DIR, 'tmp');
+    if (!fs.existsSync(tmpDir)) {
+      fs.mkdirSync(tmpDir, { recursive: true });
+    }
+
+    const tmpPath = path.join(tmpDir, filename);
+    fs.writeFileSync(tmpPath, buffer);
+
+    try {
+      const count = await this.indexFile(tmpPath);
+      return count;
+    } finally {
+      // Clean up temp file
+      if (fs.existsSync(tmpPath)) {
+        fs.unlinkSync(tmpPath);
+      }
+    }
+  }
+
+  /**
+   * Index all supported files in a directory
+   * @param {string} dirPath - Directory path
+   * @param {boolean} recursive - Recurse into subdirectories
+   * @returns {Promise<{files: number, chunks: number}>}
+   */
+  async indexDirectory(dirPath, recursive = true) {
+    const resolved = path.resolve(dirPath);
+
+    if (!fs.existsSync(resolved)) {
+      throw new Error(`Directory not found: ${dirPath}`);
+    }
+
+    const supportedExts = ['.pdf', '.docx', '.md', '.txt'];
+    let totalFiles = 0;
+    let totalChunks = 0;
+
+    const entries = fs.readdirSync(resolved, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const fullPath = path.join(resolved, entry.name);
+
+      if (entry.isDirectory() && recursive) {
+        const sub = await this.indexDirectory(fullPath, true);
+        totalFiles += sub.files;
+        totalChunks += sub.chunks;
+      } else if (entry.isFile() && supportedExts.includes(path.extname(entry.name).toLowerCase())) {
+        try {
+          const count = await this.indexFile(fullPath);
+          totalFiles++;
+          totalChunks += count;
+        } catch (err) {
+          logAudit({ action: 'index_error', file: fullPath, error: err.message });
+        }
+      }
+    }
+
+    return { files: totalFiles, chunks: totalChunks };
+  }
+
+  /**
+   * Search indexed documents
+   * @param {string} query - Search query
+   * @param {number} limit - Max results
+   * @returns {Array} - Matching chunks
+   */
+  search(query, limit = 5) {
+    return this.store.search(query, limit);
+  }
+
+  /**
+   * Get indexing stats
+   */
+  getStats() {
+    return this.store.getStats();
+  }
+
+  close() {
+    this.store.close();
+  }
+}
+
+module.exports = { DocumentIndexer };

package/src/indexer/parsers.js

@@ -0,0 +1,299 @@
+const fs = require('fs');
+const path = require('path');
+const { DocChunk } = require('./chunk');
+
+/**
+ * PDF Parser - uses pdf-parse to extract text, creates page-level chunks.
+ * Ported from aurora_context_doc.parser.pdf (PyMuPDF).
+ * pdf-parse is simpler than PyMuPDF — no TOC extraction, no font detection.
+ * We get page-level text and chunk from there.
+ */
+async function parsePDF(filePath) {
+  const pdfParse = require('pdf-parse');
+  const buffer = fs.readFileSync(filePath);
+
+  const data = await pdfParse(buffer, {
+    // Custom page renderer to get per-page text
+    pagerender: function (pageData) {
+      return pageData.getTextContent().then(function (textContent) {
+        return textContent.items.map(item => item.str).join(' ');
+      });
+    }
+  });
+
+  // pdf-parse doesn't give per-page text easily with custom renderer,
+  // but data.text has full text. Split by form feeds if available.
+  // Fallback: use numpages and split evenly, or treat as single doc.
+  const chunks = [];
+  const absPath = path.resolve(filePath);
+
+  // Try to split by page using the raw text
+  // pdf-parse joins pages - we re-parse with page tracking
+  const pdf = await pdfParse(buffer);
+  const fullText = pdf.text;
+  const numPages = pdf.numpages;
+
+  if (numPages <= 1 || fullText.length < 500) {
+    // Single chunk for small docs
+    if (fullText.trim()) {
+      chunks.push(new DocChunk({
+        filePath: absPath,
+        pageStart: 1,
+        pageEnd: numPages,
+        elementType: 'section',
+        name: path.basename(filePath),
+        content: fullText.trim(),
+        sectionPath: [path.basename(filePath)],
+        sectionLevel: 1,
+        documentType: 'pdf'
+      }));
+    }
+  } else {
+    // Split full text roughly by page count
+    // This is approximate — pdf-parse doesn't give clean page breaks
+    const avgChars = Math.ceil(fullText.length / numPages);
+    for (let i = 0; i < numPages; i++) {
+      const start = i * avgChars;
+      const end = Math.min((i + 1) * avgChars, fullText.length);
+      const pageText = fullText.slice(start, end).trim();
+
+      if (!pageText) continue;
+
+      chunks.push(new DocChunk({
+        filePath: absPath,
+        pageStart: i + 1,
+        pageEnd: i + 1,
+        elementType: 'paragraph',
+        name: `Page ${i + 1}`,
+        content: pageText,
+        sectionPath: [path.basename(filePath), `Page ${i + 1}`],
+        sectionLevel: 1,
+        documentType: 'pdf'
+      }));
+    }
+  }
+
+  return chunks;
+}
+
+/**
+ * DOCX Parser - uses mammoth to extract HTML, then parses headings for hierarchy.
+ * Ported from aurora_context_doc.parser.docx (python-docx).
+ * mammoth converts to HTML — we parse headings from that.
+ */
+async function parseDOCX(filePath) {
+  const mammoth = require('mammoth');
+  const absPath = path.resolve(filePath);
+
+  const result = await mammoth.convertToHtml({ path: filePath });
+  const html = result.value;
+
+  // Parse HTML for headings and content blocks
+  const chunks = [];
+  const sectionStack = []; // track current heading hierarchy
+
+  // Split HTML by heading tags
+  const parts = html.split(/(<h[1-6][^>]*>.*?<\/h[1-6]>)/gi);
+
+  let currentContent = '';
+  let currentName = path.basename(filePath);
+  let currentLevel = 0;
+
+  for (const part of parts) {
+    const headingMatch = part.match(/<h([1-6])[^>]*>(.*?)<\/h[1-6]>/i);
+
+    if (headingMatch) {
+      // Save previous section if it has content
+      if (currentContent.trim()) {
+        const cleanContent = stripHtml(currentContent);
+        if (cleanContent) {
+          chunks.push(new DocChunk({
+            filePath: absPath,
+            elementType: currentLevel > 0 ? 'section' : 'paragraph',
+            name: currentName,
+            content: cleanContent,
+            sectionPath: sectionStack.map(s => s.name).concat(currentLevel > 0 ? [] : [currentName]),
+            sectionLevel: currentLevel,
+            documentType: 'docx'
+          }));
+        }
+      }
+
+      // Start new section
+      const level = parseInt(headingMatch[1]);
+      const title = stripHtml(headingMatch[2]);
+
+      // Update section stack
+      while (sectionStack.length > 0 && sectionStack[sectionStack.length - 1].level >= level) {
+        sectionStack.pop();
+      }
+      sectionStack.push({ level, name: title });
+
+      currentName = title;
+      currentLevel = level;
+      currentContent = '';
+    } else {
+      currentContent += part;
+    }
+  }
+
+  // Save last section
+  if (currentContent.trim()) {
+    const cleanContent = stripHtml(currentContent);
+    if (cleanContent) {
+      chunks.push(new DocChunk({
+        filePath: absPath,
+        elementType: currentLevel > 0 ? 'section' : 'paragraph',
+        name: currentName,
+        content: cleanContent,
+        sectionPath: sectionStack.map(s => s.name),
+        sectionLevel: currentLevel,
+        documentType: 'docx'
+      }));
+    }
+  }
+
+  // If no headings found, treat as single chunk
+  if (chunks.length === 0 && html.trim()) {
+    const cleanContent = stripHtml(html);
+    if (cleanContent) {
+      chunks.push(new DocChunk({
+        filePath: absPath,
+        elementType: 'paragraph',
+        name: path.basename(filePath),
+        content: cleanContent,
+        sectionPath: [path.basename(filePath)],
+        sectionLevel: 0,
+        documentType: 'docx'
+      }));
+    }
+  }
+
+  return chunks;
+}
+
+/**
+ * Markdown Parser - native, splits by headings.
+ * No external dependency needed.
+ */
+function parseMD(filePath) {
+  const absPath = path.resolve(filePath);
+  const content = fs.readFileSync(filePath, 'utf8');
+  const lines = content.split('\n');
+  const chunks = [];
+  const sectionStack = [];
+
+  let currentContent = '';
+  let currentName = path.basename(filePath);
+  let currentLevel = 0;
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,6})\s+(.+)/);
+
+    if (headingMatch) {
+      // Save previous section
+      if (currentContent.trim()) {
+        chunks.push(new DocChunk({
+          filePath: absPath,
+          elementType: currentLevel > 0 ? 'section' : 'paragraph',
+          name: currentName,
+          content: currentContent.trim(),
+          sectionPath: sectionStack.map(s => s.name),
+          sectionLevel: currentLevel,
+          documentType: 'md'
+        }));
+      }
+
+      const level = headingMatch[1].length;
+      const title = headingMatch[2].trim();
+
+      while (sectionStack.length > 0 && sectionStack[sectionStack.length - 1].level >= level) {
+        sectionStack.pop();
+      }
+      sectionStack.push({ level, name: title });
+
+      currentName = title;
+      currentLevel = level;
+      currentContent = '';
+    } else {
+      currentContent += line + '\n';
+    }
+  }
+
+  // Save last section
+  if (currentContent.trim()) {
+    chunks.push(new DocChunk({
+      filePath: absPath,
+      elementType: currentLevel > 0 ? 'section' : 'paragraph',
+      name: currentName,
+      content: currentContent.trim(),
+      sectionPath: sectionStack.map(s => s.name),
+      sectionLevel: currentLevel,
+      documentType: 'md'
+    }));
+  }
+
+  if (chunks.length === 0 && content.trim()) {
+    chunks.push(new DocChunk({
+      filePath: absPath,
+      elementType: 'paragraph',
+      name: path.basename(filePath),
+      content: content.trim(),
+      sectionPath: [path.basename(filePath)],
+      sectionLevel: 0,
+      documentType: 'md'
+    }));
+  }
+
+  return chunks;
+}
+
+/**
+ * Plain text parser - single chunk per file
+ */
+function parseTXT(filePath) {
+  const absPath = path.resolve(filePath);
+  const content = fs.readFileSync(filePath, 'utf8');
+
+  if (!content.trim()) return [];
+
+  return [new DocChunk({
+    filePath: absPath,
+    elementType: 'paragraph',
+    name: path.basename(filePath),
+    content: content.trim(),
+    sectionPath: [path.basename(filePath)],
+    sectionLevel: 0,
+    documentType: 'txt'
+  })];
+}
+
+/**
+ * Get the appropriate parser for a file extension
+ */
+function getParser(filePath) {
+  const ext = path.extname(filePath).toLowerCase().slice(1);
+  switch (ext) {
+    case 'pdf': return parsePDF;
+    case 'docx': return parseDOCX;
+    case 'md': return parseMD;
+    case 'txt': return parseTXT;
+    default: return null;
+  }
+}
+
+/** Strip HTML tags and decode entities */
+function stripHtml(html) {
+  return html
+    .replace(/<[^>]+>/g, '')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+module.exports = { parsePDF, parseDOCX, parseMD, parseTXT, getParser };