@yesvara/svara 0.1.0 → 0.1.1

package/dist/{chunk-FEA5KIJN.mjs → chunk-WA3BIA3S.mjs}

@@ -180,6 +180,7 @@ var Chunker = class {
       id: this.chunkId(document.id, index),
       documentId: document.id,
       content: content.trim(),
+      source: document.source,
       index,
       metadata: {
         ...document.metadata,
@@ -255,7 +256,258 @@ var Chunker = class {
   }
 };
 
+// src/database/sqlite.ts
+import path2 from "path";
+import fs2 from "fs";
+
+// src/database/schema.ts
+var SCHEMA_VERSION = 1;
+var CREATE_TABLES_SQL = `
+-- Schema version tracking
+CREATE TABLE IF NOT EXISTS svara_meta (
+  key   TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+
+-- Conversation history persistence
+CREATE TABLE IF NOT EXISTS svara_messages (
+  id          TEXT PRIMARY KEY,
+  session_id  TEXT NOT NULL,
+  role        TEXT NOT NULL CHECK(role IN ('user', 'assistant', 'system', 'tool')),
+  content     TEXT NOT NULL,
+  tool_call_id TEXT,
+  created_at  INTEGER NOT NULL DEFAULT (unixepoch())
+);
+
+CREATE INDEX IF NOT EXISTS idx_messages_session
+  ON svara_messages (session_id, created_at);
+
+-- User registry
+CREATE TABLE IF NOT EXISTS svara_users (
+  id          TEXT PRIMARY KEY,
+  email       TEXT,
+  display_name TEXT,
+  first_seen  INTEGER NOT NULL DEFAULT (unixepoch()),
+  last_seen   INTEGER NOT NULL DEFAULT (unixepoch()),
+  metadata    TEXT DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_users_email
+  ON svara_users (email);
+
+-- Session metadata
+CREATE TABLE IF NOT EXISTS svara_sessions (
+  id          TEXT PRIMARY KEY,
+  user_id     TEXT NOT NULL,
+  channel     TEXT NOT NULL,
+  created_at  INTEGER NOT NULL DEFAULT (unixepoch()),
+  updated_at  INTEGER NOT NULL DEFAULT (unixepoch()),
+  metadata    TEXT DEFAULT '{}',
+  FOREIGN KEY (user_id) REFERENCES svara_users(id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_sessions_user
+  ON svara_sessions (user_id);
+
+-- Vector store chunks for RAG (per agent)
+CREATE TABLE IF NOT EXISTS svara_chunks (
+  id           TEXT PRIMARY KEY,
+  agent_name   TEXT NOT NULL,   -- Separate RAG per agent
+  document_id  TEXT NOT NULL,
+  content      TEXT NOT NULL,
+  content_hash TEXT NOT NULL,   -- MD5 hash of content for deduplication
+  chunk_index  INTEGER NOT NULL,
+  embedding    TEXT,            -- stored as JSON string of float array
+  source       TEXT NOT NULL,
+  metadata     TEXT DEFAULT '{}',
+  created_at   INTEGER NOT NULL DEFAULT (unixepoch())
+);
+
+CREATE INDEX IF NOT EXISTS idx_chunks_agent
+  ON svara_chunks (agent_name);
+
+CREATE INDEX IF NOT EXISTS idx_chunks_agent_document
+  ON svara_chunks (agent_name, document_id);
+
+CREATE INDEX IF NOT EXISTS idx_chunks_content_hash
+  ON svara_chunks (content_hash);
+
+-- Document registry
+CREATE TABLE IF NOT EXISTS svara_documents (
+  id          TEXT PRIMARY KEY,
+  source      TEXT NOT NULL UNIQUE,
+  type        TEXT NOT NULL,
+  size        INTEGER,
+  hash        TEXT,
+  indexed_at  INTEGER NOT NULL DEFAULT (unixepoch()),
+  metadata    TEXT DEFAULT '{}'
+);
+
+-- Key-value store for arbitrary agent state
+CREATE TABLE IF NOT EXISTS svara_kv (
+  key         TEXT PRIMARY KEY,
+  value       TEXT NOT NULL,
+  expires_at  INTEGER,         -- unix timestamp, NULL = no expiry
+  updated_at  INTEGER NOT NULL DEFAULT (unixepoch())
+);
+`;
+var INSERT_META_SQL = `
+INSERT OR REPLACE INTO svara_meta (key, value)
+VALUES ('schema_version', ?), ('created_at', ?);
+`;
+
+// src/database/sqlite.ts
+var KVStore = class {
+  constructor(db) {
+    this.db = db;
+  }
+  db;
+  /** Set a key-value pair, with optional TTL in seconds. */
+  set(key, value, ttlSeconds) {
+    const expiresAt = ttlSeconds ? Math.floor(Date.now() / 1e3) + ttlSeconds : null;
+    this.db.prepare(`
+      INSERT OR REPLACE INTO svara_kv (key, value, expires_at, updated_at)
+      VALUES (?, ?, ?, unixepoch())
+    `).run(key, JSON.stringify(value), expiresAt);
+  }
+  /** Get a value by key. Returns undefined if not found or expired. */
+  get(key) {
+    const row = this.db.prepare(`
+      SELECT value, expires_at FROM svara_kv
+      WHERE key = ? AND (expires_at IS NULL OR expires_at > unixepoch())
+    `).get(key);
+    if (!row) return void 0;
+    return JSON.parse(row.value);
+  }
+  /** Delete a key. */
+  delete(key) {
+    this.db.prepare("DELETE FROM svara_kv WHERE key = ?").run(key);
+  }
+  /** Check if a key exists and is not expired. */
+  has(key) {
+    return this.get(key) !== void 0;
+  }
+  /** Get all keys matching a prefix. */
+  keys(prefix = "") {
+    const rows = this.db.prepare(`
+      SELECT key FROM svara_kv
+      WHERE key LIKE ? AND (expires_at IS NULL OR expires_at > unixepoch())
+    `).all(`${prefix}%`);
+    return rows.map((r) => r.key);
+  }
+};
+var SvaraDB = class {
+  db;
+  kv;
+  constructor(dbPath = ":memory:") {
+    if (dbPath !== ":memory:") {
+      fs2.mkdirSync(path2.dirname(path2.resolve(dbPath)), { recursive: true });
+    }
+    this.db = this.openDatabase(dbPath);
+    this.configure();
+    this.migrate();
+    this.kv = new KVStore(this.db);
+  }
+  // ─── Query Helpers ────────────────────────────────────────────────────────
+  /**
+   * Run a SELECT and return all matching rows.
+   */
+  query(sql, params = []) {
+    return this.db.prepare(sql).all(...params);
+  }
+  /**
+   * Run a SELECT and return the first matching row.
+   */
+  queryOne(sql, params = []) {
+    return this.db.prepare(sql).get(...params);
+  }
+  /**
+   * Run an INSERT/UPDATE/DELETE. Returns affected row count.
+   */
+  run(sql, params = []) {
+    return this.db.prepare(sql).run(...params).changes;
+  }
+  /**
+   * Execute raw SQL (for DDL, migrations, etc.).
+   */
+  exec(sql) {
+    this.db.exec(sql);
+  }
+  /**
+   * Run multiple operations in a single transaction.
+   *
+   * @example
+   * db.transaction(() => {
+   *   db.run('INSERT INTO orders ...', [...]);
+   *   db.run('UPDATE inventory ...', [...]);
+   * });
+   */
+  transaction(fn) {
+    return this.db.transaction(fn)();
+  }
+  /**
+   * Close the database connection.
+   */
+  close() {
+    this.db.close();
+  }
+  // ─── Internal Message Storage ─────────────────────────────────────────────
+  saveMessage(params) {
+    this.db.prepare(`
+      INSERT OR REPLACE INTO svara_messages (id, session_id, role, content, tool_call_id)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(
+      params.id,
+      params.sessionId,
+      params.role,
+      params.content,
+      params.toolCallId ?? null
+    );
+  }
+  getMessages(sessionId, limit = 50) {
+    return this.db.prepare(`
+      SELECT id, role, content, tool_call_id, created_at
+      FROM svara_messages
+      WHERE session_id = ?
+      ORDER BY created_at ASC
+      LIMIT ?
+    `).all(sessionId, limit);
+  }
+  clearSession(sessionId) {
+    this.db.prepare("DELETE FROM svara_messages WHERE session_id = ?").run(sessionId);
+  }
+  // ─── Private Setup ────────────────────────────────────────────────────────
+  openDatabase(dbPath) {
+    try {
+      const Database = __require("better-sqlite3");
+      return new Database(dbPath);
+    } catch {
+      throw new Error(
+        '[SvaraJS] Database requires the "better-sqlite3" package.\nRun: npm install better-sqlite3'
+      );
+    }
+  }
+  configure() {
+    this.db.pragma("journal_mode = WAL");
+    this.db.pragma("synchronous = NORMAL");
+    this.db.pragma("foreign_keys = ON");
+  }
+  migrate() {
+    this.db.exec(CREATE_TABLES_SQL);
+    const meta = this.db.prepare(
+      "SELECT value FROM svara_meta WHERE key = 'schema_version'"
+    ).get();
+    if (!meta) {
+      this.db.prepare(INSERT_META_SQL).run(
+        String(SCHEMA_VERSION),
+        (/* @__PURE__ */ new Date()).toISOString()
+      );
+    }
+  }
+};
+
 // src/rag/retriever.ts
+import crypto3 from "crypto";
 var OpenAIEmbeddings = class {
   client;
   model;
@@ -310,25 +562,95 @@ var OllamaEmbeddings = class {
     return data.embedding;
   }
 };
-var InMemoryVectorStore = class {
-  entries = [];
-  add(chunk, embedding) {
-    const existing = this.entries.findIndex((e) => e.chunk.id === chunk.id);
-    if (existing >= 0) {
-      this.entries[existing] = { chunk, embedding };
-    } else {
-      this.entries.push({ chunk, embedding });
-    }
-  }
-  search(queryEmbedding, topK, threshold = 0) {
-    const scored = this.entries.map((entry) => ({
-      chunk: entry.chunk,
-      score: cosineSimilarity(queryEmbedding, entry.embedding)
-    }));
-    return scored.filter((s) => s.score >= threshold).sort((a, b) => b.score - a.score).slice(0, topK).map((s) => s.chunk);
+var VectorStore = class {
+  contentHash(content) {
+    return crypto3.createHash("md5").update(content).digest("hex");
   }
-  get size() {
-    return this.entries.length;
+};
+var PersistentVectorStore = class extends VectorStore {
+  constructor(db, agentName) {
+    super();
+    this.db = db;
+    this.agentName = agentName;
+  }
+  db;
+  agentName;
+  async add(chunk, embedding) {
+    const contentHash = this.contentHash(chunk.content);
+    const existing = this.db.query(
+      "SELECT id FROM svara_chunks WHERE agent_name = ? AND content_hash = ?",
+      [this.agentName, contentHash]
+    );
+    if (existing.length > 0) {
+      console.log(`[SvaraJS:RAG] Duplicate content detected for ${this.agentName}, skipping chunk ${chunk.id}`);
+      return;
+    }
+    const embeddingJson = JSON.stringify(embedding);
+    this.db.run(
+      `INSERT OR REPLACE INTO svara_chunks
+       (id, agent_name, document_id, content, content_hash, chunk_index, embedding, source, metadata)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        chunk.id,
+        this.agentName,
+        chunk.documentId,
+        chunk.content,
+        contentHash,
+        chunk.index,
+        embeddingJson,
+        chunk.source,
+        JSON.stringify(chunk.metadata)
+      ]
+    );
+  }
+  async search(queryEmbedding, topK, threshold = 0) {
+    const rows = this.db.query(
+      "SELECT id, document_id, content, chunk_index, embedding, source, metadata FROM svara_chunks WHERE agent_name = ? ORDER BY id DESC",
+      [this.agentName]
+    );
+    const scored = rows.map((row) => {
+      const embedding = JSON.parse(row.embedding);
+      return {
+        chunk: {
+          id: row.id,
+          documentId: row.document_id,
+          content: row.content,
+          index: row.chunk_index,
+          source: row.source,
+          metadata: JSON.parse(row.metadata)
+        },
+        score: cosineSimilarity(queryEmbedding, embedding)
+      };
+    }).filter((s) => s.score >= threshold).sort((a, b) => b.score - a.score).slice(0, topK);
+    return scored.map((s) => s.chunk);
+  }
+  async searchWithScores(queryEmbedding, topK, threshold = 0) {
+    const rows = this.db.query(
+      "SELECT id, document_id, content, chunk_index, embedding, source, metadata FROM svara_chunks WHERE agent_name = ? ORDER BY id DESC",
+      [this.agentName]
+    );
+    const scored = rows.map((row) => {
+      const embedding = JSON.parse(row.embedding);
+      return {
+        chunk: {
+          id: row.id,
+          documentId: row.document_id,
+          content: row.content,
+          index: row.chunk_index,
+          source: row.source,
+          metadata: JSON.parse(row.metadata)
+        },
+        score: cosineSimilarity(queryEmbedding, embedding)
+      };
+    }).filter((s) => s.score >= threshold).sort((a, b) => b.score - a.score).slice(0, topK);
+    return scored;
+  }
+  async size() {
+    const result = this.db.query(
+      "SELECT COUNT(*) as count FROM svara_chunks WHERE agent_name = ?",
+      [this.agentName]
+    );
+    return result[0]?.count ?? 0;
   }
 };
 var VectorRetriever = class {
@@ -337,13 +659,17 @@ var VectorRetriever = class {
   loader;
   chunker;
   config;
-  constructor() {
-    this.store = new InMemoryVectorStore();
+  db;
+  agentName;
+  constructor(agentName, db) {
+    this.agentName = agentName;
     this.loader = new DocumentLoader();
     this.chunker = new Chunker();
+    this.db = db || new SvaraDB("./data/svara.db");
   }
   async init(config) {
     this.config = config;
+    this.store = new PersistentVectorStore(this.db, this.agentName);
     if (config.chunking) {
       this.chunker = new Chunker({
         strategy: config.chunking.strategy ?? "sentence",
@@ -372,15 +698,17 @@ var VectorRetriever = class {
     console.log(`[SvaraJS:RAG] Embedding ${chunks.length} chunk(s)...`);
     const embeddings = await this.embedder.embed(chunks.map((c) => c.content));
     for (let i = 0; i < chunks.length; i++) {
-      this.store.add(chunks[i], embeddings[i]);
+      await this.store.add(chunks[i], embeddings[i]);
     }
-    console.log(`[SvaraJS:RAG] Vector store now has ${this.store.size} chunk(s).`);
+    const size = await this.store.size();
+    console.log(`[SvaraJS:RAG] Vector store now has ${size} chunk(s).`);
   }
   async retrieve(query, topK = 5) {
-    if (this.store.size === 0) return "";
+    const size = await this.store.size();
+    if (size === 0) return "";
     const queryEmbedding = await this.embedder.embedOne(query);
     const threshold = this.config.retrieval?.threshold ?? 0.3;
-    const chunks = this.store.search(queryEmbedding, topK, threshold);
+    const chunks = await this.store.search(queryEmbedding, topK, threshold);
     if (!chunks.length) return "";
     return chunks.map((chunk, i) => `[Context ${i + 1}]
 Source: ${String(chunk.metadata.filename ?? chunk.documentId)}
@@ -389,11 +717,11 @@ ${chunk.content}`).join("\n\n---\n\n");
   async retrieveChunks(query, topK = 5) {
     const queryEmbedding = await this.embedder.embedOne(query);
     const threshold = this.config.retrieval?.threshold ?? 0.3;
-    const chunks = this.store.search(queryEmbedding, topK, threshold);
+    const chunksWithScores = await this.store.searchWithScores(queryEmbedding, topK, threshold);
     return {
-      chunks,
+      chunks: chunksWithScores,
       query,
-      totalFound: chunks.length
+      totalFound: chunksWithScores.length
     };
   }
 };
@@ -412,6 +740,7 @@ function cosineSimilarity(a, b) {
 }
 
 export {
+  SvaraDB,
   DocumentLoader,
   Chunker,
   VectorRetriever

package/dist/index.d.mts

@@ -141,6 +141,11 @@ interface AgentRunOptions {
     userId?: string;
     metadata?: Record<string, unknown>;
 }
+interface RetrievedDocument {
+    source: string;
+    score: number;
+    excerpt: string;
+}
 interface AgentRunResult {
     response: string;
     sessionId: string;
@@ -148,6 +153,7 @@ interface AgentRunResult {
     iterations: number;
     usage: TokenUsage;
     duration: number;
+    retrievedDocuments?: RetrievedDocument[];
 }
 type DocumentType = 'text' | 'markdown' | 'pdf' | 'html' | 'json' | 'docx';
 interface Document {
@@ -166,6 +172,7 @@ interface DocumentChunk {
     id: string;
     documentId: string;
     content: string;
+    source: string;
     index: number;
     metadata: {
         filename: string;
@@ -192,8 +199,12 @@ interface RAGConfig {
         threshold?: number;
     };
 }
+interface ChunkWithScore {
+    chunk: DocumentChunk;
+    score: number;
+}
 interface RetrievedContext {
-    chunks: DocumentChunk[];
+    chunks: ChunkWithScore[];
     query: string;
     totalFound: number;
 }
@@ -432,8 +443,10 @@ declare class SvaraAgent extends EventEmitter {
     private readonly verbose;
     private channels;
     private knowledgeBase;
+    private retriever;
     private knowledgePaths;
     private isStarted;
+    private db;
     constructor(config: AgentConfig);
     /**
      * Send a message and get a reply. The simplest way to use an agent.
@@ -523,6 +536,7 @@ declare class SvaraAgent extends EventEmitter {
      * agent.addKnowledge('./new-policies.pdf');
      */
     addKnowledge(paths: string | string[]): Promise<void>;
+    private trackUserAndSession;
     /**
      * Receives a raw incoming message from a channel and processes it.
      * Called by channel handlers — not typically used directly.
@@ -910,7 +924,9 @@ declare class VectorRetriever implements RAGRetriever {
     private loader;
     private chunker;
     private config;
-    constructor();
+    private db;
+    private agentName;
+    constructor(agentName: string, db?: SvaraDB);
     init(config: RAGConfig): Promise<void>;
     addDocuments(filePaths: string[]): Promise<void>;
     retrieve(query: string, topK?: number): Promise<string>;

package/dist/index.d.ts

@@ -141,6 +141,11 @@ interface AgentRunOptions {
     userId?: string;
     metadata?: Record<string, unknown>;
 }
+interface RetrievedDocument {
+    source: string;
+    score: number;
+    excerpt: string;
+}
 interface AgentRunResult {
     response: string;
     sessionId: string;
@@ -148,6 +153,7 @@ interface AgentRunResult {
     iterations: number;
     usage: TokenUsage;
     duration: number;
+    retrievedDocuments?: RetrievedDocument[];
 }
 type DocumentType = 'text' | 'markdown' | 'pdf' | 'html' | 'json' | 'docx';
 interface Document {
@@ -166,6 +172,7 @@ interface DocumentChunk {
     id: string;
     documentId: string;
     content: string;
+    source: string;
     index: number;
     metadata: {
         filename: string;
@@ -192,8 +199,12 @@ interface RAGConfig {
         threshold?: number;
     };
 }
+interface ChunkWithScore {
+    chunk: DocumentChunk;
+    score: number;
+}
 interface RetrievedContext {
-    chunks: DocumentChunk[];
+    chunks: ChunkWithScore[];
     query: string;
     totalFound: number;
 }
@@ -432,8 +443,10 @@ declare class SvaraAgent extends EventEmitter {
     private readonly verbose;
     private channels;
     private knowledgeBase;
+    private retriever;
     private knowledgePaths;
     private isStarted;
+    private db;
     constructor(config: AgentConfig);
     /**
      * Send a message and get a reply. The simplest way to use an agent.
@@ -523,6 +536,7 @@ declare class SvaraAgent extends EventEmitter {
      * agent.addKnowledge('./new-policies.pdf');
      */
     addKnowledge(paths: string | string[]): Promise<void>;
+    private trackUserAndSession;
     /**
      * Receives a raw incoming message from a channel and processes it.
      * Called by channel handlers — not typically used directly.
@@ -910,7 +924,9 @@ declare class VectorRetriever implements RAGRetriever {
     private loader;
     private chunker;
     private config;
-    constructor();
+    private db;
+    private agentName;
+    constructor(agentName: string, db?: SvaraDB);
     init(config: RAGConfig): Promise<void>;
     addDocuments(filePaths: string[]): Promise<void>;
     retrieve(query: string, topK?: number): Promise<string>;