@yesvara/svara 0.1.0

package/dist/chunk-FEA5KIJN.mjs

@@ -0,0 +1,418 @@
+import {
+  __require
+} from "./chunk-CIESM3BP.mjs";
+
+// src/rag/loader.ts
+import fs from "fs/promises";
+import path from "path";
+import crypto from "crypto";
+var TextFileLoader = class {
+  extensions = [".txt", ".md", ".mdx", ".rst", ".csv", ".log"];
+  async load(filePath) {
+    return fs.readFile(filePath, "utf-8");
+  }
+};
+var JsonFileLoader = class {
+  extensions = [".json", ".jsonl"];
+  async load(filePath) {
+    const raw = await fs.readFile(filePath, "utf-8");
+    if (path.extname(filePath) === ".jsonl") {
+      return raw.split("\n").filter(Boolean).map((line) => {
+        const obj = JSON.parse(line);
+        return Object.values(obj).join(" ");
+      }).join("\n");
+    }
+    const data = JSON.parse(raw);
+    return JSON.stringify(data, null, 2);
+  }
+};
+var HtmlFileLoader = class {
+  extensions = [".html", ".htm"];
+  async load(filePath) {
+    const raw = await fs.readFile(filePath, "utf-8");
+    return raw.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  }
+};
+var PdfFileLoader = class {
+  extensions = [".pdf"];
+  async load(filePath) {
+    try {
+      const pdfParse = __require("pdf-parse");
+      const buffer = await fs.readFile(filePath);
+      const data = await pdfParse(buffer);
+      return data.text;
+    } catch {
+      throw new Error(
+        '[SvaraJS] PDF loading requires the "pdf-parse" package.\nRun: npm install pdf-parse'
+      );
+    }
+  }
+};
+var DocxFileLoader = class {
+  extensions = [".docx"];
+  async load(filePath) {
+    try {
+      const mammoth = __require("mammoth");
+      const result = await mammoth.extractRawText({ path: filePath });
+      return result.value;
+    } catch {
+      throw new Error(
+        '[SvaraJS] DOCX loading requires the "mammoth" package.\nRun: npm install mammoth'
+      );
+    }
+  }
+};
+var DocumentLoader = class {
+  loaders;
+  extensionMap;
+  constructor() {
+    this.loaders = [
+      new TextFileLoader(),
+      new JsonFileLoader(),
+      new HtmlFileLoader(),
+      new PdfFileLoader(),
+      new DocxFileLoader()
+    ];
+    this.extensionMap = /* @__PURE__ */ new Map();
+    for (const loader of this.loaders) {
+      for (const ext of loader.extensions) {
+        this.extensionMap.set(ext, loader);
+      }
+    }
+  }
+  /**
+   * Load a single file into a Document.
+   */
+  async load(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    const loader = this.extensionMap.get(ext);
+    if (!loader) {
+      throw new Error(
+        `[SvaraJS] Unsupported file type: "${ext}". Supported: ${[...this.extensionMap.keys()].join(", ")}`
+      );
+    }
+    const content = await loader.load(filePath);
+    const stats = await fs.stat(filePath);
+    return {
+      id: this.hashFile(filePath),
+      content,
+      type: this.detectType(ext),
+      source: filePath,
+      metadata: {
+        filename: path.basename(filePath),
+        extension: ext,
+        size: stats.size,
+        lastModified: stats.mtime.toISOString()
+      }
+    };
+  }
+  /**
+   * Load multiple files. Silently skips unreadable files with a warning.
+   */
+  async loadMany(filePaths) {
+    const results = [];
+    for (const filePath of filePaths) {
+      try {
+        const doc = await this.load(filePath);
+        results.push(doc);
+      } catch (err) {
+        console.warn(`[SvaraJS:RAG] Skipping "${filePath}": ${err.message}`);
+      }
+    }
+    return results;
+  }
+  /** Check if this loader supports a given file extension. */
+  supports(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return this.extensionMap.has(ext);
+  }
+  detectType(ext) {
+    const map = {
+      ".txt": "text",
+      ".md": "markdown",
+      ".mdx": "markdown",
+      ".pdf": "pdf",
+      ".html": "html",
+      ".htm": "html",
+      ".json": "json",
+      ".jsonl": "json",
+      ".docx": "docx"
+    };
+    return map[ext] ?? "text";
+  }
+  hashFile(filePath) {
+    return crypto.createHash("md5").update(filePath).digest("hex");
+  }
+};
+
+// src/rag/chunker.ts
+import crypto2 from "crypto";
+var Chunker = class {
+  options;
+  constructor(options = {}) {
+    this.options = {
+      strategy: options.strategy ?? "sentence",
+      size: options.size ?? 2e3,
+      overlap: options.overlap ?? 200
+    };
+  }
+  /**
+   * Split a document into overlapping chunks.
+   * Returns the document with populated `chunks` field.
+   */
+  chunk(document) {
+    const text = document.content.trim();
+    if (!text) return [];
+    let texts;
+    switch (this.options.strategy) {
+      case "fixed":
+        texts = this.fixedChunk(text);
+        break;
+      case "paragraph":
+        texts = this.paragraphChunk(text);
+        break;
+      case "sentence":
+      default:
+        texts = this.sentenceChunk(text);
+        break;
+    }
+    return texts.filter((t) => t.trim().length > 0).map((content, index) => ({
+      id: this.chunkId(document.id, index),
+      documentId: document.id,
+      content: content.trim(),
+      index,
+      metadata: {
+        ...document.metadata,
+        chunkIndex: index,
+        strategy: this.options.strategy,
+        charCount: content.length
+      }
+    }));
+  }
+  /**
+   * Chunk multiple documents at once.
+   */
+  chunkMany(documents) {
+    return documents.flatMap((doc) => this.chunk(doc));
+  }
+  // ─── Strategies ───────────────────────────────────────────────────────────
+  /** Split into fixed-size windows with overlap. Good for code and structured data. */
+  fixedChunk(text) {
+    const { size, overlap } = this.options;
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+      const end = Math.min(start + size, text.length);
+      chunks.push(text.slice(start, end));
+      start += size - overlap;
+    }
+    return chunks;
+  }
+  /**
+   * Split by sentences, grouping them until size limit.
+   * Best for prose text — preserves natural reading units.
+   */
+  sentenceChunk(text) {
+    const sentences = this.splitSentences(text);
+    return this.groupBySize(sentences);
+  }
+  /**
+   * Split by paragraphs (double newline), grouping small ones.
+   * Best for documentation, articles, and manuals.
+   */
+  paragraphChunk(text) {
+    const paragraphs = text.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean);
+    return this.groupBySize(paragraphs);
+  }
+  // ─── Helpers ──────────────────────────────────────────────────────────────
+  splitSentences(text) {
+    return text.split(/(?<=[.!?])\s+(?=[A-Z"'(])/).map((s) => s.trim()).filter(Boolean);
+  }
+  groupBySize(units) {
+    const { size, overlap } = this.options;
+    const chunks = [];
+    let current = "";
+    let overlapBuffer = "";
+    for (const unit of units) {
+      if (current.length + unit.length + 1 > size && current.length > 0) {
+        chunks.push(current);
+        current = overlapBuffer + (overlapBuffer ? " " : "") + unit;
+        overlapBuffer = "";
+      } else {
+        current += (current ? " " : "") + unit;
+      }
+      if (current.length > overlap) {
+        overlapBuffer = current.slice(-overlap);
+      } else {
+        overlapBuffer = current;
+      }
+    }
+    if (current.trim()) chunks.push(current);
+    return chunks;
+  }
+  chunkId(documentId, index) {
+    return crypto2.createHash("md5").update(`${documentId}:${index}`).digest("hex");
+  }
+};
+
+// src/rag/retriever.ts
+var OpenAIEmbeddings = class {
+  client;
+  model;
+  constructor(apiKey, model = "text-embedding-3-small") {
+    this.model = model;
+    try {
+      const { default: OpenAI } = __require("openai");
+      this.client = new OpenAI({ apiKey: apiKey ?? process.env.OPENAI_API_KEY });
+    } catch {
+      throw new Error('[SvaraJS] OpenAI embeddings require the "openai" package.');
+    }
+  }
+  async embed(texts) {
+    const client = this.client;
+    const BATCH_SIZE = 100;
+    const results = [];
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+      const batch = texts.slice(i, i + BATCH_SIZE);
+      const response = await client.embeddings.create({
+        model: this.model,
+        input: batch
+      });
+      results.push(...response.data.map((d) => d.embedding));
+    }
+    return results;
+  }
+  async embedOne(text) {
+    const [embedding] = await this.embed([text]);
+    return embedding;
+  }
+};
+var OllamaEmbeddings = class {
+  baseURL;
+  model;
+  constructor(model = "nomic-embed-text", baseURL = "http://localhost:11434") {
+    this.model = model;
+    this.baseURL = baseURL;
+  }
+  async embed(texts) {
+    return Promise.all(texts.map((t) => this.embedOne(t)));
+  }
+  async embedOne(text) {
+    const response = await fetch(`${this.baseURL}/api/embeddings`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ model: this.model, prompt: text })
+    });
+    if (!response.ok) {
+      throw new Error(`[SvaraJS] Ollama embeddings failed: ${response.statusText}`);
+    }
+    const data = await response.json();
+    return data.embedding;
+  }
+};
+var InMemoryVectorStore = class {
+  entries = [];
+  add(chunk, embedding) {
+    const existing = this.entries.findIndex((e) => e.chunk.id === chunk.id);
+    if (existing >= 0) {
+      this.entries[existing] = { chunk, embedding };
+    } else {
+      this.entries.push({ chunk, embedding });
+    }
+  }
+  search(queryEmbedding, topK, threshold = 0) {
+    const scored = this.entries.map((entry) => ({
+      chunk: entry.chunk,
+      score: cosineSimilarity(queryEmbedding, entry.embedding)
+    }));
+    return scored.filter((s) => s.score >= threshold).sort((a, b) => b.score - a.score).slice(0, topK).map((s) => s.chunk);
+  }
+  get size() {
+    return this.entries.length;
+  }
+};
+var VectorRetriever = class {
+  embedder;
+  store;
+  loader;
+  chunker;
+  config;
+  constructor() {
+    this.store = new InMemoryVectorStore();
+    this.loader = new DocumentLoader();
+    this.chunker = new Chunker();
+  }
+  async init(config) {
+    this.config = config;
+    if (config.chunking) {
+      this.chunker = new Chunker({
+        strategy: config.chunking.strategy ?? "sentence",
+        size: config.chunking.size ? config.chunking.size * 4 : 2e3,
+        // rough token→char
+        overlap: config.chunking.overlap ? config.chunking.overlap * 4 : 200
+      });
+    }
+    const emb = config.embeddings ?? { provider: "openai" };
+    switch (emb.provider) {
+      case "openai":
+        this.embedder = new OpenAIEmbeddings(emb.apiKey, emb.model);
+        break;
+      case "ollama":
+        this.embedder = new OllamaEmbeddings(emb.model);
+        break;
+      default:
+        throw new Error(`[SvaraJS] Unknown embeddings provider: "${emb.provider}"`);
+    }
+  }
+  async addDocuments(filePaths) {
+    const documents = await this.loader.loadMany(filePaths);
+    if (!documents.length) return;
+    const chunks = this.chunker.chunkMany(documents);
+    if (!chunks.length) return;
+    console.log(`[SvaraJS:RAG] Embedding ${chunks.length} chunk(s)...`);
+    const embeddings = await this.embedder.embed(chunks.map((c) => c.content));
+    for (let i = 0; i < chunks.length; i++) {
+      this.store.add(chunks[i], embeddings[i]);
+    }
+    console.log(`[SvaraJS:RAG] Vector store now has ${this.store.size} chunk(s).`);
+  }
+  async retrieve(query, topK = 5) {
+    if (this.store.size === 0) return "";
+    const queryEmbedding = await this.embedder.embedOne(query);
+    const threshold = this.config.retrieval?.threshold ?? 0.3;
+    const chunks = this.store.search(queryEmbedding, topK, threshold);
+    if (!chunks.length) return "";
+    return chunks.map((chunk, i) => `[Context ${i + 1}]
+Source: ${String(chunk.metadata.filename ?? chunk.documentId)}
+${chunk.content}`).join("\n\n---\n\n");
+  }
+  async retrieveChunks(query, topK = 5) {
+    const queryEmbedding = await this.embedder.embedOne(query);
+    const threshold = this.config.retrieval?.threshold ?? 0.3;
+    const chunks = this.store.search(queryEmbedding, topK, threshold);
+    return {
+      chunks,
+      query,
+      totalFound: chunks.length
+    };
+  }
+};
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+  return denominator === 0 ? 0 : dot / denominator;
+}
+
+export {
+  DocumentLoader,
+  Chunker,
+  VectorRetriever
+};

package/dist/cli/index.d.mts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli/index.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node