@o-lang/semantic-doc-search 1.0.6 → 1.0.8

package/bin/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
-import yargs from "yargs";
-import { hideBin } from "yargs/helpers";
-import resolver from "../src/index.js";
+const yargs = require("yargs");
+const { hideBin } = require("yargs/helpers");
+const resolver = require("../src/index.js");
 
 const argv = yargs(hideBin(process.argv))
   .usage("Usage: $0 <query> [options]")
@@ -48,44 +48,4 @@ const context = {
   } catch (err) {
     console.error("\n❌ Error running search:", err);
   }
-})();
-
-
-//   console.error("❌ Please provide a query string.");
-//   process.exit(1);
-// }
-
-// const stream = argv.stream || false;
-// const provider = argv.provider || "openai";
-
-// // Resolve doc_root if given
-// const doc_root = argv.doc_root
-//   ? path.resolve(__dirname, "..", argv.doc_root)
-//   : undefined;
-
-// // Optional runtime API keys (users pass env variables)
-// const openaiApiKey = process.env.OPENAI_API_KEY;
-// const groqApiKey = process.env.GROQ_API_KEY;
-// const anthropicApiKey = process.env.ANTHROPIC_API_KEY;
-
-// (async () => {
-//   try {
-//     const result = await resolver("search", {
-//       query,
-//       stream,
-//       doc_root,
-//       options: { provider, openaiApiKey, groqApiKey, anthropicApiKey },
-//       onToken: token => {
-//         if (stream) process.stdout.write(token);
-//       },
-//     });
-
-//     if (!stream) {
-//       console.log("\n\n✅ Result:\n");
-//       console.log(result.text);
-//       console.log("\nMeta:", result.meta);
-//     }
-//   } catch (err) {
-//     console.error("❌ Error running search:", err);
-//   }
-// })();
+})();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@o-lang/semantic-doc-search",
-  "version": "1.0.6",
+  "version": "1.0.8",
   "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
   "main": "src/index.js",
   "type": "commonjs",

package/src/embeddings/anthropic.js

@@ -1,12 +1,11 @@
 // src/embeddings/anthropic.js
-
-import Anthropic from "@anthropic-ai/sdk";
+const Anthropic = require("@anthropic-ai/sdk");
 
 /**
  * AnthropicEmbedding
  * Generates vector embeddings using Claude embeddings API.
  */
-export class AnthropicEmbedding {
+class AnthropicEmbedding {
   constructor(apiKey = process.env.ANTHROPIC_API_KEY) {
     if (!apiKey) throw new Error("Missing ANTHROPIC_API_KEY");
     this.client = new Anthropic({ apiKey });
@@ -43,3 +42,5 @@ export class AnthropicEmbedding {
     return res.data.map(item => item.embedding);
   }
 }
+
+module.exports = { AnthropicEmbedding };

package/src/embeddings/groq.js

@@ -1,6 +1,5 @@
 // src/embeddings/groq.js
-
-import Groq from "groq-sdk";
+const Groq = require("groq-sdk");
 
 /**
  * GroqEmbedding
@@ -8,7 +7,7 @@ import Groq from "groq-sdk";
  *
  * Default model: nomic-embed-text
  */
-export class GroqEmbedding {
+class GroqEmbedding {
   constructor(apiKey = process.env.GROQ_API_KEY) {
     if (!apiKey) throw new Error("Missing GROQ_API_KEY");
     this.client = new Groq({ apiKey });
@@ -45,3 +44,5 @@ export class GroqEmbedding {
     return res.data.map(item => item.embedding);
   }
 }
+
+module.exports = { GroqEmbedding };

package/src/embeddings/local.js

@@ -1,4 +1,5 @@
-import crypto from "crypto";
+// src/embeddings/local.js
+const crypto = require("crypto");
 
 /**
  * LocalEmbedding
@@ -6,7 +7,7 @@ import crypto from "crypto";
  * Each string will produce a consistent vector based on a hash.
  * Note: Not semantic, just a placeholder for testing.
  */
-export class LocalEmbedding {
+class LocalEmbedding {
   constructor(dim = 512) {
     this.dim = dim;
   }
@@ -40,7 +41,7 @@ export class LocalEmbedding {
  * Convenience function for index.js
  * Retries local embedding generation (mostly placeholder, but keeps API compatible)
  */
-export async function createEmbeddingWithRetry(text, options = {}, retries = 1) {
+async function createEmbeddingWithRetry(text, options = {}, retries = 1) {
   const embedder = new LocalEmbedding();
   for (let attempt = 1; attempt <= retries; attempt++) {
     try {
@@ -50,3 +51,5 @@ export async function createEmbeddingWithRetry(text, options = {}, retries = 1)
     }
   }
 }
+
+module.exports = { LocalEmbedding, createEmbeddingWithRetry };

package/src/embeddings/openai.js

@@ -1,6 +1,5 @@
 // src/embeddings/openai.js
-
-import OpenAI from "openai";
+const OpenAI = require("openai");
 
 /**
  * OpenAIEmbedding
@@ -8,7 +7,7 @@ import OpenAI from "openai";
  *
  * Default model: text-embedding-3-large  (best for RAG)
  */
-export class OpenAIEmbedding {
+class OpenAIEmbedding {
   constructor(apiKey = process.env.OPENAI_API_KEY) {
     if (!apiKey) throw new Error("Missing OPENAI_API_KEY");
     this.client = new OpenAI({ apiKey });
@@ -45,3 +44,5 @@ export class OpenAIEmbedding {
     return res.data.map(item => item.embedding);
   }
 }
+
+module.exports = { OpenAIEmbedding };

package/src/llm/anthropic.js

@@ -1,10 +1,10 @@
 // src/llm/anthropic.js
-import Anthropic from "@anthropic-ai/sdk";
+const Anthropic = require("@anthropic-ai/sdk");
 
 /**
  * Anthropic LLM Provider
  */
-export default class AnthropicProvider {
+class AnthropicProvider {
   constructor({ apiKey }) {
     if (!apiKey) console.warn("⚠️ WARNING: ANTHROPIC_API_KEY missing");
     this.client = new Anthropic({ apiKey });
@@ -34,3 +34,5 @@ export default class AnthropicProvider {
     return { done: true };
   }
 }
+
+module.exports = AnthropicProvider;

package/src/llm/groq.js

@@ -1,10 +1,11 @@
 // src/llm/groq.js
-// Groq LLM Provider (groq-sdk@0.5.0)
-// Multi-tenant safe, streaming-ready
+const Groq = require("groq-sdk");
 
-import Groq from "groq-sdk";
-
-export default class GroqProvider {
+/**
+ * Groq LLM Provider (groq-sdk@0.5.0)
+ * Multi-tenant safe, streaming-ready
+ */
+class GroqProvider {
   /**
    * @param {Object} options
    * @param {string} options.apiKey - User-provided Groq API key
@@ -50,3 +51,5 @@ export default class GroqProvider {
     return { done: true };
   }
 }
+
+module.exports = GroqProvider;

package/src/llm/openai.js

@@ -1,11 +1,11 @@
 // src/llm/openai.js
-import OpenAI from "openai";
+const OpenAI = require("openai");
 
 /**
  * OpenAI LLM Provider
  * Supports GPT-4, GPT-4o-mini, GPT-4o, and any Chat model
  */
-export default class OpenAIProvider {
+class OpenAIProvider {
   constructor({ apiKey }) {
     if (!apiKey) console.warn("⚠️ WARNING: OPENAI_API_KEY missing");
     this.client = new OpenAI({ apiKey });
@@ -41,3 +41,5 @@ export default class OpenAIProvider {
     return { done: true };
   }
 }
+
+module.exports = OpenAIProvider;

package/src/llm/router.js

@@ -1,8 +1,9 @@
-import OpenAIProvider from "./openai.js";
-import GroqProvider from "./groq.js";
-import AnthropicProvider from "./anthropic.js";
+// src/llm/router.js
+const OpenAIProvider = require("./openai.js");
+const GroqProvider = require("./groq.js");
+const AnthropicProvider = require("./anthropic.js");
 
-export function createLLM({ provider, openaiApiKey, groqApiKey, anthropicApiKey } = {}) {
+function createLLM({ provider, openaiApiKey, groqApiKey, anthropicApiKey } = {}) {
   switch ((provider || "").toLowerCase()) {
     case "openai":
       if (!openaiApiKey) throw new Error("Missing OpenAI API key");
@@ -20,3 +21,5 @@ export function createLLM({ provider, openaiApiKey, groqApiKey, anthropicApiKey
       throw new Error(`Unsupported LLM provider: ${provider}`);
   }
 }
+
+module.exports = { createLLM };

package/src/rerank/cohere.js

@@ -1,12 +1,11 @@
 // src/rerank/cohere.js
-
-import Cohere from "cohere-ai";
+const Cohere = require("cohere-ai");
 
 /**
  * CohereReranker
  * Uses Cohere Rerank API to reorder candidate documents/snippets
  */
-export class CohereReranker {
+class CohereReranker {
   constructor(apiKey = process.env.COHERE_API_KEY) {
     if (!apiKey) throw new Error("Missing COHERE_API_KEY");
     Cohere.init(apiKey);
@@ -37,3 +36,5 @@ export class CohereReranker {
     return ranked;
   }
 }
+
+module.exports = { CohereReranker };

package/src/rerank/groqRerank.js

@@ -1,12 +1,11 @@
 // src/rerank/groqRerank.js
-
-import Groq from "groq-sdk";
+const Groq = require("groq-sdk");
 
 /**
  * GroqReranker
  * Uses Groq LLMs to rerank candidate documents/snippets given a query
  */
-export class GroqReranker {
+class GroqReranker {
   constructor(apiKey = process.env.GROQ_API_KEY) {
     if (!apiKey) throw new Error("Missing GROQ_API_KEY");
     this.client = new Groq({ apiKey });
@@ -48,3 +47,5 @@ Return JSON array: [{"text": "...", "score": 0.95}, ...]
     }
   }
 }
+
+module.exports = { GroqReranker };

package/src/rerank/local.js

@@ -1,12 +1,11 @@
 // src/rerank/local.js
-
-import { cosine } from "../utils/similarity.js";
+const { cosine } = require("../utils/similarity.js");
 
 /**
  * LocalReranker
  * Simple fallback reranker using keyword overlap + cosine similarity
  */
-export class LocalReranker {
+class LocalReranker {
   constructor() {}
 
   /**
@@ -41,3 +40,5 @@ export class LocalReranker {
     return results;
   }
 }
+
+module.exports = { LocalReranker };

package/src/server/streamingHandler.js

@@ -1,8 +1,7 @@
 // src/server/streamingHandler.js
-
-import express from "express";
-import { initSSE, sendSSE } from "../utils/sseStream.js";
-import { llmRouter } from "../llm/router.js";
+const express = require("express");
+const { initSSE, sendSSE } = require("../utils/sseStream.js");
+const { createLLM } = require("../llm/router.js");
 
 const router = express.Router();
 
@@ -18,11 +17,19 @@ router.post("/stream", async (req, res) => {
     // Initialize SSE
     initSSE(res);
 
-    // LLM router handles streaming token-by-token
-    await llmRouter.streamQuery(query, provider, options, (token) => {
-      sendSSE(res, { token });
+    // Create LLM instance
+    const llm = createLLM({
+      provider,
+      openaiApiKey: options?.openaiApiKey,
+      groqApiKey: options?.groqApiKey,
+      anthropicApiKey: options?.anthropicApiKey
     });
 
+    // Stream the query response
+    await llm.stream({ prompt: query, onToken: (token) => {
+      sendSSE(res, { token });
+    }});
+
     // End stream
     sendSSE(res, { done: true }, true);
 
@@ -32,4 +39,4 @@ router.post("/stream", async (req, res) => {
   }
 });
 
-export { router as streamingHandler };
+module.exports = { router: router };

package/src/utils/chunker.js

@@ -1,27 +1,16 @@
 // src/utils/chunker.js
-
-/**
- * chunkText
- * Split text into overlapping chunks
- * 
- * @param {string} text - the full text
- * @param {number} chunkSize - number of characters per chunk
- * @param {number} overlap - number of characters overlap between chunks
- * @returns string[] - array of text chunks
- */
-export function chunkText(text, chunkSize = 1000, overlap = 200) {
-  if (!text || !text.trim()) return [];
-
+function chunkText(text, chunkSize, overlap) {
+  if (!text) return [];
   const chunks = [];
   let start = 0;
-
+  
   while (start < text.length) {
     const end = Math.min(start + chunkSize, text.length);
-    const chunk = text.slice(start, end);
-    chunks.push(chunk);
-
-    start += chunkSize - overlap; // move forward with overlap
+    chunks.push(text.slice(start, end));
+    start += chunkSize - overlap;
   }
-
+  
   return chunks;
 }
+
+module.exports = { chunkText };

package/src/utils/extractText.js

@@ -1,8 +1,8 @@
-import fs from "fs";
-import path from "path";
-import { JSDOM } from "jsdom";
-import pdfParse from "pdf-parse";
-import { readFile } from "fs/promises";
+// src/utils/extractText.js
+const fs = require("fs");
+const path = require("path");
+const { JSDOM } = require("jsdom");
+const pdfParse = require("pdf-parse");
 
 /**
  * extractTextFromFile
@@ -13,7 +13,7 @@ import { readFile } from "fs/promises";
  * - .pdf
  * - .docx (minimal)
  */
-export async function extractTextFromFile(filePath) {
+async function extractTextFromFile(filePath) {
   const ext = path.extname(filePath).toLowerCase();
 
   if (ext === ".txt" || ext === ".md") {
@@ -34,7 +34,7 @@ export async function extractTextFromFile(filePath) {
 
   if (ext === ".docx") {
     // Minimal extraction using ZIP (can be improved)
-    const { default: StreamZip } = await import("node-stream-zip");
+    const StreamZip = require("node-stream-zip").default;
     const zip = new StreamZip.async({ file: filePath });
     const content = await zip.entryData("word/document.xml");
     await zip.close();
@@ -49,7 +49,7 @@ export async function extractTextFromFile(filePath) {
  * Basic keyword extraction for lexical matching
  * Returns array of lowercase words, stripped of punctuation
  */
-export function extractKeywords(text = "") {
+function extractKeywords(text = "") {
   if (!text) return [];
   return text
     .toLowerCase()
@@ -57,3 +57,5 @@ export function extractKeywords(text = "") {
     .split(/\s+/)
     .filter(Boolean);
 }
+
+module.exports = { extractTextFromFile, extractKeywords };

package/src/utils/fileLoader.js

@@ -1,8 +1,7 @@
 // src/utils/fileLoader.js
-
-import fs from "fs";
-import path from "path";
-import { extractTextFromFile } from "./extractText.js";
+const fs = require("fs");
+const path = require("path");
+const { extractTextFromFile } = require("./extractText.js");
 
 /**
  * loadDocuments
@@ -13,7 +12,7 @@ import { extractTextFromFile } from "./extractText.js";
  * @param {string[]} exts - array of supported file extensions
  * @returns {Promise<Array<{ filePath: string, text: string }>>}
  */
-export async function loadDocuments(dirPath, exts = [".txt", ".md", ".pdf", ".html", ".docx"]) {
+async function loadDocuments(dirPath, exts = [".txt", ".md", ".pdf", ".html", ".docx"]) {
   if (!fs.existsSync(dirPath)) return [];
 
   const files = fs.readdirSync(dirPath, { withFileTypes: true });
@@ -37,3 +36,5 @@ export async function loadDocuments(dirPath, exts = [".txt", ".md", ".pdf", ".ht
 
   return docs;
 }
+
+module.exports = { loadDocuments };

package/src/utils/highlight.js

@@ -1,24 +1,15 @@
 // src/utils/highlight.js
-
-/**
- * highlightMatches
- * Wraps all occurrences of keywords in <mark> tags
- *
- * @param {string} text - original text
- * @param {string[]} keywords - array of keywords to highlight
- * @returns string - HTML-safe text with <mark> highlights
- */
-export function highlightMatches(text, keywords = []) {
-  if (!text || !keywords || keywords.length === 0) return text;
-
-  let highlighted = text;
-
-  // escape special regex chars in keywords
-  const escapedKeywords = keywords.map(k => k.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
-
-  const pattern = new RegExp(`\\b(${escapedKeywords.join("|")})\\b`, "gi");
-
-  highlighted = highlighted.replace(pattern, "<mark>$1</mark>");
-
-  return highlighted;
+function highlightMatches(text, keywords) {
+  if (!text || !keywords || !keywords.length) return text;
+  
+  let result = text;
+  keywords.forEach(keyword => {
+    if (!keyword) return;
+    const regex = new RegExp(`(${keyword})`, 'gi');
+    result = result.replace(regex, '**$1**');
+  });
+  
+  return result;
 }
+
+module.exports = { highlightMatches };

package/src/utils/similarity.js

@@ -7,7 +7,7 @@
  * @param {number[]} b
  * @returns {number} similarity score between -1 and 1
  */
-export function cosine(a, b) {
+function cosine(a, b) {
   if (!a || !b || a.length !== b.length) return 0;
 
   let dot = 0, magA = 0, magB = 0;
@@ -30,7 +30,7 @@ export function cosine(a, b) {
  * @param {number[]} b
  * @returns {number} Euclidean distance
  */
-export function euclidean(a, b) {
+function euclidean(a, b) {
   if (!a || !b || a.length !== b.length) return Infinity;
 
   let sum = 0;
@@ -40,3 +40,5 @@ export function euclidean(a, b) {
 
   return Math.sqrt(sum);
 }
+
+module.exports = { cosine, euclidean };

package/src/utils/sseStream.js

@@ -8,7 +8,7 @@
  * @param {string} data
  * @param {boolean} [end=false] - whether this is the final message
  */
-export function sendSSE(res, data, end = false) {
+function sendSSE(res, data, end = false) {
   res.write(`data: ${JSON.stringify(data)}\n\n`);
   if (end) res.write("event: end\ndata: [DONE]\n\n");
 }
@@ -19,7 +19,7 @@ export function sendSSE(res, data, end = false) {
  *
  * @param {import('http').ServerResponse} res
  */
-export function initSSE(res) {
+function initSSE(res) {
   res.writeHead(200, {
     "Content-Type": "text/event-stream",
     "Cache-Control": "no-cache",
@@ -27,3 +27,5 @@ export function initSSE(res) {
   });
   res.write("\n");
 }
+
+module.exports = { sendSSE, initSSE };