@o-lang/semantic-doc-search 1.0.0

package/src/adapters/inMemoryAdapter.js

@@ -0,0 +1,58 @@
+/**
+ * In-Memory Vector Store Adapter
+ * -----------------------------------
+ * Stores embeddings in RAM.
+ * Useful for local development and testing.
+ */
+
+const cosineSimilarity = (a, b) => {
+  let dot = 0,
+    magA = 0,
+    magB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+
+  if (magA === 0 || magB === 0) return 0;
+  return dot / (Math.sqrt(magA) * Math.sqrt(magB));
+};
+
+module.exports = {
+  _store: {},
+
+  async init() {
+    this._store = {}; // reset
+    return true;
+  },
+
+  async upsert(id, vector, metadata) {
+    this._store[id] = {
+      id,
+      vector,
+      metadata,
+    };
+  },
+
+  async search(queryVector, limit = 5) {
+    const scored = [];
+
+    for (const key in this._store) {
+      const entry = this._store[key];
+      const score = cosineSimilarity(queryVector, entry.vector);
+
+      scored.push({
+        id: entry.id,
+        score,
+        text: entry.metadata.text,
+        source: entry.metadata.source,
+      });
+    }
+
+    return scored
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+  },
+};

package/src/adapters/pineconeAdapter.js

@@ -0,0 +1,78 @@
+/**
+ * Pinecone Vector Store Adapter
+ *
+ * Requirements:
+ *   npm install @pinecone-database/pinecone
+ *
+ * Env:
+ *   PINECONE_API_KEY=your_key
+ *   PINECONE_INDEX=olang-docs
+ */
+
+const { Pinecone } = require("@pinecone-database/pinecone");
+
+module.exports = {
+  client: null,
+  index: null,
+
+  async init() {
+    const apiKey = process.env.PINECONE_API_KEY;
+    const indexName = process.env.PINECONE_INDEX || "olang-docs";
+
+    if (!apiKey) throw new Error("Missing PINECONE_API_KEY");
+
+    this.client = new Pinecone({ apiKey });
+
+    try {
+      // Create index if missing
+      const existingIndexes = await this.client.listIndexes();
+      if (!existingIndexes.indexes.some((i) => i.name === indexName)) {
+        console.log(`Creating Pinecone index: ${indexName} ...`);
+
+        await this.client.createIndex({
+          name: indexName,
+          dimension: 1536, // matches OpenAI text-embedding-3-small
+          metric: "cosine",
+        });
+
+        console.log("Index created. It may take a minute to be ready.");
+      }
+    } catch (err) {
+      console.error("Error checking/creating Pinecone index:", err);
+    }
+
+    this.index = this.client.index(indexName);
+    console.log("Pinecone adapter initialized ✓");
+  },
+
+  /**
+   * Upsert a vector + metadata
+   */
+  async upsert(id, vector, metadata) {
+    await this.index.upsert([
+      {
+        id: id.toString(),
+        values: vector,
+        metadata,
+      },
+    ]);
+  },
+
+  /**
+   * Search for nearest vectors
+   */
+  async search(queryVector, limit = 5) {
+    const res = await this.index.query({
+      vector: queryVector,
+      topK: limit,
+      includeMetadata: true,
+    });
+
+    return res.matches.map((m) => ({
+      id: m.id,
+      score: m.score,
+      text: m.metadata?.text,
+      source: m.metadata?.source,
+    }));
+  },
+};

package/src/adapters/redisAdapter.js

@@ -0,0 +1,103 @@
+/**
+ * Redis Vector Store Adapter
+ * Uses RediSearch vector similarity (HNSW or FLAT)
+ *
+ * Requirements:
+ * - Redis Stack installed
+ * - npm install redis
+ *
+ * Env:
+ * - REDIS_URL=redis://localhost:6379
+ */
+
+const { createClient } = require("redis");
+
+const INDEX_NAME = "olang_docs_idx";
+const VECTOR_DIM = 1536; // default for text-embedding-3-small (OpenAI)
+const VECTOR_TYPE = "FLOAT32";
+
+module.exports = {
+  client: null,
+
+  async init() {
+    this.client = createClient({
+      url: process.env.REDIS_URL || "redis://localhost:6379",
+    });
+
+    this.client.on("error", (err) =>
+      console.error("Redis connection error:", err)
+    );
+
+    await this.client.connect();
+
+    // Try creating index (ignore errors if exists)
+    try {
+      await this.client.ft.create(
+        INDEX_NAME,
+        {
+          vector: {
+            type: "VECTOR",
+            ALGORITHM: "HNSW",
+            DIM: VECTOR_DIM,
+            DISTANCE_METRIC: "COSINE",
+            TYPE: VECTOR_TYPE,
+          },
+          text: {
+            type: "TEXT",
+          },
+          source: {
+            type: "TEXT",
+          },
+        },
+        {
+          ON: "HASH",
+          PREFIX: "doc:",
+        }
+      );
+
+      console.log("Redis vector index created.");
+    } catch (err) {
+      if (err.message.includes("Index already exists")) {
+        console.log("Redis vector index already exists ✓");
+      } else {
+        console.error("Redis index creation error:", err);
+      }
+    }
+  },
+
+  async upsert(id, vector, metadata) {
+    const key = `doc:${id}`;
+
+    await this.client.hSet(key, {
+      vector: Buffer.from(new Float32Array(vector).buffer),
+      text: metadata.text,
+      source: metadata.source,
+    });
+  },
+
+  async search(queryVector, limit = 5) {
+    const queryBuffer = Buffer.from(new Float32Array(queryVector).buffer);
+
+    const results = await this.client.ft.search(
+      INDEX_NAME,
+      `*=>[KNN ${limit} @vector $BLOB AS score]`,
+      {
+        PARAMS: {
+          BLOB: queryBuffer,
+        },
+        SORTBY: {
+          BY: "score",
+          DIRECTION: "ASC",
+        },
+        RETURN: ["text", "source", "score"],
+      }
+    );
+
+    return results.documents.map((doc) => ({
+      id: doc.id,
+      score: 1 - doc.value.score, // convert distance -> similarity
+      text: doc.value.text,
+      source: doc.value.source,
+    }));
+  },
+};

package/src/embeddings/anthropic.js

@@ -0,0 +1,45 @@
+// src/embeddings/anthropic.js
+
+import Anthropic from "@anthropic-ai/sdk";
+
+/**
+ * AnthropicEmbedding
+ * Generates vector embeddings using Claude embeddings API.
+ */
+export class AnthropicEmbedding {
+  constructor(apiKey = process.env.ANTHROPIC_API_KEY) {
+    if (!apiKey) throw new Error("Missing ANTHROPIC_API_KEY");
+    this.client = new Anthropic({ apiKey });
+    this.model = process.env.ANTHROPIC_EMBED_MODEL || "claude-embed-v1";
+  }
+
+  /**
+   * Generate embedding for a single text string
+   */
+  async embed(text) {
+    if (!text || !text.trim()) return [];
+
+    const res = await this.client.embeddings.create({
+      model: this.model,
+      input: text,
+    });
+
+    return res.data[0].embedding;
+  }
+
+  /**
+   * Batch embedding for multiple texts
+   */
+  async embedBatch(textArray = []) {
+    if (!Array.isArray(textArray)) {
+      throw new Error("embedBatch expects an array");
+    }
+
+    const res = await this.client.embeddings.create({
+      model: this.model,
+      input: textArray,
+    });
+
+    return res.data.map(item => item.embedding);
+  }
+}

package/src/embeddings/groq.js

@@ -0,0 +1,47 @@
+// src/embeddings/groq.js
+
+import Groq from "groq-sdk";
+
+/**
+ * GroqEmbedding
+ * Generates vector embeddings using Groq LLM models.
+ *
+ * Default model: nomic-embed-text
+ */
+export class GroqEmbedding {
+  constructor(apiKey = process.env.GROQ_API_KEY) {
+    if (!apiKey) throw new Error("Missing GROQ_API_KEY");
+    this.client = new Groq({ apiKey });
+    this.model = process.env.GROQ_EMBED_MODEL || "nomic-embed-text";
+  }
+
+  /**
+   * Generate embedding for a single text string
+   */
+  async embed(text) {
+    if (!text || !text.trim()) return [];
+
+    const res = await this.client.embeddings.create({
+      model: this.model,
+      input: text,
+    });
+
+    return res.data[0].embedding;
+  }
+
+  /**
+   * Batch embedding for multiple texts
+   */
+  async embedBatch(textArray = []) {
+    if (!Array.isArray(textArray)) {
+      throw new Error("embedBatch expects an array");
+    }
+
+    const res = await this.client.embeddings.create({
+      model: this.model,
+      input: textArray,
+    });
+
+    return res.data.map(item => item.embedding);
+  }
+}

package/src/embeddings/local.js

@@ -0,0 +1,52 @@
+import crypto from "crypto";
+
+/**
+ * LocalEmbedding
+ * Generates deterministic "fake" embeddings for offline testing or fallback.
+ * Each string will produce a consistent vector based on a hash.
+ * Note: Not semantic, just a placeholder for testing.
+ */
+export class LocalEmbedding {
+  constructor(dim = 512) {
+    this.dim = dim;
+  }
+
+  /**
+   * Convert text → deterministic pseudo-vector
+   */
+  embed(text) {
+    if (!text || !text.trim()) return new Array(this.dim).fill(0);
+
+    const hash = crypto.createHash("sha256").update(text).digest();
+    const vector = [];
+
+    for (let i = 0; i < this.dim; i++) {
+      vector.push(hash[i % hash.length] / 255); // normalize 0–1
+    }
+
+    return vector;
+  }
+
+  /**
+   * Batch embedding for multiple strings
+   */
+  embedBatch(textArray = []) {
+    if (!Array.isArray(textArray)) throw new Error("embedBatch expects an array");
+    return textArray.map(text => this.embed(text));
+  }
+}
+
+/**
+ * Convenience function for index.js
+ * Retries local embedding generation (mostly placeholder, but keeps API compatible)
+ */
+export async function createEmbeddingWithRetry(text, options = {}, retries = 1) {
+  const embedder = new LocalEmbedding();
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      return embedder.embed(text);
+    } catch (err) {
+      if (attempt === retries) throw err;
+    }
+  }
+}

package/src/embeddings/openai.js

@@ -0,0 +1,47 @@
+// src/embeddings/openai.js
+
+import OpenAI from "openai";
+
+/**
+ * OpenAIEmbedding
+ * Generates vector embeddings using OpenAI models.
+ *
+ * Default model: text-embedding-3-large  (best for RAG)
+ */
+export class OpenAIEmbedding {
+  constructor(apiKey = process.env.OPENAI_API_KEY) {
+    if (!apiKey) throw new Error("Missing OPENAI_API_KEY");
+    this.client = new OpenAI({ apiKey });
+    this.model = process.env.OPENAI_EMBED_MODEL || "text-embedding-3-large";
+  }
+
+  /**
+   * Generate embedding for a single string
+   */
+  async embed(text) {
+    if (!text || !text.trim()) return [];
+
+    const res = await this.client.embeddings.create({
+      model: this.model,
+      input: text,
+    });
+
+    return res.data[0].embedding; // return the vector
+  }
+
+  /**
+   * Batch embedding for lists of texts
+   */
+  async embedBatch(textArray = []) {
+    if (!Array.isArray(textArray)) {
+      throw new Error("embedBatch expects an array");
+    }
+
+    const res = await this.client.embeddings.create({
+      model: this.model,
+      input: textArray,
+    });
+
+    return res.data.map(item => item.embedding);
+  }
+}

package/src/index.js

@@ -0,0 +1,181 @@
+// doc-search.js
+import fs from "fs";
+import path from "path";
+import { createLLM } from "./llm/router.js";
+import { LocalEmbedding } from "./embeddings/local.js";
+import { chunkText } from "./utils/chunker.js";
+import { extractKeywords } from "./utils/extractText.js";
+import { cosine } from "./utils/similarity.js";
+import { highlightMatches } from "./utils/highlight.js";
+
+const CACHE_PATH = path.join(process.cwd(), "embeddings.json");
+
+function safeResolve(base, userPath) {
+  const resolved = path.resolve(base, userPath);
+  if (!resolved.startsWith(path.resolve(base))) {
+    throw new Error("Path traversal detected");
+  }
+  return resolved;
+}
+
+function loadCache() {
+  try {
+    if (fs.existsSync(CACHE_PATH)) {
+      return JSON.parse(fs.readFileSync(CACHE_PATH, "utf8")) || {};
+    }
+  } catch {}
+  return {};
+}
+
+function saveCache(cache) {
+  try {
+    fs.writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2));
+  } catch {}
+}
+
+async function performDocQA(query, context = {}) {
+  const { doc_root, stream = false } = context;
+  const options = context.options || {};
+  const CHUNK_SIZE = options.chunkSize || 1200;
+  const OVERLAP = Math.floor(CHUNK_SIZE * 0.2);
+  const SEMANTIC_WEIGHT = options.semanticWeight ?? 0.75;
+  const MIN_SCORE = options.minScore ?? 0.18;
+  const model = options.model || "default";
+
+  if (!query || typeof query !== "string") {
+    return { text: "Missing required input: query" };
+  }
+
+  const baseDir = doc_root
+    ? safeResolve(process.cwd(), doc_root)
+    : path.join(process.cwd(), "docs");
+
+  if (!fs.existsSync(baseDir)) {
+    return { text: `Document directory not found: ${baseDir}` };
+  }
+
+  const files = fs.readdirSync(baseDir).filter(f => f.endsWith(".txt") || f.endsWith(".md"));
+  if (!files.length) {
+    return { text: "No documents available." };
+  }
+
+  const qLower = query.toLowerCase().trim();
+  const exact = files.find(f => path.basename(f, path.extname(f)).toLowerCase() === qLower);
+  if (exact) {
+    return {
+      text: fs.readFileSync(path.join(baseDir, exact), "utf8"),
+      meta: { file: exact, method: "exact-filename" }
+    };
+  }
+
+  const cache = loadCache();
+  const docs = [];
+  const localEmbedder = new LocalEmbedding();
+
+  for (const file of files) {
+    const raw = fs.readFileSync(path.join(baseDir, file), "utf8");
+    const chunks = chunkText(raw, CHUNK_SIZE, OVERLAP);
+    const chunkObjs = [];
+
+    for (let i = 0; i < chunks.length; i++) {
+      const key = `${file}::chunk::${i}`;
+      let emb = cache[key];
+      if (!emb) {
+        try {
+          emb = localEmbedder.embed(chunks[i]);
+          cache[key] = emb;
+          saveCache(cache);
+        } catch {
+          emb = null;
+        }
+      }
+      chunkObjs.push({ index: i, text: chunks[i], emb });
+    }
+    docs.push({ file, raw, chunks: chunkObjs });
+  }
+
+  let queryEmb = null;
+  try {
+    queryEmb = localEmbedder.embed(query);
+  } catch {}
+
+  const keywords = extractKeywords(query);
+
+  const fileScores = docs.map(doc => {
+    let bestChunk = null;
+    let bestHybrid = -Infinity;
+
+    for (const ch of doc.chunks) {
+      const semScore = queryEmb && ch.emb ? cosine(queryEmb, ch.emb) : 0;
+      const lexScore = keywords.length
+        ? keywords.reduce((acc, k) => acc + (ch.text.toLowerCase().includes(k) ? 1 : 0), 0) / keywords.length
+        : 0;
+      const hybrid = SEMANTIC_WEIGHT * semScore + (1 - SEMANTIC_WEIGHT) * lexScore;
+
+      if (hybrid > bestHybrid) {
+        bestHybrid = hybrid;
+        bestChunk = { ...ch, semScore, lexScore, hybrid };
+      }
+    }
+    return { file: doc.file, score: bestHybrid, bestChunk };
+  });
+
+  fileScores.sort((a, b) => b.score - a.score);
+  const best = fileScores[0];
+
+  if (!best || best.score < MIN_SCORE) {
+    for (const file of files) {
+      const text = fs.readFileSync(path.join(baseDir, file), "utf8").toLowerCase();
+      if (keywords.some(k => text.includes(k))) {
+        const snippetIndex = text.indexOf(keywords.find(k => text.includes(k)));
+        const start = Math.max(0, snippetIndex - 200);
+        const snippet = text.slice(start, Math.min(text.length, snippetIndex + 400));
+        return { text: snippet, meta: { file, method: "lexical-fallback" } };
+      }
+    }
+    return { text: `No document found matching: "${query}"` };
+  }
+
+  const snippet = highlightMatches(best.bestChunk.text, keywords);
+
+  if (options.provider && options.provider !== "local") {
+    const llm = createLLM({
+      provider: options.provider,
+      openaiApiKey: options.openaiApiKey,
+      groqApiKey: options.groqApiKey,
+      anthropicApiKey: options.anthropicApiKey,
+    });
+
+    if (stream && typeof context.onToken === "function") {
+      await llm.stream({ prompt: snippet, model, onToken: context.onToken });
+      return {
+        text: snippet,
+        meta: { file: best.file, chunkIndex: best.bestChunk.index, method: "hybrid-semantic-stream" }
+      };
+    } else {
+      const resp = await llm.generate({ prompt: snippet, model });
+      return {
+        text: resp.text,
+        meta: { file: best.file, chunkIndex: best.bestChunk.index, method: "hybrid-semantic" }
+      };
+    }
+  }
+
+  return {
+    text: snippet,
+    meta: { file: best.file, chunkIndex: best.bestChunk.index, method: "hybrid-semantic" }
+  };
+}
+
+// ✅ O-Lang Resolver Interface
+export default async function docSearchResolver(action, context) {
+  if (action.startsWith('Ask doc-search ')) {
+    const match = action.match(/"(.*)"|'(.*)'/);
+    const query = match ? (match[1] || match[2]) : action.replace(/^Ask doc-search\s+/, '').trim();
+    return await performDocQA(query, context);
+  }
+  return undefined;
+}
+
+// ✅ Resolver name matches package name: @o-lang/doc-search → doc-search
+docSearchResolver.resolverName = 'doc-search';

package/src/llm/anthropic.js

@@ -0,0 +1,36 @@
+// src/llm/anthropic.js
+import Anthropic from "@anthropic-ai/sdk";
+
+/**
+ * Anthropic LLM Provider
+ */
+export default class AnthropicProvider {
+  constructor({ apiKey }) {
+    if (!apiKey) console.warn("⚠️ WARNING: ANTHROPIC_API_KEY missing");
+    this.client = new Anthropic({ apiKey });
+  }
+
+  async generate({ model = "claude-v1", prompt }) {
+    const resp = await this.client.completions.create({
+      model,
+      prompt,
+      max_tokens_to_sample: 400,
+      temperature: 0.4,
+    });
+    return { text: resp.completion, raw: resp };
+  }
+
+  async stream({ model = "claude-v1", prompt, onToken }) {
+    const resp = await this.client.completions.stream({
+      model,
+      prompt,
+      max_tokens_to_sample: 400,
+      temperature: 0.4,
+    });
+
+    for await (const event of resp) {
+      if (event.type === "completion.delta" && event.delta) onToken(event.delta);
+    }
+    return { done: true };
+  }
+}

package/src/llm/groq.js

@@ -0,0 +1,52 @@
+// src/llm/groq.js
+// Groq LLM Provider (groq-sdk@0.5.0)
+// Multi-tenant safe, streaming-ready
+
+import Groq from "groq-sdk";
+
+export default class GroqProvider {
+  /**
+   * @param {Object} options
+   * @param {string} options.apiKey - User-provided Groq API key
+   */
+  constructor({ apiKey }) {
+    if (!apiKey) throw new Error("GROQ API key is required");
+    this.client = new Groq({ apiKey });
+    this.model = "llama-3.1-8b-instant"; // default model for generation
+  }
+
+  /**
+   * Generate completion from prompt
+   * @param {Object} param0
+   * @param {string} param0.prompt
+   * @returns {Promise<{text: string, raw: any}>}
+   */
+  async generate({ prompt }) {
+    const resp = await this.client.chat.completions.create({
+      messages: [{ role: "user", content: prompt }],
+      model: this.model,
+      temperature: 0.3,
+      max_tokens: 250,
+      top_p: 0.9
+    });
+
+    const text = resp.choices[0]?.message?.content?.trim() || "";
+    return { text, raw: resp };
+  }
+
+  /**
+   * Stream completion (simulated streaming)
+   * @param {Object} param0
+   * @param {string} param0.prompt
+   * @param {function} param0.onToken - callback for each chunk of text
+   * @returns {Promise<{done: boolean}>}
+   */
+  async stream({ prompt, onToken }) {
+    const { text } = await this.generate({ prompt });
+    if (onToken && typeof onToken === "function") {
+      // naive streaming: split by sentence or word
+      text.split(/(\s+)/).forEach(token => onToken(token));
+    }
+    return { done: true };
+  }
+}