@eclipse-lyra/extension-rag-system 0.0.0

package/dist/rag-service-BKBGCuO-.js

@@ -0,0 +1,1872 @@
+import { createLogger, toastWarning, subscribe, TOPIC_WORKSPACE_CONNECTED, TOPIC_WORKSPACE_CHANGED, FileContentType, File, Directory, workspaceService, rootContext } from "@eclipse-lyra/core";
+import { MLModel, inBrowserMLService, MLTask } from "@eclipse-lyra/extension-in-browser-ml/api";
+import { aiService } from "@eclipse-lyra/extension-ai-system/api";
+const logger$8 = createLogger("EmbeddingService");
+class EmbeddingService {
+  constructor() {
+    this.modelName = MLModel.FEATURE_EXTRACTION;
+    this.EMBEDDING_DIMENSION = 384;
+    this.DEFAULT_OPTIONS = {
+      pooling: "mean",
+      normalize: true
+    };
+  }
+  async initialize() {
+    if (this.pipePromise) {
+      try {
+        await this.pipePromise;
+        return;
+      } catch (error) {
+        logger$8.warn("Previous initialization failed, retrying...");
+        this.pipePromise = void 0;
+      }
+    }
+    logger$8.info(`Initializing embedding service with model: ${this.modelName}`);
+    try {
+      this.pipePromise = inBrowserMLService.getPipeline(
+        MLTask.FEATURE_EXTRACTION,
+        this.modelName,
+        { quantized: false }
+      );
+      await this.pipePromise;
+      logger$8.info("Embedding service initialized successfully");
+    } catch (error) {
+      const errorMessage = error?.message || String(error);
+      const errorDetails = error ? JSON.stringify(error) : "";
+      logger$8.error(`Failed to initialize embedding service: ${errorMessage}${errorDetails ? ` - ${errorDetails}` : ""}`);
+      this.pipePromise = void 0;
+      throw new Error(`Embedding service initialization failed: ${errorMessage}`);
+    }
+  }
+  async generateEmbedding(text, options = {}) {
+    if (!this.pipePromise) {
+      await this.initialize();
+    }
+    const pipe = await this.pipePromise;
+    const opts = { ...this.DEFAULT_OPTIONS, ...options };
+    try {
+      const output = await pipe(text, {
+        pooling: opts.pooling,
+        normalize: opts.normalize
+      });
+      const embedding = Array.from(output.data);
+      if (embedding.length !== this.EMBEDDING_DIMENSION) {
+        logger$8.warn(`Unexpected embedding dimension: ${embedding.length}, expected ${this.EMBEDDING_DIMENSION}`);
+      }
+      return embedding;
+    } catch (error) {
+      logger$8.error(`Failed to generate embedding: ${error}`);
+      throw error;
+    }
+  }
+  async generateEmbeddings(texts, options = {}) {
+    const embeddings = [];
+    for (const text of texts) {
+      const embedding = await this.generateEmbedding(text, options);
+      embeddings.push(embedding);
+    }
+    return embeddings;
+  }
+  getEmbeddingDimension() {
+    return this.EMBEDDING_DIMENSION;
+  }
+  getModelName() {
+    return this.modelName;
+  }
+}
+const embeddingService = new EmbeddingService();
+const SNIPPET_LENGTHS = {
+  PREVIEW: 500,
+  CONTEXT: 150
+};
+const RELEVANCE_WEIGHTS = {
+  FILE_NAME_MATCH: 10,
+  FILE_PATH_MATCH: 5,
+  CONTENT_MATCH: 1,
+  FILE_NAME_EXACT: 20,
+  FILE_PATH_EXACT: 10,
+  EXACT_PHRASE: 5,
+  TERM_COVERAGE: 15
+};
+const SEARCH_CONFIG = {
+  DEFAULT_LIMIT: 5,
+  MAX_LIMIT: 20
+};
+const VECTOR_SEARCH_CONFIG = {
+  SAMPLE_VECTOR_COUNT: 5
+};
+const INDEX_FIELD_NAMES = [
+  "idx0",
+  "idx1",
+  "idx2",
+  "idx3",
+  "idx4"
+];
+const CONTENT_PREVIEW_LENGTHS = {
+  LONG: 1e3
+};
+function euclideanDistance(vec1, vec2) {
+  if (vec1.length !== vec2.length) {
+    throw new Error(`Vector dimensions must match: ${vec1.length} vs ${vec2.length}`);
+  }
+  let sum = 0;
+  for (let i = 0; i < vec1.length; i++) {
+    const diff = vec1[i] - vec2[i];
+    sum += diff * diff;
+  }
+  return Math.sqrt(sum);
+}
+function generateSampleVectors(count, dimension, existingEmbeddings) {
+  const samples = [];
+  if (existingEmbeddings && existingEmbeddings.length > 0) {
+    for (let i = 0; i < count; i++) {
+      if (i < existingEmbeddings.length) {
+        samples.push({
+          vector: existingEmbeddings[i],
+          idx: i
+        });
+      } else {
+        const vector = [];
+        for (let j = 0; j < dimension; j++) {
+          vector.push(Math.random() * 2 - 1);
+        }
+        samples.push({
+          vector,
+          idx: i
+        });
+      }
+    }
+  } else {
+    for (let i = 0; i < count; i++) {
+      const vector = [];
+      for (let j = 0; j < dimension; j++) {
+        vector.push(Math.random() * 2 - 1);
+      }
+      samples.push({
+        vector,
+        idx: i
+      });
+    }
+  }
+  return samples;
+}
+function calculateIndexValues(embedding, sampleVectors) {
+  if (sampleVectors.length !== INDEX_FIELD_NAMES.length) {
+    throw new Error(
+      `Sample vectors count (${sampleVectors.length}) must match index field count (${INDEX_FIELD_NAMES.length})`
+    );
+  }
+  const result = {};
+  for (let i = 0; i < INDEX_FIELD_NAMES.length; i++) {
+    result[INDEX_FIELD_NAMES[i]] = euclideanDistance(embedding, sampleVectors[i].vector);
+  }
+  return result;
+}
+function getIndexValueArray(indexValues) {
+  return INDEX_FIELD_NAMES.map((name) => indexValues[name]);
+}
+function createIndexValuesFromArray(values) {
+  if (values.length !== INDEX_FIELD_NAMES.length) {
+    throw new Error(
+      `Values array length (${values.length}) must match index field count (${INDEX_FIELD_NAMES.length})`
+    );
+  }
+  const result = {};
+  for (let i = 0; i < INDEX_FIELD_NAMES.length; i++) {
+    result[INDEX_FIELD_NAMES[i]] = values[i];
+  }
+  return result;
+}
+function cosineSimilarity(vec1, vec2) {
+  if (vec1.length !== vec2.length) {
+    throw new Error(`Vector dimensions must match: ${vec1.length} vs ${vec2.length}`);
+  }
+  let dotProduct = 0;
+  let norm1 = 0;
+  let norm2 = 0;
+  for (let i = 0; i < vec1.length; i++) {
+    dotProduct += vec1[i] * vec2[i];
+    norm1 += vec1[i] * vec1[i];
+    norm2 += vec2[i] * vec2[i];
+  }
+  const denominator = Math.sqrt(norm1) * Math.sqrt(norm2);
+  if (denominator === 0) {
+    return 0;
+  }
+  return dotProduct / denominator;
+}
+const DEFAULT_CHUNK_SIZE$1 = 500;
+const DEFAULT_CHUNK_OVERLAP$1 = 50;
+const DEFAULT_MIN_CHUNK_SIZE = 100;
+class FallbackChunker {
+  constructor(options = {}) {
+    this.chunkSize = options.chunkSize ?? DEFAULT_CHUNK_SIZE$1;
+    this.chunkOverlap = options.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP$1;
+    this.minChunkSize = options.minChunkSize ?? DEFAULT_MIN_CHUNK_SIZE;
+  }
+  chunkDocument(documentId, content, fileName) {
+    if (content.length <= this.chunkSize) {
+      const text = `${fileName} ${content}`;
+      return [{
+        id: `${documentId}:chunk:0`,
+        documentId,
+        chunkIndex: 0,
+        text,
+        startOffset: 0,
+        endOffset: content.length
+      }];
+    }
+    const chunks = [];
+    const step = this.chunkSize - this.chunkOverlap;
+    let offset = 0;
+    let chunkIndex = 0;
+    while (offset < content.length) {
+      const endOffset = Math.min(offset + this.chunkSize, content.length);
+      const chunkText = content.substring(offset, endOffset);
+      if (chunkText.trim().length < this.minChunkSize && chunks.length > 0) {
+        break;
+      }
+      const text = chunkIndex === 0 ? `${fileName} ${chunkText}` : chunkText;
+      chunks.push({
+        id: `${documentId}:chunk:${chunkIndex}`,
+        documentId,
+        chunkIndex,
+        text,
+        startOffset: offset,
+        endOffset
+      });
+      offset += step;
+      chunkIndex++;
+    }
+    return chunks;
+  }
+  getChunkContext(chunk, allChunks) {
+    const prevChunk = chunk.chunkIndex > 0 ? allChunks[chunk.chunkIndex - 1] : null;
+    const nextChunk = chunk.chunkIndex < allChunks.length - 1 ? allChunks[chunk.chunkIndex + 1] : null;
+    let context = "";
+    if (prevChunk) {
+      context += `[Previous: ${prevChunk.text.substring(Math.max(0, prevChunk.text.length - SNIPPET_LENGTHS.CONTEXT))}]
+
+`;
+    }
+    context += chunk.text;
+    if (nextChunk) {
+      context += `
+
+[Next: ${nextChunk.text.substring(0, SNIPPET_LENGTHS.CONTEXT)}]`;
+    }
+    return context;
+  }
+}
+const logger$7 = createLogger("LangChainChunker");
+const DEFAULT_CHUNK_SIZE = 500;
+const DEFAULT_CHUNK_OVERLAP = 75;
+function countWords(text) {
+  if (!text || text.trim().length === 0) {
+    return 0;
+  }
+  return text.trim().split(/\s+/).filter((word) => word.length > 0).length;
+}
+class LangChainChunker {
+  constructor(options = {}) {
+    this.textSplitter = null;
+    this.chunkSize = options.chunkSize ?? DEFAULT_CHUNK_SIZE;
+    this.chunkOverlap = options.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP;
+  }
+  async getTextSplitter() {
+    if (!this.textSplitter) {
+      const { RecursiveCharacterTextSplitter } = await import("@langchain/textsplitters");
+      this.textSplitter = new RecursiveCharacterTextSplitter({
+        chunkSize: this.chunkSize,
+        chunkOverlap: this.chunkOverlap,
+        lengthFunction: countWords,
+        separators: [
+          "\n\n",
+          "\n",
+          ". ",
+          "! ",
+          "? ",
+          " "
+        ],
+        keepSeparator: false
+      });
+    }
+    return this.textSplitter;
+  }
+  async chunkDocument(documentId, content, fileName) {
+    try {
+      const splitter = await this.getTextSplitter();
+      const chunks = await splitter.splitText(content);
+      const documentChunks = [];
+      let currentOffset = 0;
+      for (let index = 0; index < chunks.length; index++) {
+        const chunkText = chunks[index];
+        let startOffset;
+        if (index === 0) {
+          const foundPos = content.indexOf(chunkText);
+          startOffset = foundPos !== -1 ? foundPos : 0;
+          currentOffset = startOffset;
+        } else {
+          const prevChunk = documentChunks[index - 1];
+          const overlapBuffer = Math.max(this.chunkOverlap * 10, chunkText.length);
+          const searchStart = Math.max(0, prevChunk.endOffset - overlapBuffer);
+          const searchEnd = Math.min(content.length, prevChunk.endOffset + chunkText.length);
+          const searchWindow = content.substring(searchStart, searchEnd);
+          const relativePos = searchWindow.indexOf(chunkText);
+          if (relativePos !== -1) {
+            const candidateOffset = searchStart + relativePos;
+            if (candidateOffset >= prevChunk.startOffset && candidateOffset < prevChunk.endOffset + chunkText.length) {
+              startOffset = candidateOffset;
+            } else {
+              startOffset = prevChunk.endOffset;
+            }
+          } else {
+            startOffset = prevChunk.endOffset;
+          }
+          currentOffset = startOffset;
+        }
+        const endOffset = Math.min(startOffset + chunkText.length, content.length);
+        const text = index === 0 ? `${fileName} ${chunkText}` : chunkText;
+        documentChunks.push({
+          id: `${documentId}:chunk:${index}`,
+          documentId,
+          chunkIndex: index,
+          text,
+          startOffset,
+          endOffset
+        });
+      }
+      this.validateNoWordSplitting(documentChunks, content, documentId);
+      logger$7.debug(`Document ${documentId} split into ${documentChunks.length} chunks using LangChain`);
+      return documentChunks;
+    } catch (error) {
+      logger$7.warn(`LangChain chunking failed for ${documentId}, falling back to simple chunking: ${error}`);
+      throw error;
+    }
+  }
+  validateNoWordSplitting(chunks, originalContent, documentId) {
+    for (let i = 0; i < chunks.length - 1; i++) {
+      const currentChunk = chunks[i];
+      const nextChunk = chunks[i + 1];
+      if (currentChunk.endOffset < originalContent.length && nextChunk.startOffset > currentChunk.endOffset) {
+        const gap = originalContent.substring(currentChunk.endOffset, nextChunk.startOffset);
+        const gapTrimmed = gap.trim();
+        if (gapTrimmed.length > 0 && !/^\s+$/.test(gap)) {
+          const beforeChar = originalContent[currentChunk.endOffset - 1];
+          const afterChar = originalContent[nextChunk.startOffset];
+          if (beforeChar && afterChar && /[a-zA-Z0-9]/.test(beforeChar) && /[a-zA-Z0-9]/.test(afterChar)) {
+            logger$7.warn(`Potential word split detected in document ${documentId} between chunks ${i} and ${i + 1}`);
+          }
+        }
+      }
+    }
+  }
+  getChunkContext(chunk, allChunks) {
+    const prevChunk = chunk.chunkIndex > 0 ? allChunks[chunk.chunkIndex - 1] : null;
+    const nextChunk = chunk.chunkIndex < allChunks.length - 1 ? allChunks[chunk.chunkIndex + 1] : null;
+    let context = "";
+    if (prevChunk) {
+      context += `[Previous: ${prevChunk.text.substring(Math.max(0, prevChunk.text.length - SNIPPET_LENGTHS.CONTEXT))}]
+
+`;
+    }
+    context += chunk.text;
+    if (nextChunk) {
+      context += `
+
+[Next: ${nextChunk.text.substring(0, SNIPPET_LENGTHS.CONTEXT)}]`;
+    }
+    return context;
+  }
+}
+const logger$6 = createLogger("DocumentChunker");
+class DocumentChunker {
+  constructor(options = {}) {
+    try {
+      this.chunker = new LangChainChunker(options);
+      logger$6.debug("Using LangChain chunker");
+    } catch (error) {
+      logger$6.warn(`Failed to initialize LangChain chunker, using fallback: ${error}`);
+      this.chunker = new FallbackChunker(options);
+    }
+  }
+  async chunkDocument(documentId, content, fileName) {
+    try {
+      const result = this.chunker.chunkDocument(documentId, content, fileName);
+      return await Promise.resolve(result);
+    } catch (error) {
+      logger$6.warn(`Primary chunker failed, falling back: ${error}`);
+      const fallback = new FallbackChunker();
+      return fallback.chunkDocument(documentId, content, fileName);
+    }
+  }
+  getChunkContext(chunk, allChunks) {
+    return this.chunker.getChunkContext(chunk, allChunks);
+  }
+}
+new DocumentChunker();
+const logger$5 = createLogger("PDFJSExtractor");
+class PDFJSExtractor {
+  canExtract(fileType) {
+    return fileType.toLowerCase() === "pdf";
+  }
+  async extractText(file, options = {}) {
+    try {
+      const pdfjsLib = await import("pdfjs-dist");
+      if (!pdfjsLib.GlobalWorkerOptions.workerSrc) {
+        pdfjsLib.GlobalWorkerOptions.workerSrc = `https://unpkg.com/pdfjs-dist@${pdfjsLib.version}/build/pdf.worker.min.mjs`;
+      }
+      const pdfBlob = await file.getContents({ blob: true });
+      const arrayBuffer = await pdfBlob.arrayBuffer();
+      const loadingTask = pdfjsLib.getDocument({
+        data: arrayBuffer,
+        useSystemFonts: true
+      });
+      const pdf = await loadingTask.promise;
+      const numPages = pdf.numPages;
+      const textParts = [];
+      const includePageNumbers = options.includePageNumbers !== false;
+      const pageSeparator = options.pageSeparator || "\n\n";
+      for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+        const page = await pdf.getPage(pageNum);
+        const textContent = await page.getTextContent();
+        const pageText = textContent.items.map((item) => item.str).join(" ");
+        if (pageText.trim()) {
+          if (includePageNumbers) {
+            textParts.push(`[Page ${pageNum}]
+${pageText}`);
+          } else {
+            textParts.push(pageText);
+          }
+        }
+      }
+      const extractedText = textParts.join(pageSeparator);
+      if (!extractedText || extractedText.trim().length === 0) {
+        throw new Error("PDF appears to contain no extractable text (may be image-based or scanned)");
+      }
+      logger$5.debug(`Extracted ${numPages} pages from PDF: ${file.getName()}`);
+      return extractedText;
+    } catch (error) {
+      logger$5.warn(`Failed to extract text from PDF ${file.getName()}: ${error}`);
+      throw new Error(`PDF text extraction failed: ${error}`);
+    }
+  }
+}
+const logger$4 = createLogger("LLMOCRExtractor");
+class LLMOCRExtractor {
+  canExtract(fileType) {
+    return ["pdf", "png", "jpg", "jpeg", "tiff", "tif"].includes(fileType.toLowerCase());
+  }
+  async extractText(file, options = {}) {
+    const fileName = file.getName();
+    const fileType = options?.fileType || fileName.split(".").pop()?.toLowerCase() || "pdf";
+    const providers = await aiService.getProviders();
+    const ocrProvider = providers.find((p) => {
+      const endpoint = p.parameters?.["ocrApiEndpoint"];
+      return endpoint && p.name.toLowerCase().includes("mistral");
+    });
+    const ocrEndpoint = ocrProvider?.parameters?.["ocrApiEndpoint"];
+    if (!ocrProvider || !ocrEndpoint) {
+      throw new Error("Mistral OCR provider not configured. Please add ocrApiEndpoint to the provider parameters in AI settings.");
+    }
+    try {
+      const fileBlob = await file.getContents({ blob: true });
+      const base64Content = await this.blobToBase64(fileBlob);
+      const mimeType = this.getMimeType(fileType);
+      const response = await fetch(ocrEndpoint, {
+        method: "POST",
+        headers: {
+          "Authorization": `Bearer ${ocrProvider.apiKey}`,
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify({
+          model: ocrProvider.parameters?.["ocrModel"] || ocrProvider.model || "mistral-ocr-latest",
+          document: {
+            type: "document_url",
+            document_url: `data:${mimeType};base64,${base64Content}`
+          },
+          include_image_base64: false
+        })
+      });
+      if (!response.ok) {
+        const errorText = await response.text().catch(() => "Unknown error");
+        throw new Error(`OCR request failed: HTTP ${response.status}: ${errorText}`);
+      }
+      const result = await response.json();
+      if (!result.pages || !Array.isArray(result.pages)) {
+        throw new Error("Invalid OCR response format: missing pages array");
+      }
+      const includePageNumbers = options.includePageNumbers !== false;
+      const pageSeparator = options.pageSeparator || "\n\n";
+      const textParts = result.pages.map((page, index) => {
+        const pageText = page?.markdown || page?.text || "";
+        if (!pageText.trim()) {
+          return null;
+        }
+        if (includePageNumbers) {
+          return `[Page ${index + 1}]
+${pageText}`;
+        }
+        return pageText;
+      }).filter((text) => text !== null);
+      if (textParts.length === 0) {
+        throw new Error("No text content found in OCR response");
+      }
+      const extractedText = textParts.join(pageSeparator);
+      logger$4.debug(`Extracted ${result.pages.length} pages from ${fileType} file: ${fileName}`);
+      return extractedText;
+    } catch (error) {
+      logger$4.warn(`Failed to extract text using OCR from ${fileName}: ${error}`);
+      throw new Error(`OCR text extraction failed: ${error}`);
+    }
+  }
+  async blobToBase64(blob) {
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader();
+      reader.onloadend = () => {
+        const result = reader.result;
+        const base64 = result.split(",")[1];
+        resolve(base64);
+      };
+      reader.onerror = reject;
+      reader.readAsDataURL(blob);
+    });
+  }
+  getMimeType(fileType) {
+    const mimeTypes = {
+      "pdf": "application/pdf",
+      "png": "image/png",
+      "jpg": "image/jpeg",
+      "jpeg": "image/jpeg",
+      "tiff": "image/tiff",
+      "tif": "image/tiff"
+    };
+    return mimeTypes[fileType.toLowerCase()] || "application/octet-stream";
+  }
+}
+const logger$3 = createLogger("DocumentExtractor");
+class DocumentExtractor {
+  constructor() {
+    this.extractors = [
+      new LLMOCRExtractor(),
+      new PDFJSExtractor()
+    ];
+  }
+  canExtract(fileType) {
+    return this.extractors.some((extractor) => extractor.canExtract(fileType));
+  }
+  async extractText(file, options) {
+    const fileName = file.getName();
+    const fileType = options?.fileType || fileName.split(".").pop()?.toLowerCase() || "txt";
+    const availableExtractors = this.extractors.filter((ext) => ext.canExtract(fileType));
+    if (availableExtractors.length === 0) {
+      throw new Error(`No extractor available for file type: ${fileType}`);
+    }
+    let lastError = null;
+    for (const extractor of availableExtractors) {
+      try {
+        logger$3.debug(`Using ${extractor.constructor.name} for file type: ${fileType}`);
+        return await extractor.extractText(file, { ...options, fileType });
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+        logger$3.warn(`${extractor.constructor.name} failed for ${fileName}: ${lastError.message}`);
+        if (extractor instanceof LLMOCRExtractor && availableExtractors.length > 1) {
+          const fallbackExtractor = availableExtractors.find((ext) => ext instanceof PDFJSExtractor);
+          if (fallbackExtractor) {
+            const warningMsg = `Mistral OCR extraction failed, falling back to PDF.js extractor for ${fileName}`;
+            logger$3.warn(warningMsg);
+            toastWarning(warningMsg);
+            try {
+              logger$3.debug(`Using ${fallbackExtractor.constructor.name} as fallback for file type: ${fileType}`);
+              return await fallbackExtractor.extractText(file, { ...options, fileType });
+            } catch (fallbackError) {
+              lastError = fallbackError instanceof Error ? fallbackError : new Error(String(fallbackError));
+              logger$3.warn(`Fallback extraction also failed for ${fileName}: ${lastError.message}`);
+            }
+          }
+        }
+      }
+    }
+    throw lastError || new Error(`All extractors failed for file type: ${fileType}`);
+  }
+}
+let rxdbModulesPromise = null;
+let pluginsRegistered = false;
+async function importRxdbModules() {
+  return Promise.all([
+    import("rxdb"),
+    import("rxdb/plugins/storage-dexie"),
+    import("rxdb/plugins/query-builder"),
+    import("rxdb/plugins/migration-schema"),
+    import("rxdb/plugins/update")
+  ]);
+}
+async function getRxDbModules() {
+  if (!rxdbModulesPromise) {
+    rxdbModulesPromise = importRxdbModules().then(([
+      rxdb,
+      storageDexie,
+      queryBuilder,
+      migrationSchema,
+      update
+    ]) => {
+      if (!pluginsRegistered) {
+        rxdb.addRxPlugin(queryBuilder.RxDBQueryBuilderPlugin);
+        rxdb.addRxPlugin(migrationSchema.RxDBMigrationSchemaPlugin);
+        rxdb.addRxPlugin(update.RxDBUpdatePlugin);
+        pluginsRegistered = true;
+      }
+      return {
+        rxdb,
+        storageDexie
+      };
+    });
+  }
+  return rxdbModulesPromise;
+}
+const logger$2 = createLogger("DocumentIndexService");
+class DocumentIndexService {
+  constructor() {
+    this.sampleVectors = [];
+    this.isInitialized = false;
+    this.DEFAULT_MAX_FILE_SIZE = 5 * 1024 * 1024;
+    this.chunker = new DocumentChunker();
+    this.documentExtractor = new DocumentExtractor();
+    this.DEFAULT_INDEXABLE_TYPES = [
+      "md",
+      "txt",
+      "ts",
+      "tsx",
+      "js",
+      "jsx",
+      "json",
+      "geojson",
+      "kml",
+      "gpx",
+      "py",
+      "html",
+      "css",
+      "sql",
+      "xml",
+      "yaml",
+      "yml",
+      "pdf"
+    ];
+  }
+  async initialize() {
+    if (this.isInitialized) {
+      return;
+    }
+    logger$2.info("Initializing document index service with RxDB...");
+    try {
+      const { rxdb, storageDexie } = await getRxDbModules();
+      this.db = await rxdb.createRxDatabase({
+        name: "document-index-db",
+        storage: storageDexie.getRxStorageDexie(),
+        ignoreDuplicate: true
+      });
+      const documentsSchema = {
+        version: 0,
+        primaryKey: "id",
+        type: "object",
+        properties: {
+          id: { type: "string", maxLength: 500 },
+          workspacePath: { type: "string" },
+          filePath: { type: "string" },
+          fileName: { type: "string" },
+          fileType: { type: "string" },
+          content: { type: "string" },
+          contentHash: { type: "string" },
+          metadata: {
+            type: "object",
+            properties: {
+              size: { type: "number" },
+              lastModified: { type: "number" },
+              language: { type: "string" },
+              tags: {
+                type: "array",
+                items: { type: "string" }
+              }
+            }
+          },
+          indexedAt: { type: "number" },
+          updatedAt: { type: "number" }
+        },
+        required: ["id", "workspacePath", "filePath", "content", "contentHash"],
+        indexes: ["workspacePath", "filePath", "fileType"]
+      };
+      const vectorsSchema = {
+        version: 1,
+        primaryKey: "id",
+        type: "object",
+        properties: {
+          id: { type: "string", maxLength: 500 },
+          documentId: { type: "string", maxLength: 500 },
+          embedding: {
+            type: "array",
+            items: { type: "number" }
+          },
+          idx0: { type: "number" },
+          idx1: { type: "number" },
+          idx2: { type: "number" },
+          idx3: { type: "number" },
+          idx4: { type: "number" },
+          chunkIndex: { type: "number" },
+          chunkStartOffset: { type: "number" },
+          chunkEndOffset: { type: "number" }
+        },
+        required: ["id", "documentId", "embedding", "idx0", "idx1", "idx2", "idx3", "idx4"],
+        indexes: ["documentId", "chunkIndex", "idx0", "idx1", "idx2", "idx3", "idx4"]
+      };
+      const collectionsToAdd = {
+        documents: { schema: documentsSchema },
+        vectors: { schema: vectorsSchema }
+      };
+      try {
+        await this.db.addCollections(collectionsToAdd);
+      } catch (error) {
+        if (error?.code === "DB8" || error?.message?.includes("already exists")) {
+          logger$2.debug("Collections already exist, using existing collections");
+        } else {
+          throw error;
+        }
+      }
+      this.documentsCollection = this.db.documents;
+      this.vectorsCollection = this.db.vectors;
+      await this.initializeSampleVectors();
+      await this.handleSchemaMigration();
+      this.isInitialized = true;
+      const count = await this.documentsCollection.count().exec();
+      const vectorCount = await this.vectorsCollection.count().exec();
+      logger$2.info(`Document index service initialized with ${count} documents and ${vectorCount} embeddings`);
+      subscribe(TOPIC_WORKSPACE_CONNECTED, (workspace) => {
+        if (workspace) {
+          this.handleWorkspaceChange(workspace).catch((err) => {
+            logger$2.error(`Failed to handle workspace connection: ${err}`);
+          });
+        }
+      });
+      subscribe(TOPIC_WORKSPACE_CHANGED, (workspace) => {
+        if (workspace) {
+          this.handleWorkspaceChange(workspace).catch((err) => {
+            logger$2.error(`Failed to handle workspace change: ${err}`);
+          });
+        }
+      });
+      logger$2.info("Document index service initialized");
+    } catch (error) {
+      logger$2.error(`Failed to initialize document index service: ${error}`);
+      throw error;
+    }
+  }
+  ensureInitialized() {
+    if (!this.isInitialized || !this.documentsCollection || !this.vectorsCollection) {
+      throw new Error("Document index service not initialized. Call initialize() first.");
+    }
+  }
+  async initializeSampleVectors() {
+    if (this.sampleVectors.length > 0) {
+      return;
+    }
+    const existingEmbeddings = await this.vectorsCollection.find().limit(1e3).exec();
+    const embeddingArrays = existingEmbeddings.map((v) => v.embedding);
+    this.sampleVectors = generateSampleVectors(
+      VECTOR_SEARCH_CONFIG.SAMPLE_VECTOR_COUNT,
+      embeddingService.getEmbeddingDimension(),
+      embeddingArrays.length > 0 ? embeddingArrays : void 0
+    );
+    logger$2.info(`Sample vectors initialized for index range method: ${this.sampleVectors.length} vectors, ${existingEmbeddings.length} existing embeddings`);
+  }
+  generateDocumentId(workspacePath, filePath) {
+    return `${workspacePath}:${filePath}`;
+  }
+  async computeContentHash(content) {
+    const encoder = new TextEncoder();
+    const data = encoder.encode(content);
+    const hashBuffer = await crypto.subtle.digest("SHA-256", data);
+    const hashArray = Array.from(new Uint8Array(hashBuffer));
+    return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
+  }
+  isIndexableFile(file, options) {
+    const fileName = file.getName().toLowerCase();
+    const fileTypes = options?.fileTypes || this.DEFAULT_INDEXABLE_TYPES;
+    const extension = fileName.split(".").pop();
+    if (!extension || !fileTypes.includes(extension)) {
+      return false;
+    }
+    if (options?.excludePatterns) {
+      for (const pattern of options.excludePatterns) {
+        if (fileName.includes(pattern) || file.getWorkspacePath().includes(pattern)) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+  detectLanguage(fileName) {
+    const ext = fileName.split(".").pop()?.toLowerCase();
+    const langMap = {
+      "ts": "typescript",
+      "tsx": "typescript",
+      "js": "javascript",
+      "jsx": "javascript",
+      "py": "python",
+      "md": "markdown",
+      "json": "json",
+      "geojson": "geojson",
+      "kml": "xml",
+      "gpx": "xml",
+      "html": "html",
+      "css": "css",
+      "sql": "sql",
+      "xml": "xml",
+      "yaml": "yaml",
+      "yml": "yaml",
+      "pdf": "pdf"
+    };
+    return langMap[ext || ""] || "text";
+  }
+  async indexDocument(file, options = {}) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const workspace = file.getWorkspace();
+    const workspacePath = workspace.getName();
+    const filePath = file.getWorkspacePath();
+    const fileName = file.getName();
+    const id = this.generateDocumentId(workspacePath, filePath);
+    if (!this.isIndexableFile(file, options)) {
+      throw new Error(`File type not indexable: ${fileName}`);
+    }
+    try {
+      let content;
+      const fileType = fileName.split(".").pop()?.toLowerCase() || "txt";
+      if (this.documentExtractor.canExtract(fileType)) {
+        content = await this.documentExtractor.extractText(file, { fileType });
+      } else {
+        const fileContent = await file.getContents({ contentType: FileContentType.TEXT });
+        if (typeof fileContent !== "string") {
+          throw new Error(`File content is not text: ${fileName}`);
+        }
+        content = fileContent;
+      }
+      if (!content || content.trim().length === 0) {
+        throw new Error(`File appears to be empty or text extraction failed: ${fileName}`);
+      }
+      const maxSize = options.maxFileSize || this.DEFAULT_MAX_FILE_SIZE;
+      if (content.length > maxSize) {
+        throw new Error(`File too large to index: ${fileName} (${content.length} bytes)`);
+      }
+      const contentHash = await this.computeContentHash(content);
+      const now = Date.now();
+      const existing = await this.documentsCollection.findOne(id).exec();
+      const existingDoc = existing ? existing.toJSON() : null;
+      const existingTags = existingDoc?.metadata.tags || [];
+      const newTags = options.tags || [];
+      const mergedTags = [.../* @__PURE__ */ new Set([...existingTags, ...newTags])];
+      const tagsChanged = mergedTags.length !== existingTags.length || newTags.some((tag) => !existingTags.includes(tag));
+      if (existingDoc && existingDoc.contentHash === contentHash && !tagsChanged) {
+        logger$2.debug(`Document already indexed and unchanged: ${id}`);
+        return existingDoc;
+      }
+      const language = this.detectLanguage(fileName);
+      let lastModified = now;
+      try {
+        const fileHandle = file.getHandle?.();
+        if (fileHandle) {
+          const fileInfo = await fileHandle.getFile();
+          lastModified = fileInfo.lastModified;
+        }
+      } catch (err) {
+        logger$2.debug(`Could not get file modification time: ${err}`);
+      }
+      const contentChanged = !existingDoc || existingDoc.contentHash !== contentHash;
+      const document = {
+        id,
+        workspacePath,
+        filePath,
+        fileName,
+        fileType,
+        content: options.includeContent !== false ? content : "",
+        contentHash,
+        metadata: {
+          size: content.length,
+          lastModified,
+          language,
+          tags: mergedTags
+        },
+        indexedAt: existingDoc?.indexedAt || now,
+        updatedAt: now
+      };
+      await this.documentsCollection.upsert(document);
+      if (contentChanged) {
+        await this.generateAndStoreEmbedding(document);
+      } else {
+        logger$2.debug(`Document content unchanged, skipping embedding regeneration: ${id}`);
+      }
+      logger$2.debug(`Indexed document: ${id}`);
+      return document;
+    } catch (error) {
+      logger$2.error(`Failed to index document ${id}: ${error}`);
+      throw error;
+    }
+  }
+  async getDocument(id) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const doc = await this.documentsCollection.findOne(id).exec();
+    return doc ? doc.toJSON() : null;
+  }
+  async getDocumentByPath(workspacePath, filePath) {
+    const id = this.generateDocumentId(workspacePath, filePath);
+    return this.getDocument(id);
+  }
+  async listDocuments(workspacePath) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    let query = this.documentsCollection.find();
+    if (workspacePath) {
+      query = query.where("workspacePath").eq(workspacePath);
+    }
+    const docs = await query.exec();
+    return docs.map((doc) => doc.toJSON());
+  }
+  async deleteDocument(id) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const doc = await this.documentsCollection.findOne(id).exec();
+    if (doc) {
+      await doc.remove();
+      const vectors = await this.vectorsCollection.find().where("documentId").eq(id).exec();
+      for (const vector of vectors) {
+        await vector.remove();
+      }
+      logger$2.debug(`Deleted document ${id} and ${vectors.length} associated embeddings`);
+      return true;
+    }
+    return false;
+  }
+  async deleteDocumentByPath(workspacePath, filePath) {
+    const id = this.generateDocumentId(workspacePath, filePath);
+    return this.deleteDocument(id);
+  }
+  async handleSchemaMigration() {
+    if (!this.vectorsCollection || !this.documentsCollection) {
+      return;
+    }
+    try {
+      const allVectors = await this.vectorsCollection.find().exec();
+      const vectorsWithoutChunks = allVectors.filter((v) => {
+        const data = v.toJSON();
+        return data.chunkIndex === void 0 && data.chunkStartOffset === void 0 && data.chunkEndOffset === void 0;
+      });
+      if (vectorsWithoutChunks.length === 0) {
+        logger$2.debug("No vectors need migration - all have chunk information");
+        return;
+      }
+      logger$2.info(`Detected ${vectorsWithoutChunks.length} vectors without chunk information. Invalidating and reindexing...`);
+      const documentIdsToReindex = /* @__PURE__ */ new Set();
+      for (const vector of vectorsWithoutChunks) {
+        const data = vector.toJSON();
+        documentIdsToReindex.add(data.documentId);
+        await vector.remove();
+      }
+      logger$2.info(`Removed ${vectorsWithoutChunks.length} old vectors. Reindexing ${documentIdsToReindex.size} documents...`);
+      for (const documentId of documentIdsToReindex) {
+        const doc = await this.documentsCollection.findOne(documentId).exec();
+        if (doc) {
+          const document = doc.toJSON();
+          logger$2.debug(`Reindexing document: ${document.fileName}`);
+          await this.generateAndStoreEmbedding(document);
+        }
+      }
+      logger$2.info(`Schema migration completed. Reindexed ${documentIdsToReindex.size} documents.`);
+    } catch (error) {
+      logger$2.error(`Error during schema migration: ${error}`);
+      throw error;
+    }
+  }
+  async deleteWorkspace(workspacePath) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const docs = await this.documentsCollection.find().where("workspacePath").eq(workspacePath).exec();
+    const count = docs.length;
+    for (const doc of docs) {
+      await doc.remove();
+    }
+    if (count > 0) {
+      logger$2.info(`Deleted ${count} documents for workspace: ${workspacePath}`);
+    }
+    return count;
+  }
+  async updateDocumentMetadata(id, updates) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const doc = await this.documentsCollection.findOne(id).exec();
+    if (!doc) {
+      return null;
+    }
+    const current = doc.toJSON();
+    const updated = {
+      ...current,
+      metadata: {
+        ...current.metadata,
+        ...updates.metadata
+      },
+      updatedAt: Date.now()
+    };
+    await doc.update({ $set: updated });
+    logger$2.debug(`Updated document metadata: ${id}`);
+    return updated;
+  }
+  async indexWorkspace(workspace, options = {}) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    const workspacePath = workspace.getName();
+    logger$2.info(`Starting workspace indexing: ${workspacePath}`);
+    const files = await this.collectFiles(workspace, options);
+    logger$2.info(`Found ${files.length} files to index`);
+    let indexed = 0;
+    let failed = 0;
+    const errors = [];
+    for (const file of files) {
+      try {
+        await this.indexDocument(file, options);
+        indexed++;
+      } catch (error) {
+        failed++;
+        const errorMsg = `Failed to index ${file.getName()}: ${error}`;
+        errors.push(errorMsg);
+        logger$2.warn(errorMsg);
+      }
+    }
+    logger$2.info(`Workspace indexing complete: ${indexed} indexed, ${failed} failed`);
+    return { indexed, failed, errors };
+  }
+  async collectFiles(directory, options, files = []) {
+    try {
+      const children = await directory.listChildren(false);
+      for (const child of children) {
+        if (child instanceof File) {
+          if (this.isIndexableFile(child, options)) {
+            files.push(child);
+          }
+        } else if (child instanceof Directory) {
+          await this.collectFiles(child, options, files);
+        }
+      }
+    } catch (error) {
+      logger$2.warn(`Failed to collect files from ${directory.getName()}: ${error}`);
+    }
+    return files;
+  }
+  async reindexDocument(file, options = {}) {
+    const workspace = file.getWorkspace();
+    const workspacePath = workspace.getName();
+    const filePath = file.getWorkspacePath();
+    const id = this.generateDocumentId(workspacePath, filePath);
+    const existing = await this.getDocument(id);
+    const existingTags = existing?.metadata.tags || [];
+    const newTags = options.tags || [];
+    const mergedTags = [.../* @__PURE__ */ new Set([...existingTags, ...newTags])];
+    await this.deleteDocument(id);
+    return this.indexDocument(file, {
+      ...options,
+      tags: mergedTags
+    });
+  }
+  async reindexAllDocuments(options = {}) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const allDocs = await this.listDocuments();
+    let succeeded = 0;
+    let failed = 0;
+    for (const doc of allDocs) {
+      try {
+        const workspace = await workspaceService.getWorkspace();
+        if (!workspace || workspace.getName() !== doc.workspacePath) {
+          logger$2.warn(`Workspace not found: ${doc.workspacePath}`);
+          failed++;
+          continue;
+        }
+        const resource = await workspace.getResource(doc.filePath);
+        if (!resource || !(resource instanceof File)) {
+          logger$2.warn(`File not found: ${doc.filePath}`);
+          failed++;
+          continue;
+        }
+        await this.reindexDocument(resource, options);
+        succeeded++;
+      } catch (error) {
+        logger$2.error(`Failed to reindex document ${doc.id}: ${error}`);
+        failed++;
+      }
+    }
+    return {
+      total: allDocs.length,
+      succeeded,
+      failed
+    };
+  }
+  async getStats() {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const totalDocuments = await this.documentsCollection.count().exec();
+    const allDocs = await this.documentsCollection.find().exec();
+    const byWorkspace = {};
+    for (const doc of allDocs) {
+      const docData = doc.toJSON();
+      const workspacePath = docData.workspacePath;
+      byWorkspace[workspacePath] = (byWorkspace[workspacePath] || 0) + 1;
+    }
+    return {
+      totalDocuments,
+      byWorkspace
+    };
+  }
+  async handleWorkspaceChange(workspace) {
+    logger$2.debug("Workspace changed, checking for document updates...");
+  }
+  async generateAndStoreEmbedding(document) {
+    try {
+      if (!this.vectorsCollection) {
+        logger$2.warn(`Vectors collection not initialized, cannot generate embedding for ${document.id}`);
+        return;
+      }
+      await embeddingService.initialize();
+      if (this.sampleVectors.length === 0) {
+        await this.initializeSampleVectors();
+      }
+      if (this.sampleVectors.length === 0) {
+        logger$2.warn(`Sample vectors not initialized, cannot generate embedding for ${document.id}`);
+        return;
+      }
+      const chunks = await this.chunker.chunkDocument(document.id, document.content, document.fileName);
+      logger$2.debug(`Document ${document.id} split into ${chunks.length} chunks`);
+      for (const chunk of chunks) {
+        const embedding = await embeddingService.generateEmbedding(chunk.text);
+        const indexValues = calculateIndexValues(embedding, this.sampleVectors);
+        const vectorDoc = {
+          id: chunk.id,
+          documentId: document.id,
+          chunkIndex: chunk.chunkIndex,
+          chunkStartOffset: chunk.startOffset,
+          chunkEndOffset: chunk.endOffset,
+          embedding,
+          ...indexValues
+        };
+        await this.vectorsCollection.upsert(vectorDoc);
+      }
+      logger$2.debug(`Generated and stored ${chunks.length} embeddings for document: ${document.id}`);
+    } catch (error) {
+      logger$2.warn(`Failed to generate embedding for document ${document.id}: ${error}`);
+    }
+  }
+  async searchSimilar(queryText, options = {}) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const limit = options.limit || 10;
+    const indexDistance = options.indexDistance || 2;
+    const docsPerIndexSide = options.docsPerIndexSide || 100;
+    if (!this.vectorsCollection || this.sampleVectors.length === 0) {
+      logger$2.warn("Vector search not available: vectors collection or sample vectors not initialized");
+      throw new Error("Vector search not available");
+    }
+    const totalVectors = await this.vectorsCollection.find().exec();
+    logger$2.debug(`Starting vector search with indexDistance=${indexDistance}, limit=${limit}, sampleVectors=${this.sampleVectors.length}, totalIndexedVectors=${totalVectors.length}`);
+    try {
+      await embeddingService.initialize();
+    } catch (error) {
+      logger$2.error(`Failed to initialize embedding service for vector search: ${error}`);
+      throw new Error(`Embedding service initialization failed: ${error}`);
+    }
+    let queryEmbedding;
+    try {
+      queryEmbedding = await embeddingService.generateEmbedding(queryText);
+    } catch (error) {
+      logger$2.error(`Failed to generate query embedding: ${error}`);
+      throw new Error(`Query embedding generation failed: ${error}`);
+    }
+    if (this.sampleVectors.length === 0) {
+      logger$2.warn("Sample vectors not initialized, cannot perform vector search");
+      throw new Error("Sample vectors not initialized");
+    }
+    const queryIndexValues = calculateIndexValues(queryEmbedding, this.sampleVectors);
+    logger$2.debug(`Query index values: ${JSON.stringify(queryIndexValues)}`);
+    const candidateIds = /* @__PURE__ */ new Set();
+    try {
+      for (const idxKey of INDEX_FIELD_NAMES) {
+        const queryValue = queryIndexValues[idxKey];
+        const minValue = queryValue - indexDistance;
+        const maxValue = queryValue + indexDistance;
+        logger$2.debug(`Querying index ${idxKey}: range [${minValue}, ${maxValue}]`);
+        const candidates = await this.vectorsCollection.find().where(idxKey).gte(minValue).lte(maxValue).limit(docsPerIndexSide).exec();
+        logger$2.debug(`Found ${candidates.length} candidates in index ${idxKey}`);
+        for (const candidate of candidates) {
+          candidateIds.add(candidate.documentId);
+        }
+      }
+    } catch (error) {
+      logger$2.error(`Failed to query vector index: ${error}`);
+      throw new Error(`Vector index query failed: ${error}`);
+    }
+    logger$2.debug(`Total unique candidate IDs: ${candidateIds.size} (out of ${totalVectors.length} indexed vectors)`);
+    const candidateVectors = [];
+    try {
+      for (const docId of candidateIds) {
+        const vectorDocs = await this.vectorsCollection.find().where("documentId").eq(docId).exec();
+        for (const vectorDoc of vectorDocs) {
+          const vectorData = vectorDoc.toJSON();
+          if (vectorData && vectorData.embedding) {
+            candidateVectors.push(vectorData);
+          } else {
+            logger$2.warn(`Invalid vector data for document ${docId}`);
+          }
+        }
+      }
+    } catch (error) {
+      logger$2.error(`Failed to fetch candidate vectors: ${error}`);
+      throw new Error(`Failed to fetch candidate vectors: ${error}`);
+    }
+    logger$2.debug(`Fetched ${candidateVectors.length} candidate vectors`);
+    const results = [];
+    for (const vectorDoc of candidateVectors) {
+      const cosineSim = cosineSimilarity(queryEmbedding, vectorDoc.embedding);
+      const similarity = (cosineSim + 1) / 2;
+      results.push({
+        documentId: vectorDoc.documentId,
+        similarity,
+        chunkIndex: vectorDoc.chunkIndex,
+        chunkStartOffset: vectorDoc.chunkStartOffset,
+        chunkEndOffset: vectorDoc.chunkEndOffset
+      });
+    }
+    results.sort((a, b) => b.similarity - a.similarity);
+    logger$2.debug(`Computed similarities for ${results.length} candidates, top similarity: ${results[0]?.similarity || "N/A"}`);
+    const topResults = results.slice(0, limit);
+    const documentResults = [];
+    for (const result of topResults) {
+      const doc = await this.documentsCollection.findOne(result.documentId).exec();
+      if (doc) {
+        const document = doc.toJSON();
+        if (options.workspacePath && document.workspacePath !== options.workspacePath) {
+          continue;
+        }
+        if (options.fileType && document.fileType !== options.fileType) {
+          continue;
+        }
+        documentResults.push({
+          document,
+          similarity: result.similarity,
+          chunkIndex: result.chunkIndex,
+          chunkStartOffset: result.chunkStartOffset,
+          chunkEndOffset: result.chunkEndOffset
+        });
+      }
+    }
+    return documentResults;
+  }
+  async indexFileInContext(file, context, options = {}) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const workspace = file.getWorkspace();
+    const workspacePath = workspace.getName();
+    const filePath = file.getWorkspacePath();
+    const id = this.generateDocumentId(workspacePath, filePath);
+    const existing = await this.documentsCollection.findOne(id).exec();
+    const existingDoc = existing ? existing.toJSON() : null;
+    const contextTags = context.tags || [];
+    const newTags = [...options.tags || [], ...contextTags];
+    if (existingDoc) {
+      const existingTags = existingDoc.metadata.tags || [];
+      const mergedTags = [.../* @__PURE__ */ new Set([...existingTags, ...newTags])];
+      const tagsChanged = mergedTags.length !== existingTags.length || newTags.some((tag) => !existingTags.includes(tag));
+      if (tagsChanged) {
+        await this.updateDocumentMetadata(existingDoc.id, {
+          metadata: {
+            ...existingDoc.metadata,
+            tags: mergedTags
+          }
+        });
+        logger$2.debug(`Added tags to existing document: ${id}`);
+        return {
+          ...existingDoc,
+          metadata: {
+            ...existingDoc.metadata,
+            tags: mergedTags
+          }
+        };
+      } else {
+        logger$2.debug(`Document already has all tags: ${id}`);
+        return existingDoc;
+      }
+    }
+    return this.indexDocument(file, {
+      ...options,
+      tags: newTags
+    });
+  }
+  async indexFilesInContext(files, context, options = {}) {
+    let succeeded = 0;
+    let failed = 0;
+    const contextTags = context.tags || [];
+    for (const file of files) {
+      try {
+        await this.indexDocument(file, {
+          ...options,
+          tags: [...options.tags || [], ...contextTags]
+        });
+        succeeded++;
+        logger$2.debug(`Indexed file with context tags: ${file.getWorkspacePath()}`);
+      } catch (error) {
+        logger$2.error(`Failed to index file ${file.getWorkspacePath()}: ${error}`);
+        failed++;
+      }
+    }
+    return { succeeded, failed };
+  }
+  async reindexFileInContext(file, context, options = {}) {
+    const contextTags = context.tags || [];
+    return this.reindexDocument(file, {
+      ...options,
+      tags: [...options.tags || [], ...contextTags]
+    });
+  }
+  async removeFileFromContext(file, context) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+    this.ensureInitialized();
+    const document = await this.getDocumentByPath(
+      file.getWorkspace().getName(),
+      file.getWorkspacePath()
+    );
+    if (document && context.tags && context.tags.length > 0) {
+      const contextTags = new Set(context.tags);
+      const updatedTags = (document.metadata.tags || []).filter((tag) => !contextTags.has(tag));
+      if (updatedTags.length !== document.metadata.tags?.length) {
+        await this.updateDocumentMetadata(document.id, {
+          metadata: {
+            ...document.metadata,
+            tags: updatedTags
+          }
+        });
+      }
+    }
+  }
+  async clearContext(context) {
+    if (!context.tags || context.tags.length === 0) {
+      return;
+    }
+    const contextTags = new Set(context.tags);
+    const allDocs = await this.listDocuments();
+    for (const doc of allDocs) {
+      const hasContextTag = doc.metadata.tags?.some((tag) => contextTags.has(tag));
+      if (hasContextTag) {
+        const updatedTags = doc.metadata.tags.filter((tag) => !contextTags.has(tag));
+        try {
+          const workspace = await workspaceService.getWorkspace();
+          if (workspace && workspace.getName() === doc.workspacePath) {
+            const resource = await workspace.getResource(doc.filePath);
+            if (resource instanceof File) {
+              await this.indexDocument(resource, {
+                tags: updatedTags
+              });
+            }
+          }
+        } catch (error) {
+          logger$2.warn(`Failed to clear context tags from ${doc.filePath}: ${error}`);
+        }
+      }
+    }
+  }
+  async getFilePathsInContext(context) {
+    if (!context.tags || context.tags.length === 0) {
+      return [];
+    }
+    const contextTags = new Set(context.tags);
+    const allDocs = await this.listDocuments();
+    return allDocs.filter((doc) => doc.metadata.tags?.some((tag) => contextTags.has(tag))).map((doc) => doc.filePath);
+  }
+}
+const documentIndexService = new DocumentIndexService();
+rootContext.put("documentIndexService", documentIndexService);
+const logger$1 = createLogger("WorkspaceUtils");
+async function getWorkspacePath(providedPath) {
+  const workspace = await workspaceService.getWorkspace();
+  if (!workspace) {
+    logger$1.warn("No workspace connected");
+    return null;
+  }
+  const workspacePath = providedPath || workspace.getName();
+  if (!workspacePath) {
+    logger$1.warn("No workspace path available");
+    return null;
+  }
+  return { workspace, workspacePath };
+}
+const DEFAULT_MIN_TERM_LENGTH = 2;
+function extractQueryTerms(query, options = {}) {
+  if (!query || !query.trim()) {
+    return [];
+  }
+  const minLength = options.minTermLength ?? DEFAULT_MIN_TERM_LENGTH;
+  const text = options.caseSensitive ? query : query.toLowerCase();
+  return text.split(/\s+/).filter((term) => term.length >= minLength);
+}
+function normalizeQuery(query) {
+  return query.toLowerCase().trim();
+}
+const DEFAULT_MAX_SNIPPETS = 10;
+const DEFAULT_SNIPPET_LENGTH = 400;
+const DEFAULT_MIN_GAP = 400;
+class SnippetExtractor {
+  constructor(options = {}) {
+    this.maxSnippets = options.maxSnippets ?? DEFAULT_MAX_SNIPPETS;
+    this.snippetLength = options.snippetLength ?? DEFAULT_SNIPPET_LENGTH;
+    this.minGap = options.minGap ?? DEFAULT_MIN_GAP;
+  }
+  extractSnippets(content, queryTerms, maxSnippets) {
+    const effectiveMaxSnippets = maxSnippets ?? this.maxSnippets;
+    if (queryTerms.length === 0) {
+      return [];
+    }
+    const contentLower = content.toLowerCase();
+    const snippetCandidates = [];
+    const seenSnippets = /* @__PURE__ */ new Set();
+    for (const term of queryTerms) {
+      let index = contentLower.indexOf(term);
+      while (index !== -1) {
+        const start = Math.max(0, index - this.snippetLength / 2);
+        const end = Math.min(content.length, index + term.length + this.snippetLength / 2);
+        const snippet = content.substring(start, end).trim();
+        const snippetKey = `${start}-${end}`;
+        if (snippet && !seenSnippets.has(snippetKey)) {
+          seenSnippets.add(snippetKey);
+          const score = this.calculateSnippetScore(snippet, queryTerms);
+          snippetCandidates.push({ snippet, score, start });
+        }
+        index = contentLower.indexOf(term, index + 1);
+      }
+    }
+    if (snippetCandidates.length === 0 && queryTerms.length > 0) {
+      const firstTerm = queryTerms[0];
+      const index = contentLower.indexOf(firstTerm);
+      if (index !== -1) {
+        const start = Math.max(0, index - this.snippetLength);
+        const end = Math.min(content.length, index + firstTerm.length + this.snippetLength);
+        const snippet = content.substring(start, end).trim();
+        if (snippet) {
+          snippetCandidates.push({ snippet, score: 10, start });
+        }
+      }
+    }
+    snippetCandidates.sort((a, b) => {
+      if (b.score !== a.score) {
+        return b.score - a.score;
+      }
+      return a.start - b.start;
+    });
+    return this.selectNonOverlappingSnippets(snippetCandidates, effectiveMaxSnippets);
+  }
+  calculateSnippetScore(snippet, queryTerms) {
+    const snippetLower = snippet.toLowerCase();
+    let score = 0;
+    for (const qTerm of queryTerms) {
+      const termMatches = (snippetLower.match(
+        new RegExp(qTerm.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "gi")
+      ) || []).length;
+      score += termMatches * 10;
+    }
+    const fullQuery = queryTerms.join(" ");
+    if (snippetLower.includes(fullQuery)) {
+      score += 50;
+    }
+    const uniqueTermsFound = queryTerms.filter((t) => snippetLower.includes(t)).length;
+    score += uniqueTermsFound * 20;
+    const termDensity = uniqueTermsFound / queryTerms.length;
+    score += termDensity * 30;
+    return score;
+  }
+  selectNonOverlappingSnippets(candidates, maxSnippets) {
+    const selectedSnippets = [];
+    const usedRanges = [];
+    for (const candidate of candidates) {
+      if (selectedSnippets.length >= maxSnippets) break;
+      const candidateStart = candidate.start;
+      const candidateEnd = candidate.start + candidate.snippet.length;
+      const overlaps = usedRanges.some((range) => {
+        return !(candidateEnd < range.start - this.minGap || candidateStart > range.end + this.minGap);
+      });
+      if (!overlaps) {
+        selectedSnippets.push(candidate.snippet);
+        usedRanges.push({ start: candidateStart, end: candidateEnd });
+      }
+    }
+    return selectedSnippets;
+  }
+  extractSimpleSnippet(content, maxLength = 500) {
+    const preview = content.substring(0, maxLength);
+    return preview + (content.length > maxLength ? "..." : "");
+  }
+  extractContextSnippets(content, query, contextLength = 150) {
+    const queryLower = query.toLowerCase();
+    const contentLower = content.toLowerCase();
+    const matches = [];
+    let index = contentLower.indexOf(queryLower);
+    while (index !== -1) {
+      matches.push(index);
+      index = contentLower.indexOf(queryLower, index + 1);
+    }
+    if (matches.length === 0) {
+      return [];
+    }
+    const snippets = [];
+    for (const matchIndex of matches) {
+      const start = Math.max(0, matchIndex - contextLength);
+      const end = Math.min(content.length, matchIndex + query.length + contextLength);
+      snippets.push({ start, end, matchIndex });
+    }
+    return snippets;
+  }
+}
+new SnippetExtractor();
+class RelevanceCalculator {
+  calculateRelevance(doc, query) {
+    const queryLower = normalizeQuery(query);
+    const queryTerms = extractQueryTerms(query);
+    let score = 0;
+    const fileNameLower = doc.fileName.toLowerCase();
+    const filePathLower = doc.filePath.toLowerCase();
+    const contentLower = doc.content.toLowerCase();
+    const fileNameMatches = queryTerms.filter((term) => fileNameLower.includes(term)).length;
+    const filePathMatches = queryTerms.filter((term) => filePathLower.includes(term)).length;
+    const contentMatches = queryTerms.filter((term) => contentLower.includes(term)).length;
+    score += fileNameMatches * RELEVANCE_WEIGHTS.FILE_NAME_MATCH;
+    score += filePathMatches * RELEVANCE_WEIGHTS.FILE_PATH_MATCH;
+    score += contentMatches * RELEVANCE_WEIGHTS.CONTENT_MATCH;
+    if (fileNameLower.includes(queryLower)) {
+      score += RELEVANCE_WEIGHTS.FILE_NAME_EXACT;
+    }
+    if (filePathLower.includes(queryLower)) {
+      score += RELEVANCE_WEIGHTS.FILE_PATH_EXACT;
+    }
+    const exactPhraseMatches = (contentLower.match(
+      new RegExp(queryLower.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g")
+    ) || []).length;
+    score += exactPhraseMatches * RELEVANCE_WEIGHTS.EXACT_PHRASE;
+    const totalTerms = queryTerms.length;
+    const matchedTerms = queryTerms.filter((term) => contentLower.includes(term)).length;
+    const termCoverage = totalTerms > 0 ? matchedTerms / totalTerms : 0;
+    score += termCoverage * RELEVANCE_WEIGHTS.TERM_COVERAGE;
+    return score;
+  }
+  calculateFileNameRelevance(doc, fileName) {
+    const docNameLower = doc.fileName.toLowerCase();
+    const fileNameLower = fileName.toLowerCase();
+    if (docNameLower === fileNameLower) {
+      return 100;
+    }
+    if (docNameLower.includes(fileNameLower)) {
+      return 80;
+    }
+    return 0;
+  }
+}
+const relevanceCalculator = new RelevanceCalculator();
+class RAGResultFormatter {
+  constructor(snippetExtractor2) {
+    this.snippetExtractor = snippetExtractor2;
+  }
+  formatRAGContext(results) {
+    if (results.length === 0) {
+      return "";
+    }
+    const sections = results.map((result, idx) => {
+      const doc = result.document;
+      const snippets = result.matchedSnippets.map((s, i) => `  [Snippet ${i + 1}]
+  ${s}`).join("\n\n");
+      return `[Document ${idx + 1}: ${doc.fileName} (${doc.filePath})]
+Relevance: ${result.relevance.toFixed(2)}
+${snippets.length > 0 ? `Relevant snippets:
+${snippets}` : `Content preview: ${this.snippetExtractor.extractSimpleSnippet(doc.content, SNIPPET_LENGTHS.PREVIEW)}`}
+---`;
+    }).join("\n\n");
+    return `Here are relevant documents from the workspace that might help answer the question:
+
+${sections}
+
+Use the information from these documents to provide a helpful answer. Pay special attention to numbers, percentages, dates, and specific values mentioned in the snippets. If the documents don't contain relevant information, you can still answer based on your general knowledge.`;
+  }
+  formatSearchResults(results) {
+    return results.map((r) => ({
+      file: r.document.fileName,
+      path: r.document.filePath,
+      relevance: r.relevance.toFixed(2),
+      language: r.document.metadata.language,
+      size: r.document.metadata.size,
+      snippets: r.matchedSnippets,
+      preview: this.snippetExtractor.extractSimpleSnippet(r.document.content, 200) + "..."
+    }));
+  }
+  formatCommandResults(results) {
+    return results.map((r) => ({
+      file: r.document.fileName,
+      path: r.document.filePath,
+      relevance: r.relevance,
+      snippets: r.matchedSnippets
+    }));
+  }
+}
+const logger = createLogger("RAGService");
+const snippetExtractor = new SnippetExtractor();
+const resultFormatter = new RAGResultFormatter(snippetExtractor);
+class RAGService {
+  async searchDocuments(query, options = {}) {
+    const limit = Math.min(options.limit || SEARCH_CONFIG.DEFAULT_LIMIT, SEARCH_CONFIG.MAX_LIMIT);
+    const workspaceResult = await getWorkspacePath(options.workspacePath);
+    if (!workspaceResult) {
+      logger.warn("No workspace connected, cannot search documents");
+      return [];
+    }
+    const { workspacePath } = workspaceResult;
+    if (options.filePath) {
+      const specificDoc = await documentIndexService.getDocumentByPath(workspacePath, options.filePath);
+      if (specificDoc) {
+        const queryTerms = extractQueryTerms(query);
+        const matchedSnippets = queryTerms.length > 0 ? snippetExtractor.extractSnippets(specificDoc.content, queryTerms, 3) : [snippetExtractor.extractSimpleSnippet(specificDoc.content, SNIPPET_LENGTHS.PREVIEW)];
+        return [{
+          document: specificDoc,
+          relevance: 100,
+          matchedSnippets
+        }];
+      }
+      logger.warn(`Document not found: ${options.filePath}`);
+      return [];
+    }
+    if (options.fileName) {
+      const allDocs = await documentIndexService.listDocuments(workspacePath);
+      const fileNameLower = options.fileName.toLowerCase();
+      const matchingDocs = allDocs.filter((doc) => {
+        const docNameLower = doc.fileName.toLowerCase();
+        return docNameLower === fileNameLower || docNameLower.includes(fileNameLower);
+      });
+      if (matchingDocs.length > 0) {
+        const results = [];
+        const queryTerms = extractQueryTerms(query);
+        for (const doc of matchingDocs.slice(0, limit)) {
+          const matchedSnippets = queryTerms.length > 0 ? snippetExtractor.extractSnippets(doc.content, queryTerms, 3) : [snippetExtractor.extractSimpleSnippet(doc.content, SNIPPET_LENGTHS.PREVIEW)];
+          results.push({
+            document: doc,
+            relevance: relevanceCalculator.calculateFileNameRelevance(doc, options.fileName),
+            matchedSnippets
+          });
+        }
+        return results;
+      }
+      logger.warn(`No documents found with name: ${options.fileName}`);
+      return [];
+    }
+    if (!query || !query.trim()) {
+      logger.warn("No query provided and no filePath/fileName specified");
+      return [];
+    }
+    try {
+      const vectorResults = await documentIndexService.searchSimilar(query, {
+        limit: limit * 2,
+        workspacePath,
+        fileType: options.fileType
+      });
+      const results = [];
+      for (const { document, similarity, chunkStartOffset, chunkEndOffset } of vectorResults) {
+        if (!this.matchesContextScope(document, options.documentSearchScope)) {
+          continue;
+        }
+        const relevance = similarity * 100;
+        if (options.minRelevance && relevance < options.minRelevance) {
+          continue;
+        }
+        const queryLower = normalizeQuery(query);
+        const queryTerms = extractQueryTerms(query);
+        let matchedSnippets;
+        if (chunkStartOffset !== void 0 && chunkEndOffset !== void 0) {
+          const chunkText = document.content.substring(
+            Math.max(0, chunkStartOffset),
+            Math.min(document.content.length, chunkEndOffset)
+          );
+          if (chunkText.trim().length > 0) {
+            matchedSnippets = [chunkText.trim()];
+          } else {
+            matchedSnippets = snippetExtractor.extractSnippets(document.content, queryTerms, 15);
+          }
+        } else {
+          matchedSnippets = snippetExtractor.extractSnippets(document.content, queryTerms, 15);
+        }
+        if (matchedSnippets.length === 0 && queryLower.length > 0) {
+          if (chunkStartOffset !== void 0 && chunkEndOffset !== void 0) {
+            const chunkText = document.content.substring(
+              Math.max(0, chunkStartOffset),
+              Math.min(document.content.length, chunkEndOffset)
+            );
+            if (chunkText.trim().length > 0) {
+              matchedSnippets = [chunkText.trim()];
+            } else {
+              matchedSnippets = snippetExtractor.extractSnippets(document.content, [queryLower], 10);
+            }
+          } else {
+            matchedSnippets = snippetExtractor.extractSnippets(document.content, [queryLower], 10);
+          }
+        }
+        results.push({
+          document,
+          relevance,
+          matchedSnippets
+        });
+        if (results.length >= limit) {
+          break;
+        }
+      }
+      if (results.length === 0) {
+        logger.debug("Vector search returned no results, falling back to text search");
+        return this.fallbackTextSearch(query, options);
+      }
+      return results;
+    } catch (error) {
+      logger.warn(`Vector search failed, falling back to text search: ${error}`);
+      logger.debug(`Vector search error details: ${error}`);
+      return this.fallbackTextSearch(query, options);
+    }
+  }
+  async fallbackTextSearch(query, options = {}) {
+    const limit = Math.min(options.limit || SEARCH_CONFIG.DEFAULT_LIMIT, SEARCH_CONFIG.MAX_LIMIT);
+    const workspaceResult = await getWorkspacePath(options.workspacePath);
+    if (!workspaceResult) {
+      logger.warn("No workspace connected, cannot perform text search");
+      return [];
+    }
+    const { workspacePath } = workspaceResult;
+    const allDocuments = await documentIndexService.listDocuments(workspacePath);
+    const queryTerms = extractQueryTerms(query);
+    const results = [];
+    for (const doc of allDocuments) {
+      if (options.fileType && doc.fileType !== options.fileType) {
+        continue;
+      }
+      if (!this.matchesContextScope(doc, options.documentSearchScope)) {
+        continue;
+      }
+      const relevance = relevanceCalculator.calculateRelevance(doc, query);
+      if (options.minRelevance && relevance < options.minRelevance) {
+        continue;
+      }
+      const matchedSnippets = snippetExtractor.extractSnippets(doc.content, queryTerms, 3);
+      results.push({
+        document: doc,
+        relevance,
+        matchedSnippets
+      });
+    }
+    results.sort((a, b) => b.relevance - a.relevance);
+    return results.slice(0, limit);
+  }
+  formatRAGContext(results) {
+    return resultFormatter.formatRAGContext(results);
+  }
+  matchesContextScope(doc, scope) {
+    if (!scope) {
+      return true;
+    }
+    if (scope.includePaths && scope.includePaths.length > 0) {
+      const matches = scope.includePaths.some((pattern) => {
+        if (pattern.includes("*") || pattern.includes("?")) {
+          const regex = new RegExp("^" + pattern.replace(/\*/g, ".*").replace(/\?/g, ".") + "$");
+          return regex.test(doc.filePath);
+        }
+        return doc.filePath.startsWith(pattern) || doc.filePath === pattern;
+      });
+      if (!matches) {
+        return false;
+      }
+    }
+    if (scope.excludePaths && scope.excludePaths.length > 0) {
+      const matches = scope.excludePaths.some((pattern) => {
+        if (pattern.includes("*") || pattern.includes("?")) {
+          const regex = new RegExp("^" + pattern.replace(/\*/g, ".*").replace(/\?/g, ".") + "$");
+          return regex.test(doc.filePath);
+        }
+        return doc.filePath.startsWith(pattern) || doc.filePath === pattern;
+      });
+      if (matches) {
+        return false;
+      }
+    }
+    if (scope.pathPattern) {
+      const regex = scope.pathPattern instanceof RegExp ? scope.pathPattern : new RegExp(scope.pathPattern);
+      if (!regex.test(doc.filePath)) {
+        return false;
+      }
+    }
+    if (scope.tags && scope.tags.length > 0) {
+      const docTags = doc.metadata.tags || [];
+      const hasAllTags = scope.tags.every((tag) => docTags.includes(tag));
+      if (!hasAllTags) {
+        return false;
+      }
+    }
+    if (scope.metadataFilter) {
+      if (!scope.metadataFilter(doc)) {
+        return false;
+      }
+    }
+    return true;
+  }
+}
+const ragService = new RAGService();
+async function searchWorkspaceDocuments(query, options = {}) {
+  const workspaceResult = await getWorkspacePath(options.workspacePath);
+  if (!workspaceResult) {
+    logger.warn("No workspace connected, cannot search documents");
+    return [];
+  }
+  return ragService.searchDocuments(query, {
+    ...options,
+    workspacePath: workspaceResult.workspacePath
+  });
+}
+export {
+  CONTENT_PREVIEW_LENGTHS as C,
+  RAGResultFormatter as R,
+  SnippetExtractor as S,
+  createIndexValuesFromArray as a,
+  euclideanDistance as b,
+  cosineSimilarity as c,
+  documentIndexService as d,
+  embeddingService as e,
+  getWorkspacePath as f,
+  getIndexValueArray as g,
+  SNIPPET_LENGTHS as h,
+  SEARCH_CONFIG as i,
+  ragService as r,
+  searchWorkspaceDocuments as s
+};
+//# sourceMappingURL=rag-service-BKBGCuO-.js.map