@retrivora-ai/rag-engine 1.8.0 → 1.8.2

package/src/utils/synonyms.ts

@@ -0,0 +1,78 @@
+/**
+ * Shared synonym map used by UITransformer and MessageBubble to resolve
+ * schema-agnostic field names from vector metadata.
+ */
+export const FIELD_SYNONYMS: Record<string, string[]> = {
+  name:        ['product', 'item', 'title', 'label', 'heading', 'subject', 'product name', 'item name'],
+  price:       ['cost', 'amount', 'msrp', 'price', 'rate', 'value', 'price_usd'],
+  brand:       ['manufacturer', 'vendor', 'make', 'company', 'brand_name', 'supplier'],
+  image:       ['imageUrl', 'thumbnail', 'img', 'url', 'photo', 'picture', 'media', 'image_url',
+                'main_image', 'product_image', 'thumb'],
+  stock:       ['inventory', 'quantity', 'count', 'availability', 'stock_level', 'inStock',
+                'is_available', 'in stock', 'status'],
+  description: ['summary', 'content', 'body', 'text', 'info', 'details'],
+  link:        ['url', 'href', 'product_url', 'page_url', 'link'],
+};
+
+/**
+ * Dynamically registers new synonyms into the global FIELD_SYNONYMS dictionary.
+ * You can call this during your app initialization to support custom schema fields.
+ */
+export function addSynonyms(customSynonyms: Record<string, string[]>) {
+  for (const [key, aliases] of Object.entries(customSynonyms)) {
+    if (!FIELD_SYNONYMS[key]) {
+      FIELD_SYNONYMS[key] = [];
+    }
+    const existing = new Set(FIELD_SYNONYMS[key].map(s => s.toLowerCase()));
+    
+    for (const alias of aliases) {
+      if (!existing.has(alias.toLowerCase())) {
+        FIELD_SYNONYMS[key].push(alias);
+        existing.add(alias.toLowerCase());
+      }
+    }
+  }
+}
+
+/**
+ * Robustly extract a value from metadata by checking exact, case-insensitive,
+ * and synonym-based key matches.
+ */
+export function resolveMetadataValue(
+  meta: Record<string, unknown>,
+  uiKey: string,
+): unknown {
+  const synonyms = FIELD_SYNONYMS[uiKey] ?? [];
+
+  const keys = Object.keys(meta);
+  const systemKeys = ['namespace', 'filename', 'filesize', 'filetype', 'docid', 'uploadedat', 'dimension', 'chunkindex'];
+
+  // 1. Exact case-insensitive match for primary key
+  let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
+  if (match !== undefined) return meta[match];
+
+  // 2. Exact case-insensitive match for synonyms
+  match = keys.find((k) => synonyms.map(s => s.toLowerCase()).includes(k.toLowerCase()));
+  if (match !== undefined) return meta[match];
+
+  const isBlacklisted = (kl: string) => {
+    return systemKeys.includes(kl) || kl.startsWith('option1') || kl.startsWith('option2') || kl.startsWith('option3');
+  };
+
+  // 3. Partial match (substring) strictly for primary key
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return kl.includes(uiKey.toLowerCase());
+  });
+  if (match !== undefined) return meta[match];
+
+  // 4. Partial match (substring) for synonyms
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
+  });
+  
+  return match !== undefined ? meta[match] : undefined;
+}

package/dist/DocumentChunker-C1GEEosY.d.ts

@@ -1,93 +0,0 @@
-import { C as ChatMessage, c as ChatOptions } from './index-DPsQodME.js';
-
-/**
- * Generic LLM Provider interface.
- * Covers both chat completion and embedding generation so a single
- * provider can handle both responsibilities when appropriate.
- */
-
-interface EmbedOptions {
-    /** Override model for this specific embed call */
-    model?: string;
-    /** Specify the task type for models that require prefixes (e.g. nomic-embed-text) */
-    taskType?: 'query' | 'document';
-}
-interface ILLMProvider {
-    /**
-     * Send a chat completion request.
-     * @param messages – the full conversation history
-     * @param context  – retrieved RAG context to inject
-     * @param options  – optional per-call overrides
-     * @returns        – the assistant's reply text
-     */
-    chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
-    /**
-     * Send a streaming chat completion request.
-     * @returns – an async iterable of text chunks
-     */
-    chatStream?(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string>;
-    /**
-     * Generate an embedding vector for the given text.
-     * @param text    – text to embed
-     * @param options – optional overrides
-     * @returns       – float array (the embedding)
-     */
-    embed(text: string, options?: EmbedOptions): Promise<number[]>;
-    /**
-     * Generate embedding vectors for multiple texts in a single batch.
-     * @param texts   – array of texts to embed
-     * @param options – optional overrides
-     * @returns       – array of float arrays
-     */
-    batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
-    /**
-     * Check if the provider endpoint is reachable.
-     */
-    ping(): Promise<boolean>;
-}
-
-/**
- * DocumentChunker — splits long text into overlapping chunks
- * suitable for vector embedding and retrieval.
- *
- * Strategy: sentence-aware sliding window
- *   1. Split on sentence boundaries
- *   2. Accumulate sentences into chunks up to `chunkSize` characters
- *   3. Carry over `chunkOverlap` characters from the previous chunk
- */
-interface Chunk {
-    id: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChunkOptions {
-    /** Target chunk size in characters (default 1000) */
-    chunkSize?: number;
-    /** Overlap between consecutive chunks in characters (default 200) */
-    chunkOverlap?: number;
-    /** Source document identifier used as ID prefix */
-    docId?: string | number;
-    /** Extra metadata to attach to every chunk */
-    metadata?: Record<string, unknown>;
-    /** Characters used to split text, in order of priority */
-    separators?: string[];
-}
-declare class DocumentChunker {
-    private readonly chunkSize;
-    private readonly chunkOverlap;
-    private readonly separators;
-    constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
-    /**
-     * Split a single text string into overlapping chunks using a recursive strategy.
-     * Preserves structural boundaries (Markdown headers) where possible.
-     */
-    chunk(text: string, options?: ChunkOptions): Chunk[];
-    private recursiveSplit;
-    chunkMany(documents: Array<{
-        content: string;
-        docId?: string | number;
-        metadata?: Record<string, unknown>;
-    }>): Chunk[];
-}
-
-export { type Chunk as C, DocumentChunker as D, type EmbedOptions as E, type ILLMProvider as I, type ChunkOptions as a };

package/dist/DocumentChunker-CFEiRopR.d.mts

@@ -1,93 +0,0 @@
-import { C as ChatMessage, c as ChatOptions } from './index-DPsQodME.mjs';
-
-/**
- * Generic LLM Provider interface.
- * Covers both chat completion and embedding generation so a single
- * provider can handle both responsibilities when appropriate.
- */
-
-interface EmbedOptions {
-    /** Override model for this specific embed call */
-    model?: string;
-    /** Specify the task type for models that require prefixes (e.g. nomic-embed-text) */
-    taskType?: 'query' | 'document';
-}
-interface ILLMProvider {
-    /**
-     * Send a chat completion request.
-     * @param messages – the full conversation history
-     * @param context  – retrieved RAG context to inject
-     * @param options  – optional per-call overrides
-     * @returns        – the assistant's reply text
-     */
-    chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
-    /**
-     * Send a streaming chat completion request.
-     * @returns – an async iterable of text chunks
-     */
-    chatStream?(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string>;
-    /**
-     * Generate an embedding vector for the given text.
-     * @param text    – text to embed
-     * @param options – optional overrides
-     * @returns       – float array (the embedding)
-     */
-    embed(text: string, options?: EmbedOptions): Promise<number[]>;
-    /**
-     * Generate embedding vectors for multiple texts in a single batch.
-     * @param texts   – array of texts to embed
-     * @param options – optional overrides
-     * @returns       – array of float arrays
-     */
-    batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
-    /**
-     * Check if the provider endpoint is reachable.
-     */
-    ping(): Promise<boolean>;
-}
-
-/**
- * DocumentChunker — splits long text into overlapping chunks
- * suitable for vector embedding and retrieval.
- *
- * Strategy: sentence-aware sliding window
- *   1. Split on sentence boundaries
- *   2. Accumulate sentences into chunks up to `chunkSize` characters
- *   3. Carry over `chunkOverlap` characters from the previous chunk
- */
-interface Chunk {
-    id: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChunkOptions {
-    /** Target chunk size in characters (default 1000) */
-    chunkSize?: number;
-    /** Overlap between consecutive chunks in characters (default 200) */
-    chunkOverlap?: number;
-    /** Source document identifier used as ID prefix */
-    docId?: string | number;
-    /** Extra metadata to attach to every chunk */
-    metadata?: Record<string, unknown>;
-    /** Characters used to split text, in order of priority */
-    separators?: string[];
-}
-declare class DocumentChunker {
-    private readonly chunkSize;
-    private readonly chunkOverlap;
-    private readonly separators;
-    constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
-    /**
-     * Split a single text string into overlapping chunks using a recursive strategy.
-     * Preserves structural boundaries (Markdown headers) where possible.
-     */
-    chunk(text: string, options?: ChunkOptions): Chunk[];
-    private recursiveSplit;
-    chunkMany(documents: Array<{
-        content: string;
-        docId?: string | number;
-        metadata?: Record<string, unknown>;
-    }>): Chunk[];
-}
-
-export { type Chunk as C, DocumentChunker as D, type EmbedOptions as E, type ILLMProvider as I, type ChunkOptions as a };

package/dist/PostgreSQLProvider-BMOETDZA.mjs

@@ -1,8 +0,0 @@
-import {
-  PostgreSQLProvider
-} from "./chunk-FLOSGE6A.mjs";
-import "./chunk-IMP6FUCY.mjs";
-import "./chunk-X4TOT24V.mjs";
-export {
-  PostgreSQLProvider
-};

package/dist/chunk-FLOSGE6A.mjs

@@ -1,202 +0,0 @@
-import {
-  BaseVectorProvider
-} from "./chunk-IMP6FUCY.mjs";
-
-// src/providers/vectordb/PostgreSQLProvider.ts
-import { Pool } from "pg";
-var PostgreSQLProvider = class extends BaseVectorProvider {
-  constructor(config) {
-    var _a;
-    super(config);
-    this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
-    const opts = config.options;
-    if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
-    this.connectionString = opts.connectionString;
-    this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
-  }
-  static getValidator() {
-    return {
-      validate(config) {
-        const errors = [];
-        const opts = config.options || {};
-        if (!opts.connectionString) {
-          errors.push({
-            field: "vectorDb.options.connectionString",
-            message: "PostgreSQL connection string is required",
-            suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
-            severity: "error"
-          });
-        }
-        if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
-          errors.push({
-            field: "vectorDb.options.tables",
-            message: "PostgreSQL tables must be a string or a string array",
-            severity: "error"
-          });
-        }
-        return errors;
-      }
-    };
-  }
-  static getHealthChecker() {
-    return {
-      async check(config) {
-        const opts = config.options || {};
-        const timestamp = Date.now();
-        try {
-          const { Client } = await import("pg");
-          const client = new Client({ connectionString: opts.connectionString });
-          await client.connect();
-          const result = await client.query(`
-            SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
-          `);
-          const hasVector = result.rows[0].exists;
-          await client.end();
-          return {
-            healthy: true,
-            provider: "postgresql",
-            capabilities: { pgvectorInstalled: hasVector },
-            timestamp
-          };
-        } catch (error) {
-          return {
-            healthy: false,
-            provider: "postgresql",
-            error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
-            timestamp
-          };
-        }
-      }
-    };
-  }
-  async initialize() {
-    this.pool = new Pool({ connectionString: this.connectionString });
-    const client = await this.pool.connect();
-    try {
-      await client.query("CREATE EXTENSION IF NOT EXISTS vector");
-      await client.query(`
-        CREATE TABLE IF NOT EXISTS ${this.tableName} (
-          id          TEXT PRIMARY KEY,
-          namespace   TEXT NOT NULL DEFAULT '',
-          content     TEXT NOT NULL,
-          metadata    JSONB,
-          embedding   VECTOR(${this.dimensions})
-        )
-      `);
-      await client.query(`
-        CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
-        ON ${this.tableName}
-        USING hnsw (embedding vector_cosine_ops)
-      `);
-    } finally {
-      client.release();
-    }
-  }
-  async upsert(doc, namespace = "") {
-    var _a;
-    const vectorLiteral = `[${doc.vector.join(",")}]`;
-    await this.pool.query(
-      `INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
-       VALUES ($1, $2, $3, $4, $5::vector)
-       ON CONFLICT (id) DO UPDATE
-         SET namespace = EXCLUDED.namespace,
-             content   = EXCLUDED.content,
-             metadata  = EXCLUDED.metadata,
-             embedding = EXCLUDED.embedding`,
-      [doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
-    );
-  }
-  async batchUpsert(docs, namespace = "") {
-    if (docs.length === 0) return;
-    const client = await this.pool.connect();
-    try {
-      await client.query("BEGIN");
-      const BATCH_SIZE = 50;
-      for (let i = 0; i < docs.length; i += BATCH_SIZE) {
-        const batch = docs.slice(i, i + BATCH_SIZE);
-        const values = [];
-        const valuePlaceholders = batch.map((doc, idx) => {
-          var _a;
-          const offset = idx * 5;
-          values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
-          return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
-        }).join(", ");
-        const query = `
-          INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
-          VALUES ${valuePlaceholders}
-          ON CONFLICT (id) DO UPDATE
-            SET namespace = EXCLUDED.namespace,
-                content   = EXCLUDED.content,
-                metadata  = EXCLUDED.metadata,
-                embedding = EXCLUDED.embedding
-        `;
-        await client.query(query, values);
-      }
-      await client.query("COMMIT");
-    } catch (error) {
-      await client.query("ROLLBACK");
-      throw error;
-    } finally {
-      client.release();
-    }
-  }
-  async query(vector, topK, namespace, filter) {
-    const vectorLiteral = `[${vector.join(",")}]`;
-    let whereClause = namespace ? `WHERE namespace = $3` : "";
-    const params = [vectorLiteral, topK];
-    if (namespace) params.push(namespace);
-    const publicFilter = this.sanitizeFilter(filter);
-    if (Object.keys(publicFilter).length > 0) {
-      const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
-        const paramIdx = params.length + 1;
-        params.push(JSON.stringify(val));
-        return `metadata->>'${key}' = $${paramIdx}`;
-      }).join(" AND ");
-      whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
-    }
-    const client = await this.pool.connect();
-    try {
-      const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
-      await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
-      const result = await client.query(
-        `SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
-         FROM ${this.tableName}
-         ${whereClause}
-         ORDER BY embedding <=> $1::vector
-         LIMIT $2`,
-        params
-      );
-      return result.rows.map((row) => ({
-        id: String(row["id"]),
-        score: parseFloat(String(row["score"])),
-        content: String(row["content"]),
-        metadata: row["metadata"]
-      }));
-    } finally {
-      client.release();
-    }
-  }
-  async delete(id, namespace) {
-    const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
-    const params = namespace ? [id, namespace] : [id];
-    await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
-  }
-  async deleteNamespace(namespace) {
-    await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
-  }
-  async ping() {
-    try {
-      await this.pool.query("SELECT 1");
-      return true;
-    } catch (e) {
-      return false;
-    }
-  }
-  async disconnect() {
-    await this.pool.end();
-  }
-};
-
-export {
-  PostgreSQLProvider
-};