@retrivora-ai/rag-engine 0.1.0

package/src/vectordb/adapters/PgVectorAdapter.ts

@@ -0,0 +1,159 @@
+/**
+ * pgVector Adapter
+ *
+ * Stores embeddings in a PostgreSQL table using the pgvector extension.
+ * Required options in VectorDBConfig.options:
+ *   - connectionString: string  – PostgreSQL connection string
+ *   - dimensions?:      number  – vector dimensions (default 1536 for OpenAI ada-002)
+ *
+ * The adapter auto-creates the table and index on initialize().
+ */
+
+import { Pool, PoolClient } from 'pg';
+import { IVectorDB, VectorMatch, UpsertDocument } from '../IVectorDB';
+import { VectorDBConfig } from '../../config/RagConfig';
+
+export class PgVectorAdapter implements IVectorDB {
+  private pool!: Pool;
+  private readonly tableName: string;
+  private readonly dimensions: number;
+  private readonly connectionString: string;
+
+  constructor(config: VectorDBConfig) {
+    this.tableName = config.indexName.replace(/[^a-z0-9_]/gi, '_'); // sanitize
+    const opts = config.options as Record<string, unknown>;
+    if (!opts.connectionString) {
+      throw new Error('[PgVectorAdapter] options.connectionString is required');
+    }
+    this.connectionString = opts.connectionString as string;
+    this.dimensions = (opts.dimensions as number) ?? 1536;
+  }
+
+  async initialize(): Promise<void> {
+    this.pool = new Pool({ connectionString: this.connectionString });
+
+    const client: PoolClient = await this.pool.connect();
+    try {
+      // Enable pgvector extension
+      await client.query('CREATE EXTENSION IF NOT EXISTS vector');
+
+      // Create the chunks table if it doesn't exist
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.tableName} (
+          id          TEXT PRIMARY KEY,
+          namespace   TEXT NOT NULL DEFAULT '',
+          content     TEXT NOT NULL,
+          metadata    JSONB,
+          embedding   VECTOR(${this.dimensions})
+        )
+      `);
+
+      // Create HNSW index for fast approximate nearest-neighbour search
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
+        ON ${this.tableName}
+        USING hnsw (embedding vector_cosine_ops)
+      `);
+    } finally {
+      client.release();
+    }
+  }
+
+  async upsert(doc: UpsertDocument, namespace = ''): Promise<void> {
+    const vectorLiteral = `[${doc.vector.join(',')}]`;
+    await this.pool.query(
+      `INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
+       VALUES ($1, $2, $3, $4, $5::vector)
+       ON CONFLICT (id) DO UPDATE
+         SET namespace = EXCLUDED.namespace,
+             content   = EXCLUDED.content,
+             metadata  = EXCLUDED.metadata,
+             embedding = EXCLUDED.embedding`,
+      [doc.id, namespace, doc.content, JSON.stringify(doc.metadata ?? {}), vectorLiteral]
+    );
+  }
+
+  async batchUpsert(docs: UpsertDocument[], namespace = ''): Promise<void> {
+    // Sequential for simplicity; swap for COPY or multi-row INSERT for perf
+    for (const doc of docs) {
+      await this.upsert(doc, namespace);
+    }
+  }
+
+  async query(
+    vector: number[],
+    topK: number,
+    namespace?: string,
+    filter?: Record<string, unknown>
+  ): Promise<VectorMatch[]> {
+    const vectorLiteral = `[${vector.join(',')}]`;
+
+    let whereClause = namespace ? `WHERE namespace = $3` : '';
+    const params: unknown[] = [vectorLiteral, topK];
+    if (namespace) params.push(namespace);
+
+    // Append metadata filter conditions if provided
+    if (filter && Object.keys(filter).length > 0) {
+      const filterConditions = Object.entries(filter)
+        .map(([key, val]) => {
+          const paramIdx = params.length + 1;
+          params.push(JSON.stringify(val));
+          return `metadata->>'${key}' = $${paramIdx}`;
+        })
+        .join(' AND ');
+      whereClause = whereClause
+        ? `${whereClause} AND ${filterConditions}`
+        : `WHERE ${filterConditions}`;
+    }
+
+    const result = await this.pool.query(
+      `SELECT id,
+              content,
+              metadata,
+              1 - (embedding <=> $1::vector) AS score
+       FROM   ${this.tableName}
+       ${whereClause}
+       ORDER  BY embedding <=> $1::vector
+       LIMIT  $2`,
+      params
+    );
+
+    return result.rows.map((row: Record<string, unknown>) => ({
+      id: String(row.id),
+      score: parseFloat(String(row.score)),
+      content: String(row.content),
+      metadata: row.metadata as Record<string, unknown>,
+    }));
+  }
+
+  async delete(id: string, namespace?: string): Promise<void> {
+    const where = namespace ? 'WHERE id = $1 AND namespace = $2' : 'WHERE id = $1';
+    const params = namespace ? [id, namespace] : [id];
+    await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
+  }
+
+  async deleteNamespace(namespace: string): Promise<void> {
+    await this.pool.query(
+      `DELETE FROM ${this.tableName} WHERE namespace = $1`,
+      [namespace]
+    );
+  }
+
+  async ping(): Promise<boolean> {
+    try {
+      await this.pool.query('SELECT 1');
+      return true;
+    } catch (err) {
+      console.error('[PgVectorAdapter] Ping failed:', err);
+      return false;
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    try {
+      await this.pool.end();
+    } catch (err) {
+      console.error('[PgVectorAdapter] Failed to disconnect:', err);
+    }
+  }
+}

package/src/vectordb/adapters/PineconeAdapter.ts

@@ -0,0 +1,115 @@
+/**
+ * Pinecone Vector DB Adapter
+ *
+ * Required options in VectorDBConfig.options:
+ *   - apiKey:       string  – Pinecone API key
+ *   - environment?: string  – Pinecone environment (legacy, optional for new projects)
+ */
+
+import { Pinecone, RecordMetadata } from '@pinecone-database/pinecone';
+import { IVectorDB, VectorMatch, UpsertDocument } from '../IVectorDB';
+import { VectorDBConfig } from '../../config/RagConfig';
+
+export class PineconeAdapter implements IVectorDB {
+  private client!: Pinecone;
+  private readonly indexName: string;
+  private readonly apiKey: string;
+
+  constructor(config: VectorDBConfig) {
+    this.indexName = config.indexName;
+    const opts = config.options as Record<string, string>;
+    if (!opts.apiKey) throw new Error('[PineconeAdapter] options.apiKey is required');
+    this.apiKey = opts.apiKey;
+  }
+
+  async initialize(): Promise<void> {
+    this.client = new Pinecone({ apiKey: this.apiKey });
+    // Verify index exists
+    const indexes = await this.client.listIndexes();
+    const names = indexes.indexes?.map((i) => i.name) ?? [];
+    if (!names.includes(this.indexName)) {
+      throw new Error(
+        `[PineconeAdapter] Index "${this.indexName}" not found. ` +
+          `Available: ${names.join(', ')}`
+      );
+    }
+  }
+
+  private index(namespace?: string) {
+    const idx = this.client.index(this.indexName);
+    return namespace ? idx.namespace(namespace) : idx.namespace('');
+  }
+
+  async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
+    await this.index(namespace).upsert({
+      records: [
+        {
+          id: doc.id,
+          values: doc.vector,
+          metadata: {
+            content: doc.content,
+            ...(doc.metadata ?? {}),
+          } as RecordMetadata,
+        },
+      ],
+    });
+  }
+
+  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
+    const BATCH = 100;
+    for (let i = 0; i < docs.length; i += BATCH) {
+      const records = docs.slice(i, i + BATCH).map((d) => ({
+        id: d.id,
+        values: d.vector,
+        metadata: {
+          content: d.content,
+          ...(d.metadata ?? {}),
+        } as RecordMetadata,
+      }));
+      await this.index(namespace).upsert({ records });
+    }
+  }
+
+  async query(
+    vector: number[],
+    topK: number,
+    namespace?: string,
+    filter?: Record<string, unknown>
+  ): Promise<VectorMatch[]> {
+    const result = await this.index(namespace).query({
+      vector,
+      topK,
+      includeMetadata: true,
+      ...(filter ? { filter } : {}),
+    });
+
+    return (result.matches ?? []).map((m) => ({
+      id: m.id,
+      score: m.score ?? 0,
+      content: (m.metadata?.content as string) ?? '',
+      metadata: m.metadata as Record<string, unknown>,
+    }));
+  }
+
+  async delete(id: string, namespace?: string): Promise<void> {
+    await this.index(namespace).deleteOne({ id });
+  }
+
+  async deleteNamespace(namespace: string): Promise<void> {
+    await this.client.index(this.indexName).namespace(namespace).deleteAll();
+  }
+
+  async ping(): Promise<boolean> {
+    try {
+      await this.client.listIndexes();
+      return true;
+    } catch (err) {
+      console.error('[PineconeAdapter] Ping failed:', err);
+      return false;
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    // Pinecone SDK is stateless REST, no persistent connection to close.
+  }
+}

package/src/vectordb/adapters/UniversalVectorDBAdapter.ts

@@ -0,0 +1,177 @@
+import axios, { AxiosInstance } from 'axios';
+import { IVectorDB, VectorMatch, UpsertDocument } from '../IVectorDB';
+import { VectorDBConfig } from '../../config/RagConfig';
+import { resolvePath, buildPayload } from '../../utils/templateUtils';
+import { VECTOR_PROFILES } from '../../config/UniversalProfiles';
+
+interface UniversalVectorDBOptions {
+  baseUrl?: string;
+  headers?: Record<string, string>;
+  queryPath?: string;
+  upsertPath?: string;
+  deletePath?: string;
+  deleteNamespacePath?: string;
+  pingPath?: string;
+  queryPayloadTemplate?: string;
+  upsertPayloadTemplate?: string;
+  batchUpsertPayloadTemplate?: string;
+  responseExtractPath?: string;
+  idPath?: string;
+  scorePath?: string;
+  contentPath?: string;
+  metadataPath?: string;
+  timeout?: number;
+}
+
+export class UniversalVectorDBAdapter implements IVectorDB {
+  private readonly http: AxiosInstance;
+  private readonly indexName: string;
+  private readonly opts: UniversalVectorDBOptions;
+
+  constructor(config: VectorDBConfig) {
+    this.indexName = config.indexName;
+    
+    // Merge profile defaults if specified
+    const options = (config.options as Record<string, unknown>) ?? {};
+    let profile: Record<string, unknown> = {};
+
+    if (typeof options.profile === 'string') {
+      profile = (VECTOR_PROFILES as Record<string, Record<string, unknown>>)[options.profile] || {};
+    } else if (typeof options.profile === 'object' && options.profile !== null) {
+      profile = options.profile as Record<string, unknown>;
+    }
+
+    this.opts = { ...profile, ...(config.options ?? {}) } as UniversalVectorDBOptions;
+
+    if (!this.opts.baseUrl) {
+      throw new Error('[UniversalVectorDBAdapter] options.baseUrl is required');
+    }
+
+    this.http = axios.create({
+      baseURL: this.opts.baseUrl,
+      headers: {
+        'Content-Type': 'application/json',
+        ...(this.opts.headers || {}),
+      },
+      timeout: this.opts.timeout ?? 30_000,
+    });
+  }
+
+  async initialize(): Promise<void> {
+    if (this.opts.pingPath) {
+      const ok = await this.ping();
+      if (!ok) {
+        throw new Error(`[UniversalVectorDBAdapter] Could not reach health endpoint: ${this.opts.pingPath}`);
+      }
+    }
+  }
+
+  async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
+    const path = this.opts.upsertPath ?? '/upsert';
+    
+    if (this.opts.upsertPayloadTemplate) {
+      const payload = buildPayload(this.opts.upsertPayloadTemplate, {
+        id: doc.id,
+        vector: doc.vector,
+        content: doc.content,
+        metadata: doc.metadata ?? {},
+        namespace: namespace ?? '',
+        index: this.indexName,
+      });
+      await this.http.post(path, payload);
+    } else {
+      // Fallback to standard generic shape
+      await this.http.post(path, {
+        index: this.indexName,
+        namespace,
+        ...doc,
+      });
+    }
+  }
+
+  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
+    const path = this.opts.upsertPath ?? '/upsert';
+    
+    // For batch upsert with templating, if the user provided batchUpsertPayloadTemplate
+    if (this.opts.batchUpsertPayloadTemplate) {
+       const payload = buildPayload(this.opts.batchUpsertPayloadTemplate, {
+          vectors: docs,
+          namespace: namespace ?? '',
+          index: this.indexName,
+       });
+       await this.http.post(path, payload);
+    } else {
+       // Fallback to standard loop or generic shape
+       await Promise.all(docs.map((doc) => this.upsert(doc, namespace)));
+    }
+  }
+
+  async query(
+    vector: number[],
+    topK: number,
+    namespace?: string,
+    filter?: Record<string, unknown>
+  ): Promise<VectorMatch[]> {
+    const path = this.opts.queryPath ?? '/query';
+
+    let payload: unknown;
+    if (this.opts.queryPayloadTemplate) {
+      payload = buildPayload(this.opts.queryPayloadTemplate, {
+        vector,
+        topK,
+        namespace: namespace ?? '',
+        index: this.indexName,
+        filter: filter ?? {},
+      });
+    } else {
+      payload = { index: this.indexName, namespace, vector, topK, filter };
+    }
+
+    const { data } = await this.http.post(path, payload);
+
+    const extractPath = this.opts.responseExtractPath ?? 'matches';
+    const matchesRaw = resolvePath(data, extractPath);
+
+    if (!Array.isArray(matchesRaw)) {
+       throw new Error(`[UniversalVectorDBAdapter] Expected an array at responseExtractPath '${extractPath}', but got ${typeof matchesRaw}`);
+    }
+
+    const idPath = this.opts.idPath ?? 'id';
+    const scorePath = this.opts.scorePath ?? 'score';
+    const contentPath = this.opts.contentPath ?? 'content';
+    const metadataPath = this.opts.metadataPath ?? 'metadata';
+
+    return matchesRaw.map((match) => ({
+      id: String(resolvePath(match, idPath) || ''),
+      score: Number(resolvePath(match, scorePath) || 0),
+      content: String(resolvePath(match, contentPath) || ''),
+      metadata: (resolvePath(match, metadataPath) as Record<string, unknown> | undefined) || {},
+    }));
+  }
+
+  async delete(id: string, namespace?: string): Promise<void> {
+    const path = this.opts.deletePath ?? '/delete';
+    await this.http.post(path, { index: this.indexName, namespace, id });
+  }
+
+  async deleteNamespace(namespace: string): Promise<void> {
+    const path = this.opts.deleteNamespacePath ?? '/delete-namespace';
+    await this.http.post(path, { index: this.indexName, namespace });
+  }
+
+  async ping(): Promise<boolean> {
+    try {
+      if (this.opts.pingPath) {
+        await this.http.get(this.opts.pingPath);
+      }
+      return true;
+    } catch (err) {
+      console.error('[UniversalVectorDBAdapter] Ping failed:', err);
+      return false;
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    // Axios based adapter is stateless per request.
+  }
+}