@opensaas/stack-rag 0.1.6

package/src/storage/index.ts

@@ -0,0 +1,86 @@
+import type { VectorStorage } from './types.js'
+import type { VectorStorageConfig } from '../config/types.js'
+import { createJsonStorage } from './json.js'
+import { createPgVectorStorage } from './pgvector.js'
+import { createSqliteVssStorage } from './sqlite-vss.js'
+
+/**
+ * Storage backend factory registry
+ * Maps storage types to factory functions
+ */
+const storageFactories = new Map<string, (config: VectorStorageConfig) => VectorStorage>()
+
+/**
+ * Register the built-in storage backends
+ */
+storageFactories.set('json', () => createJsonStorage())
+storageFactories.set('pgvector', (config) => {
+  if (config.type !== 'pgvector') {
+    throw new Error('Invalid config type for pgvector storage')
+  }
+  return createPgVectorStorage(config as import('../config/types.js').PgVectorStorageConfig)
+})
+storageFactories.set('sqlite-vss', (config) => {
+  if (config.type !== 'sqlite-vss') {
+    throw new Error('Invalid config type for sqlite-vss storage')
+  }
+  return createSqliteVssStorage(config as import('../config/types.js').SqliteVssStorageConfig)
+})
+
+/**
+ * Register a custom vector storage backend
+ * Use this to add support for custom storage backends (e.g., Pinecone, Qdrant)
+ *
+ * @example
+ * ```typescript
+ * import { registerVectorStorage } from '@opensaas/stack-rag/storage'
+ *
+ * registerVectorStorage('pinecone', (config) => {
+ *   return new PineconeVectorStorage(config)
+ * })
+ * ```
+ */
+export function registerVectorStorage(
+  type: string,
+  factory: (config: VectorStorageConfig) => VectorStorage,
+): void {
+  storageFactories.set(type, factory)
+}
+
+/**
+ * Create a vector storage instance from configuration
+ * Automatically selects the correct backend based on config.type
+ *
+ * @example
+ * ```typescript
+ * import { createVectorStorage } from '@opensaas/stack-rag/storage'
+ *
+ * const storage = createVectorStorage({
+ *   type: 'pgvector',
+ *   distanceFunction: 'cosine'
+ * })
+ *
+ * const results = await storage.search('Article', 'contentEmbedding', queryVector, {
+ *   limit: 10,
+ *   context
+ * })
+ * ```
+ */
+export function createVectorStorage(config: VectorStorageConfig): VectorStorage {
+  const factory = storageFactories.get(config.type)
+
+  if (!factory) {
+    throw new Error(
+      `Unknown vector storage type: ${config.type}. ` +
+        `Available backends: ${Array.from(storageFactories.keys()).join(', ')}`,
+    )
+  }
+
+  return factory(config)
+}
+
+// Export types and individual storage backends
+export * from './types.js'
+export { JsonVectorStorage, createJsonStorage } from './json.js'
+export { PgVectorStorage, createPgVectorStorage } from './pgvector.js'
+export { SqliteVssStorage, createSqliteVssStorage } from './sqlite-vss.js'

package/src/storage/json.ts

@@ -0,0 +1,106 @@
+import type { VectorStorage, SearchOptions } from './types.js'
+import type { SearchResult, StoredEmbedding } from '../config/types.js'
+import { cosineSimilarity as calculateCosineSimilarity } from './types.js'
+import { getDbKey } from '@opensaas/stack-core'
+
+/**
+ * JSON-based vector storage
+ * Stores vectors as JSON in the database and performs similarity search in JavaScript
+ * Good for development and small datasets, doesn't require special database extensions
+ */
+export class JsonVectorStorage implements VectorStorage {
+  readonly type = 'json'
+
+  /**
+   * Search for similar vectors using JavaScript-based cosine similarity
+   */
+  async search<T = unknown>(
+    listKey: string,
+    fieldName: string,
+    queryVector: number[],
+    options: SearchOptions,
+  ): Promise<SearchResult<T>[]> {
+    const { limit = 10, minScore = 0.0, context, where = {} } = options
+
+    const dbKey = getDbKey(listKey)
+    const model = context.db[dbKey]
+
+    if (!model) {
+      throw new Error(`List '${listKey}' not found in context.db`)
+    }
+
+    // Fetch all items with embeddings (access control applied via context)
+    // We need to fetch all items because we're doing similarity scoring in JS
+    const items = await model.findMany({
+      where: {
+        ...where,
+        [fieldName]: {
+          not: null,
+        },
+      },
+    })
+
+    // Calculate similarity for each item
+    const results: Array<{ item: T; score: number; distance: number }> = []
+
+    for (const item of items) {
+      const embeddingData = item[fieldName] as StoredEmbedding | null
+
+      if (!embeddingData || !embeddingData.vector) {
+        continue
+      }
+
+      const storedVector = embeddingData.vector
+
+      // Validate vector dimensions
+      if (storedVector.length !== queryVector.length) {
+        console.warn(
+          `Vector dimension mismatch for ${listKey}.${item.id}.${fieldName}: ` +
+            `expected ${queryVector.length}, got ${storedVector.length}. Skipping.`,
+        )
+        continue
+      }
+
+      // Calculate cosine similarity
+      const score = this.cosineSimilarity(queryVector, storedVector)
+
+      if (score >= minScore) {
+        results.push({
+          item: item as T,
+          score,
+          distance: 1 - score, // Convert similarity to distance
+        })
+      }
+    }
+
+    // Sort by score (descending) and limit results
+    results.sort((a, b) => b.score - a.score)
+
+    return results.slice(0, limit)
+  }
+
+  /**
+   * Calculate cosine similarity between two vectors
+   */
+  cosineSimilarity(a: number[], b: number[]): number {
+    return calculateCosineSimilarity(a, b)
+  }
+}
+
+/**
+ * Create a JSON vector storage instance
+ *
+ * @example
+ * ```typescript
+ * import { createJsonStorage } from '@opensaas/stack-rag/storage'
+ *
+ * const storage = createJsonStorage()
+ * const results = await storage.search('Article', 'contentEmbedding', queryVector, {
+ *   limit: 10,
+ *   context
+ * })
+ * ```
+ */
+export function createJsonStorage(): JsonVectorStorage {
+  return new JsonVectorStorage()
+}

package/src/storage/pgvector.ts

@@ -0,0 +1,206 @@
+import type { VectorStorage, SearchOptions } from './types.js'
+import type { SearchResult } from '../config/types.js'
+import type { PgVectorStorageConfig } from '../config/types.js'
+import { cosineSimilarity as calculateCosineSimilarity } from './types.js'
+import { getDbKey } from '@opensaas/stack-core'
+
+/**
+ * pgvector storage backend
+ * Uses PostgreSQL with pgvector extension for efficient vector similarity search
+ * Requires: CREATE EXTENSION vector;
+ */
+export class PgVectorStorage implements VectorStorage {
+  readonly type = 'pgvector'
+  private distanceFunction: 'cosine' | 'l2' | 'inner_product'
+
+  constructor(config: PgVectorStorageConfig) {
+    this.distanceFunction = config.distanceFunction || 'cosine'
+  }
+
+  /**
+   * Get the appropriate distance operator for pgvector
+   */
+  private getDistanceOperator(): string {
+    switch (this.distanceFunction) {
+      case 'cosine':
+        return '<=>' // Cosine distance
+      case 'l2':
+        return '<->' // L2 distance
+      case 'inner_product':
+        return '<#>' // Inner product (negative, so smaller is more similar)
+      default:
+        return '<=>' // Default to cosine
+    }
+  }
+
+  /**
+   * Convert distance to similarity score (0-1, higher is more similar)
+   */
+  private distanceToScore(distance: number): number {
+    switch (this.distanceFunction) {
+      case 'cosine':
+        // Cosine distance is 1 - similarity, so similarity = 1 - distance
+        return 1 - distance
+      case 'l2':
+        // L2 distance: convert to similarity using 1 / (1 + distance)
+        return 1 / (1 + distance)
+      case 'inner_product':
+        // Inner product: larger (less negative) is more similar
+        // Convert to 0-1 range
+        return -distance
+      default:
+        return 1 - distance
+    }
+  }
+
+  /**
+   * Search for similar vectors using pgvector
+   */
+  async search<T = unknown>(
+    listKey: string,
+    fieldName: string,
+    queryVector: number[],
+    options: SearchOptions,
+  ): Promise<SearchResult<T>[]> {
+    const { limit = 10, minScore = 0.0, context, where = {} } = options
+
+    const dbKey = getDbKey(listKey)
+    const model = context.db[dbKey]
+
+    if (!model) {
+      throw new Error(`List '${listKey}' not found in context.db`)
+    }
+
+    const distanceOp = this.getDistanceOperator()
+
+    // Build the vector string for Prisma raw query
+    // pgvector expects vectors in format: '[1,2,3]'
+    const vectorString = `[${queryVector.join(',')}]`
+
+    // We need to use Prisma.$queryRaw to access pgvector operators
+    // The access-controlled context.db doesn't expose $queryRaw directly,
+    // so we need to use a two-step approach:
+    // 1. Get all matching IDs using raw query
+    // 2. Fetch full items via access-controlled context
+
+    try {
+      // Get the underlying Prisma client
+      // Note: This bypasses access control for the similarity search,
+      // but we enforce it in the second query
+      const prisma = context.prisma
+
+      if (!prisma) {
+        // Fallback: if we can't access Prisma directly, use JSON storage approach
+        console.warn(
+          'pgvector: Could not access Prisma client directly. ' +
+            'Falling back to JSON-based search. ' +
+            'For full pgvector support, ensure the context exposes _prisma.',
+        )
+        return this.fallbackSearch(listKey, fieldName, queryVector, options)
+      }
+
+      // Raw query to get IDs and distances
+      // We extract the vector from the JSON field and cast it to vector type
+      const tableName = listKey.toLowerCase() // Prisma table names are lowercase
+      const results = (await prisma.$queryRawUnsafe(`
+        SELECT id,
+               (("${fieldName}"->>'vector')::vector ${distanceOp} '${vectorString}'::vector) as distance
+        FROM "${tableName}"
+        WHERE "${fieldName}" IS NOT NULL
+          AND "${fieldName}"->>'vector' IS NOT NULL
+        ORDER BY distance
+        LIMIT ${limit * 2}
+      `)) as Array<{ id: string; distance: string }>
+
+      // Get IDs of items within score threshold
+      const itemIds = results
+        .map((r) => ({
+          id: r.id,
+          distance: Number(r.distance),
+          score: this.distanceToScore(Number(r.distance)),
+        }))
+        .filter((r) => r.score >= minScore)
+        .slice(0, limit)
+        .map((r) => ({ id: r.id, distance: r.distance, score: r.score }))
+
+      if (itemIds.length === 0) {
+        return []
+      }
+
+      // Fetch full items via access-controlled context
+      const items = await model.findMany({
+        where: {
+          ...where,
+          id: {
+            in: itemIds.map((r) => r.id),
+          },
+        },
+      })
+
+      // Match items with their scores and sort by score
+      const searchResults: SearchResult<T>[] = []
+      for (const idInfo of itemIds) {
+        const item = items.find(
+          (i: Record<string, unknown>) => (i as { id: string }).id === idInfo.id,
+        )
+        if (item) {
+          searchResults.push({
+            item: item as T,
+            score: idInfo.score,
+            distance: idInfo.distance,
+          })
+        }
+      }
+
+      return searchResults
+    } catch (error) {
+      throw new Error(
+        `pgvector search failed: ${(error as Error).message}\n` +
+          'Ensure pgvector extension is installed: CREATE EXTENSION vector;',
+      )
+    }
+  }
+
+  /**
+   * Fallback to JSON-based search if we can't access Prisma directly
+   */
+  private async fallbackSearch<T = unknown>(
+    listKey: string,
+    fieldName: string,
+    queryVector: number[],
+    options: SearchOptions,
+  ): Promise<SearchResult<T>[]> {
+    const { JsonVectorStorage } = await import('./json.js')
+    const jsonStorage = new JsonVectorStorage()
+    return jsonStorage.search(listKey, fieldName, queryVector, options)
+  }
+
+  /**
+   * Calculate cosine similarity between two vectors
+   */
+  cosineSimilarity(a: number[], b: number[]): number {
+    return calculateCosineSimilarity(a, b)
+  }
+}
+
+/**
+ * Create a pgvector storage instance
+ *
+ * @example
+ * ```typescript
+ * import { createPgVectorStorage } from '@opensaas/stack-rag/storage'
+ *
+ * const storage = createPgVectorStorage({
+ *   type: 'pgvector',
+ *   distanceFunction: 'cosine'
+ * })
+ *
+ * const results = await storage.search('Article', 'contentEmbedding', queryVector, {
+ *   limit: 10,
+ *   context
+ * })
+ * ```
+ */
+export function createPgVectorStorage(config: PgVectorStorageConfig): PgVectorStorage {
+  return new PgVectorStorage(config)
+}

package/src/storage/sqlite-vss.ts

@@ -0,0 +1,193 @@
+import type { VectorStorage, SearchOptions } from './types.js'
+import type { SearchResult } from '../config/types.js'
+import type { SqliteVssStorageConfig } from '../config/types.js'
+import { cosineSimilarity as calculateCosineSimilarity } from './types.js'
+import { getDbKey } from '@opensaas/stack-core'
+
+/**
+ * SQLite VSS storage backend
+ * Uses sqlite-vss extension for vector similarity search
+ * Requires: sqlite-vss extension to be loaded
+ */
+export class SqliteVssStorage implements VectorStorage {
+  readonly type = 'sqlite-vss'
+  private distanceFunction: 'cosine' | 'l2'
+
+  constructor(config: SqliteVssStorageConfig) {
+    this.distanceFunction = config.distanceFunction || 'cosine'
+  }
+
+  /**
+   * Convert distance to similarity score (0-1, higher is more similar)
+   */
+  private distanceToScore(distance: number): number {
+    if (this.distanceFunction === 'cosine') {
+      // Cosine distance is 1 - similarity
+      return 1 - distance
+    } else {
+      // L2 distance: convert to similarity using 1 / (1 + distance)
+      return 1 / (1 + distance)
+    }
+  }
+
+  /**
+   * Search for similar vectors using sqlite-vss
+   */
+  async search<T = unknown>(
+    listKey: string,
+    fieldName: string,
+    queryVector: number[],
+    options: SearchOptions,
+  ): Promise<SearchResult<T>[]> {
+    const { limit = 10, minScore = 0.0, context, where = {} } = options
+
+    const dbKey = getDbKey(listKey)
+    const model = context.db[dbKey]
+
+    if (!model) {
+      throw new Error(`List '${listKey}' not found in context.db`)
+    }
+
+    try {
+      // Get the underlying Prisma client
+      const prisma = context.prisma
+
+      if (!prisma) {
+        // Fallback: if we can't access Prisma directly, use JSON storage approach
+        console.warn(
+          'sqlite-vss: Could not access Prisma client directly. ' +
+            'Falling back to JSON-based search. ' +
+            'For full sqlite-vss support, ensure the context exposes _prisma.',
+        )
+        return this.fallbackSearch(listKey, fieldName, queryVector, options)
+      }
+
+      // Build JSON array string for the vector
+      // Note: vectorString would be used for native sqlite-vss queries
+      // Currently using fallback JS-based search
+      // const vectorString = JSON.stringify(queryVector)
+
+      // Table name (Prisma uses PascalCase in schema but lowercases in DB)
+      // Note: tableName would be used for native sqlite-vss queries
+      // const tableName = listKey
+
+      // SQLite VSS query
+      // We need to create a virtual table for VSS search
+      // For now, we'll use a simpler approach: extract vectors and compute in JS
+      // Full sqlite-vss integration would require creating virtual tables at schema generation time
+
+      // Query to get all items with embeddings
+      const items = await model.findMany({
+        where: {
+          ...where,
+          [fieldName]: {
+            not: null,
+          },
+        },
+      })
+
+      // Calculate similarity for each item (JavaScript fallback)
+      const results: Array<{ item: T; score: number; distance: number }> = []
+
+      for (const item of items) {
+        const embeddingData = item[fieldName] as
+          | import('../config/types.js').StoredEmbedding
+          | null
+          | undefined
+
+        if (!embeddingData || !embeddingData.vector) {
+          continue
+        }
+
+        const storedVector = embeddingData.vector
+
+        // Validate vector dimensions
+        if (storedVector.length !== queryVector.length) {
+          console.warn(
+            `Vector dimension mismatch for ${listKey}.${item.id}.${fieldName}: ` +
+              `expected ${queryVector.length}, got ${storedVector.length}. Skipping.`,
+          )
+          continue
+        }
+
+        // Calculate similarity
+        let distance: number
+        if (this.distanceFunction === 'cosine') {
+          const similarity = this.cosineSimilarity(queryVector, storedVector)
+          distance = 1 - similarity
+        } else {
+          // L2 distance
+          distance = Math.sqrt(
+            storedVector.reduce((sum: number, val: number, i: number) => {
+              const diff = val - queryVector[i]
+              return sum + diff * diff
+            }, 0),
+          )
+        }
+
+        const score = this.distanceToScore(distance)
+
+        if (score >= minScore) {
+          results.push({
+            item: item as T,
+            score,
+            distance,
+          })
+        }
+      }
+
+      // Sort by score (descending) and limit results
+      results.sort((a, b) => b.score - a.score)
+
+      return results.slice(0, limit)
+    } catch (error) {
+      throw new Error(
+        `sqlite-vss search failed: ${(error as Error).message}\n` +
+          'Ensure sqlite-vss extension is loaded in your SQLite connection.',
+      )
+    }
+  }
+
+  /**
+   * Fallback to JSON-based search if we can't access Prisma directly
+   */
+  private async fallbackSearch<T = unknown>(
+    listKey: string,
+    fieldName: string,
+    queryVector: number[],
+    options: SearchOptions,
+  ): Promise<SearchResult<T>[]> {
+    const { JsonVectorStorage } = await import('./json.js')
+    const jsonStorage = new JsonVectorStorage()
+    return jsonStorage.search(listKey, fieldName, queryVector, options)
+  }
+
+  /**
+   * Calculate cosine similarity between two vectors
+   */
+  cosineSimilarity(a: number[], b: number[]): number {
+    return calculateCosineSimilarity(a, b)
+  }
+}
+
+/**
+ * Create a SQLite VSS storage instance
+ *
+ * @example
+ * ```typescript
+ * import { createSqliteVssStorage } from '@opensaas/stack-rag/storage'
+ *
+ * const storage = createSqliteVssStorage({
+ *   type: 'sqlite-vss',
+ *   distanceFunction: 'cosine'
+ * })
+ *
+ * const results = await storage.search('Article', 'contentEmbedding', queryVector, {
+ *   limit: 10,
+ *   context
+ * })
+ * ```
+ */
+export function createSqliteVssStorage(config: SqliteVssStorageConfig): SqliteVssStorage {
+  return new SqliteVssStorage(config)
+}