@opensaas/stack-rag 0.1.6

package/src/runtime/embeddings.ts

@@ -0,0 +1,380 @@
+/**
+ * High-level embedding generation utilities
+ */
+
+import type { EmbeddingProvider } from '../providers/types.js'
+import type { StoredEmbedding, EmbeddingMetadata } from '../config/types.js'
+import { chunkText, type ChunkingOptions, type TextChunk } from './chunking.js'
+import { createHash } from 'node:crypto'
+
+export interface GenerateEmbeddingOptions {
+  /**
+   * Embedding provider to use
+   */
+  provider: EmbeddingProvider
+
+  /**
+   * Text to embed
+   */
+  text: string
+
+  /**
+   * Whether to enable text chunking for long documents
+   * @default false
+   */
+  enableChunking?: boolean
+
+  /**
+   * Chunking configuration (only used if enableChunking is true)
+   */
+  chunking?: ChunkingOptions
+
+  /**
+   * Whether to include source hash in metadata for change detection
+   * @default true
+   */
+  includeSourceHash?: boolean
+
+  /**
+   * Additional metadata to include
+   */
+  metadata?: Record<string, unknown>
+}
+
+export interface ChunkedEmbedding {
+  /**
+   * The chunk information
+   */
+  chunk: TextChunk
+
+  /**
+   * The stored embedding for this chunk
+   */
+  embedding: StoredEmbedding
+}
+
+/**
+ * Generate embedding for text with automatic chunking support
+ *
+ * For single embeddings (no chunking), returns a StoredEmbedding.
+ * For chunked text, returns an array of ChunkedEmbeddings.
+ *
+ * @example
+ * ```typescript
+ * // Simple embedding
+ * const embedding = await generateEmbedding({
+ *   provider: createEmbeddingProvider({ type: 'openai', apiKey: '...' }),
+ *   text: 'Hello world',
+ * })
+ *
+ * // Chunked embedding for long text
+ * const chunks = await generateEmbedding({
+ *   provider: createEmbeddingProvider({ type: 'openai', apiKey: '...' }),
+ *   text: longDocument,
+ *   enableChunking: true,
+ *   chunking: { chunkSize: 1000, chunkOverlap: 200 },
+ * })
+ * ```
+ */
+// Overload signatures
+
+export function generateEmbedding(
+  options: GenerateEmbeddingOptions & { enableChunking: true },
+): Promise<ChunkedEmbedding[]>
+// eslint-disable-next-line no-redeclare
+export function generateEmbedding(
+  options: GenerateEmbeddingOptions & { enableChunking?: false },
+): Promise<StoredEmbedding>
+// eslint-disable-next-line no-redeclare
+export function generateEmbedding(
+  options: GenerateEmbeddingOptions,
+): Promise<StoredEmbedding | ChunkedEmbedding[]>
+// Implementation
+// eslint-disable-next-line no-redeclare
+export async function generateEmbedding(
+  options: GenerateEmbeddingOptions,
+): Promise<StoredEmbedding | ChunkedEmbedding[]> {
+  const {
+    provider,
+    text,
+    enableChunking = false,
+    chunking,
+    includeSourceHash = true,
+    metadata: additionalMetadata,
+  } = options
+
+  const sourceHash = includeSourceHash ? hashText(text) : undefined
+
+  // Generate base metadata
+  const baseMetadata: EmbeddingMetadata = {
+    model: provider.model,
+    provider: provider.type,
+    dimensions: provider.dimensions,
+    generatedAt: new Date().toISOString(),
+    sourceHash,
+  }
+
+  // Without chunking, generate single embedding
+  if (!enableChunking) {
+    const vector = await provider.embed(text)
+
+    return {
+      vector,
+      metadata: {
+        ...baseMetadata,
+        ...additionalMetadata,
+      },
+    }
+  }
+
+  // With chunking, split text and generate embeddings for each chunk
+  const chunks = chunkText(text, chunking)
+
+  // Extract chunk texts
+  const chunkTexts = chunks.map((c) => c.text)
+
+  // Generate embeddings for all chunks in batch
+  const vectors = await provider.embedBatch(chunkTexts)
+
+  // Combine chunks with their embeddings
+  const chunkedEmbeddings: ChunkedEmbedding[] = chunks.map((chunk, index) => ({
+    chunk,
+    embedding: {
+      vector: vectors[index],
+      metadata: {
+        ...baseMetadata,
+        ...additionalMetadata,
+        chunkIndex: index,
+        chunkStart: chunk.start,
+        chunkEnd: chunk.end,
+      },
+    },
+  }))
+
+  return chunkedEmbeddings
+}
+
+export interface GenerateEmbeddingsOptions {
+  /**
+   * Embedding provider to use
+   */
+  provider: EmbeddingProvider
+
+  /**
+   * Array of texts to embed
+   */
+  texts: string[]
+
+  /**
+   * Whether to include source hash in metadata for change detection
+   * @default true
+   */
+  includeSourceHash?: boolean
+
+  /**
+   * Additional metadata to include for all embeddings
+   */
+  metadata?: Record<string, unknown>
+
+  /**
+   * Batch size for embedding generation
+   * @default 10
+   */
+  batchSize?: number
+}
+
+/**
+ * Generate embeddings for multiple texts in batches
+ *
+ * More efficient than calling generateEmbedding() multiple times.
+ * Automatically batches requests to respect API limits.
+ *
+ * @example
+ * ```typescript
+ * const embeddings = await generateEmbeddings({
+ *   provider: createEmbeddingProvider({ type: 'openai', apiKey: '...' }),
+ *   texts: ['text 1', 'text 2', 'text 3'],
+ *   batchSize: 10,
+ * })
+ * ```
+ */
+export async function generateEmbeddings(
+  options: GenerateEmbeddingsOptions,
+): Promise<StoredEmbedding[]> {
+  const {
+    provider,
+    texts,
+    includeSourceHash = true,
+    metadata: additionalMetadata,
+    batchSize = 10,
+  } = options
+
+  const baseMetadata: Omit<EmbeddingMetadata, 'sourceHash'> = {
+    model: provider.model,
+    provider: provider.type,
+    dimensions: provider.dimensions,
+    generatedAt: new Date().toISOString(),
+  }
+
+  const embeddings: StoredEmbedding[] = []
+
+  // Process in batches
+  for (let i = 0; i < texts.length; i += batchSize) {
+    const batch = texts.slice(i, i + batchSize)
+
+    // Generate embeddings for batch
+    const vectors = await provider.embedBatch(batch)
+
+    // Create StoredEmbedding objects
+    for (let j = 0; j < batch.length; j++) {
+      const text = batch[j]
+      const vector = vectors[j]
+      const sourceHash = includeSourceHash ? hashText(text) : undefined
+
+      embeddings.push({
+        vector,
+        metadata: {
+          ...baseMetadata,
+          sourceHash,
+          ...additionalMetadata,
+        },
+      })
+    }
+  }
+
+  return embeddings
+}
+
+/**
+ * Check if an embedding needs regeneration based on source text changes
+ *
+ * @param sourceText - Current source text
+ * @param currentEmbedding - Existing embedding (if any)
+ * @returns true if embedding needs regeneration
+ */
+export function shouldRegenerateEmbedding(
+  sourceText: string,
+  currentEmbedding: StoredEmbedding | null | undefined,
+): boolean {
+  // No existing embedding, needs generation
+  if (!currentEmbedding) {
+    return true
+  }
+
+  // No source hash in metadata, can't detect changes
+  if (!currentEmbedding.metadata.sourceHash) {
+    return false // Conservative: don't regenerate if we can't tell
+  }
+
+  // Compare source hash
+  const currentHash = hashText(sourceText)
+  return currentHash !== currentEmbedding.metadata.sourceHash
+}
+
+/**
+ * Hash text for change detection
+ * Uses SHA-256 for consistent hashing
+ */
+export function hashText(text: string): string {
+  return createHash('sha256').update(text).digest('hex')
+}
+
+/**
+ * Validate that embedding dimensions match expected dimensions
+ *
+ * @param embedding - The embedding to validate
+ * @param expectedDimensions - Expected number of dimensions
+ * @throws Error if dimensions don't match
+ */
+export function validateEmbeddingDimensions(
+  embedding: StoredEmbedding,
+  expectedDimensions: number,
+): void {
+  const actualDimensions = embedding.vector.length
+
+  if (actualDimensions !== expectedDimensions) {
+    throw new Error(
+      `Embedding dimension mismatch: expected ${expectedDimensions}, got ${actualDimensions}. ` +
+        `Provider: ${embedding.metadata.provider}, Model: ${embedding.metadata.model}`,
+    )
+  }
+
+  if (embedding.metadata.dimensions !== actualDimensions) {
+    throw new Error(
+      `Embedding metadata dimension mismatch: metadata says ${embedding.metadata.dimensions}, ` +
+        `but vector has ${actualDimensions} dimensions`,
+    )
+  }
+}
+
+/**
+ * Merge multiple embeddings into a single embedding
+ * Uses average pooling by default
+ *
+ * Useful for combining chunk embeddings into a single document embedding.
+ *
+ * @param embeddings - Array of embeddings to merge
+ * @param method - Merge method ('average' or 'max')
+ * @returns Merged embedding
+ */
+export function mergeEmbeddings(
+  embeddings: StoredEmbedding[],
+  method: 'average' | 'max' = 'average',
+): StoredEmbedding {
+  if (embeddings.length === 0) {
+    throw new Error('Cannot merge empty array of embeddings')
+  }
+
+  if (embeddings.length === 1) {
+    return embeddings[0]
+  }
+
+  // Validate all embeddings have same dimensions
+  const dimensions = embeddings[0].vector.length
+  for (const emb of embeddings) {
+    if (emb.vector.length !== dimensions) {
+      throw new Error(
+        `Cannot merge embeddings with different dimensions: ${dimensions} vs ${emb.vector.length}`,
+      )
+    }
+  }
+
+  let mergedVector: number[]
+
+  if (method === 'average') {
+    // Average pooling
+    mergedVector = new Array(dimensions).fill(0)
+
+    for (const emb of embeddings) {
+      for (let i = 0; i < dimensions; i++) {
+        mergedVector[i] += emb.vector[i]
+      }
+    }
+
+    for (let i = 0; i < dimensions; i++) {
+      mergedVector[i] /= embeddings.length
+    }
+  } else {
+    // Max pooling
+    mergedVector = new Array(dimensions).fill(-Infinity)
+
+    for (const emb of embeddings) {
+      for (let i = 0; i < dimensions; i++) {
+        mergedVector[i] = Math.max(mergedVector[i], emb.vector[i])
+      }
+    }
+  }
+
+  // Merge metadata (use first embedding's metadata)
+  const firstMetadata = embeddings[0].metadata
+
+  return {
+    vector: mergedVector,
+    metadata: {
+      ...firstMetadata,
+      generatedAt: new Date().toISOString(),
+      mergedFrom: embeddings.length,
+      mergeMethod: method,
+    } as EmbeddingMetadata,
+  }
+}

package/src/runtime/index.ts

@@ -0,0 +1,51 @@
+/**
+ * Runtime utilities for RAG operations
+ *
+ * This module provides high-level APIs for:
+ * - Text chunking
+ * - Embedding generation
+ * - Semantic search
+ * - Batch processing with rate limiting
+ */
+
+// Text chunking
+export {
+  chunkText,
+  estimateTokenCount,
+  mergeSmallChunks,
+  type ChunkingStrategy,
+  type ChunkingOptions,
+  type TextChunk,
+} from './chunking.js'
+
+// Embedding generation
+export {
+  generateEmbedding,
+  generateEmbeddings,
+  shouldRegenerateEmbedding,
+  hashText,
+  validateEmbeddingDimensions,
+  mergeEmbeddings,
+  type GenerateEmbeddingOptions,
+  type GenerateEmbeddingsOptions,
+  type ChunkedEmbedding,
+} from './embeddings.js'
+
+// Semantic search
+export {
+  semanticSearch,
+  findSimilar,
+  type SemanticSearchOptions,
+  type FindSimilarOptions,
+} from './search.js'
+
+// Batch processing
+export {
+  batchProcess,
+  RateLimiter,
+  ProcessingQueue,
+  type BatchProcessOptions,
+  type BatchProgress,
+  type BatchError,
+  type BatchProcessResult,
+} from './batch.js'

package/src/runtime/search.ts

@@ -0,0 +1,243 @@
+/**
+ * High-level semantic search APIs
+ */
+
+import type { AccessContext } from '@opensaas/stack-core'
+import type { SearchResult } from '../config/types.js'
+import type { EmbeddingProvider } from '../providers/types.js'
+import type { VectorStorage } from '../storage/types.js'
+
+export interface SemanticSearchOptions {
+  /**
+   * List key to search (e.g., 'Article', 'Post')
+   */
+  listKey: string
+
+  /**
+   * Field name containing embeddings
+   */
+  fieldName: string
+
+  /**
+   * Natural language query text
+   */
+  query: string
+
+  /**
+   * Embedding provider to use for query embedding
+   */
+  provider: EmbeddingProvider
+
+  /**
+   * Vector storage backend to use for search
+   */
+  storage: VectorStorage
+
+  /**
+   * Access context for enforcing access control
+   */
+  context: AccessContext
+
+  /**
+   * Maximum number of results to return
+   * @default 10
+   */
+  limit?: number
+
+  /**
+   * Minimum similarity score (0-1)
+   * @default 0.0
+   */
+  minScore?: number
+
+  /**
+   * Additional Prisma where clause to filter results
+   */
+  where?: Record<string, unknown>
+}
+
+/**
+ * Perform semantic search using natural language query
+ *
+ * This is a high-level API that:
+ * 1. Generates embedding for the query text
+ * 2. Searches for similar vectors in the database
+ * 3. Enforces access control
+ *
+ * @example
+ * ```typescript
+ * const results = await semanticSearch({
+ *   listKey: 'Article',
+ *   fieldName: 'contentEmbedding',
+ *   query: 'articles about machine learning',
+ *   provider: createEmbeddingProvider({ type: 'openai', apiKey: '...' }),
+ *   storage: createVectorStorage({ type: 'pgvector' }),
+ *   context: await getContext(),
+ *   limit: 10,
+ *   minScore: 0.7,
+ * })
+ * ```
+ */
+export async function semanticSearch<T = unknown>(
+  options: SemanticSearchOptions,
+): Promise<SearchResult<T>[]> {
+  const {
+    listKey,
+    fieldName,
+    query,
+    provider,
+    storage,
+    context,
+    limit = 10,
+    minScore = 0.0,
+    where,
+  } = options
+
+  // Generate embedding for query
+  const queryVector = await provider.embed(query)
+
+  // Search for similar vectors
+  const results = await storage.search<T>(listKey, fieldName, queryVector, {
+    limit,
+    minScore,
+    context,
+    where,
+  })
+
+  return results
+}
+
+export interface FindSimilarOptions {
+  /**
+   * List key to search (e.g., 'Article', 'Post')
+   */
+  listKey: string
+
+  /**
+   * Field name containing embeddings
+   */
+  fieldName: string
+
+  /**
+   * ID of the item to find similar items for
+   */
+  itemId: string
+
+  /**
+   * Vector storage backend to use for search
+   */
+  storage: VectorStorage
+
+  /**
+   * Access context for enforcing access control
+   */
+  context: AccessContext
+
+  /**
+   * Maximum number of results to return
+   * @default 10
+   */
+  limit?: number
+
+  /**
+   * Minimum similarity score (0-1)
+   * @default 0.0
+   */
+  minScore?: number
+
+  /**
+   * Whether to exclude the source item from results
+   * @default true
+   */
+  excludeSelf?: boolean
+
+  /**
+   * Additional Prisma where clause to filter results
+   */
+  where?: Record<string, unknown>
+}
+
+/**
+ * Find items similar to a given item by ID
+ *
+ * This is a high-level API that:
+ * 1. Fetches the embedding of the source item
+ * 2. Searches for similar vectors in the database
+ * 3. Enforces access control
+ * 4. Optionally excludes the source item from results
+ *
+ * @example
+ * ```typescript
+ * const similar = await findSimilar({
+ *   listKey: 'Article',
+ *   fieldName: 'contentEmbedding',
+ *   itemId: 'article-123',
+ *   storage: createVectorStorage({ type: 'pgvector' }),
+ *   context: await getContext(),
+ *   limit: 5,
+ *   excludeSelf: true,
+ * })
+ * ```
+ */
+export async function findSimilar<T = unknown>(
+  options: FindSimilarOptions,
+): Promise<SearchResult<T>[]> {
+  const {
+    listKey,
+    fieldName,
+    itemId,
+    storage,
+    context,
+    limit = 10,
+    minScore = 0.0,
+    excludeSelf = true,
+    where = {},
+  } = options
+
+  // Fetch the source item's embedding
+  // We need to access the database through the context
+  const dbKey = getDbKey(listKey)
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const model = (context.db as any)[dbKey]
+
+  if (!model) {
+    throw new Error(`List "${listKey}" not found in database`)
+  }
+
+  const item = await model.findUnique({
+    where: { id: itemId },
+    select: { [fieldName]: true },
+  })
+
+  if (!item) {
+    throw new Error(`Item with id "${itemId}" not found in list "${listKey}"`)
+  }
+
+  const embedding = item[fieldName]
+  if (!embedding || !embedding.vector) {
+    throw new Error(`Item "${itemId}" does not have an embedding in field "${fieldName}"`)
+  }
+
+  const queryVector = embedding.vector
+
+  // Build where clause
+  const searchWhere = excludeSelf ? { ...where, id: { not: itemId } } : where
+
+  // Search for similar vectors
+  const results = await storage.search<T>(listKey, fieldName, queryVector, {
+    limit,
+    minScore,
+    context,
+    where: searchWhere,
+  })
+
+  return results
+}
+
+/**
+ * Convert list key (PascalCase) to database key (camelCase)
+ * Same logic as in core package
+ */
+function getDbKey(listKey: string): string {
+  return listKey.charAt(0).toLowerCase() + listKey.slice(1)
+}