@opensaas/stack-rag 0.1.6

package/src/config/plugin.ts

@@ -0,0 +1,257 @@
+import type { Plugin } from '@opensaas/stack-core'
+import type { RAGConfig, NormalizedRAGConfig } from './types.js'
+import { normalizeRAGConfig } from './index.js'
+import { createEmbeddingProvider } from '../providers/index.js'
+
+/**
+ * RAG plugin for OpenSaas Stack
+ * Provides vector embeddings, semantic search, and automatic embedding generation
+ *
+ * @example
+ * ```typescript
+ * import { config, list } from '@opensaas/stack-core'
+ * import { text } from '@opensaas/stack-core/fields'
+ * import { ragPlugin, openaiEmbeddings, pgvectorStorage } from '@opensaas/stack-rag'
+ * import { embedding } from '@opensaas/stack-rag/fields'
+ *
+ * export default config({
+ *   plugins: [
+ *     ragPlugin({
+ *       provider: openaiEmbeddings({ apiKey: process.env.OPENAI_API_KEY }),
+ *       storage: pgvectorStorage()
+ *     })
+ *   ],
+ *   db: { provider: 'postgresql', url: process.env.DATABASE_URL },
+ *   lists: {
+ *     Article: list({
+ *       fields: {
+ *         content: text(),
+ *         contentEmbedding: embedding({
+ *           sourceField: 'content',
+ *           provider: 'openai',
+ *           autoGenerate: true
+ *         })
+ *       }
+ *     })
+ *   }
+ * })
+ * ```
+ */
+export function ragPlugin(config: RAGConfig): Plugin {
+  const normalized = normalizeRAGConfig(config)
+
+  return {
+    name: 'rag',
+    version: '0.1.0',
+
+    init: async (context) => {
+      // Find all embedding fields with autoGenerate enabled
+      for (const [listName, listConfig] of Object.entries(context.config.lists)) {
+        for (const [fieldName, fieldConfig] of Object.entries(listConfig.fields)) {
+          if (
+            fieldConfig.type === 'embedding' &&
+            (fieldConfig as { autoGenerate?: boolean }).autoGenerate
+          ) {
+            const embeddingConfig = fieldConfig as {
+              sourceField?: string
+              provider?: string
+              dimensions?: number
+            }
+
+            const sourceField = embeddingConfig.sourceField
+            if (!sourceField) {
+              throw new Error(
+                `RAG plugin: Field "${listName}.${fieldName}" has autoGenerate enabled but no sourceField specified`,
+              )
+            }
+
+            // Inject afterOperation hook to auto-generate embeddings
+            context.extendList(listName, {
+              hooks: {
+                resolveInput: async (args) => {
+                  // Skip if item is missing
+                  if (!args.resolvedData)
+                    throw new Error('RAG plugin: Missing resolvedData in resolveInput hook')
+
+                  const sourceText = args.resolvedData[sourceField] as string | undefined
+                  const currentEmbedding = args.resolvedData[fieldName] as {
+                    vector: number[]
+                    metadata: { sourceHash?: string }
+                  } | null
+
+                  // Skip if source text is empty
+                  if (!sourceText) return args.resolvedData
+
+                  // Check if we need to regenerate (source text changed)
+                  const sourceHash = await hashText(sourceText)
+                  if (currentEmbedding && currentEmbedding.metadata.sourceHash === sourceHash) {
+                    // Source text hasn't changed, skip regeneration
+                    return args.resolvedData
+                  }
+
+                  // Get provider for this field
+                  const providerName = embeddingConfig.provider || 'default'
+                  const providerConfig =
+                    providerName === 'default'
+                      ? normalized.provider
+                      : normalized.providers[providerName] || normalized.provider
+
+                  if (!providerConfig) {
+                    console.warn(
+                      `RAG plugin: No provider configured for field "${listName}.${fieldName}"`,
+                    )
+                    return args.resolvedData
+                  }
+
+                  // Generate embedding
+                  const provider = createEmbeddingProvider(providerConfig)
+                  const vector = await provider.embed(sourceText)
+
+                  return {
+                    ...args.resolvedData,
+                    [fieldName]: {
+                      vector,
+                      metadata: {
+                        model: provider.model,
+                        provider: provider.type,
+                        dimensions: provider.dimensions,
+                        generatedAt: new Date().toISOString(),
+                        sourceHash,
+                      },
+                    },
+                  }
+                },
+              },
+            })
+          }
+        }
+      }
+
+      // Register MCP tools if enabled
+      if (normalized.enableMcpTools && context.registerMcpTool) {
+        // Find all lists with embedding fields
+        for (const [listName, listConfig] of Object.entries(context.config.lists)) {
+          const embeddingFields = Object.entries(listConfig.fields).filter(
+            ([, fieldConfig]) => fieldConfig.type === 'embedding',
+          )
+
+          if (embeddingFields.length > 0) {
+            // Register semantic search tool for this list
+            const toolName = `semantic_search_${listName.toLowerCase()}`
+
+            context.registerMcpTool({
+              name: toolName,
+              description: `Search ${listName} using natural language (semantic search)`,
+              inputSchema: {
+                type: 'object',
+                properties: {
+                  query: { type: 'string', description: 'Natural language search query' },
+                  limit: { type: 'number', description: 'Maximum results', default: 10 },
+                  minScore: {
+                    type: 'number',
+                    description: 'Minimum similarity score (0-1)',
+                    default: 0.5,
+                  },
+                  field: {
+                    type: 'string',
+                    description: 'Embedding field to search',
+                    default: embeddingFields[0][0],
+                    enum: embeddingFields.map(([name]) => name),
+                  },
+                },
+                required: ['query'],
+              },
+              handler: async ({ input, context }) => {
+                const { query, limit = 10, minScore = 0.5, field = embeddingFields[0][0] } = input
+
+                // Get provider
+                const providerConfig = normalized.provider
+                if (!providerConfig) {
+                  throw new Error('RAG plugin: No default provider configured')
+                }
+
+                // Generate query embedding
+                const provider = createEmbeddingProvider(providerConfig)
+                const queryVector = await provider.embed(query)
+
+                // Search using configured storage backend
+                // Note: This is a simplified implementation
+                // Full implementation would use VectorStorage interface
+                const dbKey = listName.charAt(0).toLowerCase() + listName.slice(1)
+                const allItems = await context.db[dbKey].findMany()
+
+                // Calculate cosine similarity for each item
+                const results = allItems
+                  .map((item: { [key: string]: { vector: number[] } | null }) => {
+                    const embedding = item[field]
+                    if (!embedding || !embedding.vector) return null
+
+                    const score = cosineSimilarity(queryVector, embedding.vector)
+                    return { item, score }
+                  })
+                  .filter((r: { score: number } | null) => r !== null && r.score >= minScore)
+                  .sort((a: { score: number }, b: { score: number }) => b.score - a.score)
+                  .slice(0, limit)
+
+                return {
+                  results: results.map((r: { item: unknown; score: number }) => {
+                    // Ensure item is an object before spreading
+                    const item = r.item as Record<string, unknown>
+                    return {
+                      ...item,
+                      _similarity: r.score,
+                    }
+                  }),
+                  count: results.length,
+                }
+              },
+            })
+          }
+        }
+      }
+
+      // Store RAG config for runtime access
+      // Access at runtime via: config._pluginData.rag
+      context.setPluginData<NormalizedRAGConfig>('rag', normalized)
+    },
+  }
+}
+
+/**
+ * Hash text using SHA-256 for change detection
+ */
+async function hashText(text: string): Promise<string> {
+  // Simple hash implementation
+  // In production, could use crypto.subtle.digest for better performance
+  let hash = 0
+  for (let i = 0; i < text.length; i++) {
+    const char = text.charCodeAt(i)
+    hash = (hash << 5) - hash + char
+    hash = hash & hash // Convert to 32-bit integer
+  }
+  return hash.toString(36)
+}
+
+/**
+ * Calculate cosine similarity between two vectors
+ */
+function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have same dimensions')
+  }
+
+  let dotProduct = 0
+  let normA = 0
+  let normB = 0
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i]
+    normA += a[i] * a[i]
+    normB += b[i] * b[i]
+  }
+
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB)
+  if (denominator === 0) return 0
+
+  return dotProduct / denominator
+}

package/src/config/types.ts

@@ -0,0 +1,283 @@
+/**
+ * RAG (Retrieval-Augmented Generation) configuration types
+ */
+
+/**
+ * Text chunking strategies for splitting documents
+ */
+export type ChunkingStrategy = 'none' | 'recursive' | 'sentence' | 'sliding-window'
+
+/**
+ * Chunking configuration options
+ */
+export type ChunkingConfig = {
+  /**
+   * Chunking strategy to use
+   * @default 'recursive'
+   */
+  strategy?: ChunkingStrategy
+  /**
+   * Maximum tokens per chunk
+   * @default 500
+   */
+  maxTokens?: number
+  /**
+   * Overlap between chunks (tokens)
+   * Only applies to 'recursive' and 'sliding-window' strategies
+   * @default 50
+   */
+  overlap?: number
+}
+
+/**
+ * Embedding provider names
+ */
+export type EmbeddingProviderName = 'openai' | 'ollama' | string
+
+/**
+ * OpenAI embedding model options
+ */
+export type OpenAIEmbeddingModel =
+  | 'text-embedding-3-small'
+  | 'text-embedding-3-large'
+  | 'text-embedding-ada-002'
+
+/**
+ * OpenAI embedding provider configuration
+ */
+export type OpenAIEmbeddingConfig = {
+  type: 'openai'
+  /**
+   * OpenAI API key
+   */
+  apiKey: string
+  /**
+   * Model to use for embeddings
+   * @default 'text-embedding-3-small'
+   */
+  model?: OpenAIEmbeddingModel
+  /**
+   * Organization ID (optional)
+   */
+  organization?: string
+  /**
+   * Base URL for API requests (optional, for Azure or custom endpoints)
+   */
+  baseURL?: string
+}
+
+/**
+ * Ollama embedding provider configuration
+ */
+export type OllamaEmbeddingConfig = {
+  type: 'ollama'
+  /**
+   * Ollama API endpoint
+   * @default 'http://localhost:11434'
+   */
+  baseURL?: string
+  /**
+   * Model to use for embeddings
+   * @default 'nomic-embed-text'
+   */
+  model?: string
+}
+
+/**
+ * Generic embedding provider configuration
+ * For custom or third-party providers
+ */
+export type CustomEmbeddingConfig = {
+  type: string
+  [key: string]: unknown
+}
+
+/**
+ * Embedding provider configuration union
+ */
+export type EmbeddingProviderConfig =
+  | OpenAIEmbeddingConfig
+  | OllamaEmbeddingConfig
+  | CustomEmbeddingConfig
+
+/**
+ * Vector storage backend types
+ */
+export type VectorStorageBackend = 'pgvector' | 'sqlite-vss' | 'json' | string
+
+/**
+ * pgvector storage configuration
+ */
+export type PgVectorStorageConfig = {
+  type: 'pgvector'
+  /**
+   * Distance function for similarity search
+   * @default 'cosine'
+   */
+  distanceFunction?: 'cosine' | 'l2' | 'inner_product'
+}
+
+/**
+ * SQLite VSS storage configuration
+ */
+export type SqliteVssStorageConfig = {
+  type: 'sqlite-vss'
+  /**
+   * Distance function for similarity search
+   * @default 'cosine'
+   */
+  distanceFunction?: 'cosine' | 'l2'
+}
+
+/**
+ * JSON-based storage (in-memory search)
+ * Stores vectors as JSON and uses JavaScript for similarity search
+ * Good for development and small datasets
+ */
+export type JsonStorageConfig = {
+  type: 'json'
+}
+
+/**
+ * Generic storage configuration for custom backends
+ */
+export type CustomStorageConfig = {
+  type: string
+  [key: string]: unknown
+}
+
+/**
+ * Vector storage configuration union
+ */
+export type VectorStorageConfig =
+  | PgVectorStorageConfig
+  | SqliteVssStorageConfig
+  | JsonStorageConfig
+  | CustomStorageConfig
+
+/**
+ * Main RAG configuration
+ */
+export type RAGConfig = {
+  /**
+   * Default embedding provider
+   * Can be overridden per field
+   */
+  provider?: EmbeddingProviderConfig
+
+  /**
+   * Named embedding providers
+   * Allows using different providers for different fields
+   *
+   * @example
+   * ```typescript
+   * providers: {
+   *   openai: openaiEmbeddings({ apiKey: '...' }),
+   *   ollama: ollamaEmbeddings({ model: 'nomic-embed-text' })
+   * }
+   * ```
+   */
+  providers?: Record<string, EmbeddingProviderConfig>
+
+  /**
+   * Vector storage backend
+   * @default { type: 'json' }
+   */
+  storage?: VectorStorageConfig
+
+  /**
+   * Default chunking configuration
+   * Can be overridden per field
+   */
+  chunking?: ChunkingConfig
+
+  /**
+   * Whether to enable MCP tools for semantic search
+   * Requires MCP to be enabled in main config
+   * @default true
+   */
+  enableMcpTools?: boolean
+
+  /**
+   * Batch size for embedding generation
+   * @default 10
+   */
+  batchSize?: number
+
+  /**
+   * Rate limit (requests per minute) for embedding API calls
+   * @default 100
+   */
+  rateLimit?: number
+}
+
+/**
+ * Normalized RAG configuration with defaults applied
+ */
+export type NormalizedRAGConfig = {
+  provider: EmbeddingProviderConfig | null
+  providers: Record<string, EmbeddingProviderConfig>
+  storage: VectorStorageConfig
+  chunking: Required<ChunkingConfig>
+  enableMcpTools: boolean
+  batchSize: number
+  rateLimit: number
+}
+
+/**
+ * Embedding metadata stored alongside vectors
+ */
+export type EmbeddingMetadata = {
+  /**
+   * Embedding model used
+   */
+  model: string
+  /**
+   * Provider type (openai, ollama, etc.)
+   */
+  provider: string
+  /**
+   * Vector dimensions
+   */
+  dimensions: number
+  /**
+   * When the embedding was generated
+   */
+  generatedAt: string
+  /**
+   * Hash of source text (for detecting changes)
+   */
+  sourceHash?: string
+}
+
+/**
+ * Stored embedding with vector and metadata
+ */
+export type StoredEmbedding = {
+  /**
+   * The embedding vector
+   */
+  vector: number[]
+  /**
+   * Metadata about the embedding
+   */
+  metadata: EmbeddingMetadata
+}
+
+/**
+ * Semantic search result
+ */
+export type SearchResult<T = unknown> = {
+  /**
+   * The matching item
+   */
+  item: T
+  /**
+   * Similarity score (0-1, higher is more similar)
+   */
+  score: number
+  /**
+   * Distance metric (depends on storage backend)
+   */
+  distance: number
+}