@opensaas/stack-rag 0.1.6

package/src/fields/embedding.test.ts

@@ -0,0 +1,408 @@
+import { describe, it, expect } from 'vitest'
+import { embedding } from './embedding.js'
+import type { EmbeddingField } from './embedding.js'
+import { z } from 'zod'
+
+describe('Embedding Field', () => {
+  describe('embedding field builder', () => {
+    it('should create embedding field with default options', () => {
+      const field = embedding()
+
+      expect(field.type).toBe('embedding')
+      expect(field.dimensions).toBe(1536) // Default OpenAI dimensions
+      expect(field.autoGenerate).toBe(false) // No sourceField
+    })
+
+    it('should create embedding field with custom dimensions', () => {
+      const field = embedding({ dimensions: 3072 })
+
+      expect(field.dimensions).toBe(3072)
+    })
+
+    it('should set autoGenerate to true when sourceField is provided', () => {
+      const field = embedding({ sourceField: 'content' })
+
+      expect(field.autoGenerate).toBe(true)
+      expect(field.sourceField).toBe('content')
+    })
+
+    it('should allow explicit autoGenerate override', () => {
+      const field = embedding({ sourceField: 'content', autoGenerate: false })
+
+      expect(field.autoGenerate).toBe(false)
+      expect(field.sourceField).toBe('content')
+    })
+
+    it('should support provider configuration', () => {
+      const field = embedding({ provider: 'openai' })
+
+      expect(field.provider).toBe('openai')
+    })
+
+    it('should support chunking configuration', () => {
+      const field = embedding({
+        sourceField: 'content',
+        chunking: {
+          strategy: 'recursive',
+          maxTokens: 500,
+          overlap: 50,
+        },
+      })
+
+      expect(field.chunking).toEqual({
+        strategy: 'recursive',
+        maxTokens: 500,
+        overlap: 50,
+      })
+    })
+
+    it('should support UI configuration', () => {
+      const field = embedding({
+        ui: {
+          showVector: true,
+          showMetadata: true,
+        },
+      })
+
+      expect(field.ui).toEqual({
+        showVector: true,
+        showMetadata: true,
+      })
+    })
+  })
+
+  describe('getZodSchema', () => {
+    it('should generate valid Zod schema', () => {
+      const field = embedding({ dimensions: 3 })
+      const schema = field.getZodSchema!('embedding', 'create')
+
+      expect(schema).toBeDefined()
+      expect(schema instanceof z.ZodType).toBe(true)
+    })
+
+    it('should validate correct embedding structure', () => {
+      const field = embedding({ dimensions: 3 })
+      const schema = field.getZodSchema!('embedding', 'create')
+
+      const validEmbedding = {
+        vector: [0.1, 0.2, 0.3],
+        metadata: {
+          model: 'text-embedding-3-small',
+          provider: 'openai',
+          dimensions: 3,
+          generatedAt: new Date().toISOString(),
+        },
+      }
+
+      expect(() => schema.parse(validEmbedding)).not.toThrow()
+    })
+
+    it('should accept null embedding', () => {
+      const field = embedding({ dimensions: 3 })
+      const schema = field.getZodSchema!('embedding', 'create')
+
+      expect(() => schema.parse(null)).not.toThrow()
+    })
+
+    it('should accept undefined embedding', () => {
+      const field = embedding({ dimensions: 3 })
+      const schema = field.getZodSchema!('embedding', 'create')
+
+      expect(() => schema.parse(undefined)).not.toThrow()
+    })
+
+    it('should reject incorrect vector dimensions', () => {
+      const field = embedding({ dimensions: 3 })
+      const schema = field.getZodSchema!('embedding', 'create')
+
+      const invalidEmbedding = {
+        vector: [0.1, 0.2], // Only 2 dimensions, expected 3
+        metadata: {
+          model: 'test',
+          provider: 'test',
+          dimensions: 2,
+          generatedAt: new Date().toISOString(),
+        },
+      }
+
+      expect(() => schema.parse(invalidEmbedding)).toThrow(/exactly 3 dimensions/)
+    })
+
+    it('should validate metadata structure', () => {
+      const field = embedding({ dimensions: 3 })
+      const schema = field.getZodSchema!('embedding', 'create')
+
+      const invalidEmbedding = {
+        vector: [0.1, 0.2, 0.3],
+        metadata: {
+          // Missing required fields
+          model: 'test',
+        },
+      }
+
+      expect(() => schema.parse(invalidEmbedding)).toThrow()
+    })
+
+    it('should accept optional sourceHash in metadata', () => {
+      const field = embedding({ dimensions: 3 })
+      const schema = field.getZodSchema!('embedding', 'create')
+
+      const embeddingWithHash = {
+        vector: [0.1, 0.2, 0.3],
+        metadata: {
+          model: 'test',
+          provider: 'test',
+          dimensions: 3,
+          generatedAt: new Date().toISOString(),
+          sourceHash: 'abc123',
+        },
+      }
+
+      expect(() => schema.parse(embeddingWithHash)).not.toThrow()
+    })
+
+    it('should work for both create and update operations', () => {
+      const field = embedding({ dimensions: 3 })
+
+      const createSchema = field.getZodSchema!('embedding', 'create')
+      const updateSchema = field.getZodSchema!('embedding', 'update')
+
+      const validEmbedding = {
+        vector: [0.1, 0.2, 0.3],
+        metadata: {
+          model: 'test',
+          provider: 'test',
+          dimensions: 3,
+          generatedAt: new Date().toISOString(),
+        },
+      }
+
+      expect(() => createSchema.parse(validEmbedding)).not.toThrow()
+      expect(() => updateSchema.parse(validEmbedding)).not.toThrow()
+    })
+  })
+
+  describe('getPrismaType', () => {
+    it('should return Json type', () => {
+      const field = embedding()
+      const prismaType = field.getPrismaType!('embedding')
+
+      expect(prismaType.type).toBe('Json')
+    })
+
+    it('should return optional modifier', () => {
+      const field = embedding()
+      const prismaType = field.getPrismaType!('embedding')
+
+      expect(prismaType.modifiers).toBe('?')
+    })
+
+    it('should work regardless of field options', () => {
+      const field1 = embedding()
+      const field2 = embedding({ dimensions: 3072, provider: 'openai' })
+
+      const type1 = field1.getPrismaType!('embedding')
+      const type2 = field2.getPrismaType!('embedding')
+
+      expect(type1).toEqual(type2)
+      expect(type1.type).toBe('Json')
+      expect(type1.modifiers).toBe('?')
+    })
+  })
+
+  describe('getTypeScriptType', () => {
+    it('should return StoredEmbedding type', () => {
+      const field = embedding()
+      const tsType = field.getTypeScriptType!()
+
+      expect(tsType.type).toBe('StoredEmbedding | null')
+    })
+
+    it('should be optional', () => {
+      const field = embedding()
+      const tsType = field.getTypeScriptType!()
+
+      expect(tsType.optional).toBe(true)
+    })
+
+    it('should work regardless of field options', () => {
+      const field1 = embedding()
+      const field2 = embedding({ dimensions: 3072, provider: 'openai' })
+
+      const type1 = field1.getTypeScriptType!()
+      const type2 = field2.getTypeScriptType!()
+
+      expect(type1).toEqual(type2)
+      expect(type1.type).toBe('StoredEmbedding | null')
+      expect(type1.optional).toBe(true)
+    })
+  })
+
+  describe('Field configuration combinations', () => {
+    it('should support manual embedding storage', () => {
+      const field = embedding({
+        dimensions: 1536,
+        provider: 'openai',
+      })
+
+      expect(field.autoGenerate).toBe(false)
+      expect(field.sourceField).toBeUndefined()
+      expect(field.dimensions).toBe(1536)
+      expect(field.provider).toBe('openai')
+    })
+
+    it('should support automatic embedding generation', () => {
+      const field = embedding({
+        sourceField: 'content',
+        provider: 'openai',
+        dimensions: 1536,
+        autoGenerate: true,
+      })
+
+      expect(field.autoGenerate).toBe(true)
+      expect(field.sourceField).toBe('content')
+      expect(field.dimensions).toBe(1536)
+      expect(field.provider).toBe('openai')
+    })
+
+    it('should support chunking for long documents', () => {
+      const field = embedding({
+        sourceField: 'content',
+        chunking: {
+          strategy: 'recursive',
+          maxTokens: 500,
+          overlap: 50,
+        },
+      })
+
+      expect(field.chunking?.strategy).toBe('recursive')
+      expect(field.chunking?.maxTokens).toBe(500)
+      expect(field.chunking?.overlap).toBe(50)
+    })
+
+    it('should support multiple providers via provider name', () => {
+      const openaiField = embedding({ provider: 'openai' })
+      const ollamaField = embedding({ provider: 'ollama' })
+
+      expect(openaiField.provider).toBe('openai')
+      expect(ollamaField.provider).toBe('ollama')
+    })
+  })
+
+  describe('Field type conformance', () => {
+    it('should conform to BaseFieldConfig interface', () => {
+      const field: EmbeddingField = embedding()
+
+      expect(field).toHaveProperty('type')
+      expect(field).toHaveProperty('getZodSchema')
+      expect(field).toHaveProperty('getPrismaType')
+      expect(field).toHaveProperty('getTypeScriptType')
+    })
+
+    it('should have all required builder methods', () => {
+      const field = embedding()
+
+      expect(typeof field.getZodSchema).toBe('function')
+      expect(typeof field.getPrismaType).toBe('function')
+      expect(typeof field.getTypeScriptType).toBe('function')
+    })
+
+    it('should return consistent types across methods', () => {
+      const field = embedding({ dimensions: 1536 })
+
+      const zodSchema = field.getZodSchema!('embedding', 'create')
+      const prismaType = field.getPrismaType!('embedding')
+      const tsType = field.getTypeScriptType!()
+
+      // All methods should be callable and return valid results
+      expect(zodSchema).toBeDefined()
+      expect(prismaType).toBeDefined()
+      expect(tsType).toBeDefined()
+
+      // Types should be consistent (Json in Prisma, StoredEmbedding in TS)
+      expect(prismaType.type).toBe('Json')
+      expect(tsType.type).toContain('StoredEmbedding')
+    })
+  })
+
+  describe('UI configuration', () => {
+    it('should default to hiding vector display', () => {
+      const field = embedding()
+
+      expect(field.ui?.showVector).toBeUndefined()
+    })
+
+    it('should allow showing vector in UI', () => {
+      const field = embedding({
+        ui: { showVector: true },
+      })
+
+      expect(field.ui?.showVector).toBe(true)
+    })
+
+    it('should default to showing metadata', () => {
+      const field = embedding()
+
+      expect(field.ui?.showMetadata).toBeUndefined()
+    })
+
+    it('should allow customizing metadata display', () => {
+      const field = embedding({
+        ui: { showMetadata: false },
+      })
+
+      expect(field.ui?.showMetadata).toBe(false)
+    })
+  })
+
+  describe('Edge cases', () => {
+    it('should handle very large dimensions', () => {
+      const field = embedding({ dimensions: 10000 })
+
+      expect(field.dimensions).toBe(10000)
+
+      const schema = field.getZodSchema!('embedding', 'create')
+      const largeVector = Array(10000).fill(0.1)
+
+      const validEmbedding = {
+        vector: largeVector,
+        metadata: {
+          model: 'test',
+          provider: 'test',
+          dimensions: 10000,
+          generatedAt: new Date().toISOString(),
+        },
+      }
+
+      expect(() => schema.parse(validEmbedding)).not.toThrow()
+    })
+
+    it('should handle minimum dimensions', () => {
+      const field = embedding({ dimensions: 1 })
+
+      expect(field.dimensions).toBe(1)
+
+      const schema = field.getZodSchema!('embedding', 'create')
+      const validEmbedding = {
+        vector: [0.5],
+        metadata: {
+          model: 'test',
+          provider: 'test',
+          dimensions: 1,
+          generatedAt: new Date().toISOString(),
+        },
+      }
+
+      expect(() => schema.parse(validEmbedding)).not.toThrow()
+    })
+
+    it('should handle empty sourceField gracefully', () => {
+      const field = embedding({ sourceField: '' })
+
+      expect(field.sourceField).toBe('')
+      // Empty string is truthy (it's a defined value), so autoGenerate will be true
+      // unless explicitly set to false
+      expect(field.autoGenerate).toBe(true)
+    })
+  })
+})

package/src/fields/embedding.ts

@@ -0,0 +1,150 @@
+import { z } from 'zod'
+import type { BaseFieldConfig } from '@opensaas/stack-core'
+import type { StoredEmbedding, EmbeddingProviderName, ChunkingConfig } from '../config/types.js'
+
+/**
+ * Embedding field configuration
+ * Stores vector embeddings as JSON with metadata
+ */
+export type EmbeddingField = BaseFieldConfig<StoredEmbedding | null, StoredEmbedding | null> & {
+  type: 'embedding'
+
+  /**
+   * Source field name to generate embeddings from
+   * When this field changes, embeddings will be automatically regenerated
+   *
+   * @example 'content' or 'title'
+   */
+  sourceField?: string
+
+  /**
+   * Embedding provider to use
+   * References a provider name from RAG config
+   * Falls back to default provider if not specified
+   *
+   * @example 'openai' or 'ollama'
+   */
+  provider?: EmbeddingProviderName
+
+  /**
+   * Vector dimensions
+   * Must match the provider's output dimensions
+   * @default 1536 (OpenAI text-embedding-3-small)
+   */
+  dimensions?: number
+
+  /**
+   * Chunking configuration for long texts
+   * Only applies if sourceField is set
+   */
+  chunking?: ChunkingConfig
+
+  /**
+   * Whether to automatically generate embeddings
+   * @default true if sourceField is set
+   */
+  autoGenerate?: boolean
+
+  /**
+   * UI configuration
+   */
+  ui?: {
+    /**
+     * Whether to show the embedding vector in the UI
+     * @default false (usually too large to display)
+     */
+    showVector?: boolean
+
+    /**
+     * Whether to show embedding metadata
+     * @default true
+     */
+    showMetadata?: boolean
+  }
+}
+
+/**
+ * Embedding field builder
+ * Creates a field that stores vector embeddings with metadata
+ *
+ * @example
+ * ```typescript
+ * import { embedding } from '@opensaas/stack-rag/fields'
+ *
+ * // Manual embedding storage
+ * fields: {
+ *   embedding: embedding({
+ *     dimensions: 1536,
+ *     provider: 'openai'
+ *   })
+ * }
+ *
+ * // Automatic embedding generation from source field
+ * fields: {
+ *   content: text(),
+ *   contentEmbedding: embedding({
+ *     sourceField: 'content',
+ *     provider: 'openai',
+ *     dimensions: 1536,
+ *     autoGenerate: true
+ *   })
+ * }
+ * ```
+ */
+export function embedding(options?: Omit<EmbeddingField, 'type'>): EmbeddingField {
+  const dimensions = options?.dimensions || 1536
+  const autoGenerate = options?.autoGenerate ?? options?.sourceField != null
+
+  return {
+    type: 'embedding',
+    ...options,
+    dimensions,
+    autoGenerate,
+
+    getZodSchema: (_fieldName: string, _operation: 'create' | 'update') => {
+      // Embedding schema validation
+      const embeddingSchema = z.object({
+        vector: z.array(z.number()).length(dimensions, {
+          message: `Embedding vector must have exactly ${dimensions} dimensions`,
+        }),
+        metadata: z.object({
+          model: z.string(),
+          provider: z.string(),
+          dimensions: z.number(),
+          generatedAt: z.string(),
+          sourceHash: z.string().optional(),
+        }),
+      })
+
+      // Embeddings are always optional in input
+      // They are generated automatically via hooks if sourceField is set
+      return embeddingSchema.nullable().optional() as unknown as z.ZodTypeAny
+    },
+
+    getPrismaType: (_fieldName: string) => {
+      // Store as JSON in database
+      // For pgvector, we could use vector() type, but JSON is more portable
+      return {
+        type: 'Json',
+        modifiers: '?', // Always optional
+      }
+    },
+
+    getTypeScriptType: () => {
+      return {
+        type: 'StoredEmbedding | null',
+        optional: true,
+      }
+    },
+
+    getTypeScriptImports: () => {
+      return [
+        {
+          names: ['StoredEmbedding'],
+          from: '@opensaas/stack-rag',
+          typeOnly: true,
+        },
+      ]
+    },
+  }
+}

package/src/fields/index.ts

@@ -0,0 +1,6 @@
+/**
+ * RAG field types
+ */
+
+export { embedding } from './embedding.js'
+export type { EmbeddingField } from './embedding.js'

package/src/index.ts

@@ -0,0 +1,33 @@
+/**
+ * @opensaas/stack-rag
+ * RAG and AI embeddings integration for OpenSaas Stack
+ */
+
+// Config exports
+export {
+  openaiEmbeddings,
+  ollamaEmbeddings,
+  pgvectorStorage,
+  sqliteVssStorage,
+  jsonStorage,
+} from './config/index.js'
+
+// Plugin export
+export { ragPlugin } from './config/plugin.js'
+
+export type {
+  RAGConfig,
+  NormalizedRAGConfig,
+  EmbeddingProviderConfig,
+  OpenAIEmbeddingConfig,
+  OllamaEmbeddingConfig,
+  VectorStorageConfig,
+  PgVectorStorageConfig,
+  SqliteVssStorageConfig,
+  JsonStorageConfig,
+  ChunkingConfig,
+  ChunkingStrategy,
+  EmbeddingMetadata,
+  StoredEmbedding,
+  SearchResult,
+} from './config/types.js'

package/src/mcp/index.ts

@@ -0,0 +1,21 @@
+/**
+ * MCP (Model Context Protocol) integration for RAG
+ *
+ * MCP tools for semantic search are automatically generated by the ragPlugin.
+ * This export is provided for future extensibility and custom MCP tool generation.
+ */
+
+// Re-export types that may be useful for MCP tool development
+export type { SearchResult } from '../config/types.js'
+export type { SemanticSearchOptions } from '../runtime/search.js'
+
+/**
+ * MCP tools are automatically generated by ragPlugin when:
+ * - MCP is enabled in main config
+ * - ragPlugin has enableMcpTools: true (default)
+ *
+ * Generated tools:
+ * - semantic_search_{listKey} - Search by natural language query
+ *
+ * See packages/rag/src/config/plugin.ts for implementation details.
+ */

package/src/providers/index.ts

@@ -0,0 +1,81 @@
+import type { EmbeddingProvider } from './types.js'
+import type { EmbeddingProviderConfig } from '../config/types.js'
+import { createOpenAIProvider } from './openai.js'
+import { createOllamaProvider } from './ollama.js'
+
+/**
+ * Provider factory registry
+ * Maps provider types to factory functions
+ */
+const providerFactories = new Map<string, (config: EmbeddingProviderConfig) => EmbeddingProvider>()
+
+/**
+ * Register the built-in providers
+ */
+providerFactories.set('openai', (config) => {
+  if (config.type !== 'openai') {
+    throw new Error('Invalid config type for OpenAI provider')
+  }
+  return createOpenAIProvider(config as import('../config/types.js').OpenAIEmbeddingConfig)
+})
+providerFactories.set('ollama', (config) => {
+  if (config.type !== 'ollama') {
+    throw new Error('Invalid config type for Ollama provider')
+  }
+  return createOllamaProvider(config as import('../config/types.js').OllamaEmbeddingConfig)
+})
+
+/**
+ * Register a custom embedding provider factory
+ * Use this to add support for custom embedding providers
+ *
+ * @example
+ * ```typescript
+ * import { registerEmbeddingProvider } from '@opensaas/stack-rag/providers'
+ *
+ * registerEmbeddingProvider('custom', (config) => {
+ *   return new CustomEmbeddingProvider(config)
+ * })
+ * ```
+ */
+export function registerEmbeddingProvider(
+  type: string,
+  factory: (config: EmbeddingProviderConfig) => EmbeddingProvider,
+): void {
+  providerFactories.set(type, factory)
+}
+
+/**
+ * Create an embedding provider instance from configuration
+ * Automatically selects the correct provider based on config.type
+ *
+ * @example
+ * ```typescript
+ * import { createEmbeddingProvider } from '@opensaas/stack-rag/providers'
+ *
+ * const provider = createEmbeddingProvider({
+ *   type: 'openai',
+ *   apiKey: process.env.OPENAI_API_KEY!,
+ *   model: 'text-embedding-3-small'
+ * })
+ *
+ * const embedding = await provider.embed('Hello world')
+ * ```
+ */
+export function createEmbeddingProvider(config: EmbeddingProviderConfig): EmbeddingProvider {
+  const factory = providerFactories.get(config.type)
+
+  if (!factory) {
+    throw new Error(
+      `Unknown embedding provider type: ${config.type}. ` +
+        `Available providers: ${Array.from(providerFactories.keys()).join(', ')}`,
+    )
+  }
+
+  return factory(config)
+}
+
+// Export types and individual providers
+export * from './types.js'
+export { OpenAIEmbeddingProvider, createOpenAIProvider } from './openai.js'
+export { OllamaEmbeddingProvider, createOllamaProvider } from './ollama.js'