@opensaas/stack-rag 0.1.7 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opensaas/stack-rag",
-  "version": "0.1.7",
+  "version": "0.3.0",
   "description": "RAG and AI embeddings integration for OpenSaas Stack",
   "type": "module",
   "main": "./dist/index.js",
@@ -51,8 +51,8 @@
     "url": "https://github.com/OpenSaasAU/stack/issues"
   },
   "peerDependencies": {
-    "openai": "^6.8.0",
-    "@opensaas/stack-core": "0.1.7"
+    "@opensaas/stack-core": "^0",
+    "openai": "^6.8.0"
   },
   "peerDependenciesMeta": {
     "openai": {
@@ -60,16 +60,17 @@
     }
   },
   "dependencies": {
+    "dotenv": "^16.4.7",
     "zod": "^4.1.12"
   },
   "devDependencies": {
     "@types/node": "^24.7.2",
     "@vitest/coverage-v8": "^4.0.4",
     "@vitest/ui": "^4.0.0",
-    "openai": "^6.0.0",
+    "openai": "^6.8.0",
     "typescript": "^5.9.3",
     "vitest": "^4.0.0",
-    "@opensaas/stack-core": "0.1.7"
+    "@opensaas/stack-core": "0.3.0"
   },
   "scripts": {
     "build": "tsc",

package/src/config/index.ts

@@ -19,6 +19,15 @@ export function normalizeRAGConfig(config: RAGConfig): NormalizedRAGConfig {
       maxTokens: config.chunking?.maxTokens || 500,
       overlap: config.chunking?.overlap || 50,
     },
+    buildTime: config.buildTime
+      ? {
+          enabled: config.buildTime.enabled,
+          outputPath: config.buildTime.outputPath || '.embeddings/embeddings.json',
+          chunkSize: config.buildTime.chunkSize || 500,
+          chunkOverlap: config.buildTime.chunkOverlap || 50,
+          differential: config.buildTime.differential ?? true,
+        }
+      : null,
     enableMcpTools: config.enableMcpTools ?? true,
     batchSize: config.batchSize || 10,
     rateLimit: config.rateLimit || 100,

package/src/config/plugin.test.ts

@@ -78,7 +78,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -122,7 +126,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -158,7 +166,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -197,7 +209,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {},
         },
         setPluginData: vi.fn(),
@@ -231,7 +247,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -275,7 +295,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -324,7 +348,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -366,7 +394,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -401,7 +433,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             User: {
               fields: {
@@ -435,7 +471,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             BlogPost: {
               fields: {
@@ -483,7 +523,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -537,7 +581,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -574,7 +622,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {
             Article: {
               fields: {
@@ -613,7 +665,11 @@ describe('RAG Plugin', () => {
 
       const mockContext = {
         config: {
-          db: { provider: 'sqlite', url: 'file:./test.db' },
+          db: {
+            provider: 'sqlite',
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            prismaClientConstructor: (() => null) as any,
+          },
           lists: {},
         },
         setPluginData: vi.fn(),

package/src/config/plugin.ts

@@ -45,6 +45,11 @@ export function ragPlugin(config: RAGConfig): Plugin {
     name: 'rag',
     version: '0.1.0',
 
+    runtimeServiceTypes: {
+      import: "import type { RAGRuntimeServices } from '@opensaas/stack-rag'",
+      typeName: 'RAGRuntimeServices',
+    },
+
     init: async (context) => {
       // First pass: Scan for searchable() wrapped fields and inject embedding fields
       for (const [listName, listConfig] of Object.entries(context.config.lists)) {
@@ -247,6 +252,38 @@ export function ragPlugin(config: RAGConfig): Plugin {
       // Access at runtime via: config._pluginData.rag
       context.setPluginData<NormalizedRAGConfig>('rag', normalized)
     },
+
+    runtime: () => {
+      // Provide RAG-related utilities at runtime
+      return {
+        /**
+         * Generate embedding for a given text
+         * Uses the configured embedding provider
+         */
+        generateEmbedding: async (text: string) => {
+          const ragConfig = normalized
+          if (!ragConfig || !ragConfig.provider) {
+            throw new Error('RAG plugin not configured')
+          }
+
+          const provider = createEmbeddingProvider(ragConfig.provider)
+          return await provider.embed(text)
+        },
+
+        /**
+         * Generate embeddings for multiple texts (batch)
+         */
+        generateEmbeddings: async (texts: string[]) => {
+          const ragConfig = normalized
+          if (!ragConfig || !ragConfig.provider) {
+            throw new Error('RAG plugin not configured')
+          }
+
+          const provider = createEmbeddingProvider(ragConfig.provider)
+          return await provider.embedBatch(texts)
+        },
+      }
+    },
   }
 }
 

package/src/config/types.ts

@@ -155,6 +155,42 @@ export type VectorStorageConfig =
   | JsonStorageConfig
   | CustomStorageConfig
 
+/**
+ * Build-time embedding generation configuration
+ */
+export type BuildTimeConfig = {
+  /**
+   * Enable build-time embedding generation
+   */
+  enabled: boolean
+
+  /**
+   * Output path for embeddings JSON file
+   * Relative to project root
+   * @default '.embeddings/embeddings.json'
+   */
+  outputPath?: string
+
+  /**
+   * Chunk size for text splitting (in characters)
+   * @default 500
+   */
+  chunkSize?: number
+
+  /**
+   * Overlap between chunks (in characters)
+   * @default 50
+   */
+  chunkOverlap?: number
+
+  /**
+   * Whether to enable differential updates
+   * Only regenerate embeddings for changed content
+   * @default true
+   */
+  differential?: boolean
+}
+
 /**
  * Main RAG configuration
  */
@@ -191,6 +227,13 @@ export type RAGConfig = {
    */
   chunking?: ChunkingConfig
 
+  /**
+   * Build-time embedding generation configuration
+   * When enabled, embeddings are generated at build time and stored in a JSON file
+   * instead of being generated at runtime via hooks
+   */
+  buildTime?: BuildTimeConfig
+
   /**
    * Whether to enable MCP tools for semantic search
    * Requires MCP to be enabled in main config
@@ -219,6 +262,7 @@ export type NormalizedRAGConfig = {
   providers: Record<string, EmbeddingProviderConfig>
   storage: VectorStorageConfig
   chunking: Required<ChunkingConfig>
+  buildTime: Required<BuildTimeConfig> | null
   enableMcpTools: boolean
   batchSize: number
   rateLimit: number
@@ -340,3 +384,117 @@ export type SearchableMetadata = {
    */
   chunking?: ChunkingConfig
 }
+
+/**
+ * A chunk of text with its embedding
+ * Used in build-time generation output
+ */
+export type EmbeddingChunk = {
+  /**
+   * The text content of this chunk
+   */
+  text: string
+
+  /**
+   * The embedding vector for this chunk
+   */
+  embedding: number[]
+
+  /**
+   * Metadata about the chunk
+   */
+  metadata: {
+    /**
+     * Index of this chunk within the document
+     */
+    chunkIndex: number
+
+    /**
+     * Start character position in original text
+     */
+    startOffset: number
+
+    /**
+     * End character position in original text
+     */
+    endOffset: number
+
+    /**
+     * Whether this chunk represents a document title
+     * Title chunks receive boosted scoring during search
+     */
+    isTitle?: boolean
+
+    /**
+     * Additional custom metadata
+     */
+    [key: string]: unknown
+  }
+}
+
+/**
+ * Document with embeddings
+ * Used in build-time generation output
+ */
+export type EmbeddedDocument = {
+  /**
+   * Document ID or slug
+   */
+  id: string
+
+  /**
+   * Document title
+   */
+  title?: string
+
+  /**
+   * The chunks of this document with embeddings
+   */
+  chunks: EmbeddingChunk[]
+
+  /**
+   * Embedding metadata
+   */
+  embeddingMetadata: EmbeddingMetadata
+
+  /**
+   * When the embeddings were generated
+   */
+  generatedAt: string
+
+  /**
+   * Hash of the source content (for differential updates)
+   */
+  contentHash: string
+}
+
+/**
+ * Build-time embeddings index file format
+ */
+export type EmbeddingsIndex = {
+  /**
+   * Version of the embeddings format
+   */
+  version: string
+
+  /**
+   * Embedding configuration used to generate these embeddings
+   */
+  config: {
+    provider: string
+    model: string
+    dimensions: number
+    chunkSize: number
+    chunkOverlap: number
+  }
+
+  /**
+   * Documents with embeddings
+   */
+  documents: Record<string, EmbeddedDocument>
+
+  /**
+   * When the index was generated
+   */
+  generatedAt: string
+}

package/src/index.ts

@@ -15,6 +15,9 @@ export {
 // Plugin export
 export { ragPlugin } from './config/plugin.js'
 
+// Runtime type exports
+export type { RAGRuntimeServices } from './runtime/types.js'
+
 export type {
   RAGConfig,
   NormalizedRAGConfig,
@@ -30,4 +33,7 @@ export type {
   EmbeddingMetadata,
   StoredEmbedding,
   SearchResult,
+  EmbeddingsIndex,
+  EmbeddedDocument,
+  EmbeddingChunk,
 } from './config/types.js'

package/src/providers/openai.ts

@@ -17,7 +17,7 @@ async function getOpenAI() {
   try {
     const module = await import('openai')
     return module.default
-  } catch (error) {
+  } catch {
     throw new Error(
       'OpenAI package not found. Install it with: npm install openai\n' +
         'Make sure to run: pnpm install openai',

package/src/runtime/build-time.ts

@@ -0,0 +1,216 @@
+/**
+ * Build-time utilities for generating and managing embeddings
+ * Used by CLI tools and custom build scripts
+ */
+
+import { readFileSync, existsSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import type { EmbeddingProvider } from '../providers/types.js'
+import type { EmbeddingsIndex, EmbeddedDocument, EmbeddingChunk } from '../config/types.js'
+
+/**
+ * Simple character-based text chunking for build-time generation
+ *
+ * Simpler than the runtime chunking strategies, optimized for build-time batch processing.
+ * Splits text into fixed-size chunks with overlap.
+ *
+ * @param text - Text to chunk
+ * @param chunkSize - Size of each chunk in characters
+ * @param overlap - Overlap between chunks in characters
+ * @returns Array of text chunks
+ *
+ * @example
+ * ```typescript
+ * import { simpleChunkText } from '@opensaas/stack-rag/runtime'
+ *
+ * const chunks = simpleChunkText("Long document...", 500, 50)
+ * ```
+ */
+export function simpleChunkText(text: string, chunkSize: number, overlap: number): string[] {
+  const chunks: string[] = []
+  let start = 0
+
+  while (start < text.length) {
+    const end = Math.min(start + chunkSize, text.length)
+    chunks.push(text.slice(start, end))
+    start += chunkSize - overlap
+  }
+
+  return chunks
+}
+
+/**
+ * Compute SHA256 hash of content for change detection
+ *
+ * @param content - Content to hash
+ * @returns Hexadecimal hash string
+ *
+ * @example
+ * ```typescript
+ * import { hashContent } from '@opensaas/stack-rag/runtime'
+ *
+ * const hash = hashContent("document content")
+ * ```
+ */
+export function hashContent(content: string): string {
+  return createHash('sha256').update(content).digest('hex')
+}
+
+/**
+ * Load existing embeddings index from file
+ *
+ * Used for differential updates - only regenerate embeddings for changed content.
+ *
+ * @param filePath - Path to embeddings JSON file
+ * @returns Loaded index or null if file doesn't exist or can't be loaded
+ *
+ * @example
+ * ```typescript
+ * import { loadExistingIndex } from '@opensaas/stack-rag/runtime'
+ *
+ * const existing = loadExistingIndex('.embeddings/docs.json')
+ * if (existing) {
+ *   console.log(`Found ${Object.keys(existing.documents).length} existing documents`)
+ * }
+ * ```
+ */
+export function loadExistingIndex(filePath: string): EmbeddingsIndex | null {
+  if (!existsSync(filePath)) {
+    return null
+  }
+
+  try {
+    const content = readFileSync(filePath, 'utf-8')
+    return JSON.parse(content) as EmbeddingsIndex
+  } catch {
+    console.warn(`Warning: Could not load existing embeddings from ${filePath}`)
+    return null
+  }
+}
+
+/**
+ * Generate embeddings for a document with chunking
+ *
+ * Main utility for build-time embedding generation. Chunks the document,
+ * generates embeddings for each chunk, and returns a complete EmbeddedDocument.
+ *
+ * @param documentId - Unique identifier for the document
+ * @param content - Document content (plain text)
+ * @param provider - Embedding provider instance
+ * @param options - Generation options
+ * @returns Complete embedded document ready to be added to index
+ *
+ * @example
+ * ```typescript
+ * import { generateDocumentEmbeddings } from '@opensaas/stack-rag/runtime'
+ * import { createEmbeddingProvider } from '@opensaas/stack-rag/providers'
+ *
+ * const provider = createEmbeddingProvider({
+ *   type: 'openai',
+ *   apiKey: process.env.OPENAI_API_KEY
+ * })
+ *
+ * const doc = await generateDocumentEmbeddings(
+ *   'docs/getting-started',
+ *   'Document content here...',
+ *   provider,
+ *   {
+ *     title: 'Getting Started',
+ *     chunkSize: 500,
+ *     chunkOverlap: 50,
+ *     metadata: { section: 'guides' }
+ *   }
+ * )
+ * ```
+ */
+export async function generateDocumentEmbeddings(
+  documentId: string,
+  content: string,
+  provider: EmbeddingProvider,
+  options: {
+    title?: string
+    chunkSize: number
+    chunkOverlap: number
+    metadata?: Record<string, unknown>
+  },
+): Promise<EmbeddedDocument> {
+  const { title, chunkSize, chunkOverlap, metadata = {} } = options
+
+  // Hash content for differential updates
+  const contentHash = hashContent(content)
+
+  // Prepare all text chunks to embed
+  const allTextChunks: string[] = []
+  const chunkTypes: Array<'title' | 'content'> = []
+
+  // Add title chunk first if title exists
+  if (title) {
+    allTextChunks.push(title)
+    chunkTypes.push('title')
+  }
+
+  // Chunk the content
+  const contentChunks = simpleChunkText(content, chunkSize, chunkOverlap)
+  allTextChunks.push(...contentChunks)
+  contentChunks.forEach(() => chunkTypes.push('content'))
+
+  // Generate embeddings in batch for all chunks
+  const allEmbeddings = await provider.embedBatch(allTextChunks)
+
+  // Build chunks with embeddings
+  const chunks: EmbeddingChunk[] = []
+
+  let embeddingIndex = 0
+  let contentChunkIndex = 0
+
+  for (let i = 0; i < chunkTypes.length; i++) {
+    const type = chunkTypes[i]
+
+    if (type === 'title') {
+      // Title chunk
+      chunks.push({
+        text: allTextChunks[embeddingIndex],
+        embedding: allEmbeddings[embeddingIndex],
+        metadata: {
+          chunkIndex: -1, // Special index for title
+          startOffset: 0,
+          endOffset: 0,
+          isTitle: true,
+          ...metadata,
+        },
+      })
+    } else {
+      // Content chunk
+      chunks.push({
+        text: allTextChunks[embeddingIndex],
+        embedding: allEmbeddings[embeddingIndex],
+        metadata: {
+          chunkIndex: contentChunkIndex,
+          startOffset: contentChunkIndex * (chunkSize - chunkOverlap),
+          endOffset: Math.min(
+            (contentChunkIndex + 1) * chunkSize - contentChunkIndex * chunkOverlap,
+            content.length,
+          ),
+          ...metadata,
+        },
+      })
+      contentChunkIndex++
+    }
+
+    embeddingIndex++
+  }
+
+  return {
+    id: documentId,
+    title,
+    chunks,
+    embeddingMetadata: {
+      model: provider.model,
+      provider: provider.type,
+      dimensions: provider.dimensions,
+      generatedAt: new Date().toISOString(),
+    },
+    generatedAt: new Date().toISOString(),
+    contentHash,
+  }
+}