@opensaas/stack-rag 0.1.7 → 0.4.0

package/src/config/types.ts

@@ -155,6 +155,42 @@ export type VectorStorageConfig =
   | JsonStorageConfig
   | CustomStorageConfig
 
+/**
+ * Build-time embedding generation configuration
+ */
+export type BuildTimeConfig = {
+  /**
+   * Enable build-time embedding generation
+   */
+  enabled: boolean
+
+  /**
+   * Output path for embeddings JSON file
+   * Relative to project root
+   * @default '.embeddings/embeddings.json'
+   */
+  outputPath?: string
+
+  /**
+   * Chunk size for text splitting (in characters)
+   * @default 500
+   */
+  chunkSize?: number
+
+  /**
+   * Overlap between chunks (in characters)
+   * @default 50
+   */
+  chunkOverlap?: number
+
+  /**
+   * Whether to enable differential updates
+   * Only regenerate embeddings for changed content
+   * @default true
+   */
+  differential?: boolean
+}
+
 /**
  * Main RAG configuration
  */
@@ -191,6 +227,13 @@ export type RAGConfig = {
    */
   chunking?: ChunkingConfig
 
+  /**
+   * Build-time embedding generation configuration
+   * When enabled, embeddings are generated at build time and stored in a JSON file
+   * instead of being generated at runtime via hooks
+   */
+  buildTime?: BuildTimeConfig
+
   /**
    * Whether to enable MCP tools for semantic search
    * Requires MCP to be enabled in main config
@@ -219,6 +262,7 @@ export type NormalizedRAGConfig = {
   providers: Record<string, EmbeddingProviderConfig>
   storage: VectorStorageConfig
   chunking: Required<ChunkingConfig>
+  buildTime: Required<BuildTimeConfig> | null
   enableMcpTools: boolean
   batchSize: number
   rateLimit: number
@@ -340,3 +384,117 @@ export type SearchableMetadata = {
    */
   chunking?: ChunkingConfig
 }
+
+/**
+ * A chunk of text with its embedding
+ * Used in build-time generation output
+ */
+export type EmbeddingChunk = {
+  /**
+   * The text content of this chunk
+   */
+  text: string
+
+  /**
+   * The embedding vector for this chunk
+   */
+  embedding: number[]
+
+  /**
+   * Metadata about the chunk
+   */
+  metadata: {
+    /**
+     * Index of this chunk within the document
+     */
+    chunkIndex: number
+
+    /**
+     * Start character position in original text
+     */
+    startOffset: number
+
+    /**
+     * End character position in original text
+     */
+    endOffset: number
+
+    /**
+     * Whether this chunk represents a document title
+     * Title chunks receive boosted scoring during search
+     */
+    isTitle?: boolean
+
+    /**
+     * Additional custom metadata
+     */
+    [key: string]: unknown
+  }
+}
+
+/**
+ * Document with embeddings
+ * Used in build-time generation output
+ */
+export type EmbeddedDocument = {
+  /**
+   * Document ID or slug
+   */
+  id: string
+
+  /**
+   * Document title
+   */
+  title?: string
+
+  /**
+   * The chunks of this document with embeddings
+   */
+  chunks: EmbeddingChunk[]
+
+  /**
+   * Embedding metadata
+   */
+  embeddingMetadata: EmbeddingMetadata
+
+  /**
+   * When the embeddings were generated
+   */
+  generatedAt: string
+
+  /**
+   * Hash of the source content (for differential updates)
+   */
+  contentHash: string
+}
+
+/**
+ * Build-time embeddings index file format
+ */
+export type EmbeddingsIndex = {
+  /**
+   * Version of the embeddings format
+   */
+  version: string
+
+  /**
+   * Embedding configuration used to generate these embeddings
+   */
+  config: {
+    provider: string
+    model: string
+    dimensions: number
+    chunkSize: number
+    chunkOverlap: number
+  }
+
+  /**
+   * Documents with embeddings
+   */
+  documents: Record<string, EmbeddedDocument>
+
+  /**
+   * When the index was generated
+   */
+  generatedAt: string
+}

package/src/fields/embedding.ts

@@ -1,12 +1,12 @@
 import { z } from 'zod'
-import type { BaseFieldConfig } from '@opensaas/stack-core'
-import type { StoredEmbedding, EmbeddingProviderName, ChunkingConfig } from '../config/types.js'
+import type { BaseFieldConfig, TypeInfo } from '@opensaas/stack-core'
+import type { EmbeddingProviderName, ChunkingConfig } from '../config/types.js'
 
 /**
  * Embedding field configuration
  * Stores vector embeddings as JSON with metadata
  */
-export type EmbeddingField = BaseFieldConfig<StoredEmbedding | null, StoredEmbedding | null> & {
+export type EmbeddingField<TTypeInfo extends TypeInfo = TypeInfo> = BaseFieldConfig<TTypeInfo> & {
   type: 'embedding'
 
   /**
@@ -91,7 +91,9 @@ export type EmbeddingField = BaseFieldConfig<StoredEmbedding | null, StoredEmbed
  * }
  * ```
  */
-export function embedding(options?: Omit<EmbeddingField, 'type'>): EmbeddingField {
+export function embedding<TTypeInfo extends TypeInfo = TypeInfo>(
+  options?: Omit<EmbeddingField<TTypeInfo>, 'type'>,
+): EmbeddingField<TTypeInfo> {
   const dimensions = options?.dimensions || 1536
   const autoGenerate = options?.autoGenerate ?? options?.sourceField != null
 

package/src/fields/searchable.test.ts

@@ -4,7 +4,8 @@ import type { BaseFieldConfig } from '@opensaas/stack-core'
 import type { SearchableOptions } from '../config/types.js'
 
 // Mock text field for testing
-function mockTextField(): BaseFieldConfig {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Mock function for testing
+function mockTextField(): BaseFieldConfig<any> {
   return {
     type: 'text',
     // eslint-disable-next-line @typescript-eslint/no-explicit-any

package/src/fields/searchable.ts

@@ -37,7 +37,8 @@ import type { SearchableOptions, SearchableMetadata } from '../config/types.js'
  * @param options - Embedding configuration options
  * @returns The same field with searchable metadata attached
  */
-export function searchable<T extends BaseFieldConfig>(
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Searchable must accept any field config
+export function searchable<T extends BaseFieldConfig<any>>(
   field: T,
   options: SearchableOptions = {},
 ): T & { _searchable: SearchableMetadata } {

package/src/index.ts

@@ -15,6 +15,9 @@ export {
 // Plugin export
 export { ragPlugin } from './config/plugin.js'
 
+// Runtime type exports
+export type { RAGRuntimeServices } from './runtime/types.js'
+
 export type {
   RAGConfig,
   NormalizedRAGConfig,
@@ -30,4 +33,7 @@ export type {
   EmbeddingMetadata,
   StoredEmbedding,
   SearchResult,
+  EmbeddingsIndex,
+  EmbeddedDocument,
+  EmbeddingChunk,
 } from './config/types.js'

package/src/providers/ollama.ts

@@ -117,11 +117,6 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider {
       return []
     }
 
-    // Ensure dimensions are initialized
-    if (!this.dimensionsInitialized) {
-      await this.initializeDimensions()
-    }
-
     // Filter out empty texts and keep track of indices
     const validTexts: string[] = []
     const validIndices: number[] = []
@@ -137,6 +132,11 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider {
       throw new Error('Cannot generate embeddings for all empty texts')
     }
 
+    // Ensure dimensions are initialized (only after validating we have valid texts)
+    if (!this.dimensionsInitialized) {
+      await this.initializeDimensions()
+    }
+
     try {
       // Make parallel requests (Ollama doesn't have batch API)
       const embeddingPromises = validTexts.map((text) => this.embed(text))

package/src/providers/openai.ts

@@ -17,7 +17,7 @@ async function getOpenAI() {
   try {
     const module = await import('openai')
     return module.default
-  } catch (error) {
+  } catch {
     throw new Error(
       'OpenAI package not found. Install it with: npm install openai\n' +
         'Make sure to run: pnpm install openai',

package/src/runtime/build-time.ts

@@ -0,0 +1,216 @@
+/**
+ * Build-time utilities for generating and managing embeddings
+ * Used by CLI tools and custom build scripts
+ */
+
+import { readFileSync, existsSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import type { EmbeddingProvider } from '../providers/types.js'
+import type { EmbeddingsIndex, EmbeddedDocument, EmbeddingChunk } from '../config/types.js'
+
+/**
+ * Simple character-based text chunking for build-time generation
+ *
+ * Simpler than the runtime chunking strategies, optimized for build-time batch processing.
+ * Splits text into fixed-size chunks with overlap.
+ *
+ * @param text - Text to chunk
+ * @param chunkSize - Size of each chunk in characters
+ * @param overlap - Overlap between chunks in characters
+ * @returns Array of text chunks
+ *
+ * @example
+ * ```typescript
+ * import { simpleChunkText } from '@opensaas/stack-rag/runtime'
+ *
+ * const chunks = simpleChunkText("Long document...", 500, 50)
+ * ```
+ */
+export function simpleChunkText(text: string, chunkSize: number, overlap: number): string[] {
+  const chunks: string[] = []
+  let start = 0
+
+  while (start < text.length) {
+    const end = Math.min(start + chunkSize, text.length)
+    chunks.push(text.slice(start, end))
+    start += chunkSize - overlap
+  }
+
+  return chunks
+}
+
+/**
+ * Compute SHA256 hash of content for change detection
+ *
+ * @param content - Content to hash
+ * @returns Hexadecimal hash string
+ *
+ * @example
+ * ```typescript
+ * import { hashContent } from '@opensaas/stack-rag/runtime'
+ *
+ * const hash = hashContent("document content")
+ * ```
+ */
+export function hashContent(content: string): string {
+  return createHash('sha256').update(content).digest('hex')
+}
+
+/**
+ * Load existing embeddings index from file
+ *
+ * Used for differential updates - only regenerate embeddings for changed content.
+ *
+ * @param filePath - Path to embeddings JSON file
+ * @returns Loaded index or null if file doesn't exist or can't be loaded
+ *
+ * @example
+ * ```typescript
+ * import { loadExistingIndex } from '@opensaas/stack-rag/runtime'
+ *
+ * const existing = loadExistingIndex('.embeddings/docs.json')
+ * if (existing) {
+ *   console.log(`Found ${Object.keys(existing.documents).length} existing documents`)
+ * }
+ * ```
+ */
+export function loadExistingIndex(filePath: string): EmbeddingsIndex | null {
+  if (!existsSync(filePath)) {
+    return null
+  }
+
+  try {
+    const content = readFileSync(filePath, 'utf-8')
+    return JSON.parse(content) as EmbeddingsIndex
+  } catch {
+    console.warn(`Warning: Could not load existing embeddings from ${filePath}`)
+    return null
+  }
+}
+
+/**
+ * Generate embeddings for a document with chunking
+ *
+ * Main utility for build-time embedding generation. Chunks the document,
+ * generates embeddings for each chunk, and returns a complete EmbeddedDocument.
+ *
+ * @param documentId - Unique identifier for the document
+ * @param content - Document content (plain text)
+ * @param provider - Embedding provider instance
+ * @param options - Generation options
+ * @returns Complete embedded document ready to be added to index
+ *
+ * @example
+ * ```typescript
+ * import { generateDocumentEmbeddings } from '@opensaas/stack-rag/runtime'
+ * import { createEmbeddingProvider } from '@opensaas/stack-rag/providers'
+ *
+ * const provider = createEmbeddingProvider({
+ *   type: 'openai',
+ *   apiKey: process.env.OPENAI_API_KEY
+ * })
+ *
+ * const doc = await generateDocumentEmbeddings(
+ *   'docs/getting-started',
+ *   'Document content here...',
+ *   provider,
+ *   {
+ *     title: 'Getting Started',
+ *     chunkSize: 500,
+ *     chunkOverlap: 50,
+ *     metadata: { section: 'guides' }
+ *   }
+ * )
+ * ```
+ */
+export async function generateDocumentEmbeddings(
+  documentId: string,
+  content: string,
+  provider: EmbeddingProvider,
+  options: {
+    title?: string
+    chunkSize: number
+    chunkOverlap: number
+    metadata?: Record<string, unknown>
+  },
+): Promise<EmbeddedDocument> {
+  const { title, chunkSize, chunkOverlap, metadata = {} } = options
+
+  // Hash content for differential updates
+  const contentHash = hashContent(content)
+
+  // Prepare all text chunks to embed
+  const allTextChunks: string[] = []
+  const chunkTypes: Array<'title' | 'content'> = []
+
+  // Add title chunk first if title exists
+  if (title) {
+    allTextChunks.push(title)
+    chunkTypes.push('title')
+  }
+
+  // Chunk the content
+  const contentChunks = simpleChunkText(content, chunkSize, chunkOverlap)
+  allTextChunks.push(...contentChunks)
+  contentChunks.forEach(() => chunkTypes.push('content'))
+
+  // Generate embeddings in batch for all chunks
+  const allEmbeddings = await provider.embedBatch(allTextChunks)
+
+  // Build chunks with embeddings
+  const chunks: EmbeddingChunk[] = []
+
+  let embeddingIndex = 0
+  let contentChunkIndex = 0
+
+  for (let i = 0; i < chunkTypes.length; i++) {
+    const type = chunkTypes[i]
+
+    if (type === 'title') {
+      // Title chunk
+      chunks.push({
+        text: allTextChunks[embeddingIndex],
+        embedding: allEmbeddings[embeddingIndex],
+        metadata: {
+          chunkIndex: -1, // Special index for title
+          startOffset: 0,
+          endOffset: 0,
+          isTitle: true,
+          ...metadata,
+        },
+      })
+    } else {
+      // Content chunk
+      chunks.push({
+        text: allTextChunks[embeddingIndex],
+        embedding: allEmbeddings[embeddingIndex],
+        metadata: {
+          chunkIndex: contentChunkIndex,
+          startOffset: contentChunkIndex * (chunkSize - chunkOverlap),
+          endOffset: Math.min(
+            (contentChunkIndex + 1) * chunkSize - contentChunkIndex * chunkOverlap,
+            content.length,
+          ),
+          ...metadata,
+        },
+      })
+      contentChunkIndex++
+    }
+
+    embeddingIndex++
+  }
+
+  return {
+    id: documentId,
+    title,
+    chunks,
+    embeddingMetadata: {
+      model: provider.model,
+      provider: provider.type,
+      dimensions: provider.dimensions,
+      generatedAt: new Date().toISOString(),
+    },
+    generatedAt: new Date().toISOString(),
+    contentHash,
+  }
+}

package/src/runtime/index.ts

@@ -49,3 +49,21 @@ export {
   type BatchError,
   type BatchProcessResult,
 } from './batch.js'
+
+// Build-time utilities
+export {
+  simpleChunkText,
+  hashContent,
+  loadExistingIndex,
+  generateDocumentEmbeddings,
+} from './build-time.js'
+
+// Markdown processing
+export { stripMarkdown, extractMarkdownText } from './markdown.js'
+
+// Provider helpers
+export {
+  createProviderFromEnv,
+  getProviderConfigFromEnv,
+  type ProviderType,
+} from './provider-helpers.js'

package/src/runtime/markdown.ts

@@ -0,0 +1,119 @@
+/**
+ * Markdown processing utilities for content preparation
+ */
+
+/**
+ * Strip markdown formatting for cleaner text suitable for embeddings
+ *
+ * Removes code blocks, formatting markers, links, images, and HTML tags
+ * while preserving the actual content.
+ *
+ * @param markdown - Markdown text to process
+ * @returns Plain text with markdown removed
+ *
+ * @example
+ * ```typescript
+ * import { stripMarkdown } from '@opensaas/stack-rag/runtime'
+ *
+ * const markdown = '# Hello\n\nThis is **bold** text with a [link](url).'
+ * const plain = stripMarkdown(markdown)
+ * // Returns: 'Hello\n\nThis is bold text with a link.'
+ * ```
+ */
+export function stripMarkdown(markdown: string): string {
+  let text = markdown
+
+  // Remove code blocks
+  text = text.replace(/```[\s\S]*?```/g, '')
+  text = text.replace(/`[^`]+`/g, '')
+
+  // Remove headings markers but keep text
+  text = text.replace(/^#+\s+/gm, '')
+
+  // Remove bold/italic markers
+  text = text.replace(/\*\*([^*]+)\*\*/g, '$1')
+  text = text.replace(/\*([^*]+)\*/g, '$1')
+  text = text.replace(/__([^_]+)__/g, '$1')
+  text = text.replace(/_([^_]+)_/g, '$1')
+
+  // Remove links but keep text
+  text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+
+  // Remove images
+  text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+
+  // Remove HTML tags
+  text = text.replace(/<[^>]+>/g, '')
+
+  // Normalize whitespace
+  text = text.replace(/\n{3,}/g, '\n\n')
+  text = text.replace(/[ \t]+/g, ' ')
+
+  return text.trim()
+}
+
+/**
+ * Extract text content from common markdown structures
+ *
+ * More aggressive than stripMarkdown - extracts only text content,
+ * removes all structural elements.
+ *
+ * @param markdown - Markdown text
+ * @returns Extracted plain text
+ */
+export function extractMarkdownText(markdown: string): string {
+  let text = markdown
+
+  // Remove YAML frontmatter
+  text = text.replace(/^---[\s\S]*?---\n/m, '')
+
+  // Remove code blocks entirely (including content)
+  text = text.replace(/```[\s\S]*?```/g, '')
+
+  // Remove inline code
+  text = text.replace(/`[^`]+`/g, '')
+
+  // Remove horizontal rules
+  text = text.replace(/^[-*_]{3,}$/gm, '')
+
+  // Remove blockquotes markers
+  text = text.replace(/^>\s+/gm, '')
+
+  // Remove list markers
+  text = text.replace(/^[\s]*[-*+]\s+/gm, '')
+  text = text.replace(/^[\s]*\d+\.\s+/gm, '')
+
+  // Remove headings markers
+  text = text.replace(/^#+\s+/gm, '')
+
+  // Remove emphasis markers
+  text = text.replace(/\*\*([^*]+)\*\*/g, '$1')
+  text = text.replace(/\*([^*]+)\*/g, '$1')
+  text = text.replace(/__([^_]+)__/g, '$1')
+  text = text.replace(/_([^_]+)_/g, '$1')
+
+  // Remove strikethrough
+  text = text.replace(/~~([^~]+)~~/g, '$1')
+
+  // Remove links but keep text
+  text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+
+  // Remove reference-style links
+  text = text.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1')
+
+  // Remove images
+  text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+
+  // Remove HTML tags
+  text = text.replace(/<[^>]+>/g, '')
+
+  // Remove HTML entities
+  text = text.replace(/&[a-z]+;/gi, '')
+
+  // Normalize whitespace
+  text = text.replace(/\n{3,}/g, '\n\n')
+  text = text.replace(/[ \t]+/g, ' ')
+  text = text.replace(/^\s+|\s+$/gm, '')
+
+  return text.trim()
+}