@opensaas/stack-rag 0.1.6 → 0.3.0

package/src/runtime/build-time.ts

@@ -0,0 +1,216 @@
+/**
+ * Build-time utilities for generating and managing embeddings
+ * Used by CLI tools and custom build scripts
+ */
+
+import { readFileSync, existsSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import type { EmbeddingProvider } from '../providers/types.js'
+import type { EmbeddingsIndex, EmbeddedDocument, EmbeddingChunk } from '../config/types.js'
+
+/**
+ * Simple character-based text chunking for build-time generation
+ *
+ * Simpler than the runtime chunking strategies, optimized for build-time batch processing.
+ * Splits text into fixed-size chunks with overlap.
+ *
+ * @param text - Text to chunk
+ * @param chunkSize - Size of each chunk in characters
+ * @param overlap - Overlap between chunks in characters
+ * @returns Array of text chunks
+ *
+ * @example
+ * ```typescript
+ * import { simpleChunkText } from '@opensaas/stack-rag/runtime'
+ *
+ * const chunks = simpleChunkText("Long document...", 500, 50)
+ * ```
+ */
+export function simpleChunkText(text: string, chunkSize: number, overlap: number): string[] {
+  const chunks: string[] = []
+  let start = 0
+
+  while (start < text.length) {
+    const end = Math.min(start + chunkSize, text.length)
+    chunks.push(text.slice(start, end))
+    start += chunkSize - overlap
+  }
+
+  return chunks
+}
+
+/**
+ * Compute SHA256 hash of content for change detection
+ *
+ * @param content - Content to hash
+ * @returns Hexadecimal hash string
+ *
+ * @example
+ * ```typescript
+ * import { hashContent } from '@opensaas/stack-rag/runtime'
+ *
+ * const hash = hashContent("document content")
+ * ```
+ */
+export function hashContent(content: string): string {
+  return createHash('sha256').update(content).digest('hex')
+}
+
+/**
+ * Load existing embeddings index from file
+ *
+ * Used for differential updates - only regenerate embeddings for changed content.
+ *
+ * @param filePath - Path to embeddings JSON file
+ * @returns Loaded index or null if file doesn't exist or can't be loaded
+ *
+ * @example
+ * ```typescript
+ * import { loadExistingIndex } from '@opensaas/stack-rag/runtime'
+ *
+ * const existing = loadExistingIndex('.embeddings/docs.json')
+ * if (existing) {
+ *   console.log(`Found ${Object.keys(existing.documents).length} existing documents`)
+ * }
+ * ```
+ */
+export function loadExistingIndex(filePath: string): EmbeddingsIndex | null {
+  if (!existsSync(filePath)) {
+    return null
+  }
+
+  try {
+    const content = readFileSync(filePath, 'utf-8')
+    return JSON.parse(content) as EmbeddingsIndex
+  } catch {
+    console.warn(`Warning: Could not load existing embeddings from ${filePath}`)
+    return null
+  }
+}
+
+/**
+ * Generate embeddings for a document with chunking
+ *
+ * Main utility for build-time embedding generation. Chunks the document,
+ * generates embeddings for each chunk, and returns a complete EmbeddedDocument.
+ *
+ * @param documentId - Unique identifier for the document
+ * @param content - Document content (plain text)
+ * @param provider - Embedding provider instance
+ * @param options - Generation options
+ * @returns Complete embedded document ready to be added to index
+ *
+ * @example
+ * ```typescript
+ * import { generateDocumentEmbeddings } from '@opensaas/stack-rag/runtime'
+ * import { createEmbeddingProvider } from '@opensaas/stack-rag/providers'
+ *
+ * const provider = createEmbeddingProvider({
+ *   type: 'openai',
+ *   apiKey: process.env.OPENAI_API_KEY
+ * })
+ *
+ * const doc = await generateDocumentEmbeddings(
+ *   'docs/getting-started',
+ *   'Document content here...',
+ *   provider,
+ *   {
+ *     title: 'Getting Started',
+ *     chunkSize: 500,
+ *     chunkOverlap: 50,
+ *     metadata: { section: 'guides' }
+ *   }
+ * )
+ * ```
+ */
+export async function generateDocumentEmbeddings(
+  documentId: string,
+  content: string,
+  provider: EmbeddingProvider,
+  options: {
+    title?: string
+    chunkSize: number
+    chunkOverlap: number
+    metadata?: Record<string, unknown>
+  },
+): Promise<EmbeddedDocument> {
+  const { title, chunkSize, chunkOverlap, metadata = {} } = options
+
+  // Hash content for differential updates
+  const contentHash = hashContent(content)
+
+  // Prepare all text chunks to embed
+  const allTextChunks: string[] = []
+  const chunkTypes: Array<'title' | 'content'> = []
+
+  // Add title chunk first if title exists
+  if (title) {
+    allTextChunks.push(title)
+    chunkTypes.push('title')
+  }
+
+  // Chunk the content
+  const contentChunks = simpleChunkText(content, chunkSize, chunkOverlap)
+  allTextChunks.push(...contentChunks)
+  contentChunks.forEach(() => chunkTypes.push('content'))
+
+  // Generate embeddings in batch for all chunks
+  const allEmbeddings = await provider.embedBatch(allTextChunks)
+
+  // Build chunks with embeddings
+  const chunks: EmbeddingChunk[] = []
+
+  let embeddingIndex = 0
+  let contentChunkIndex = 0
+
+  for (let i = 0; i < chunkTypes.length; i++) {
+    const type = chunkTypes[i]
+
+    if (type === 'title') {
+      // Title chunk
+      chunks.push({
+        text: allTextChunks[embeddingIndex],
+        embedding: allEmbeddings[embeddingIndex],
+        metadata: {
+          chunkIndex: -1, // Special index for title
+          startOffset: 0,
+          endOffset: 0,
+          isTitle: true,
+          ...metadata,
+        },
+      })
+    } else {
+      // Content chunk
+      chunks.push({
+        text: allTextChunks[embeddingIndex],
+        embedding: allEmbeddings[embeddingIndex],
+        metadata: {
+          chunkIndex: contentChunkIndex,
+          startOffset: contentChunkIndex * (chunkSize - chunkOverlap),
+          endOffset: Math.min(
+            (contentChunkIndex + 1) * chunkSize - contentChunkIndex * chunkOverlap,
+            content.length,
+          ),
+          ...metadata,
+        },
+      })
+      contentChunkIndex++
+    }
+
+    embeddingIndex++
+  }
+
+  return {
+    id: documentId,
+    title,
+    chunks,
+    embeddingMetadata: {
+      model: provider.model,
+      provider: provider.type,
+      dimensions: provider.dimensions,
+      generatedAt: new Date().toISOString(),
+    },
+    generatedAt: new Date().toISOString(),
+    contentHash,
+  }
+}

package/src/runtime/index.ts

@@ -49,3 +49,21 @@ export {
   type BatchError,
   type BatchProcessResult,
 } from './batch.js'
+
+// Build-time utilities
+export {
+  simpleChunkText,
+  hashContent,
+  loadExistingIndex,
+  generateDocumentEmbeddings,
+} from './build-time.js'
+
+// Markdown processing
+export { stripMarkdown, extractMarkdownText } from './markdown.js'
+
+// Provider helpers
+export {
+  createProviderFromEnv,
+  getProviderConfigFromEnv,
+  type ProviderType,
+} from './provider-helpers.js'

package/src/runtime/markdown.ts

@@ -0,0 +1,119 @@
+/**
+ * Markdown processing utilities for content preparation
+ */
+
+/**
+ * Strip markdown formatting for cleaner text suitable for embeddings
+ *
+ * Removes code blocks, formatting markers, links, images, and HTML tags
+ * while preserving the actual content.
+ *
+ * @param markdown - Markdown text to process
+ * @returns Plain text with markdown removed
+ *
+ * @example
+ * ```typescript
+ * import { stripMarkdown } from '@opensaas/stack-rag/runtime'
+ *
+ * const markdown = '# Hello\n\nThis is **bold** text with a [link](url).'
+ * const plain = stripMarkdown(markdown)
+ * // Returns: 'Hello\n\nThis is bold text with a link.'
+ * ```
+ */
+export function stripMarkdown(markdown: string): string {
+  let text = markdown
+
+  // Remove code blocks
+  text = text.replace(/```[\s\S]*?```/g, '')
+  text = text.replace(/`[^`]+`/g, '')
+
+  // Remove headings markers but keep text
+  text = text.replace(/^#+\s+/gm, '')
+
+  // Remove bold/italic markers
+  text = text.replace(/\*\*([^*]+)\*\*/g, '$1')
+  text = text.replace(/\*([^*]+)\*/g, '$1')
+  text = text.replace(/__([^_]+)__/g, '$1')
+  text = text.replace(/_([^_]+)_/g, '$1')
+
+  // Remove links but keep text
+  text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+
+  // Remove images
+  text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+
+  // Remove HTML tags
+  text = text.replace(/<[^>]+>/g, '')
+
+  // Normalize whitespace
+  text = text.replace(/\n{3,}/g, '\n\n')
+  text = text.replace(/[ \t]+/g, ' ')
+
+  return text.trim()
+}
+
+/**
+ * Extract text content from common markdown structures
+ *
+ * More aggressive than stripMarkdown - extracts only text content,
+ * removes all structural elements.
+ *
+ * @param markdown - Markdown text
+ * @returns Extracted plain text
+ */
+export function extractMarkdownText(markdown: string): string {
+  let text = markdown
+
+  // Remove YAML frontmatter
+  text = text.replace(/^---[\s\S]*?---\n/m, '')
+
+  // Remove code blocks entirely (including content)
+  text = text.replace(/```[\s\S]*?```/g, '')
+
+  // Remove inline code
+  text = text.replace(/`[^`]+`/g, '')
+
+  // Remove horizontal rules
+  text = text.replace(/^[-*_]{3,}$/gm, '')
+
+  // Remove blockquotes markers
+  text = text.replace(/^>\s+/gm, '')
+
+  // Remove list markers
+  text = text.replace(/^[\s]*[-*+]\s+/gm, '')
+  text = text.replace(/^[\s]*\d+\.\s+/gm, '')
+
+  // Remove headings markers
+  text = text.replace(/^#+\s+/gm, '')
+
+  // Remove emphasis markers
+  text = text.replace(/\*\*([^*]+)\*\*/g, '$1')
+  text = text.replace(/\*([^*]+)\*/g, '$1')
+  text = text.replace(/__([^_]+)__/g, '$1')
+  text = text.replace(/_([^_]+)_/g, '$1')
+
+  // Remove strikethrough
+  text = text.replace(/~~([^~]+)~~/g, '$1')
+
+  // Remove links but keep text
+  text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+
+  // Remove reference-style links
+  text = text.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1')
+
+  // Remove images
+  text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+
+  // Remove HTML tags
+  text = text.replace(/<[^>]+>/g, '')
+
+  // Remove HTML entities
+  text = text.replace(/&[a-z]+;/gi, '')
+
+  // Normalize whitespace
+  text = text.replace(/\n{3,}/g, '\n\n')
+  text = text.replace(/[ \t]+/g, ' ')
+  text = text.replace(/^\s+|\s+$/gm, '')
+
+  return text.trim()
+}

package/src/runtime/provider-helpers.ts

@@ -0,0 +1,115 @@
+/**
+ * Helper utilities for working with embedding providers
+ * Simplifies provider creation from environment variables
+ */
+
+import { createEmbeddingProvider } from '../providers/index.js'
+import type { EmbeddingProvider } from '../providers/types.js'
+import type { EmbeddingProviderConfig } from '../config/types.js'
+import 'dotenv/config'
+
+/**
+ * Provider type from environment or configuration
+ */
+export type ProviderType = 'openai' | 'ollama'
+
+/**
+ * Create an embedding provider from environment variables
+ *
+ * Reads configuration from environment variables:
+ * - EMBEDDING_PROVIDER: 'openai' or 'ollama' (default: 'openai')
+ * - OPENAI_API_KEY: Required if using OpenAI
+ * - OLLAMA_BASE_URL: Ollama endpoint (default: 'http://localhost:11434')
+ *
+ * @param overrides - Optional overrides for environment config
+ * @returns Configured embedding provider
+ *
+ * @example
+ * ```typescript
+ * import { createProviderFromEnv } from '@opensaas/stack-rag/runtime'
+ *
+ * // Uses EMBEDDING_PROVIDER and OPENAI_API_KEY from env
+ * const provider = createProviderFromEnv()
+ *
+ * // Override provider type
+ * const ollamaProvider = createProviderFromEnv({ provider: 'ollama' })
+ * ```
+ */
+export function createProviderFromEnv(overrides?: {
+  provider?: ProviderType
+  openaiApiKey?: string
+  ollamaBaseUrl?: string
+  model?: string
+}): EmbeddingProvider {
+  const providerType =
+    overrides?.provider || (process.env.EMBEDDING_PROVIDER as ProviderType) || 'openai'
+
+  let config: EmbeddingProviderConfig
+
+  if (providerType === 'openai') {
+    const apiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY
+
+    if (!apiKey) {
+      throw new Error('OPENAI_API_KEY environment variable is required when using OpenAI provider')
+    }
+
+    config = {
+      type: 'openai',
+      apiKey,
+      model:
+        (overrides?.model as 'text-embedding-3-small' | 'text-embedding-3-large') ||
+        'text-embedding-3-small',
+    }
+  } else if (providerType === 'ollama') {
+    config = {
+      type: 'ollama',
+      baseURL: overrides?.ollamaBaseUrl || process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
+      model: overrides?.model || 'nomic-embed-text',
+    }
+  } else {
+    throw new Error(`Unknown provider type: ${providerType}. Supported: openai, ollama`)
+  }
+
+  return createEmbeddingProvider(config)
+}
+
+/**
+ * Get provider configuration from environment
+ *
+ * Useful for inspecting what provider would be used without creating it.
+ *
+ * @returns Provider configuration object
+ *
+ * @example
+ * ```typescript
+ * import { getProviderConfigFromEnv } from '@opensaas/stack-rag/runtime'
+ *
+ * const config = getProviderConfigFromEnv()
+ * console.log(`Using ${config.type} provider`)
+ * ```
+ */
+export function getProviderConfigFromEnv(): EmbeddingProviderConfig {
+  const providerType = (process.env.EMBEDDING_PROVIDER as ProviderType) || 'openai'
+
+  if (providerType === 'openai') {
+    const apiKey = process.env.OPENAI_API_KEY
+
+    if (!apiKey) {
+      throw new Error('OPENAI_API_KEY environment variable is required')
+    }
+
+    return {
+      type: 'openai',
+      apiKey,
+      model: 'text-embedding-3-small',
+    }
+  } else if (providerType === 'ollama') {
+    return {
+      type: 'ollama',
+      baseURL: process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
+      model: 'nomic-embed-text',
+    }
+  } else {
+    throw new Error(`Unknown provider type: ${providerType}`)
+  }
+}

package/src/runtime/types.ts

@@ -0,0 +1,30 @@
+/**
+ * Type definitions for RAG plugin runtime services
+ * These types are used for type-safe access to context.plugins.rag
+ */
+
+/**
+ * Runtime services provided by the RAG plugin
+ * Available via context.plugins.rag
+ */
+export interface RAGRuntimeServices {
+  /**
+   * Generate embedding for a given text
+   * Uses the configured embedding provider
+   *
+   * @param text - The text to generate an embedding for
+   * @param providerName - Optional provider name if multiple providers are configured
+   * @returns Vector embedding as array of numbers
+   */
+  generateEmbedding: (text: string, providerName?: string) => Promise<number[]>
+
+  /**
+   * Generate embeddings for multiple texts (batch)
+   * More efficient than calling generateEmbedding multiple times
+   *
+   * @param texts - Array of texts to generate embeddings for
+   * @param providerName - Optional provider name if multiple providers are configured
+   * @returns Array of vector embeddings
+   */
+  generateEmbeddings: (texts: string[], providerName?: string) => Promise<number[][]>
+}