@opensaas/stack-rag 0.1.7 → 0.3.0

package/src/runtime/index.ts

@@ -49,3 +49,21 @@ export {
   type BatchError,
   type BatchProcessResult,
 } from './batch.js'
+
+// Build-time utilities
+export {
+  simpleChunkText,
+  hashContent,
+  loadExistingIndex,
+  generateDocumentEmbeddings,
+} from './build-time.js'
+
+// Markdown processing
+export { stripMarkdown, extractMarkdownText } from './markdown.js'
+
+// Provider helpers
+export {
+  createProviderFromEnv,
+  getProviderConfigFromEnv,
+  type ProviderType,
+} from './provider-helpers.js'

package/src/runtime/markdown.ts

@@ -0,0 +1,119 @@
+/**
+ * Markdown processing utilities for content preparation
+ */
+
+/**
+ * Strip markdown formatting for cleaner text suitable for embeddings
+ *
+ * Removes code blocks, formatting markers, links, images, and HTML tags
+ * while preserving the actual content.
+ *
+ * @param markdown - Markdown text to process
+ * @returns Plain text with markdown removed
+ *
+ * @example
+ * ```typescript
+ * import { stripMarkdown } from '@opensaas/stack-rag/runtime'
+ *
+ * const markdown = '# Hello\n\nThis is **bold** text with a [link](url).'
+ * const plain = stripMarkdown(markdown)
+ * // Returns: 'Hello\n\nThis is bold text with a link.'
+ * ```
+ */
+export function stripMarkdown(markdown: string): string {
+  let text = markdown
+
+  // Remove code blocks
+  text = text.replace(/```[\s\S]*?```/g, '')
+  text = text.replace(/`[^`]+`/g, '')
+
+  // Remove headings markers but keep text
+  text = text.replace(/^#+\s+/gm, '')
+
+  // Remove bold/italic markers
+  text = text.replace(/\*\*([^*]+)\*\*/g, '$1')
+  text = text.replace(/\*([^*]+)\*/g, '$1')
+  text = text.replace(/__([^_]+)__/g, '$1')
+  text = text.replace(/_([^_]+)_/g, '$1')
+
+  // Remove links but keep text
+  text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+
+  // Remove images
+  text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+
+  // Remove HTML tags
+  text = text.replace(/<[^>]+>/g, '')
+
+  // Normalize whitespace
+  text = text.replace(/\n{3,}/g, '\n\n')
+  text = text.replace(/[ \t]+/g, ' ')
+
+  return text.trim()
+}
+
+/**
+ * Extract text content from common markdown structures
+ *
+ * More aggressive than stripMarkdown - extracts only text content,
+ * removes all structural elements.
+ *
+ * @param markdown - Markdown text
+ * @returns Extracted plain text
+ */
+export function extractMarkdownText(markdown: string): string {
+  let text = markdown
+
+  // Remove YAML frontmatter
+  text = text.replace(/^---[\s\S]*?---\n/m, '')
+
+  // Remove code blocks entirely (including content)
+  text = text.replace(/```[\s\S]*?```/g, '')
+
+  // Remove inline code
+  text = text.replace(/`[^`]+`/g, '')
+
+  // Remove horizontal rules
+  text = text.replace(/^[-*_]{3,}$/gm, '')
+
+  // Remove blockquotes markers
+  text = text.replace(/^>\s+/gm, '')
+
+  // Remove list markers
+  text = text.replace(/^[\s]*[-*+]\s+/gm, '')
+  text = text.replace(/^[\s]*\d+\.\s+/gm, '')
+
+  // Remove headings markers
+  text = text.replace(/^#+\s+/gm, '')
+
+  // Remove emphasis markers
+  text = text.replace(/\*\*([^*]+)\*\*/g, '$1')
+  text = text.replace(/\*([^*]+)\*/g, '$1')
+  text = text.replace(/__([^_]+)__/g, '$1')
+  text = text.replace(/_([^_]+)_/g, '$1')
+
+  // Remove strikethrough
+  text = text.replace(/~~([^~]+)~~/g, '$1')
+
+  // Remove links but keep text
+  text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+
+  // Remove reference-style links
+  text = text.replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1')
+
+  // Remove images
+  text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+
+  // Remove HTML tags
+  text = text.replace(/<[^>]+>/g, '')
+
+  // Remove HTML entities
+  text = text.replace(/&[a-z]+;/gi, '')
+
+  // Normalize whitespace
+  text = text.replace(/\n{3,}/g, '\n\n')
+  text = text.replace(/[ \t]+/g, ' ')
+  text = text.replace(/^\s+|\s+$/gm, '')
+
+  return text.trim()
+}

package/src/runtime/provider-helpers.ts

@@ -0,0 +1,115 @@
+/**
+ * Helper utilities for working with embedding providers
+ * Simplifies provider creation from environment variables
+ */
+
+import { createEmbeddingProvider } from '../providers/index.js'
+import type { EmbeddingProvider } from '../providers/types.js'
+import type { EmbeddingProviderConfig } from '../config/types.js'
+import 'dotenv/config'
+
+/**
+ * Provider type from environment or configuration
+ */
+export type ProviderType = 'openai' | 'ollama'
+
+/**
+ * Create an embedding provider from environment variables
+ *
+ * Reads configuration from environment variables:
+ * - EMBEDDING_PROVIDER: 'openai' or 'ollama' (default: 'openai')
+ * - OPENAI_API_KEY: Required if using OpenAI
+ * - OLLAMA_BASE_URL: Ollama endpoint (default: 'http://localhost:11434')
+ *
+ * @param overrides - Optional overrides for environment config
+ * @returns Configured embedding provider
+ *
+ * @example
+ * ```typescript
+ * import { createProviderFromEnv } from '@opensaas/stack-rag/runtime'
+ *
+ * // Uses EMBEDDING_PROVIDER and OPENAI_API_KEY from env
+ * const provider = createProviderFromEnv()
+ *
+ * // Override provider type
+ * const ollamaProvider = createProviderFromEnv({ provider: 'ollama' })
+ * ```
+ */
+export function createProviderFromEnv(overrides?: {
+  provider?: ProviderType
+  openaiApiKey?: string
+  ollamaBaseUrl?: string
+  model?: string
+}): EmbeddingProvider {
+  const providerType =
+    overrides?.provider || (process.env.EMBEDDING_PROVIDER as ProviderType) || 'openai'
+
+  let config: EmbeddingProviderConfig
+
+  if (providerType === 'openai') {
+    const apiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY
+
+    if (!apiKey) {
+      throw new Error('OPENAI_API_KEY environment variable is required when using OpenAI provider')
+    }
+
+    config = {
+      type: 'openai',
+      apiKey,
+      model:
+        (overrides?.model as 'text-embedding-3-small' | 'text-embedding-3-large') ||
+        'text-embedding-3-small',
+    }
+  } else if (providerType === 'ollama') {
+    config = {
+      type: 'ollama',
+      baseURL: overrides?.ollamaBaseUrl || process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
+      model: overrides?.model || 'nomic-embed-text',
+    }
+  } else {
+    throw new Error(`Unknown provider type: ${providerType}. Supported: openai, ollama`)
+  }
+
+  return createEmbeddingProvider(config)
+}
+
+/**
+ * Get provider configuration from environment
+ *
+ * Useful for inspecting what provider would be used without creating it.
+ *
+ * @returns Provider configuration object
+ *
+ * @example
+ * ```typescript
+ * import { getProviderConfigFromEnv } from '@opensaas/stack-rag/runtime'
+ *
+ * const config = getProviderConfigFromEnv()
+ * console.log(`Using ${config.type} provider`)
+ * ```
+ */
+export function getProviderConfigFromEnv(): EmbeddingProviderConfig {
+  const providerType = (process.env.EMBEDDING_PROVIDER as ProviderType) || 'openai'
+
+  if (providerType === 'openai') {
+    const apiKey = process.env.OPENAI_API_KEY
+
+    if (!apiKey) {
+      throw new Error('OPENAI_API_KEY environment variable is required')
+    }
+
+    return {
+      type: 'openai',
+      apiKey,
+      model: 'text-embedding-3-small',
+    }
+  } else if (providerType === 'ollama') {
+    return {
+      type: 'ollama',
+      baseURL: process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
+      model: 'nomic-embed-text',
+    }
+  } else {
+    throw new Error(`Unknown provider type: ${providerType}`)
+  }
+}

package/src/runtime/types.ts

@@ -0,0 +1,30 @@
+/**
+ * Type definitions for RAG plugin runtime services
+ * These types are used for type-safe access to context.plugins.rag
+ */
+
+/**
+ * Runtime services provided by the RAG plugin
+ * Available via context.plugins.rag
+ */
+export interface RAGRuntimeServices {
+  /**
+   * Generate embedding for a given text
+   * Uses the configured embedding provider
+   *
+   * @param text - The text to generate an embedding for
+   * @param providerName - Optional provider name if multiple providers are configured
+   * @returns Vector embedding as array of numbers
+   */
+  generateEmbedding: (text: string, providerName?: string) => Promise<number[]>
+
+  /**
+   * Generate embeddings for multiple texts (batch)
+   * More efficient than calling generateEmbedding multiple times
+   *
+   * @param texts - Array of texts to generate embeddings for
+   * @param providerName - Optional provider name if multiple providers are configured
+   * @returns Array of vector embeddings
+   */
+  generateEmbeddings: (texts: string[], providerName?: string) => Promise<number[][]>
+}

package/src/storage/index.ts

@@ -82,6 +82,7 @@ export function createVectorStorage(config: VectorStorageConfig): VectorStorage
 // Export types and individual storage backends
 export * from './types.js'
 export { JsonVectorStorage, createJsonStorage } from './json.js'
+export { JsonFileStorage, createJsonFileStorage } from './json-file.js'
 export { PgVectorStorage, createPgVectorStorage } from './pgvector.js'
 export { SqliteVssStorage, createSqliteVssStorage } from './sqlite-vss.js'
 

package/src/storage/json-file.ts

@@ -0,0 +1,157 @@
+import { readFileSync, existsSync } from 'node:fs'
+import type { VectorStorage, SearchOptions } from './types.js'
+import type { SearchResult, EmbeddingsIndex } from '../config/types.js'
+import { cosineSimilarity as calculateCosineSimilarity } from './types.js'
+
+/**
+ * JSON file-based vector storage
+ * Loads embeddings from a JSON file generated at build time
+ * Performs similarity search in JavaScript without database queries
+ * Ideal for static sites, documentation, and build-time generated embeddings
+ */
+export class JsonFileStorage implements VectorStorage {
+  readonly type = 'json-file'
+
+  private index: EmbeddingsIndex | null = null
+  private filePath: string
+
+  constructor(filePath: string) {
+    this.filePath = filePath
+  }
+
+  /**
+   * Load embeddings index from file
+   * Caches the result in memory
+   */
+  private loadIndex(): EmbeddingsIndex {
+    if (this.index) {
+      return this.index
+    }
+
+    if (!existsSync(this.filePath)) {
+      throw new Error(
+        `Embeddings file not found: ${this.filePath}. Run embeddings generation first.`,
+      )
+    }
+
+    try {
+      const content = readFileSync(this.filePath, 'utf-8')
+      this.index = JSON.parse(content) as EmbeddingsIndex
+      return this.index
+    } catch (error) {
+      throw new Error(`Failed to load embeddings from ${this.filePath}: ${error}`)
+    }
+  }
+
+  /**
+   * Search for similar documents using JavaScript-based cosine similarity
+   * Note: listKey parameter is included for interface compatibility but not used
+   * since this storage is typically used for standalone content (e.g., docs)
+   */
+  async search<T = unknown>(
+    _listKey: string,
+    _fieldName: string,
+    queryVector: number[],
+    options: SearchOptions,
+  ): Promise<SearchResult<T>[]> {
+    const { limit = 10, minScore = 0.0, where = {} } = options
+
+    const index = this.loadIndex()
+
+    // Validate query vector dimensions against index config
+    if (queryVector.length !== index.config.dimensions) {
+      throw new Error(
+        `Query vector dimensions (${queryVector.length}) don't match index dimensions (${index.config.dimensions})`,
+      )
+    }
+
+    const results: Array<{
+      item: T
+      score: number
+      distance: number
+      documentId: string
+      chunkIndex: number
+    }> = []
+
+    // Search through all documents and chunks
+    for (const [documentId, document] of Object.entries(index.documents)) {
+      // Apply where filters if provided (simple equality check)
+      if (where && Object.keys(where).length > 0) {
+        let matches = true
+        for (const [key, value] of Object.entries(where)) {
+          if ((document as Record<string, unknown>)[key] !== value) {
+            matches = false
+            break
+          }
+        }
+        if (!matches) continue
+      }
+
+      // Search through each chunk
+      for (const chunk of document.chunks) {
+        const score = this.cosineSimilarity(queryVector, chunk.embedding)
+
+        if (score >= minScore) {
+          results.push({
+            item: {
+              documentId,
+              title: document.title,
+              content: chunk.text,
+              chunkIndex: chunk.metadata.chunkIndex,
+              metadata: chunk.metadata,
+            } as T,
+            score,
+            distance: 1 - score,
+            documentId,
+            chunkIndex: chunk.metadata.chunkIndex,
+          })
+        }
+      }
+    }
+
+    // Sort by score (descending) and limit results
+    results.sort((a, b) => b.score - a.score)
+
+    return results.slice(0, limit)
+  }
+
+  /**
+   * Calculate cosine similarity between two vectors
+   */
+  cosineSimilarity(a: number[], b: number[]): number {
+    return calculateCosineSimilarity(a, b)
+  }
+
+  /**
+   * Get the loaded index for inspection
+   */
+  getIndex(): EmbeddingsIndex | null {
+    return this.index
+  }
+
+  /**
+   * Force reload the index from disk
+   */
+  reloadIndex(): void {
+    this.index = null
+    this.loadIndex()
+  }
+}
+
+/**
+ * Create a JSON file vector storage instance
+ *
+ * @example
+ * ```typescript
+ * import { createJsonFileStorage } from '@opensaas/stack-rag/storage'
+ *
+ * const storage = createJsonFileStorage('.embeddings/docs.json')
+ * const results = await storage.search('', '', queryVector, {
+ *   limit: 10,
+ *   minScore: 0.7
+ * })
+ * ```
+ */
+export function createJsonFileStorage(filePath: string): JsonFileStorage {
+  return new JsonFileStorage(filePath)
+}

package/src/storage/storage.test.ts

@@ -13,6 +13,7 @@ function createMockContext(dbOverrides: Record<string, unknown> = {}): AccessCon
     sudo: vi.fn(),
     prisma: {} as unknown,
     storage: {} as unknown,
+    plugins: {},
     _isSudo: false,
   } as AccessContext<unknown>
 }