@strav/rag 0.1.0

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@strav/rag",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Vector retrieval framework for RAG in the Strav framework",
+  "license": "MIT",
+  "exports": {
+    ".": "./src/index.ts",
+    "./*": "./src/*.ts"
+  },
+  "strav": {
+    "commands": "src/commands"
+  },
+  "files": [
+    "src/",
+    "stubs/",
+    "package.json",
+    "tsconfig.json"
+  ],
+  "peerDependencies": {
+    "@strav/kernel": "0.1.0",
+    "@strav/brain": "0.1.0",
+    "@strav/database": "0.1.0",
+    "@strav/cli": "0.1.0"
+  },
+  "scripts": {
+    "test": "bun test tests/",
+    "typecheck": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "commander": "^14.0.3"
+  }
+}

package/src/chunking/chunker.ts

@@ -0,0 +1,14 @@
+import type { Chunker, ChunkingConfig } from '../types.ts'
+import { FixedSizeChunker } from './fixed_size_chunker.ts'
+import { RecursiveChunker } from './recursive_chunker.ts'
+
+export function createChunker(config: ChunkingConfig): Chunker {
+  switch (config.strategy) {
+    case 'fixed':
+      return new FixedSizeChunker(config.chunkSize, config.overlap)
+    case 'recursive':
+      return new RecursiveChunker(config.chunkSize, config.overlap, config.separators)
+    default:
+      return new RecursiveChunker(config.chunkSize, config.overlap)
+  }
+}

package/src/chunking/fixed_size_chunker.ts

@@ -0,0 +1,33 @@
+import type { Chunk, Chunker } from '../types.ts'
+
+export class FixedSizeChunker implements Chunker {
+  constructor(
+    private readonly chunkSize: number = 512,
+    private readonly overlap: number = 64
+  ) {}
+
+  chunk(content: string): Chunk[] {
+    if (!content) return []
+
+    const chunks: Chunk[] = []
+    const step = Math.max(1, this.chunkSize - this.overlap)
+
+    let start = 0
+    let index = 0
+
+    while (start < content.length) {
+      const end = Math.min(start + this.chunkSize, content.length)
+      chunks.push({
+        content: content.slice(start, end),
+        index,
+        startOffset: start,
+        endOffset: end,
+      })
+      index++
+      start += step
+      if (end === content.length) break
+    }
+
+    return chunks
+  }
+}

package/src/chunking/recursive_chunker.ts

@@ -0,0 +1,83 @@
+import type { Chunk, Chunker } from '../types.ts'
+
+const DEFAULT_SEPARATORS = ['\n\n', '\n', '. ', ' ']
+
+export class RecursiveChunker implements Chunker {
+  private readonly separators: string[]
+
+  constructor(
+    private readonly chunkSize: number = 512,
+    private readonly overlap: number = 64,
+    separators?: string[]
+  ) {
+    this.separators = separators ?? DEFAULT_SEPARATORS
+  }
+
+  chunk(content: string): Chunk[] {
+    if (!content) return []
+    const pieces = this.splitRecursive(content, 0)
+    return this.buildChunks(content, pieces)
+  }
+
+  private splitRecursive(text: string, separatorIndex: number): string[] {
+    if (text.length <= this.chunkSize) return [text]
+
+    const separator = this.separators[separatorIndex]
+    if (!separator) {
+      const result: string[] = []
+      for (let i = 0; i < text.length; i += this.chunkSize) {
+        result.push(text.slice(i, i + this.chunkSize))
+      }
+      return result
+    }
+
+    const parts = text.split(separator)
+    const merged: string[] = []
+    let current = ''
+
+    for (const part of parts) {
+      const candidate = current ? current + separator + part : part
+
+      if (candidate.length <= this.chunkSize) {
+        current = candidate
+      } else {
+        if (current) merged.push(current)
+        if (part.length > this.chunkSize) {
+          merged.push(...this.splitRecursive(part, separatorIndex + 1))
+          current = ''
+        } else {
+          current = part
+        }
+      }
+    }
+    if (current) merged.push(current)
+
+    return merged
+  }
+
+  private buildChunks(original: string, pieces: string[]): Chunk[] {
+    const chunks: Chunk[] = []
+    let searchFrom = 0
+
+    for (let i = 0; i < pieces.length; i++) {
+      const piece = pieces[i]!
+      const foundAt = original.indexOf(piece, searchFrom)
+      const startOffset = foundAt >= 0 ? foundAt : searchFrom
+      const pieceEnd = startOffset + piece.length
+
+      const overlapEnd = Math.min(pieceEnd + this.overlap, original.length)
+      const chunkContent = original.slice(startOffset, overlapEnd)
+
+      chunks.push({
+        content: chunkContent,
+        index: i,
+        startOffset,
+        endOffset: overlapEnd,
+      })
+
+      searchFrom = pieceEnd
+    }
+
+    return chunks
+  }
+}

package/src/commands/rag_flush.ts

@@ -0,0 +1,41 @@
+import type { Command } from 'commander'
+import chalk from 'chalk'
+import { bootstrap, shutdown } from '@stravigor/cli'
+import { BaseModel } from '@stravigor/database'
+import RagManager from '../rag_manager.ts'
+
+export function register(program: Command): void {
+  program
+    .command('rag:flush <model>')
+    .description("Flush all vectors from a model's vector collection")
+    .action(async (modelPath: string) => {
+      let db
+      try {
+        const { db: database, config } = await bootstrap()
+        db = database
+
+        new BaseModel(db)
+        new RagManager(config)
+
+        const resolved = require.resolve(`${process.cwd()}/${modelPath}`)
+        const module = await import(resolved)
+        const ModelClass = module.default ?? (Object.values(module)[0] as any)
+
+        if (typeof ModelClass?.flushVectors !== 'function') {
+          console.error(chalk.red(`Model "${modelPath}" does not use the retrievable() mixin.`))
+          process.exit(1)
+        }
+
+        const collectionName = ModelClass.retrievableAs()
+        console.log(chalk.dim(`Flushing "${collectionName}"...`))
+
+        await ModelClass.flushVectors()
+        console.log(chalk.green(`Flushed all vectors from "${collectionName}".`))
+      } catch (err) {
+        console.error(chalk.red(`Error: ${err instanceof Error ? err.message : err}`))
+        process.exit(1)
+      } finally {
+        if (db) await shutdown(db)
+      }
+    })
+}

package/src/commands/rag_ingest.ts

@@ -0,0 +1,45 @@
+import type { Command } from 'commander'
+import chalk from 'chalk'
+import { bootstrap, shutdown } from '@stravigor/cli'
+import { BaseModel } from '@stravigor/database'
+import { BrainManager } from '@stravigor/brain'
+import RagManager from '../rag_manager.ts'
+
+export function register(program: Command): void {
+  program
+    .command('rag:ingest <model>')
+    .description('Vectorize all records for a model into the vector store')
+    .option('--chunk <size>', 'Records per batch', '100')
+    .action(async (modelPath: string, options: { chunk: string }) => {
+      let db
+      try {
+        const { db: database, config } = await bootstrap()
+        db = database
+
+        new BaseModel(db)
+        new RagManager(config)
+        new BrainManager(config)
+
+        const resolved = require.resolve(`${process.cwd()}/${modelPath}`)
+        const module = await import(resolved)
+        const ModelClass = module.default ?? (Object.values(module)[0] as any)
+
+        if (typeof ModelClass?.importAll !== 'function') {
+          console.error(chalk.red(`Model "${modelPath}" does not use the retrievable() mixin.`))
+          process.exit(1)
+        }
+
+        const chunkSize = parseInt(options.chunk, 10)
+        const collectionName = ModelClass.retrievableAs()
+        console.log(chalk.dim(`Vectorizing ${ModelClass.name} into "${collectionName}"...`))
+
+        const count = await ModelClass.importAll(chunkSize)
+        console.log(chalk.green(`Vectorized ${count} record(s) into "${collectionName}".`))
+      } catch (err) {
+        console.error(chalk.red(`Error: ${err instanceof Error ? err.message : err}`))
+        process.exit(1)
+      } finally {
+        if (db) await shutdown(db)
+      }
+    })
+}

package/src/drivers/memory_driver.ts

@@ -0,0 +1,135 @@
+import type { VectorStore } from '../vector_store.ts'
+import type { VectorDocument, QueryOptions, QueryResult, VectorMatch } from '../types.ts'
+
+export class MemoryDriver implements VectorStore {
+  readonly name = 'memory'
+  private collections = new Map<string, VectorDocument[]>()
+
+  async createCollection(collection: string, _dimension: number): Promise<void> {
+    if (!this.collections.has(collection)) {
+      this.collections.set(collection, [])
+    }
+  }
+
+  async deleteCollection(collection: string): Promise<void> {
+    this.collections.delete(collection)
+  }
+
+  async upsert(collection: string, documents: VectorDocument[]): Promise<void> {
+    let docs = this.collections.get(collection)
+    if (!docs) {
+      docs = []
+      this.collections.set(collection, docs)
+    }
+
+    for (const doc of documents) {
+      if (doc.id != null) {
+        const existingIndex = docs.findIndex(d => d.id === doc.id)
+        if (existingIndex >= 0) {
+          docs[existingIndex] = doc
+        } else {
+          docs.push(doc)
+        }
+      } else {
+        docs.push(doc)
+      }
+    }
+  }
+
+  async delete(collection: string, ids: (string | number)[]): Promise<void> {
+    const docs = this.collections.get(collection)
+    if (!docs) return
+
+    const idSet = new Set(ids.map(String))
+    this.collections.set(
+      collection,
+      docs.filter(d => !idSet.has(String(d.id)))
+    )
+  }
+
+  async deleteBySource(collection: string, sourceId: string | number): Promise<void> {
+    const docs = this.collections.get(collection)
+    if (!docs) return
+
+    const sourceStr = String(sourceId)
+    this.collections.set(
+      collection,
+      docs.filter(d => String(d.sourceId) !== sourceStr)
+    )
+  }
+
+  async flush(collection: string): Promise<void> {
+    if (this.collections.has(collection)) {
+      this.collections.set(collection, [])
+    }
+  }
+
+  async query(
+    collection: string,
+    vector: number[],
+    options?: QueryOptions
+  ): Promise<QueryResult> {
+    const start = performance.now()
+    const docs = this.collections.get(collection)
+    if (!docs || docs.length === 0) {
+      return { matches: [], processingTimeMs: performance.now() - start }
+    }
+
+    const topK = options?.topK ?? 5
+    const threshold = options?.threshold ?? 0
+
+    let scored: VectorMatch[] = docs.map(doc => ({
+      id: doc.id ?? 0,
+      content: doc.content,
+      score: cosineSimilarity(vector, doc.embedding),
+      metadata: doc.metadata ?? {},
+    }))
+
+    if (options?.filter) {
+      scored = scored.filter(m => matchesFilter(m.metadata, options.filter!))
+    }
+
+    if (threshold > 0) {
+      scored = scored.filter(m => m.score >= threshold)
+    }
+
+    scored.sort((a, b) => b.score - a.score)
+    const matches = scored.slice(0, topK)
+
+    return {
+      matches,
+      processingTimeMs: performance.now() - start,
+    }
+  }
+
+  getCollection(collection: string): VectorDocument[] {
+    return this.collections.get(collection) ?? []
+  }
+}
+
+function cosineSimilarity(a: number[], b: number[]): number {
+  let dot = 0
+  let magA = 0
+  let magB = 0
+
+  for (let i = 0; i < a.length; i++) {
+    const ai = a[i]!
+    const bi = b[i]!
+    dot += ai * bi
+    magA += ai * ai
+    magB += bi * bi
+  }
+
+  const denom = Math.sqrt(magA) * Math.sqrt(magB)
+  return denom === 0 ? 0 : dot / denom
+}
+
+function matchesFilter(
+  metadata: Record<string, unknown>,
+  filter: Record<string, unknown>
+): boolean {
+  for (const [key, value] of Object.entries(filter)) {
+    if (metadata[key] !== value) return false
+  }
+  return true
+}

package/src/drivers/null_driver.ts

@@ -0,0 +1,21 @@
+import type { VectorStore } from '../vector_store.ts'
+import type { VectorDocument, QueryOptions, QueryResult } from '../types.ts'
+
+export class NullDriver implements VectorStore {
+  readonly name = 'null'
+
+  async createCollection(_collection: string, _dimension: number): Promise<void> {}
+  async deleteCollection(_collection: string): Promise<void> {}
+  async upsert(_collection: string, _documents: VectorDocument[]): Promise<void> {}
+  async delete(_collection: string, _ids: (string | number)[]): Promise<void> {}
+  async deleteBySource(_collection: string, _sourceId: string | number): Promise<void> {}
+  async flush(_collection: string): Promise<void> {}
+
+  async query(
+    _collection: string,
+    _vector: number[],
+    _options?: QueryOptions
+  ): Promise<QueryResult> {
+    return { matches: [] }
+  }
+}

package/src/drivers/pgvector_driver.ts

@@ -0,0 +1,157 @@
+import { Database } from '@stravigor/database'
+import type { VectorStore } from '../vector_store.ts'
+import type { VectorDocument, QueryOptions, QueryResult, VectorMatch, StoreConfig } from '../types.ts'
+import { VectorQueryError } from '../errors.ts'
+
+export class PgvectorDriver implements VectorStore {
+  readonly name = 'pgvector'
+  private initialized = false
+
+  constructor(_config: StoreConfig) {}
+
+  async createCollection(collection: string, dimension: number): Promise<void> {
+    await this.ensureTable(dimension)
+
+    const indexName = `idx_strav_vectors_hnsw_${collection.replace(/[^a-z0-9_]/gi, '_')}`
+    try {
+      await Database.raw.unsafe(
+        `CREATE INDEX IF NOT EXISTS "${indexName}"
+         ON _strav_vectors USING hnsw (embedding vector_cosine_ops)
+         WHERE collection = '${collection}'`
+      )
+    } catch {
+      // Index may already exist
+    }
+  }
+
+  async deleteCollection(collection: string): Promise<void> {
+    await Database.raw.unsafe(
+      `DELETE FROM _strav_vectors WHERE collection = $1`,
+      [collection]
+    )
+  }
+
+  async upsert(collection: string, documents: VectorDocument[]): Promise<void> {
+    const sql = Database.raw
+
+    for (const doc of documents) {
+      const embeddingStr = `[${doc.embedding.join(',')}]`
+      const metadata = JSON.stringify(doc.metadata ?? {})
+      const id = doc.id != null ? String(doc.id) : crypto.randomUUID()
+      const sourceId = doc.sourceId != null ? String(doc.sourceId) : null
+
+      await sql.unsafe(
+        `INSERT INTO _strav_vectors (collection, source_id, content, metadata, embedding)
+         VALUES ($1, $2, $3, $4::jsonb, $5::vector)`,
+        [collection, sourceId, doc.content, metadata, embeddingStr]
+      )
+    }
+  }
+
+  async delete(collection: string, ids: (string | number)[]): Promise<void> {
+    if (ids.length === 0) return
+    const placeholders = ids.map((_, i) => `$${i + 2}`).join(', ')
+    await Database.raw.unsafe(
+      `DELETE FROM _strav_vectors WHERE collection = $1 AND id IN (${placeholders})`,
+      [collection, ...ids]
+    )
+  }
+
+  async deleteBySource(collection: string, sourceId: string | number): Promise<void> {
+    await Database.raw.unsafe(
+      `DELETE FROM _strav_vectors WHERE collection = $1 AND source_id = $2`,
+      [collection, String(sourceId)]
+    )
+  }
+
+  async flush(collection: string): Promise<void> {
+    await Database.raw.unsafe(
+      `DELETE FROM _strav_vectors WHERE collection = $1`,
+      [collection]
+    )
+  }
+
+  async query(
+    collection: string,
+    vector: number[],
+    options?: QueryOptions
+  ): Promise<QueryResult> {
+    const start = performance.now()
+    const topK = options?.topK ?? 5
+    const threshold = options?.threshold ?? 0
+    const embeddingStr = `[${vector.join(',')}]`
+
+    let whereClause = 'collection = $1'
+    const params: unknown[] = [collection]
+    let paramIndex = 2
+
+    if (options?.filter) {
+      for (const [key, value] of Object.entries(options.filter)) {
+        whereClause += ` AND metadata->>'${key}' = $${paramIndex}`
+        params.push(String(value))
+        paramIndex++
+      }
+    }
+
+    if (threshold > 0) {
+      whereClause += ` AND (embedding <=> $${paramIndex}::vector) <= $${paramIndex + 1}`
+      params.push(embeddingStr, 1 - threshold)
+      paramIndex += 2
+    }
+
+    try {
+      const rows = (await Database.raw.unsafe(
+        `SELECT id, source_id, content, metadata,
+                1 - (embedding <=> $${paramIndex}::vector) AS score
+         FROM _strav_vectors
+         WHERE ${whereClause}
+         ORDER BY embedding <=> $${paramIndex}::vector
+         LIMIT $${paramIndex + 1}`,
+        [...params, embeddingStr, topK]
+      )) as any[]
+
+      const matches: VectorMatch[] = rows.map((row: any) => ({
+        id: row.source_id ?? row.id,
+        content: row.content,
+        score: parseFloat(row.score),
+        metadata: typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata ?? {},
+      }))
+
+      return {
+        matches,
+        processingTimeMs: performance.now() - start,
+      }
+    } catch (err) {
+      throw new VectorQueryError(collection, err instanceof Error ? err.message : String(err))
+    }
+  }
+
+  private async ensureTable(dimension: number): Promise<void> {
+    if (this.initialized) return
+
+    const sql = Database.raw
+
+    await sql.unsafe(`CREATE EXTENSION IF NOT EXISTS vector`)
+
+    await sql.unsafe(`
+      CREATE TABLE IF NOT EXISTS _strav_vectors (
+        id BIGSERIAL PRIMARY KEY,
+        collection VARCHAR(255) NOT NULL,
+        source_id VARCHAR(255),
+        content TEXT NOT NULL,
+        metadata JSONB DEFAULT '{}',
+        embedding vector(${dimension}),
+        created_at TIMESTAMPTZ DEFAULT NOW()
+      )
+    `)
+
+    await sql.unsafe(
+      `CREATE INDEX IF NOT EXISTS idx_strav_vectors_collection ON _strav_vectors(collection)`
+    )
+    await sql.unsafe(
+      `CREATE INDEX IF NOT EXISTS idx_strav_vectors_source ON _strav_vectors(collection, source_id)`
+    )
+
+    this.initialized = true
+  }
+}

package/src/errors.ts

@@ -0,0 +1,21 @@
+import { StravError } from '@stravigor/kernel'
+
+export class RagError extends StravError {}
+
+export class CollectionNotFoundError extends RagError {
+  constructor(collection: string) {
+    super(`Vector collection "${collection}" not found.`)
+  }
+}
+
+export class VectorQueryError extends RagError {
+  constructor(collection: string, cause?: string) {
+    super(`Vector query on "${collection}" failed${cause ? `: ${cause}` : ''}.`)
+  }
+}
+
+export class EmbeddingError extends RagError {
+  constructor(cause?: string) {
+    super(`Embedding generation failed${cause ? `: ${cause}` : ''}.`)
+  }
+}

package/src/helpers.ts

@@ -0,0 +1,160 @@
+import { brain } from '@stravigor/brain'
+import RagManager from './rag_manager.ts'
+import type { VectorStore } from './vector_store.ts'
+import type {
+  RetrieveOptions,
+  RetrieveResult,
+  RetrievedDocument,
+  VectorDocument,
+  StoreConfig,
+} from './types.ts'
+import { createChunker } from './chunking/chunker.ts'
+import { EmbeddingError } from './errors.ts'
+
+export interface IngestOptions {
+  metadata?: Record<string, unknown>
+  sourceId?: string | number
+  chunkSize?: number
+  overlap?: number
+  strategy?: string
+}
+
+export const rag = {
+  store(name?: string): VectorStore {
+    return RagManager.store(name)
+  },
+
+  extend(name: string, factory: (config: StoreConfig) => VectorStore): void {
+    RagManager.extend(name, factory)
+  },
+
+  async ingest(
+    collection: string,
+    content: string,
+    options: IngestOptions = {}
+  ): Promise<string[]> {
+    const config = RagManager.config
+    const fullCollection = RagManager.collectionName(collection)
+
+    const chunkerConfig = {
+      strategy: options.strategy ?? config.chunking.strategy,
+      chunkSize: options.chunkSize ?? config.chunking.chunkSize,
+      overlap: options.overlap ?? config.chunking.overlap,
+      separators: config.chunking.separators,
+    }
+    const chunker = createChunker(chunkerConfig)
+    const chunks = chunker.chunk(content)
+
+    if (chunks.length === 0) return []
+
+    const chunkTexts = chunks.map(c => c.content)
+    let embeddings: number[][]
+    try {
+      embeddings = await brain.embed(chunkTexts, {
+        provider: config.embedding.provider,
+        model: config.embedding.model,
+      })
+    } catch (err) {
+      throw new EmbeddingError(err instanceof Error ? err.message : String(err))
+    }
+
+    const baseId = crypto.randomUUID()
+    const documents: VectorDocument[] = chunks.map((chunk, i) => ({
+      id: `${baseId}_${i}`,
+      sourceId: options.sourceId,
+      content: chunk.content,
+      embedding: embeddings[i]!,
+      metadata: {
+        ...options.metadata,
+        chunkIndex: chunk.index,
+        startOffset: chunk.startOffset,
+        endOffset: chunk.endOffset,
+      },
+    }))
+
+    await RagManager.store().upsert(fullCollection, documents)
+    return documents.map(d => String(d.id))
+  },
+
+  async retrieve(query: string, options: RetrieveOptions = {}): Promise<RetrieveResult> {
+    const start = performance.now()
+    const config = RagManager.config
+    const collection = RagManager.collectionName(options.collection ?? 'default')
+
+    let queryVector: number[]
+    try {
+      const vectors = await brain.embed(query, {
+        provider: config.embedding.provider,
+        model: config.embedding.model,
+      })
+      queryVector = vectors[0]!
+    } catch (err) {
+      throw new EmbeddingError(err instanceof Error ? err.message : String(err))
+    }
+
+    const queryResult = await RagManager.store().query(collection, queryVector, {
+      topK: options.topK,
+      threshold: options.threshold,
+      filter: options.filter,
+    })
+
+    let matches: RetrievedDocument[] = queryResult.matches.map(m => ({
+      id: m.id,
+      content: m.content,
+      score: m.score,
+      similarity: m.score,
+      metadata: m.metadata,
+    }))
+
+    if (options.rerank) {
+      const {
+        similarityWeight = 0.6,
+        authorityWeight = 0.2,
+        recencyWeight = 0.2,
+      } = options.rerank
+
+      matches = matches.map(m => {
+        const authority =
+          typeof m.metadata.authority === 'number' ? m.metadata.authority : 0
+        const createdAt = m.metadata.createdAt
+        const recencyScore = createdAt
+          ? 1 / (1 + daysSince(new Date(createdAt as string)) / 30)
+          : 0.5
+
+        const finalScore =
+          m.similarity * similarityWeight +
+          authority * authorityWeight +
+          recencyScore * recencyWeight
+
+        return { ...m, score: finalScore }
+      })
+
+      matches.sort((a, b) => b.score - a.score)
+    }
+
+    return {
+      matches,
+      query,
+      processingTimeMs: performance.now() - start,
+    }
+  },
+
+  async delete(collection: string, ids: (string | number)[]): Promise<void> {
+    const fullCollection = RagManager.collectionName(collection)
+    await RagManager.store().delete(fullCollection, ids)
+  },
+
+  async deleteBySource(collection: string, sourceId: string | number): Promise<void> {
+    const fullCollection = RagManager.collectionName(collection)
+    await RagManager.store().deleteBySource(fullCollection, sourceId)
+  },
+
+  async flush(collection: string): Promise<void> {
+    const fullCollection = RagManager.collectionName(collection)
+    await RagManager.store().flush(fullCollection)
+  },
+}
+
+function daysSince(date: Date): number {
+  return (Date.now() - date.getTime()) / (1000 * 60 * 60 * 24)
+}

package/src/index.ts

@@ -0,0 +1,48 @@
+// Manager
+export { default, default as RagManager } from './rag_manager.ts'
+
+// Provider
+export { default as RagProvider } from './rag_provider.ts'
+
+// Store interface
+export type { VectorStore } from './vector_store.ts'
+
+// Drivers
+export { NullDriver } from './drivers/null_driver.ts'
+export { MemoryDriver } from './drivers/memory_driver.ts'
+export { PgvectorDriver } from './drivers/pgvector_driver.ts'
+
+// Mixin
+export { retrievable } from './retrievable.ts'
+export type { RetrievableInstance, RetrievableModel } from './retrievable.ts'
+
+// Helper
+export { rag } from './helpers.ts'
+
+// Chunking
+export { createChunker } from './chunking/chunker.ts'
+export { FixedSizeChunker } from './chunking/fixed_size_chunker.ts'
+export { RecursiveChunker } from './chunking/recursive_chunker.ts'
+
+// Errors
+export { RagError, CollectionNotFoundError, VectorQueryError, EmbeddingError } from './errors.ts'
+
+// Types
+export type {
+  RagConfig,
+  StoreConfig,
+  EmbeddingConfig,
+  ChunkingConfig,
+  VectorDocument,
+  QueryOptions,
+  QueryResult,
+  VectorMatch,
+  RetrieveOptions,
+  RerankOptions,
+  RetrieveResult,
+  RetrievedDocument,
+  Chunk,
+  Chunker,
+} from './types.ts'
+
+export type { IngestOptions } from './helpers.ts'

package/src/rag_manager.ts

@@ -0,0 +1,98 @@
+import { inject, Configuration, ConfigurationError } from '@stravigor/kernel'
+import type { VectorStore } from './vector_store.ts'
+import type { RagConfig, StoreConfig, EmbeddingConfig, ChunkingConfig } from './types.ts'
+import { NullDriver } from './drivers/null_driver.ts'
+import { MemoryDriver } from './drivers/memory_driver.ts'
+import { PgvectorDriver } from './drivers/pgvector_driver.ts'
+
+@inject
+export default class RagManager {
+  private static _config: RagConfig
+  private static _stores = new Map<string, VectorStore>()
+  private static _extensions = new Map<string, (config: StoreConfig) => VectorStore>()
+
+  constructor(config: Configuration) {
+    RagManager._config = {
+      default: config.get('rag.default', 'null') as string,
+      prefix: config.get('rag.prefix', '') as string,
+      embedding: config.get('rag.embedding', {
+        provider: 'openai',
+        model: 'text-embedding-3-small',
+        dimension: 1536,
+      }) as EmbeddingConfig,
+      chunking: config.get('rag.chunking', {
+        strategy: 'recursive',
+        chunkSize: 512,
+        overlap: 64,
+      }) as ChunkingConfig,
+      stores: config.get('rag.stores', {}) as Record<string, StoreConfig>,
+    }
+  }
+
+  static get config(): RagConfig {
+    if (!RagManager._config) {
+      throw new ConfigurationError(
+        'RagManager not configured. Resolve it through the container first.'
+      )
+    }
+    return RagManager._config
+  }
+
+  static store(name?: string): VectorStore {
+    const key = name ?? RagManager.config.default
+
+    let store = RagManager._stores.get(key)
+    if (store) return store
+
+    const storeConfig = RagManager.config.stores[key]
+    if (!storeConfig) {
+      throw new ConfigurationError(`RAG store "${key}" is not configured.`)
+    }
+
+    store = RagManager.createStore(key, storeConfig)
+    RagManager._stores.set(key, store)
+    return store
+  }
+
+  static get prefix(): string {
+    return RagManager._config?.prefix ?? ''
+  }
+
+  static collectionName(name: string): string {
+    return RagManager.prefix ? `${RagManager.prefix}${name}` : name
+  }
+
+  static extend(name: string, factory: (config: StoreConfig) => VectorStore): void {
+    RagManager._extensions.set(name, factory)
+  }
+
+  static useStore(store: VectorStore): void {
+    RagManager._stores.set(store.name, store)
+  }
+
+  static reset(): void {
+    RagManager._stores.clear()
+    RagManager._extensions.clear()
+    RagManager._config = undefined as any
+  }
+
+  private static createStore(name: string, config: StoreConfig): VectorStore {
+    const driverName = config.driver ?? name
+
+    const extension = RagManager._extensions.get(driverName)
+    if (extension) return extension(config)
+
+    switch (driverName) {
+      case 'pgvector':
+        return new PgvectorDriver(config)
+      case 'memory':
+        return new MemoryDriver()
+      case 'null':
+        return new NullDriver()
+      default:
+        throw new ConfigurationError(
+          `Unknown RAG driver "${driverName}". Register it with RagManager.extend().`
+        )
+    }
+  }
+}

package/src/rag_provider.ts

@@ -0,0 +1,16 @@
+import { ServiceProvider } from '@stravigor/kernel'
+import type { Application } from '@stravigor/kernel'
+import RagManager from './rag_manager.ts'
+
+export default class RagProvider extends ServiceProvider {
+  readonly name = 'rag'
+  override readonly dependencies = ['config']
+
+  override register(app: Application): void {
+    app.singleton(RagManager)
+  }
+
+  override boot(app: Application): void {
+    app.resolve(RagManager)
+  }
+}

package/src/retrievable.ts

@@ -0,0 +1,179 @@
+import type { BaseModel } from '@stravigor/database'
+import type { NormalizeConstructor } from '@stravigor/kernel'
+import { Emitter } from '@stravigor/kernel'
+import { brain } from '@stravigor/brain'
+import RagManager from './rag_manager.ts'
+import { createChunker } from './chunking/chunker.ts'
+import type { VectorDocument, RetrieveOptions, RetrieveResult } from './types.ts'
+
+export function retrievable<T extends NormalizeConstructor<typeof BaseModel>>(Base: T) {
+  return class Retrievable extends Base {
+    private static _retrievalBooted = false
+
+    static retrievableAs(): string {
+      return (this as unknown as typeof BaseModel).tableName
+    }
+
+    toRetrievableContent(): string {
+      const parts: string[] = []
+      for (const key of Object.keys(this)) {
+        if (key.startsWith('_')) continue
+        const val = (this as any)[key]
+        if (typeof val === 'string' && val.length > 0) parts.push(val)
+      }
+      return parts.join('\n')
+    }
+
+    toRetrievableMetadata(): Record<string, unknown> {
+      return {}
+    }
+
+    shouldBeRetrievable(): boolean {
+      return true
+    }
+
+    // ── Instance methods ──────────────────────────────────────────────
+
+    async vectorize(): Promise<void> {
+      if (!this.shouldBeRetrievable()) return
+
+      const ctor = this.constructor as typeof Retrievable
+      const collection = RagManager.collectionName(ctor.retrievableAs())
+      const config = RagManager.config
+      const pkProp = (ctor as unknown as typeof BaseModel).primaryKeyProperty
+      const id = (this as any)[pkProp]
+
+      const content = this.toRetrievableContent()
+      if (!content) return
+
+      // Remove existing chunks for this model instance
+      await RagManager.store().deleteBySource(collection, id)
+
+      const chunker = createChunker(config.chunking)
+      const chunks = chunker.chunk(content)
+      if (chunks.length === 0) return
+
+      const texts = chunks.map(c => c.content)
+      const embeddings = await brain.embed(texts, {
+        provider: config.embedding.provider,
+        model: config.embedding.model,
+      })
+
+      const metadata = this.toRetrievableMetadata()
+      const documents: VectorDocument[] = chunks.map((chunk, i) => ({
+        id: `${id}_${i}`,
+        sourceId: id,
+        content: chunk.content,
+        embedding: embeddings[i]!,
+        metadata: {
+          ...metadata,
+          modelId: id,
+          chunkIndex: chunk.index,
+        },
+      }))
+
+      await RagManager.store().upsert(collection, documents)
+    }
+
+    async vectorRemove(): Promise<void> {
+      const ctor = this.constructor as typeof Retrievable
+      const collection = RagManager.collectionName(ctor.retrievableAs())
+      const pkProp = (ctor as unknown as typeof BaseModel).primaryKeyProperty
+      const id = (this as any)[pkProp]
+      await RagManager.store().deleteBySource(collection, id)
+    }
+
+    // ── Static methods ────────────────────────────────────────────────
+
+    static async retrieve(query: string, options?: RetrieveOptions): Promise<RetrieveResult> {
+      const { rag } = await import('./helpers.ts')
+      return rag.retrieve(query, {
+        ...options,
+        collection: options?.collection ?? this.retrievableAs(),
+      })
+    }
+
+    static async importAll(chunkSize: number = 100): Promise<number> {
+      const ModelCtor = this as unknown as typeof BaseModel & typeof Retrievable
+      const collection = RagManager.collectionName(this.retrievableAs())
+      const config = RagManager.config
+      const db = ModelCtor.db
+      const table = ModelCtor.tableName
+      const pkCol = ModelCtor.primaryKeyColumn
+
+      await RagManager.store().createCollection(collection, config.embedding.dimension)
+
+      let imported = 0
+      let offset = 0
+
+      while (true) {
+        const rows = (await db.sql.unsafe(
+          `SELECT * FROM "${table}" ORDER BY "${pkCol}" LIMIT $1 OFFSET $2`,
+          [chunkSize, offset]
+        )) as Record<string, unknown>[]
+
+        if (rows.length === 0) break
+
+        for (const row of rows) {
+          const instance = ModelCtor.hydrate(row) as InstanceType<typeof Retrievable>
+          if (instance.shouldBeRetrievable()) {
+            try {
+              await instance.vectorize()
+              imported++
+            } catch {
+              // Vectorization is secondary — continue on failure
+            }
+          }
+        }
+
+        offset += chunkSize
+        if (rows.length < chunkSize) break
+      }
+
+      return imported
+    }
+
+    static async flushVectors(): Promise<void> {
+      const collection = RagManager.collectionName(this.retrievableAs())
+      await RagManager.store().flush(collection)
+    }
+
+    static async createVectorCollection(): Promise<void> {
+      const collection = RagManager.collectionName(this.retrievableAs())
+      await RagManager.store().createCollection(collection, RagManager.config.embedding.dimension)
+    }
+
+    static bootRetrieval(eventPrefix: string): void {
+      if (this._retrievalBooted) return
+      this._retrievalBooted = true
+
+      const vectorizeFn = async (model: unknown) => {
+        if (model && typeof (model as any).vectorize === 'function') {
+          try {
+            await (model as any).vectorize()
+          } catch {
+            // Vectorization is secondary — failures should not break the event pipeline
+          }
+        }
+      }
+
+      const removeFn = async (model: unknown) => {
+        if (model && typeof (model as any).vectorRemove === 'function') {
+          try {
+            await (model as any).vectorRemove()
+          } catch {
+            // Vector removal is secondary
+          }
+        }
+      }
+
+      Emitter.on(`${eventPrefix}.created`, vectorizeFn)
+      Emitter.on(`${eventPrefix}.updated`, vectorizeFn)
+      Emitter.on(`${eventPrefix}.synced`, vectorizeFn)
+      Emitter.on(`${eventPrefix}.deleted`, removeFn)
+    }
+  }
+}
+
+export type RetrievableInstance = InstanceType<ReturnType<typeof retrievable>>
+export type RetrievableModel = ReturnType<typeof retrievable>

package/src/types.ts

@@ -0,0 +1,100 @@
+// ── Vector Documents ─────────────────────────────────────────────────────
+
+export interface VectorDocument {
+  id?: string | number
+  sourceId?: string | number
+  content: string
+  embedding: number[]
+  metadata?: Record<string, unknown>
+}
+
+// ── Query Options & Results ──────────────────────────────────────────────
+
+export interface QueryOptions {
+  topK?: number
+  threshold?: number
+  filter?: Record<string, unknown>
+}
+
+export interface QueryResult {
+  matches: VectorMatch[]
+  processingTimeMs?: number
+}
+
+export interface VectorMatch {
+  id: string | number
+  content: string
+  score: number
+  metadata: Record<string, unknown>
+}
+
+// ── Retrieval (high-level pipeline) ──────────────────────────────────────
+
+export interface RetrieveOptions {
+  collection?: string
+  topK?: number
+  threshold?: number
+  filter?: Record<string, unknown>
+  rerank?: RerankOptions
+}
+
+export interface RerankOptions {
+  authorityWeight?: number
+  recencyWeight?: number
+  similarityWeight?: number
+}
+
+export interface RetrieveResult {
+  matches: RetrievedDocument[]
+  query: string
+  processingTimeMs: number
+}
+
+export interface RetrievedDocument {
+  id: string | number
+  content: string
+  score: number
+  similarity: number
+  metadata: Record<string, unknown>
+}
+
+// ── Chunking ─────────────────────────────────────────────────────────────
+
+export interface Chunk {
+  content: string
+  index: number
+  startOffset: number
+  endOffset: number
+}
+
+export interface Chunker {
+  chunk(content: string): Chunk[]
+}
+
+// ── Configuration ────────────────────────────────────────────────────────
+
+export interface RagConfig {
+  default: string
+  prefix: string
+  embedding: EmbeddingConfig
+  chunking: ChunkingConfig
+  stores: Record<string, StoreConfig>
+}
+
+export interface EmbeddingConfig {
+  provider: string
+  model: string
+  dimension: number
+}
+
+export interface ChunkingConfig {
+  strategy: string
+  chunkSize: number
+  overlap: number
+  separators?: string[]
+}
+
+export interface StoreConfig {
+  driver: string
+  [key: string]: unknown
+}

package/src/vector_store.ts

@@ -0,0 +1,15 @@
+import type { VectorDocument, QueryOptions, QueryResult } from './types.ts'
+
+export interface VectorStore {
+  readonly name: string
+
+  createCollection(collection: string, dimension: number): Promise<void>
+  deleteCollection(collection: string): Promise<void>
+
+  upsert(collection: string, documents: VectorDocument[]): Promise<void>
+  delete(collection: string, ids: (string | number)[]): Promise<void>
+  deleteBySource(collection: string, sourceId: string | number): Promise<void>
+  flush(collection: string): Promise<void>
+
+  query(collection: string, vector: number[], options?: QueryOptions): Promise<QueryResult>
+}

package/stubs/config/rag.ts

@@ -0,0 +1,33 @@
+import { env } from '@stravigor/kernel'
+
+export default {
+  default: env('RAG_DRIVER', 'pgvector'),
+
+  prefix: env('RAG_PREFIX', ''),
+
+  embedding: {
+    provider: env('RAG_EMBEDDING_PROVIDER', 'openai'),
+    model: env('RAG_EMBEDDING_MODEL', 'text-embedding-3-small'),
+    dimension: 1536,
+  },
+
+  chunking: {
+    strategy: 'recursive',
+    chunkSize: 512,
+    overlap: 64,
+  },
+
+  stores: {
+    pgvector: {
+      driver: 'pgvector',
+    },
+
+    memory: {
+      driver: 'memory',
+    },
+
+    null: {
+      driver: 'null',
+    },
+  },
+}

package/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "tests"]
+}