@strav/rag 0.4.31 → 1.0.0-alpha.20

package/src/commands/rag_flush.ts

@@ -1,41 +0,0 @@
-import type { Command } from 'commander'
-import chalk from 'chalk'
-import { bootstrap, shutdown } from '@strav/cli'
-import { BaseModel } from '@strav/database'
-import RagManager from '../rag_manager.ts'
-
-export function register(program: Command): void {
-  program
-    .command('rag:flush <model>')
-    .description("Flush all vectors from a model's vector collection")
-    .action(async (modelPath: string) => {
-      let db
-      try {
-        const { db: database, config } = await bootstrap()
-        db = database
-
-        new BaseModel(db)
-        new RagManager(config)
-
-        const resolved = require.resolve(`${process.cwd()}/${modelPath}`)
-        const module = await import(resolved)
-        const ModelClass = module.default ?? (Object.values(module)[0] as any)
-
-        if (typeof ModelClass?.flushVectors !== 'function') {
-          console.error(chalk.red(`Model "${modelPath}" does not use the retrievable() mixin.`))
-          process.exit(1)
-        }
-
-        const collectionName = ModelClass.retrievableAs()
-        console.log(chalk.dim(`Flushing "${collectionName}"...`))
-
-        await ModelClass.flushVectors()
-        console.log(chalk.green(`Flushed all vectors from "${collectionName}".`))
-      } catch (err) {
-        console.error(chalk.red(`Error: ${err instanceof Error ? err.message : err}`))
-        process.exit(1)
-      } finally {
-        if (db) await shutdown(db)
-      }
-    })
-}

package/src/commands/rag_ingest.ts

@@ -1,45 +0,0 @@
-import type { Command } from 'commander'
-import chalk from 'chalk'
-import { bootstrap, shutdown } from '@strav/cli'
-import { BaseModel } from '@strav/database'
-import { BrainManager } from '@strav/brain'
-import RagManager from '../rag_manager.ts'
-
-export function register(program: Command): void {
-  program
-    .command('rag:ingest <model>')
-    .description('Vectorize all records for a model into the vector store')
-    .option('--chunk <size>', 'Records per batch', '100')
-    .action(async (modelPath: string, options: { chunk: string }) => {
-      let db
-      try {
-        const { db: database, config } = await bootstrap()
-        db = database
-
-        new BaseModel(db)
-        new RagManager(config)
-        new BrainManager(config)
-
-        const resolved = require.resolve(`${process.cwd()}/${modelPath}`)
-        const module = await import(resolved)
-        const ModelClass = module.default ?? (Object.values(module)[0] as any)
-
-        if (typeof ModelClass?.importAll !== 'function') {
-          console.error(chalk.red(`Model "${modelPath}" does not use the retrievable() mixin.`))
-          process.exit(1)
-        }
-
-        const chunkSize = parseInt(options.chunk, 10)
-        const collectionName = ModelClass.retrievableAs()
-        console.log(chalk.dim(`Vectorizing ${ModelClass.name} into "${collectionName}"...`))
-
-        const count = await ModelClass.importAll(chunkSize)
-        console.log(chalk.green(`Vectorized ${count} record(s) into "${collectionName}".`))
-      } catch (err) {
-        console.error(chalk.red(`Error: ${err instanceof Error ? err.message : err}`))
-        process.exit(1)
-      } finally {
-        if (db) await shutdown(db)
-      }
-    })
-}

package/src/drivers/null_driver.ts

@@ -1,21 +0,0 @@
-import type { VectorStore } from '../vector_store.ts'
-import type { VectorDocument, QueryOptions, QueryResult } from '../types.ts'
-
-export class NullDriver implements VectorStore {
-  readonly name = 'null'
-
-  async createCollection(_collection: string, _dimension: number): Promise<void> {}
-  async deleteCollection(_collection: string): Promise<void> {}
-  async upsert(_collection: string, _documents: VectorDocument[]): Promise<void> {}
-  async delete(_collection: string, _ids: (string | number)[]): Promise<void> {}
-  async deleteBySource(_collection: string, _sourceId: string | number): Promise<void> {}
-  async flush(_collection: string): Promise<void> {}
-
-  async query(
-    _collection: string,
-    _vector: number[],
-    _options?: QueryOptions
-  ): Promise<QueryResult> {
-    return { matches: [] }
-  }
-}

package/src/errors.ts

@@ -1,21 +0,0 @@
-import { StravError } from '@strav/kernel'
-
-export class RagError extends StravError {}
-
-export class CollectionNotFoundError extends RagError {
-  constructor(collection: string) {
-    super(`Vector collection "${collection}" not found.`)
-  }
-}
-
-export class VectorQueryError extends RagError {
-  constructor(collection: string, cause?: string) {
-    super(`Vector query on "${collection}" failed${cause ? `: ${cause}` : ''}.`)
-  }
-}
-
-export class EmbeddingError extends RagError {
-  constructor(cause?: string) {
-    super(`Embedding generation failed${cause ? `: ${cause}` : ''}.`)
-  }
-}

package/src/helpers.ts

@@ -1,186 +0,0 @@
-import { brain } from '@strav/brain'
-import RagManager from './rag_manager.ts'
-import type { VectorStore } from './vector_store.ts'
-import type {
-  RetrieveOptions,
-  RetrieveResult,
-  RetrievedDocument,
-  VectorDocument,
-  StoreConfig,
-} from './types.ts'
-import { createChunker } from './chunking/chunker.ts'
-import { EmbeddingError } from './errors.ts'
-
-export interface IngestOptions {
-  metadata?: Record<string, unknown>
-  sourceId?: string | number
-  chunkSize?: number
-  overlap?: number
-  strategy?: string
-  /**
-   * Optional per-chunk sanitizer applied AFTER chunking, BEFORE
-   * embedding. Use to scrub PII, secrets, or prompt-injection markers
-   * out of untrusted source content before it lands in the vector
-   * store. Return `null` to drop a chunk; otherwise return the
-   * (possibly modified) text.
-   *
-   * The hook is the caller's escape valve — RAG cannot judge what's
-   * sensitive in your domain. See `docs/rag/rag.md` "Content trust
-   * model" for the threat surface (prompt injection at retrieval
-   * time, indexed PII, accidental secret indexing).
-   */
-  sanitize?: (chunk: { content: string; index: number }) => string | null | Promise<string | null>
-}
-
-export const rag = {
-  store(name?: string): VectorStore {
-    return RagManager.store(name)
-  },
-
-  extend(name: string, factory: (config: StoreConfig) => VectorStore): void {
-    RagManager.extend(name, factory)
-  },
-
-  async ingest(
-    collection: string,
-    content: string,
-    options: IngestOptions = {}
-  ): Promise<string[]> {
-    const config = RagManager.config
-    const fullCollection = RagManager.collectionName(collection)
-
-    const chunkerConfig = {
-      strategy: options.strategy ?? config.chunking.strategy,
-      chunkSize: options.chunkSize ?? config.chunking.chunkSize,
-      overlap: options.overlap ?? config.chunking.overlap,
-      separators: config.chunking.separators,
-    }
-    const chunker = createChunker(chunkerConfig)
-    let chunks = chunker.chunk(content)
-
-    if (chunks.length === 0) return []
-
-    // Apply the optional sanitize hook before embedding. Drops chunks
-    // where the hook returns null (e.g., a chunk that's all PII).
-    if (options.sanitize) {
-      const sanitized: typeof chunks = []
-      for (const chunk of chunks) {
-        const result = await options.sanitize({ content: chunk.content, index: chunk.index })
-        if (result === null) continue
-        sanitized.push({ ...chunk, content: result })
-      }
-      chunks = sanitized
-      if (chunks.length === 0) return []
-    }
-
-    const chunkTexts = chunks.map(c => c.content)
-    let embeddings: number[][]
-    try {
-      embeddings = await brain.embed(chunkTexts, {
-        provider: config.embedding.provider,
-        model: config.embedding.model,
-      })
-    } catch (err) {
-      throw new EmbeddingError(err instanceof Error ? err.message : String(err))
-    }
-
-    const baseId = crypto.randomUUID()
-    const documents: VectorDocument[] = chunks.map((chunk, i) => ({
-      id: `${baseId}_${i}`,
-      sourceId: options.sourceId,
-      content: chunk.content,
-      embedding: embeddings[i]!,
-      metadata: {
-        ...options.metadata,
-        chunkIndex: chunk.index,
-        startOffset: chunk.startOffset,
-        endOffset: chunk.endOffset,
-      },
-    }))
-
-    await RagManager.store().upsert(fullCollection, documents)
-    return documents.map(d => String(d.id))
-  },
-
-  async retrieve(query: string, options: RetrieveOptions = {}): Promise<RetrieveResult> {
-    const start = performance.now()
-    const config = RagManager.config
-    const collection = RagManager.collectionName(options.collection ?? 'default')
-
-    let queryVector: number[]
-    try {
-      const vectors = await brain.embed(query, {
-        provider: config.embedding.provider,
-        model: config.embedding.model,
-      })
-      queryVector = vectors[0]!
-    } catch (err) {
-      throw new EmbeddingError(err instanceof Error ? err.message : String(err))
-    }
-
-    const queryResult = await RagManager.store().query(collection, queryVector, {
-      topK: options.topK,
-      threshold: options.threshold,
-      filter: options.filter,
-    })
-
-    let matches: RetrievedDocument[] = queryResult.matches.map(m => ({
-      id: m.id,
-      content: m.content,
-      score: m.score,
-      similarity: m.score,
-      metadata: m.metadata,
-    }))
-
-    if (options.rerank) {
-      const {
-        similarityWeight = 0.6,
-        authorityWeight = 0.2,
-        recencyWeight = 0.2,
-      } = options.rerank
-
-      matches = matches.map(m => {
-        const authority =
-          typeof m.metadata.authority === 'number' ? m.metadata.authority : 0
-        const createdAt = m.metadata.createdAt
-        const recencyScore = createdAt
-          ? 1 / (1 + daysSince(new Date(createdAt as string)) / 30)
-          : 0.5
-
-        const finalScore =
-          m.similarity * similarityWeight +
-          authority * authorityWeight +
-          recencyScore * recencyWeight
-
-        return { ...m, score: finalScore }
-      })
-
-      matches.sort((a, b) => b.score - a.score)
-    }
-
-    return {
-      matches,
-      query,
-      processingTimeMs: performance.now() - start,
-    }
-  },
-
-  async delete(collection: string, ids: (string | number)[]): Promise<void> {
-    const fullCollection = RagManager.collectionName(collection)
-    await RagManager.store().delete(fullCollection, ids)
-  },
-
-  async deleteBySource(collection: string, sourceId: string | number): Promise<void> {
-    const fullCollection = RagManager.collectionName(collection)
-    await RagManager.store().deleteBySource(fullCollection, sourceId)
-  },
-
-  async flush(collection: string): Promise<void> {
-    const fullCollection = RagManager.collectionName(collection)
-    await RagManager.store().flush(fullCollection)
-  },
-}
-
-function daysSince(date: Date): number {
-  return (Date.now() - date.getTime()) / (1000 * 60 * 60 * 24)
-}

package/stubs/config/rag.ts

@@ -1,33 +0,0 @@
-import { env } from '@strav/kernel'
-
-export default {
-  default: env('RAG_DRIVER', 'pgvector'),
-
-  prefix: env('RAG_PREFIX', ''),
-
-  embedding: {
-    provider: env('RAG_EMBEDDING_PROVIDER', 'openai'),
-    model: env('RAG_EMBEDDING_MODEL', 'text-embedding-3-small'),
-    dimension: 1536,
-  },
-
-  chunking: {
-    strategy: 'recursive',
-    chunkSize: 512,
-    overlap: 64,
-  },
-
-  stores: {
-    pgvector: {
-      driver: 'pgvector',
-    },
-
-    memory: {
-      driver: 'memory',
-    },
-
-    null: {
-      driver: 'null',
-    },
-  },
-}

package/tsconfig.json

@@ -1,5 +0,0 @@
-{
-  "extends": "../../tsconfig.json",
-  "include": ["src/**/*.ts"],
-  "exclude": ["node_modules", "tests"]
-}