@strav/rag 0.4.31 → 1.0.0-alpha.20

package/package.json

@@ -1,33 +1,31 @@
 {
   "name": "@strav/rag",
-  "version": "0.4.31",
+  "version": "1.0.0-alpha.20",
+  "description": "Strav RAG module — vector store abstraction, pgvector + in-memory drivers, chunking strategies. Composes with @strav/brain for embeddings and @strav/database for persistence.",
   "type": "module",
-  "description": "Vector retrieval framework for RAG in the Strav framework",
-  "license": "MIT",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
   "exports": {
-    ".": "./src/index.ts",
-    "./*": "./src/*.ts"
-  },
-  "strav": {
-    "commands": "src/commands"
+    ".": "./src/index.ts"
   },
   "files": [
-    "src/",
-    "stubs/",
-    "package.json",
-    "tsconfig.json"
+    "src",
+    "README.md"
   ],
-  "peerDependencies": {
-    "@strav/kernel": "0.4.31",
-    "@strav/brain": "0.4.31",
-    "@strav/database": "0.4.31",
-    "@strav/cli": "0.4.31"
+  "engines": {
+    "bun": ">=1.3.14"
+  },
+  "publishConfig": {
+    "access": "public"
   },
-  "scripts": {
-    "test": "bun test tests/",
-    "typecheck": "tsc --noEmit"
+  "dependencies": {
+    "@strav/brain": "1.0.0-alpha.20",
+    "@strav/cli": "1.0.0-alpha.20",
+    "@strav/database": "1.0.0-alpha.20",
+    "@strav/kernel": "1.0.0-alpha.20"
+  },
+  "peerDependencies": {
+    "@types/bun": ">=1.3.14"
   },
-  "devDependencies": {
-    "commander": "^14.0.3"
-  }
+  "devDependencies": null
 }

package/src/chunking/chunker.ts

@@ -1,3 +1,10 @@
+/**
+ * `createChunker(config)` — factory that returns the right chunker
+ * for a `ChunkingConfig`. Apps that want a custom strategy build
+ * their own `Chunker` implementation and pass it directly into
+ * `rag.ingest({ chunker })` instead of going through config.
+ */
+
 import type { Chunker, ChunkingConfig } from '../types.ts'
 import { FixedSizeChunker } from './fixed_size_chunker.ts'
 import { RecursiveChunker } from './recursive_chunker.ts'
@@ -8,7 +15,5 @@ export function createChunker(config: ChunkingConfig): Chunker {
       return new FixedSizeChunker(config.chunkSize, config.overlap)
     case 'recursive':
       return new RecursiveChunker(config.chunkSize, config.overlap, config.separators)
-    default:
-      return new RecursiveChunker(config.chunkSize, config.overlap)
   }
 }

package/src/chunking/fixed_size_chunker.ts

@@ -1,23 +1,40 @@
+/**
+ * `FixedSizeChunker` — mechanical character-window chunking.
+ *
+ * Walks the content with a fixed window of `chunkSize` characters
+ * and steps forward by `chunkSize - overlap` each iteration. Cheap,
+ * predictable, agnostic to structure — best for content where
+ * paragraph / sentence boundaries don't carry meaning (logs, code
+ * tokens, raw transcript text).
+ *
+ * Apps with prose-style content should prefer `RecursiveChunker`,
+ * which respects paragraph and sentence boundaries.
+ */
+
 import type { Chunk, Chunker } from '../types.ts'
 
 export class FixedSizeChunker implements Chunker {
   constructor(
     private readonly chunkSize: number = 512,
-    private readonly overlap: number = 64
-  ) {}
+    private readonly overlap: number = 64,
+  ) {
+    if (chunkSize <= 0) throw new RangeError('FixedSizeChunker: chunkSize must be > 0.')
+    if (overlap < 0 || overlap >= chunkSize) {
+      throw new RangeError('FixedSizeChunker: overlap must satisfy 0 <= overlap < chunkSize.')
+    }
+  }
 
   chunk(content: string): Chunk[] {
     if (!content) return []
 
-    const chunks: Chunk[] = []
-    const step = Math.max(1, this.chunkSize - this.overlap)
+    const out: Chunk[] = []
+    const step = this.chunkSize - this.overlap
 
     let start = 0
     let index = 0
-
     while (start < content.length) {
       const end = Math.min(start + this.chunkSize, content.length)
-      chunks.push({
+      out.push({
         content: content.slice(start, end),
         index,
         startOffset: start,
@@ -27,7 +44,6 @@ export class FixedSizeChunker implements Chunker {
       start += step
       if (end === content.length) break
     }
-
-    return chunks
+    return out
   }
 }

package/src/chunking/recursive_chunker.ts

@@ -1,15 +1,43 @@
+/**
+ * `RecursiveChunker` — splits on paragraph / sentence / word
+ * boundaries before falling back to fixed-size cuts. Better for
+ * prose and Markdown content than `FixedSizeChunker` because
+ * semantic boundaries survive.
+ *
+ * Strategy:
+ *
+ *   1. If the text fits in one chunk, return it whole.
+ *   2. Otherwise split on the first separator that produces
+ *      pieces small enough to fit (defaults: paragraph → line →
+ *      sentence → word).
+ *   3. Merge adjacent pieces greedily up to `chunkSize`.
+ *   4. Compute `startOffset` / `endOffset` by walking the merged
+ *      pieces against the original content.
+ *   5. Apply a sliding overlap pass at the end so consecutive
+ *      chunks share `overlap` characters of context — important
+ *      for retrieval recall around chunk boundaries.
+ *
+ * Offsets are byte-accurate against the original content so apps
+ * that highlight retrieved passages in the source can slice
+ * directly with `content.slice(chunk.startOffset, chunk.endOffset)`.
+ */
+
 import type { Chunk, Chunker } from '../types.ts'
 
-const DEFAULT_SEPARATORS = ['\n\n', '\n', '. ', ' ']
+const DEFAULT_SEPARATORS = ['\n\n', '\n', '. ', ' '] as const
 
 export class RecursiveChunker implements Chunker {
-  private readonly separators: string[]
+  private readonly separators: readonly string[]
 
   constructor(
     private readonly chunkSize: number = 512,
     private readonly overlap: number = 64,
-    separators?: string[]
+    separators?: readonly string[],
   ) {
+    if (chunkSize <= 0) throw new RangeError('RecursiveChunker: chunkSize must be > 0.')
+    if (overlap < 0 || overlap >= chunkSize) {
+      throw new RangeError('RecursiveChunker: overlap must satisfy 0 <= overlap < chunkSize.')
+    }
     this.separators = separators ?? DEFAULT_SEPARATORS
   }
 
@@ -19,25 +47,30 @@ export class RecursiveChunker implements Chunker {
     return this.buildChunks(content, pieces)
   }
 
+  /**
+   * Recursive split. At each separator level, split the text and
+   * try to merge adjacent pieces back together greedily without
+   * exceeding `chunkSize`. Pieces that don't fit at this level
+   * recurse one separator deeper.
+   */
   private splitRecursive(text: string, separatorIndex: number): string[] {
     if (text.length <= this.chunkSize) return [text]
 
     const separator = this.separators[separatorIndex]
     if (!separator) {
-      const result: string[] = []
+      // Out of separators — hard-cut to `chunkSize`.
+      const out: string[] = []
       for (let i = 0; i < text.length; i += this.chunkSize) {
-        result.push(text.slice(i, i + this.chunkSize))
+        out.push(text.slice(i, i + this.chunkSize))
       }
-      return result
+      return out
     }
 
     const parts = text.split(separator)
     const merged: string[] = []
     let current = ''
-
     for (const part of parts) {
       const candidate = current ? current + separator + part : part
-
       if (candidate.length <= this.chunkSize) {
         current = candidate
       } else {
@@ -51,33 +84,61 @@ export class RecursiveChunker implements Chunker {
       }
     }
     if (current) merged.push(current)
-
     return merged
   }
 
-  private buildChunks(original: string, pieces: string[]): Chunk[] {
-    const chunks: Chunk[] = []
-    let searchFrom = 0
-
-    for (let i = 0; i < pieces.length; i++) {
-      const piece = pieces[i]!
-      const foundAt = original.indexOf(piece, searchFrom)
-      const startOffset = foundAt >= 0 ? foundAt : searchFrom
-      const pieceEnd = startOffset + piece.length
+  /**
+   * Map merged pieces back onto offsets in the original content,
+   * then apply a sliding overlap so adjacent chunks share
+   * `overlap` characters of trailing context.
+   */
+  private buildChunks(content: string, pieces: readonly string[]): Chunk[] {
+    if (pieces.length === 0) return []
+
+    // Walk the original content looking for each piece. The piece
+    // contents are substrings of the source; `indexOf(piece, cursor)`
+    // is sufficient because the recursive split preserves textual
+    // order.
+    const rawSpans: Array<{ start: number; end: number }> = []
+    let cursor = 0
+    for (const piece of pieces) {
+      const start = content.indexOf(piece, cursor)
+      if (start === -1) {
+        // Should never happen — splitRecursive only emits substrings —
+        // but guard against pathological input by falling back to
+        // appending at the cursor with the piece's literal length.
+        rawSpans.push({ start: cursor, end: cursor + piece.length })
+        cursor += piece.length
+        continue
+      }
+      const end = start + piece.length
+      rawSpans.push({ start, end })
+      cursor = end
+    }
 
-      const overlapEnd = Math.min(pieceEnd + this.overlap, original.length)
-      const chunkContent = original.slice(startOffset, overlapEnd)
+    if (this.overlap === 0) {
+      return rawSpans.map((s, i) => ({
+        content: content.slice(s.start, s.end),
+        index: i,
+        startOffset: s.start,
+        endOffset: s.end,
+      }))
+    }
 
-      chunks.push({
-        content: chunkContent,
+    // Apply trailing overlap: each chunk after the first extends
+    // backward by `overlap` characters into the previous span so
+    // boundary context is duplicated.
+    const out: Chunk[] = []
+    for (let i = 0; i < rawSpans.length; i++) {
+      const span = rawSpans[i]!
+      const start = i === 0 ? span.start : Math.max(0, span.start - this.overlap)
+      out.push({
+        content: content.slice(start, span.end),
         index: i,
-        startOffset,
-        endOffset: overlapEnd,
+        startOffset: start,
+        endOffset: span.end,
       })
-
-      searchFrom = pieceEnd
     }
-
-    return chunks
+    return out
   }
 }

package/src/console/index.ts

@@ -0,0 +1,3 @@
+export { RagConsoleProvider } from './rag_console_provider.ts'
+export { RagFlush } from './rag_flush.ts'
+export { RagList } from './rag_list.ts'

package/src/console/rag_console_provider.ts

@@ -0,0 +1,17 @@
+/**
+ * `RagConsoleProvider` — declares the rag console commands.
+ *
+ * Apps add it to `bootstrap/providers.ts` alongside `RagProvider`.
+ * Separate provider (mirrors `QueueConsoleProvider`) so apps
+ * that don't use the CLI don't pay the cost of resolving the
+ * commands at boot.
+ */
+
+import { ConsoleProvider } from '@strav/cli'
+import { RagFlush } from './rag_flush.ts'
+import { RagList } from './rag_list.ts'
+
+export class RagConsoleProvider extends ConsoleProvider {
+  override readonly name = 'console.rag'
+  override readonly commands = [RagFlush, RagList] as const
+}

package/src/console/rag_flush.ts

@@ -0,0 +1,51 @@
+/**
+ * `bun strav rag:flush <collection> [--store=name] [--force]` —
+ * drop every vector in a collection on the active (or named)
+ * store.
+ *
+ * Use cases:
+ *
+ *   - Wiping a corrupted index before re-ingest.
+ *   - Cleaning up a dev / staging environment.
+ *   - Recovering after a dimension / model change.
+ *
+ * The command confirms before running unless `--force` is set.
+ * Doesn't touch the source data — apps run their own re-ingest
+ * afterward, typically via `retrievable` repo's `reindexAll()`.
+ */
+
+import { Command, type ExecuteArgs, ExitCode } from '@strav/cli'
+import { RagManager } from '../rag_manager.ts'
+
+export class RagFlush extends Command {
+  static signature = 'rag:flush {collection} {--store=} {--force}'
+  static description = 'Delete every vector in a collection (on the active or --store= named store).'
+  static providers = ['config', 'logger', 'brain', 'rag']
+
+  override async execute({ args, flags }: ExecuteArgs): Promise<number> {
+    const collection = args.collection as string
+    const storeName = typeof flags.store === 'string' && flags.store.length > 0
+      ? flags.store
+      : undefined
+
+    const manager = this.app.resolve(RagManager)
+    const fullCollection = manager.collectionName(collection)
+    const storeLabel = storeName ?? manager.config.default
+
+    if (flags.force !== true) {
+      const ok = await this.confirm(
+        `Delete every vector in collection "${fullCollection}" on store "${storeLabel}"? This is irreversible.`,
+      )
+      if (!ok) {
+        this.info('Aborted.')
+        return ExitCode.Success
+      }
+    }
+
+    await manager.store(storeName).flush(fullCollection)
+    this.success(
+      `Flushed collection "${fullCollection}" on store "${storeLabel}".`,
+    )
+    return ExitCode.Success
+  }
+}

package/src/console/rag_list.ts

@@ -0,0 +1,48 @@
+/**
+ * `bun strav rag:list` — print the configured RAG stores +
+ * chunker + embedding setup.
+ *
+ * Diagnostic only — no mutations. Useful for verifying that
+ * `config/rag.ts` parses correctly and that the registered
+ * driver names match what's expected.
+ */
+
+import { Command, type ExecuteArgs, ExitCode } from '@strav/cli'
+import { RagManager } from '../rag_manager.ts'
+
+export class RagList extends Command {
+  static signature = 'rag:list'
+  static description = 'List configured RAG stores + embedding + chunking settings.'
+  static providers = ['config', 'logger', 'brain', 'rag']
+
+  override async execute(_args: ExecuteArgs): Promise<number> {
+    const manager = this.app.resolve(RagManager)
+    const config = manager.config
+
+    this.info(`Default store: ${config.default}`)
+    if (config.prefix) this.info(`Collection prefix: ${config.prefix}`)
+
+    this.info('')
+    this.info('Stores:')
+    for (const [name, store] of Object.entries(config.stores)) {
+      const flag = name === config.default ? ' (default)' : ''
+      this.info(`  ${name}${flag}: driver=${store.driver}`)
+    }
+
+    this.info('')
+    this.info('Embedding:')
+    this.info(`  provider: ${config.embedding.provider}`)
+    this.info(`  model:    ${config.embedding.model}`)
+    this.info(`  dim:      ${config.embedding.dimension}`)
+
+    this.info('')
+    this.info('Chunking:')
+    this.info(`  strategy:  ${config.chunking.strategy}`)
+    this.info(`  chunkSize: ${config.chunking.chunkSize}`)
+    this.info(`  overlap:   ${config.chunking.overlap}`)
+    if (config.chunking.separators) {
+      this.info(`  separators: ${JSON.stringify(config.chunking.separators)}`)
+    }
+    return ExitCode.Success
+  }
+}

package/src/drivers/memory_driver.ts

@@ -1,135 +1,160 @@
+/**
+ * `MemoryDriver` — in-process `VectorStore` backed by `Map`s.
+ *
+ * Two real use cases:
+ *
+ *   1. **Tests.** Apps test their retrieval logic without booting
+ *      Postgres + pgvector. Reset between tests via
+ *      `new MemoryDriver()`.
+ *   2. **Local dev.** Faster boot, no migration to run. Apps
+ *      flip to `pgvector` for production via
+ *      `config.rag.default`.
+ *
+ * Out of scope:
+ *
+ *   - **Multitenancy.** No tenant scoping; everything in the
+ *     same Map. Apps that test tenant isolation use pgvector
+ *     against a real Postgres.
+ *   - **Persistence.** Vectors die with the process.
+ *   - **Performance.** O(N) scan per query — fine for thousands
+ *     of vectors, painful past tens of thousands.
+ */
+
+import { CollectionNotFoundError } from '../rag_error.ts'
+import type {
+  QueryOptions,
+  QueryResult,
+  VectorDocument,
+  VectorMatch,
+} from '../types.ts'
 import type { VectorStore } from '../vector_store.ts'
-import type { VectorDocument, QueryOptions, QueryResult, VectorMatch } from '../types.ts'
+
+interface StoredDoc {
+  id: string
+  sourceId: string | null
+  content: string
+  embedding: readonly number[]
+  metadata: Record<string, unknown>
+}
 
 export class MemoryDriver implements VectorStore {
   readonly name = 'memory'
-  private collections = new Map<string, VectorDocument[]>()
 
-  async createCollection(collection: string, _dimension: number): Promise<void> {
+  private readonly collections = new Map<string, Map<string, StoredDoc>>()
+  private readonly dimensions = new Map<string, number>()
+
+  async createCollection(collection: string, dimension: number): Promise<void> {
     if (!this.collections.has(collection)) {
-      this.collections.set(collection, [])
+      this.collections.set(collection, new Map())
+      this.dimensions.set(collection, dimension)
     }
   }
 
   async deleteCollection(collection: string): Promise<void> {
     this.collections.delete(collection)
+    this.dimensions.delete(collection)
   }
 
-  async upsert(collection: string, documents: VectorDocument[]): Promise<void> {
-    let docs = this.collections.get(collection)
-    if (!docs) {
-      docs = []
-      this.collections.set(collection, docs)
-    }
-
+  async upsert(
+    collection: string,
+    documents: readonly VectorDocument[],
+  ): Promise<void> {
+    const bucket = this.requireBucket(collection)
     for (const doc of documents) {
-      if (doc.id != null) {
-        const existingIndex = docs.findIndex(d => d.id === doc.id)
-        if (existingIndex >= 0) {
-          docs[existingIndex] = doc
-        } else {
-          docs.push(doc)
-        }
-      } else {
-        docs.push(doc)
-      }
+      const id = doc.id ?? crypto.randomUUID()
+      bucket.set(id, {
+        id,
+        sourceId: doc.sourceId ?? null,
+        content: doc.content,
+        embedding: [...doc.embedding],
+        metadata: doc.metadata ?? {},
+      })
     }
   }
 
-  async delete(collection: string, ids: (string | number)[]): Promise<void> {
-    const docs = this.collections.get(collection)
-    if (!docs) return
-
-    const idSet = new Set(ids.map(String))
-    this.collections.set(
-      collection,
-      docs.filter(d => !idSet.has(String(d.id)))
-    )
+  async delete(collection: string, ids: readonly string[]): Promise<void> {
+    const bucket = this.requireBucket(collection)
+    for (const id of ids) bucket.delete(id)
   }
 
-  async deleteBySource(collection: string, sourceId: string | number): Promise<void> {
-    const docs = this.collections.get(collection)
-    if (!docs) return
-
-    const sourceStr = String(sourceId)
-    this.collections.set(
-      collection,
-      docs.filter(d => String(d.sourceId) !== sourceStr)
-    )
+  async deleteBySource(collection: string, sourceId: string): Promise<void> {
+    const bucket = this.requireBucket(collection)
+    for (const [id, doc] of bucket) {
+      if (doc.sourceId === sourceId) bucket.delete(id)
+    }
   }
 
   async flush(collection: string): Promise<void> {
-    if (this.collections.has(collection)) {
-      this.collections.set(collection, [])
-    }
+    const bucket = this.collections.get(collection)
+    if (bucket) bucket.clear()
   }
 
   async query(
     collection: string,
-    vector: number[],
-    options?: QueryOptions
+    vector: readonly number[],
+    options: QueryOptions = {},
   ): Promise<QueryResult> {
     const start = performance.now()
-    const docs = this.collections.get(collection)
-    if (!docs || docs.length === 0) {
-      return { matches: [], processingTimeMs: performance.now() - start }
-    }
-
-    const topK = options?.topK ?? 5
-    const threshold = options?.threshold ?? 0
-
-    let scored: VectorMatch[] = docs.map(doc => ({
-      id: doc.id ?? 0,
-      content: doc.content,
-      score: cosineSimilarity(vector, doc.embedding),
-      metadata: doc.metadata ?? {},
-    }))
-
-    if (options?.filter) {
-      scored = scored.filter(m => matchesFilter(m.metadata, options.filter!))
-    }
-
-    if (threshold > 0) {
-      scored = scored.filter(m => m.score >= threshold)
+    const bucket = this.requireBucket(collection)
+    const topK = options.topK ?? 5
+    const threshold = options.threshold ?? 0
+    const filter = options.filter
+
+    const scored: VectorMatch[] = []
+    for (const doc of bucket.values()) {
+      if (filter && !matchesFilter(doc.metadata, filter)) continue
+      const score = cosineSimilarity(vector, doc.embedding)
+      if (score < threshold) continue
+      scored.push({
+        id: doc.id,
+        content: doc.content,
+        score,
+        metadata: doc.metadata,
+        sourceId: doc.sourceId,
+      })
     }
 
     scored.sort((a, b) => b.score - a.score)
     const matches = scored.slice(0, topK)
-
-    return {
-      matches,
-      processingTimeMs: performance.now() - start,
-    }
+    return { matches, processingTimeMs: performance.now() - start }
   }
 
-  getCollection(collection: string): VectorDocument[] {
-    return this.collections.get(collection) ?? []
+  private requireBucket(collection: string): Map<string, StoredDoc> {
+    const bucket = this.collections.get(collection)
+    if (!bucket) throw new CollectionNotFoundError(collection, this.name)
+    return bucket
   }
 }
 
-function cosineSimilarity(a: number[], b: number[]): number {
+/**
+ * Cosine similarity in [-1, 1] mapped to [0, 1] by `(s + 1) / 2`.
+ * Matches pgvector's `1 - (a <=> b)` semantic so MemoryDriver and
+ * PgvectorDriver scores compare like-for-like.
+ */
+function cosineSimilarity(a: readonly number[], b: readonly number[]): number {
+  const len = Math.min(a.length, b.length)
   let dot = 0
-  let magA = 0
-  let magB = 0
-
-  for (let i = 0; i < a.length; i++) {
+  let normA = 0
+  let normB = 0
+  for (let i = 0; i < len; i++) {
     const ai = a[i]!
     const bi = b[i]!
     dot += ai * bi
-    magA += ai * ai
-    magB += bi * bi
+    normA += ai * ai
+    normB += bi * bi
   }
-
-  const denom = Math.sqrt(magA) * Math.sqrt(magB)
-  return denom === 0 ? 0 : dot / denom
+  if (normA === 0 || normB === 0) return 0
+  const cos = dot / (Math.sqrt(normA) * Math.sqrt(normB))
+  return (cos + 1) / 2
 }
 
+/** Flat AND match — every key in `filter` must equal the corresponding `metadata` key. */
 function matchesFilter(
   metadata: Record<string, unknown>,
-  filter: Record<string, unknown>
+  filter: Record<string, unknown>,
 ): boolean {
-  for (const [key, value] of Object.entries(filter)) {
-    if (metadata[key] !== value) return false
+  for (const key of Object.keys(filter)) {
+    if (metadata[key] !== filter[key]) return false
   }
   return true
 }