@dudousxd/nestjs-agent-rag 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,300 @@
+import { Passage, Retriever, EmbeddingProvider, RetrieveOptions, Reranker, ToolSpec, ToolHandler } from '@dudousxd/nestjs-agent-core';
+
+interface ChunkOptions {
+    /** Target max characters per chunk. Default 800. */
+    chunkSize?: number;
+    /** Characters of overlap carried from the end of one chunk into the next. Default 100. */
+    overlap?: number;
+}
+/**
+ * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break
+ * on a paragraph → sentence → word boundary in the back half of the window (so a chunk rarely cuts
+ * mid-sentence), then carries `overlap` characters into the next chunk to preserve context across
+ * the seam. Pure and deterministic.
+ */
+declare function chunkText(text: string, options?: ChunkOptions): string[];
+
+/** A stored, embedded chunk. `embedding` length must match the store's configured dimensions. */
+interface VectorRecord {
+    id: string;
+    text: string;
+    embedding: number[];
+    /** Citation-facing origin (document title, URL, row id). */
+    source?: string;
+    metadata?: Record<string, unknown>;
+}
+interface VectorSearchOptions {
+    topK: number;
+    /** Exact-match metadata filter (all keys must equal). Passed through from `RetrieveOptions.filter`. */
+    filter?: Record<string, unknown>;
+}
+/**
+ * The write + search side of RAG storage. `@dudousxd/nestjs-agent-rag` ships `MemoryVectorStore`
+ * (in-JS cosine, tests + small scale) and `PgVectorStore` (pgvector); any impl works. Pair one with
+ * an `EmbeddingProvider` via {@link import('./embedding-retriever.js').EmbeddingRetriever} to get a
+ * core `Retriever`.
+ */
+interface VectorStore {
+    upsert(records: VectorRecord[]): Promise<void>;
+    search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
+    /**
+     * Delete every chunk belonging to a source document — all records whose id is `${documentId}`
+     * or `${documentId}#<n>` (the id scheme {@link import('./ingest.js').chunkDocuments} produces).
+     * Use it to drop a document from the index, and before re-ingesting one: `upsert` overwrites
+     * matching ids but can't remove chunks a shorter new version no longer produces, so a re-ingest
+     * without a preceding `remove` leaves the old tail orphaned.
+     */
+    remove(documentId: string): Promise<void>;
+    /**
+     * List the distinct source documents currently indexed (chunk ids collapsed back to their document
+     * by stripping the trailing `#<n>`), each with a representative chunk's `metadata`, optionally
+     * narrowed by a metadata `filter`. The enumeration seam for reconciliation: diff this against your
+     * source of truth to find documents to ingest (missing), re-ingest (a stored fingerprint like
+     * `size` changed), or {@link VectorStore.remove remove} (orphaned).
+     */
+    listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
+}
+/** A distinct source document as seen by the index — its id plus a representative chunk's metadata. */
+interface IndexedDocument {
+    id: string;
+    metadata?: Record<string, unknown>;
+}
+/** Collapse a chunk id (`${documentId}#<n>`) back to its source document id. */
+declare function documentIdOf(chunkId: string): string;
+
+/**
+ * An in-process {@link VectorStore} — cosine similarity over a Map, no infra. The reference adapter
+ * for tests and small/embedded corpora; for production scale use `PgVectorStore` (or your own).
+ */
+declare class MemoryVectorStore implements VectorStore {
+    private readonly records;
+    upsert(records: VectorRecord[]): Promise<void>;
+    remove(documentId: string): Promise<void>;
+    listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
+    search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
+}
+
+/**
+ * The minimal Postgres surface {@link PgVectorStore} needs — adapt your own `pg` / `postgres.js`
+ * client to it, so this package pulls in NO driver (bring your own, like the store adapters take an
+ * ORM handle and the Redis transport takes a client). `query` runs parameterized SQL (`$1,$2,…`) and
+ * resolves the result rows.
+ */
+interface PgClient {
+    query<Row = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<Row[]>;
+}
+interface PgVectorStoreOptions {
+    /** Table name. Default `agent_rag_chunks`. */
+    table?: string;
+    /** Embedding width — must match your model (e.g. 1536 for text-embedding-3-small). Default 1536. */
+    dimensions?: number;
+}
+/**
+ * A pgvector-backed {@link VectorStore} — the production reference adapter. Cosine distance via the
+ * `<=>` operator over an HNSW index; metadata in a `jsonb` column filtered with `@>`. Call
+ * {@link PgVectorStore.ensureSchema} once at boot to create the extension, table, and index.
+ */
+declare class PgVectorStore implements VectorStore {
+    private readonly client;
+    private readonly table;
+    private readonly dimensions;
+    constructor(client: PgClient, options?: PgVectorStoreOptions);
+    /** Idempotent DDL: the `vector` extension, the chunks table, and the cosine HNSW index. */
+    ensureSchema(): Promise<void>;
+    upsert(records: VectorRecord[]): Promise<void>;
+    remove(documentId: string): Promise<void>;
+    listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
+    search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
+}
+
+/**
+ * The minimal Redis surface {@link RedisVectorStore} needs: send a raw command and get the reply.
+ * Both `node-redis` (`client.sendCommand(args)`) and `ioredis` (wrap `redis.call`) satisfy it — so
+ * this package pulls in NO Redis driver (bring your own, like the other adapters). Requires the
+ * RediSearch module (Redis Stack / Redis 8+).
+ */
+interface RedisSearchClient {
+    sendCommand(args: (string | Buffer)[]): Promise<unknown>;
+}
+interface RedisVectorStoreOptions {
+    /** RediSearch index name. Default `agent_rag_idx`. */
+    index?: string;
+    /** Key prefix for the per-chunk hashes. Default `agent_rag:`. */
+    prefix?: string;
+    /** Embedding width — must match your model (e.g. 1536). Default 1536. */
+    dimensions?: number;
+    /**
+     * Metadata keys to index as filterable TAG fields. RediSearch needs filter fields declared up
+     * front, so only keys listed here can be passed in `search`'s `filter`. Default none.
+     */
+    filterableFields?: string[];
+}
+/**
+ * A RediSearch-backed {@link VectorStore} — HNSW + cosine over Redis, the ecosystem-native option
+ * for anyone already running Redis (see `-transport-redis`). Chunks are hashes under `prefix`, the
+ * embedding a FLOAT32 vector field; `search` is a KNN query. Call {@link RedisVectorStore.ensureSchema}
+ * once at boot to create the index.
+ */
+declare class RedisVectorStore implements VectorStore {
+    private readonly client;
+    private readonly index;
+    private readonly prefix;
+    private readonly dimensions;
+    private readonly filterableFields;
+    constructor(client: RedisSearchClient, options?: RedisVectorStoreOptions);
+    /** Idempotent: create the HNSW/cosine index if `FT.INFO` says it doesn't exist yet. */
+    ensureSchema(): Promise<void>;
+    upsert(records: VectorRecord[]): Promise<void>;
+    remove(documentId: string): Promise<void>;
+    listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
+    search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
+}
+
+/**
+ * Bridges an {@link EmbeddingProvider} + {@link VectorStore} into the core {@link Retriever} SPI:
+ * embed the query, then vector-search. This is what you wire as the agent's retriever (as a tool
+ * via `createRetrievalTool`, or into `AgentModule.forRoot({ retrieval })` for inject mode).
+ */
+declare class EmbeddingRetriever implements Retriever {
+    private readonly embedder;
+    private readonly store;
+    constructor(embedder: EmbeddingProvider, store: VectorStore);
+    retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
+}
+
+/** A source document to ingest. `id` scopes the chunk ids (`${id}#${n}`); `source` is the citation. */
+interface IngestDocument {
+    id: string;
+    text: string;
+    source?: string;
+    metadata?: Record<string, unknown>;
+}
+/** A chunked, not-yet-embedded record. Feed the same array to a vector store AND a keyword index so
+ * both retrievers key on identical chunk ids (which is what lets hybrid fusion line them up). */
+interface ChunkRecord {
+    id: string;
+    text: string;
+    source?: string;
+    metadata?: Record<string, unknown>;
+}
+interface IngestChunksOptions {
+    embedder: EmbeddingProvider;
+    store: VectorStore;
+}
+interface IngestOptions extends ChunkOptions, IngestChunksOptions {
+}
+/**
+ * Split documents into chunk records. Chunk ids are `${doc.id}#${index}`, so re-chunking the same
+ * document produces the same ids (upsert overwrites in place rather than duplicating).
+ */
+declare function chunkDocuments(documents: IngestDocument[], options?: ChunkOptions): ChunkRecord[];
+/** Embed pre-chunked records (one batched `embed`) and upsert them. Returns the record count. */
+declare function ingestChunks(chunks: ChunkRecord[], options: IngestChunksOptions): Promise<number>;
+/** Chunk → embed → upsert in one call. For hybrid search, use {@link chunkDocuments} +
+ * {@link ingestChunks} and feed the same chunks to a {@link import('./keyword-retriever.js').KeywordRetriever}. */
+declare function ingestDocuments(documents: IngestDocument[], options: IngestOptions): Promise<number>;
+
+interface KeywordRetrieverOptions {
+    /** BM25 term-frequency saturation. Default 1.5. */
+    k1?: number;
+    /** BM25 length normalization, 0 (off) … 1 (full). Default 0.75. */
+    b?: number;
+}
+/**
+ * An in-memory BM25 lexical retriever — the keyword half of hybrid search. Feed it the SAME
+ * {@link ChunkRecord}s you upsert into the vector store (from `chunkDocuments`) so their chunk ids
+ * line up and `HybridRetriever` can fuse the two rankings. Full BM25: idf, term-frequency saturation
+ * (`k1`), and document-length normalization (`b`).
+ */
+declare class KeywordRetriever implements Retriever {
+    private readonly docs;
+    private readonly documentFrequency;
+    private totalLength;
+    private readonly k1;
+    private readonly b;
+    constructor(options?: KeywordRetrieverOptions);
+    /** Index (or re-index) records. Re-adding an existing id replaces its posting and updates stats. */
+    add(records: ChunkRecord[]): void;
+    private remove;
+    retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
+}
+
+interface HybridRetrieverOptions {
+    /** RRF constant — larger flattens the rank weighting. Default 60 (the value from the original paper). */
+    k?: number;
+    /** Candidates pulled from each retriever before fusion. Default 20. */
+    fetchTopK?: number;
+    /** Per-retriever multipliers on each RRF contribution (same order as `retrievers`). Default all 1. */
+    weights?: number[];
+}
+/**
+ * Fuses several retrievers with Reciprocal Rank Fusion — the robust way to combine, say, dense
+ * vector search with lexical BM25. RRF scores each passage by `Σ weight / (k + rank)` across the
+ * lists it appears in, so it needs no score normalization between incompatible scales (cosine vs
+ * BM25). Deduplicates by passage id, so aligned chunk ids from the two retrievers reinforce.
+ */
+declare class HybridRetriever implements Retriever {
+    private readonly retrievers;
+    private readonly options;
+    constructor(retrievers: Retriever[], options?: HybridRetrieverOptions);
+    retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
+}
+
+interface RerankingRetrieverOptions {
+    /** How many candidates to pull from the base retriever before reranking. Default 20. */
+    fetchTopK?: number;
+}
+/**
+ * Two-stage retrieval: over-fetch cheap candidates from a base `Retriever`, then reorder them with a
+ * stronger {@link Reranker} and keep the top few. The standard precision boost — a fast first stage
+ * casts a wide net, a slow accurate second stage sharpens it. Composes over ANY retriever (vector,
+ * hybrid, keyword), and is itself a `Retriever`.
+ */
+declare class RerankingRetriever implements Retriever {
+    private readonly base;
+    private readonly reranker;
+    private readonly options;
+    constructor(base: Retriever, reranker: Reranker, options?: RerankingRetrieverOptions);
+    retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
+}
+
+/**
+ * Wraps a `Retriever` so every `retrieve` call ANDs a fixed metadata `filter` on top of whatever the
+ * caller passes — the owner/tenant-scoping primitive. `new FilteredRetriever(base, { ownerId })` only
+ * ever returns that owner's passages, so a shared vector store serves per-user / per-tenant RAG
+ * without the agent (or a crafted query) reaching across the boundary.
+ *
+ * The fixed filter takes precedence on key conflicts: a caller can narrow further but cannot widen
+ * past the scope. Composes over ANY retriever (vector, hybrid, reranked) and is itself a `Retriever`.
+ */
+declare class FilteredRetriever implements Retriever {
+    private readonly base;
+    private readonly filter;
+    constructor(base: Retriever, filter: Record<string, unknown>);
+    retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
+}
+
+interface RetrievalToolOptions {
+    /** Tool name the model sees. Default `search_knowledge`. */
+    name?: string;
+    description?: string;
+    /** Passages per search. Default lets the retriever decide (typically 5). */
+    topK?: number;
+}
+/** A functional tool (`{ spec, handler }`) — pass it to nestjs `provideAgentTool`. */
+interface RetrievalTool {
+    spec: ToolSpec;
+    handler: ToolHandler;
+}
+/**
+ * Builds the agentic-retrieval tool: a `read`-kind `search_knowledge(query)` the model calls when it
+ * decides it needs context. Its output `{ passages }` persists as the tool call's result, so sources
+ * flow to the frontend and telescope through the normal tool-call machinery (the citation surface).
+ *
+ * ```ts
+ * providers: [provideAgentTool(createRetrievalTool(retriever))]
+ * ```
+ */
+declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
+
+export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, MemoryVectorStore, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, type VectorRecord, type VectorSearchOptions, type VectorStore, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments };