@chatbot-packages/rag 0.1.0

package/dist/types-CjnplPJD.d.ts

@@ -0,0 +1,242 @@
+/**
+ * RAG System Types
+ */
+interface Document {
+    id: string;
+    sourceId: string;
+    path: string;
+    title: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+    createdAt: Date;
+    updatedAt: Date;
+}
+interface DocumentChunk {
+    id: string;
+    documentId: string;
+    text: string;
+    embedding?: number[];
+    metadata: ChunkMetadata;
+    createdAt: Date;
+}
+interface ChunkMetadata {
+    /** Section path (e.g., "Chapter 1 > Section 2 > Subsection A") */
+    sectionPath: string;
+    /** H1 heading */
+    headingH1?: string;
+    /** H2 heading */
+    headingH2?: string;
+    /** H3 heading */
+    headingH3?: string;
+    /** Start position in original document */
+    startOffset?: number;
+    /** End position in original document */
+    endOffset?: number;
+    /** Source file path */
+    sourcePath?: string;
+    /** Page number (if applicable) */
+    pageNumber?: number;
+    /** Custom metadata */
+    [key: string]: unknown;
+}
+type ExtractorType = 'chm' | 'html' | 'markdown' | 'text';
+interface ExtractorOptions {
+    /** Source file or directory path */
+    sourcePath: string;
+    /** Output directory for extracted content */
+    outputDir?: string;
+    /** Whether to preserve directory structure */
+    preserveStructure?: boolean;
+}
+interface ExtractedDocument {
+    path: string;
+    title: string;
+    content: string;
+    format: ExtractorType;
+    metadata?: Record<string, unknown>;
+}
+interface ChunkingOptions {
+    /** Target chunk size in tokens */
+    chunkSize?: number;
+    /** Overlap between chunks in tokens */
+    chunkOverlap?: number;
+    /** Minimum chunk size in tokens */
+    minChunkSize?: number;
+    /** Maximum chunk size in tokens */
+    maxChunkSize?: number;
+    /** Whether to respect heading boundaries */
+    respectHeadings?: boolean;
+    /** Heading levels to split on (e.g., [1, 2]) */
+    splitOnHeadings?: number[];
+}
+interface ChunkResult {
+    chunks: DocumentChunk[];
+    stats: {
+        totalChunks: number;
+        avgChunkSize: number;
+        minChunkSize: number;
+        maxChunkSize: number;
+    };
+}
+type EmbeddingProvider = 'local' | 'openai' | 'huggingface';
+interface EmbeddingOptions {
+    /** Embedding provider */
+    provider: EmbeddingProvider;
+    /** Model name (for local: 'bge-large-en-v1.5', for OpenAI: 'text-embedding-3-small') */
+    model?: string;
+    /** API key (for cloud providers) */
+    apiKey?: string;
+    /** Batch size for embedding generation */
+    batchSize?: number;
+    /** Embedding dimensions (for providers that support it) */
+    dimensions?: number;
+}
+interface EmbeddingResult {
+    embedding: number[];
+    tokens?: number;
+}
+interface EmbeddingBackend {
+    /** Embed a single text */
+    embed(text: string): Promise<EmbeddingResult>;
+    /** Embed multiple texts */
+    embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
+    /** Get embedding dimensions */
+    getDimensions(): number;
+    /** Get model name */
+    getModel(): string;
+}
+type VectorStoreType = 'sqlite' | 'postgres' | 'memory';
+interface VectorStoreOptions {
+    /** Store type */
+    type: VectorStoreType;
+    /** Database path (for SQLite) or connection string (for PostgreSQL) */
+    connectionString?: string;
+    /** Table/collection name */
+    tableName?: string;
+    /** Embedding dimensions */
+    dimensions: number;
+}
+interface SearchResult {
+    chunk: DocumentChunk;
+    score: number;
+    /** Search type that found this result */
+    searchType: 'dense' | 'sparse' | 'hybrid';
+}
+interface VectorStore {
+    /** Initialize the store (create tables, indexes) */
+    initialize(): Promise<void>;
+    /** Insert chunks with embeddings */
+    insert(chunks: DocumentChunk[]): Promise<void>;
+    /** Dense (vector) search */
+    denseSearch(embedding: number[], topK: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    /** Sparse (BM25/text) search */
+    sparseSearch(query: string, topK: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    /** Delete chunks by document ID */
+    deleteByDocumentId(documentId: string): Promise<number>;
+    /** Get chunk by ID */
+    getById(id: string): Promise<DocumentChunk | null>;
+    /** Close connection */
+    close(): Promise<void>;
+}
+interface RetrievalOptions {
+    /** Number of results to return */
+    topK?: number;
+    /** Dense search weight (0-1) */
+    denseWeight?: number;
+    /** Sparse search weight (0-1) */
+    sparseWeight?: number;
+    /** RRF k parameter */
+    rrfK?: number;
+    /** Whether to use reranking */
+    useReranking?: boolean;
+    /** Number of candidates to rerank */
+    rerankTopK?: number;
+    /** Metadata filter */
+    filter?: Record<string, unknown>;
+}
+interface RetrievalResult {
+    results: SearchResult[];
+    stats: {
+        denseCount: number;
+        sparseCount: number;
+        rerankingApplied: boolean;
+        totalTime: number;
+    };
+}
+interface RAGOptions {
+    /** Path to documents (file or directory) */
+    documents: string;
+    /** Vector store configuration */
+    vectorStore: VectorStoreOptions;
+    /** Embedding configuration */
+    embeddings: EmbeddingOptions;
+    /** LLM configuration */
+    llm: {
+        provider: 'openai' | 'anthropic' | 'cerebras' | 'groq';
+        apiKey?: string;
+        model?: string;
+        baseUrl?: string;
+    };
+    /** Chunking options */
+    chunking?: ChunkingOptions;
+    /** Retrieval options */
+    retrieval?: RetrievalOptions;
+    /** System prompt for the LLM */
+    systemPrompt?: string;
+}
+interface Citation {
+    /** Citation index (1-based) */
+    index: number;
+    /** Chunk ID */
+    chunkId: string;
+    /** Section path */
+    sectionPath: string;
+    /** Heading (if available) */
+    heading?: string;
+    /** Relevant text snippet */
+    snippet?: string;
+}
+interface RAGResponse {
+    /** The question that was asked */
+    question: string;
+    /** The generated answer */
+    answer: string;
+    /** Citations from source documents */
+    citations: Citation[];
+    /** Retrieved context chunks */
+    context: DocumentChunk[];
+    /** Response metadata */
+    metadata: {
+        /** Total processing time in ms */
+        totalTime: number;
+        /** Retrieval time in ms */
+        retrievalTime: number;
+        /** Generation time in ms */
+        generationTime: number;
+        /** Whether response was cached */
+        cached: boolean;
+        /** Model used */
+        model: string;
+    };
+}
+interface RAGService {
+    /** Index documents from a path */
+    index(path: string, options?: {
+        sourceId?: string;
+    }): Promise<{
+        documentsIndexed: number;
+        chunksCreated: number;
+    }>;
+    /** Ask a question */
+    ask(question: string, options?: RetrievalOptions): Promise<RAGResponse>;
+    /** Get indexed document count */
+    getDocumentCount(): Promise<number>;
+    /** Get chunk count */
+    getChunkCount(): Promise<number>;
+    /** Clear all indexed data */
+    clear(): Promise<void>;
+    /** Close connections */
+    close(): Promise<void>;
+}
+
+export type { ChunkMetadata as C, Document as D, EmbeddingBackend as E, RAGService as R, SearchResult as S, VectorStore as V, RAGOptions as a, RetrievalOptions as b, RAGResponse as c, ChunkResult as d, ChunkingOptions as e, Citation as f, DocumentChunk as g, EmbeddingOptions as h, EmbeddingResult as i, ExtractedDocument as j, RetrievalResult as k, VectorStoreOptions as l, ExtractorOptions as m, ExtractorType as n };

package/dist/vectorstore/index.d.ts

@@ -0,0 +1,109 @@
+import { V as VectorStore, g as DocumentChunk, S as SearchResult, l as VectorStoreOptions } from '../types-CjnplPJD.js';
+import { PoolConfig } from 'pg';
+
+/**
+ * SQLite Vector Store
+ *
+ * Simple vector store using SQLite with better-sqlite3.
+ * Stores embeddings as JSON arrays and performs similarity search in JavaScript.
+ * Best for development, small datasets, or when PostgreSQL isn't available.
+ */
+
+interface SQLiteVectorStoreOptions {
+    /** Database file path (use ':memory:' for in-memory) */
+    path?: string;
+    /** Table name (default: 'chunks') */
+    tableName?: string;
+    /** Embedding dimensions */
+    dimensions: number;
+}
+declare class SQLiteVectorStore implements VectorStore {
+    private db;
+    private tableName;
+    private dimensions;
+    constructor(options: SQLiteVectorStoreOptions);
+    initialize(): Promise<void>;
+    insert(chunks: DocumentChunk[]): Promise<void>;
+    denseSearch(embedding: number[], topK: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    sparseSearch(query: string, topK: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    deleteByDocumentId(documentId: string): Promise<number>;
+    getById(id: string): Promise<DocumentChunk | null>;
+    close(): Promise<void>;
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    private cosineSimilarity;
+    /**
+     * Convert database row to DocumentChunk
+     */
+    private rowToChunk;
+    /**
+     * Escape FTS query to prevent syntax errors
+     */
+    private escapeFTSQuery;
+}
+
+/**
+ * PostgreSQL Vector Store with pgvector
+ *
+ * Production-ready vector store using PostgreSQL with pgvector extension.
+ * Supports HNSW indexing for fast similarity search.
+ */
+
+interface PostgresVectorStoreOptions {
+    /** Connection string or pool config */
+    connectionString?: string;
+    poolConfig?: PoolConfig;
+    /** Table name (default: 'chunks') */
+    tableName?: string;
+    /** Embedding dimensions */
+    dimensions: number;
+    /** Schema name (default: 'public') */
+    schema?: string;
+}
+declare class PostgresVectorStore implements VectorStore {
+    private pool;
+    private tableName;
+    private schema;
+    private dimensions;
+    private fullTableName;
+    constructor(options: PostgresVectorStoreOptions);
+    initialize(): Promise<void>;
+    insert(chunks: DocumentChunk[]): Promise<void>;
+    denseSearch(embedding: number[], topK: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    sparseSearch(query: string, topK: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    deleteByDocumentId(documentId: string): Promise<number>;
+    getById(id: string): Promise<DocumentChunk | null>;
+    close(): Promise<void>;
+    /**
+     * Get chunk and document counts
+     */
+    getStats(): Promise<{
+        chunks: number;
+        documents: number;
+    }>;
+    /**
+     * Insert or update a document
+     */
+    upsertDocument(doc: {
+        id: string;
+        sourceId: string;
+        path: string;
+        title: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    private rowToChunk;
+}
+
+/**
+ * Vector Store Implementations
+ *
+ * Store and search document chunks using vector similarity.
+ */
+
+/**
+ * Create a vector store based on options
+ */
+declare function createVectorStore(options: VectorStoreOptions): VectorStore;
+
+export { PostgresVectorStore, type PostgresVectorStoreOptions, SQLiteVectorStore, type SQLiteVectorStoreOptions, createVectorStore };