@revealui/ai 0.1.0

package/dist/ingestion/file-parsers.js

@@ -0,0 +1,247 @@
+/**
+ * File Parsers
+ *
+ * Zero external dependencies — pure Node.js + regex/string manipulation.
+ * PDF extraction uses a hand-rolled parser that reads text streams directly
+ * from the binary PDF format (BT/ET blocks, Tj/TJ operators).
+ */
+// =============================================================================
+// Plain Text
+// =============================================================================
+export class PlainTextParser {
+    parse(input) {
+        return { text: input.replace(/\r\n/g, '\n').replace(/\r/g, '\n').trim() };
+    }
+}
+// =============================================================================
+// Markdown
+// =============================================================================
+const FRONTMATTER_RE = /^---[\s\S]*?---\n?/;
+const CODE_FENCE_RE = /```[\s\S]*?```/g;
+const INLINE_CODE_RE = /`[^`]*`/g;
+const HEADING_RE = /^#{1,6}\s+/gm;
+const EMPHASIS_RE = /[*_]{1,3}([^*_]+)[*_]{1,3}/g;
+const LINK_RE = /\[([^\]]*)\]\([^)]*\)/g;
+const IMAGE_RE = /!\[[^\]]*\]\([^)]*\)/g;
+const HTML_TAG_IN_MD_RE = /<[^>]+>/g;
+const HORIZONTAL_RULE_RE = /^[-*_]{3,}\s*$/gm;
+export class MarkdownParser {
+    parse(input) {
+        const text = input
+            // Strip frontmatter
+            .replace(FRONTMATTER_RE, '')
+            // Strip code fences (preserve content inside)
+            .replace(CODE_FENCE_RE, (m) => m.replace(/```\w*\n?/g, '').replace(/```/g, ''))
+            // Strip inline code backticks but keep content
+            .replace(INLINE_CODE_RE, (m) => m.slice(1, -1))
+            // Strip heading markers
+            .replace(HEADING_RE, '')
+            // Unwrap bold/italic — keep inner text
+            .replace(EMPHASIS_RE, '$1')
+            // Unwrap links — keep link text
+            .replace(LINK_RE, '$1')
+            // Remove images entirely
+            .replace(IMAGE_RE, '')
+            // Strip inline HTML
+            .replace(HTML_TAG_IN_MD_RE, '')
+            // Remove horizontal rules
+            .replace(HORIZONTAL_RULE_RE, '')
+            // Normalize CRLF
+            .replace(/\r\n/g, '\n')
+            .replace(/\r/g, '\n')
+            // Collapse 3+ blank lines to 2
+            .replace(/\n{3,}/g, '\n\n')
+            .trim();
+        return { text };
+    }
+}
+// =============================================================================
+// HTML
+// =============================================================================
+const SCRIPT_STYLE_RE = /<(script|style)[^>]*>[\s\S]*?<\/\1>/gi;
+const HTML_COMMENT_RE = /<!--[\s\S]*?-->/g;
+const HTML_ALL_TAGS_RE = /<[^>]+>/g;
+const HTML_ENTITIES = {
+    '&amp;': '&',
+    '&lt;': '<',
+    '&gt;': '>',
+    '&quot;': '"',
+    '&#39;': "'",
+    '&nbsp;': ' ',
+    '&apos;': "'",
+};
+function decodeEntities(text) {
+    return text
+        .replace(/&[a-z]+;|&#\d+;/gi, (entity) => HTML_ENTITIES[entity] ?? entity)
+        .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)));
+}
+export class HtmlParser {
+    parse(input) {
+        const text = decodeEntities(input
+            .replace(SCRIPT_STYLE_RE, '')
+            .replace(HTML_COMMENT_RE, '')
+            .replace(HTML_ALL_TAGS_RE, ' ')
+            .replace(/\r\n/g, '\n')
+            .replace(/\r/g, '\n')
+            .replace(/[ \t]+/g, ' ')
+            .replace(/\n{3,}/g, '\n\n')
+            .trim());
+        return { text };
+    }
+}
+// =============================================================================
+// PDF
+// =============================================================================
+/**
+ * Decode a PDF literal string token.
+ *
+ * PDF literal strings are enclosed in `(…)` and support:
+ * - Escape sequences: \n \r \t \b \f \\ \( \)
+ * - Octal escapes: \nnn
+ * - Line continuation: \ at end of line (ignored)
+ */
+function decodePdfLiteral(raw) {
+    return raw.replace(/\\([\\()nrtbf]|[0-7]{1,3}|\n)/g, (_, esc) => {
+        if (esc === '\n')
+            return '';
+        if (esc === 'n')
+            return '\n';
+        if (esc === 'r')
+            return '\r';
+        if (esc === 't')
+            return '\t';
+        if (esc === 'b')
+            return '\b';
+        if (esc === 'f')
+            return '\f';
+        if (esc === '\\')
+            return '\\';
+        if (esc === '(')
+            return '(';
+        if (esc === ')')
+            return ')';
+        // Octal
+        return String.fromCharCode(Number.parseInt(esc, 8));
+    });
+}
+/**
+ * Extract text from a single BT…ET block.
+ * Handles: (string)Tj  [(array)]TJ  (string)'  (string)"
+ */
+function extractTextFromBlock(block) {
+    const parts = [];
+    // Tj — show string: (text) Tj
+    const tjRe = /\(([^)]*(?:\\.[^)]*)*)\)\s*Tj/g;
+    let m;
+    // biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
+    while ((m = tjRe.exec(block)) !== null) {
+        if (m[1] !== undefined)
+            parts.push(decodePdfLiteral(m[1]));
+    }
+    // TJ — show array: [(t1)(t2) num …] TJ
+    const tjArrayRe = /\[([^\]]*)\]\s*TJ/g;
+    // biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
+    while ((m = tjArrayRe.exec(block)) !== null) {
+        const inner = m[1] ?? '';
+        const strRe = /\(([^)]*(?:\\.[^)]*)*)\)/g;
+        // biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
+        while ((m = strRe.exec(inner)) !== null) {
+            if (m[1] !== undefined)
+                parts.push(decodePdfLiteral(m[1]));
+        }
+    }
+    // ' and " operators (move-to-next-line-then-show)
+    const quoteRe = /\(([^)]*(?:\\.[^)]*)*)\)\s*['"]/g;
+    // biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
+    while ((m = quoteRe.exec(block)) !== null) {
+        if (m[1] !== undefined)
+            parts.push(decodePdfLiteral(m[1]));
+    }
+    return parts.join(' ');
+}
+/**
+ * Hand-rolled PDF text extractor.
+ *
+ * Reads text content directly from the binary PDF format without any
+ * external dependencies. Works for text-based PDFs (the vast majority).
+ * Encrypted or image-only PDFs return an empty string.
+ *
+ * Supports:
+ * - BT/ET text blocks with Tj, TJ, ' and " operators
+ * - Literal string decoding (escape sequences + octal)
+ * - Page count from /Count dictionary entry
+ * - Document title from /Title dictionary entry
+ *
+ */
+export class PdfParser {
+    /** Parse a PDF string (for API compatibility; use parseBuffer for binary data). */
+    parse(input) {
+        return this.parseBuffer(Buffer.from(input, 'binary'));
+    }
+    /** Parse a PDF from a Buffer. */
+    parseBuffer(input) {
+        const src = input.toString('binary');
+        // Extract all BT…ET text blocks
+        const btEtRe = /BT\s([\s\S]*?)\sET/g;
+        const blocks = [];
+        let m;
+        // biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
+        while ((m = btEtRe.exec(src)) !== null) {
+            if (m[1] !== undefined)
+                blocks.push(m[1]);
+        }
+        const rawText = blocks.map(extractTextFromBlock).filter(Boolean).join('\n');
+        const text = rawText
+            .replace(/[ \t]+/g, ' ')
+            .replace(/\n{3,}/g, '\n\n')
+            .trim();
+        // Page count from /Count N
+        const countMatch = /\/Count\s+(\d+)/.exec(src);
+        const pageCount = countMatch ? Number.parseInt(countMatch[1] ?? '0', 10) : 0;
+        // Document title from /Title (string) or /Title <hex>
+        let title;
+        const titleMatch = /\/Title\s*\(([^)]*(?:\\.[^)]*)*)\)/.exec(src);
+        if (titleMatch) {
+            title = decodePdfLiteral(titleMatch[1] ?? '').trim() || undefined;
+        }
+        return {
+            text,
+            metadata: {
+                pageCount,
+                ...(title ? { title } : {}),
+            },
+        };
+    }
+}
+// =============================================================================
+// JSON
+// =============================================================================
+export class JsonParser {
+    parse(input) {
+        try {
+            const parsed = JSON.parse(input);
+            return { text: JSON.stringify(parsed, null, 2) };
+        }
+        catch {
+            // Not valid JSON — return as-is
+            return { text: input.trim() };
+        }
+    }
+}
+export function createParser(mimeType) {
+    const normalized = mimeType.toLowerCase().split(';')[0]?.trim() ?? '';
+    if (normalized === 'text/markdown' || normalized === 'text/x-markdown') {
+        return new MarkdownParser();
+    }
+    if (normalized === 'text/html' || normalized === 'application/xhtml+xml') {
+        return new HtmlParser();
+    }
+    if (normalized === 'application/json') {
+        return new JsonParser();
+    }
+    if (normalized === 'application/pdf') {
+        return new PdfParser();
+    }
+    // Default: plain text (covers text/plain and unknown types)
+    return new PlainTextParser();
+}

package/dist/ingestion/hybrid-search.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Hybrid Search — Vector + BM25 with Reciprocal Rank Fusion
+ *
+ * speed mode: cosine vector search only (fast, default)
+ * accuracy mode: fetch 50 vector candidates → BM25 on those → RRF fusion → optional LLM rerank
+ *
+ * RRF formula: score(d) = Σ 1 / (k + rank(d))  where k = 60
+ */
+import type { Database } from '@revealui/db/client';
+import type { LLMClient } from '../llm/client.js';
+import type { RagSearchOptions, RagSearchResult } from './rag-vector-service.js';
+export interface HybridSearchOptions extends RagSearchOptions {
+    /** speed = vector only, accuracy = BM25 hybrid + optional rerank (default: 'speed') */
+    mode?: 'speed' | 'accuracy';
+    /** Number of vector candidates to fetch before BM25 filtering (accuracy mode only) */
+    vectorCandidates?: number;
+    /** Whether to apply LLM re-ranking in accuracy mode (default: false) */
+    rerank?: boolean;
+    llmClient?: LLMClient;
+}
+export declare function hybridSearch(query: string, _db: Database, embeddingFn: (text: string) => Promise<number[]>, options: HybridSearchOptions): Promise<RagSearchResult[]>;
+//# sourceMappingURL=hybrid-search.d.ts.map

package/dist/ingestion/hybrid-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-search.d.ts","sourceRoot":"","sources":["../../src/ingestion/hybrid-search.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAEjD,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AAIhF,MAAM,WAAW,mBAAoB,SAAQ,gBAAgB;IAC3D,uFAAuF;IACvF,IAAI,CAAC,EAAE,OAAO,GAAG,UAAU,CAAA;IAC3B,sFAAsF;IACtF,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,wEAAwE;IACxE,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,SAAS,CAAC,EAAE,SAAS,CAAA;CACtB;AAID,wBAAsB,YAAY,CAChC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,QAAQ,EACb,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,EAChD,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,eAAe,EAAE,CAAC,CA+D5B"}

package/dist/ingestion/hybrid-search.js

@@ -0,0 +1,63 @@
+/**
+ * Hybrid Search — Vector + BM25 with Reciprocal Rank Fusion
+ *
+ * speed mode: cosine vector search only (fast, default)
+ * accuracy mode: fetch 50 vector candidates → BM25 on those → RRF fusion → optional LLM rerank
+ *
+ * RRF formula: score(d) = Σ 1 / (k + rank(d))  where k = 60
+ */
+import { BM25 } from './bm25.js';
+import { RagVectorService } from './rag-vector-service.js';
+import { rerankChunks } from './reranker.js';
+const RRF_K = 60;
+export async function hybridSearch(query, _db, embeddingFn, options) {
+    const service = new RagVectorService();
+    const mode = options.mode ?? 'speed';
+    const limit = options.limit ?? 5;
+    // Speed mode: pure vector search
+    if (mode === 'speed') {
+        const embedding = await embeddingFn(query);
+        return service.searchSimilar(embedding, { ...options, limit });
+    }
+    // Accuracy mode: over-fetch vector candidates, BM25 on them, RRF fusion
+    const candidateCount = options.vectorCandidates ?? 50;
+    const embedding = await embeddingFn(query);
+    const vectorResults = await service.searchSimilar(embedding, {
+        ...options,
+        limit: candidateCount,
+        threshold: 0.3, // lower threshold to get more candidates
+    });
+    if (vectorResults.length === 0)
+        return [];
+    // Build BM25 index from vector candidates
+    const bm25 = new BM25();
+    bm25.index(vectorResults.map((r) => ({
+        id: r.chunk.id,
+        text: r.chunk.content,
+    })));
+    const bm25Results = bm25.search(query, candidateCount);
+    // Build rank maps
+    const vectorRankMap = new Map(vectorResults.map((r, i) => [r.chunk.id, i + 1]));
+    const bm25RankMap = new Map(bm25Results.map((r, i) => [r.id, i + 1]));
+    // RRF fusion
+    const allIds = new Set([...vectorRankMap.keys(), ...bm25RankMap.keys()]);
+    const rrfScores = new Map();
+    for (const id of allIds) {
+        const vectorRank = vectorRankMap.get(id) ?? candidateCount + 1;
+        const bm25Rank = bm25RankMap.get(id) ?? candidateCount + 1;
+        const rrf = 1 / (RRF_K + vectorRank) + 1 / (RRF_K + bm25Rank);
+        rrfScores.set(id, rrf);
+    }
+    // Sort by RRF score and map back to RagSearchResult
+    const resultMap = new Map(vectorResults.map((r) => [r.chunk.id, r]));
+    const sorted = Array.from(rrfScores.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, limit)
+        .map(([id]) => resultMap.get(id))
+        .filter((r) => r !== undefined);
+    // Optional LLM re-ranking
+    if (options.rerank && options.llmClient) {
+        return rerankChunks(query, sorted, options.llmClient, limit);
+    }
+    return sorted;
+}

package/dist/ingestion/index.d.ts

@@ -0,0 +1,9 @@
+export * from './bm25.js';
+export * from './cms-indexer.js';
+export * from './file-parsers.js';
+export * from './hybrid-search.js';
+export * from './pipeline.js';
+export * from './rag-vector-service.js';
+export * from './reranker.js';
+export * from './text-splitter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/ingestion/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ingestion/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAA;AACzB,cAAc,kBAAkB,CAAA;AAChC,cAAc,mBAAmB,CAAA;AACjC,cAAc,oBAAoB,CAAA;AAClC,cAAc,eAAe,CAAA;AAC7B,cAAc,yBAAyB,CAAA;AACvC,cAAc,eAAe,CAAA;AAC7B,cAAc,oBAAoB,CAAA"}

package/dist/ingestion/index.js

@@ -0,0 +1,8 @@
+export * from './bm25.js';
+export * from './cms-indexer.js';
+export * from './file-parsers.js';
+export * from './hybrid-search.js';
+export * from './pipeline.js';
+export * from './rag-vector-service.js';
+export * from './reranker.js';
+export * from './text-splitter.js';

package/dist/ingestion/pipeline.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Ingestion Pipeline
+ *
+ * Orchestrates: parse → split → embed → store.
+ * One document produces N chunks, each with a 768-dim embedding.
+ */
+import type { Database } from '@revealui/db/client';
+export interface IngestRequest {
+    workspaceId: string;
+    sourceType: 'cms_collection' | 'url' | 'file' | 'text';
+    sourceId?: string;
+    sourceCollection?: string;
+    title?: string;
+    mimeType?: string;
+    rawContent: string;
+    chunkSize?: number;
+    chunkOverlap?: number;
+}
+export interface IngestResult {
+    documentId: string;
+    chunkCount: number;
+    status: 'indexed' | 'failed';
+    error?: string;
+}
+export declare class IngestionPipeline {
+    private db;
+    private embeddingFn;
+    private splitter;
+    constructor(db: Database, embeddingFn: (text: string) => Promise<number[]>);
+    ingest(req: IngestRequest): Promise<IngestResult>;
+    ingestBatch(docs: IngestRequest[]): Promise<IngestResult[]>;
+    deleteDocument(documentId: string): Promise<void>;
+    deleteBySource(workspaceId: string, sourceCollection: string, sourceId: string): Promise<void>;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/ingestion/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAMnD,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,gBAAgB,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAA;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAcD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,EAAE,CAAU;IACpB,OAAO,CAAC,WAAW,CAAqC;IACxD,OAAO,CAAC,QAAQ,CAA4B;gBAEhC,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;IAMpE,MAAM,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAsEjD,WAAW,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAa3D,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjD,cAAc,CAClB,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,EACxB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC;CAiBjB"}

package/dist/ingestion/pipeline.js

@@ -0,0 +1,114 @@
+/**
+ * Ingestion Pipeline
+ *
+ * Orchestrates: parse → split → embed → store.
+ * One document produces N chunks, each with a 768-dim embedding.
+ */
+import { ragChunks, ragDocuments } from '@revealui/db/schema/rag';
+import { and, eq } from 'drizzle-orm';
+import { createParser } from './file-parsers.js';
+import { RecursiveCharacterSplitter } from './text-splitter.js';
+function generateId(prefix) {
+    return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
+}
+function estimateWordCount(text) {
+    return text.split(/\s+/).filter((w) => w.length > 0).length;
+}
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+export class IngestionPipeline {
+    db;
+    embeddingFn;
+    splitter;
+    constructor(db, embeddingFn) {
+        this.db = db;
+        this.embeddingFn = embeddingFn;
+        this.splitter = new RecursiveCharacterSplitter();
+    }
+    async ingest(req) {
+        const docId = generateId('rdoc');
+        const now = new Date();
+        // 1. Insert document row with status='processing'
+        await this.db.insert(ragDocuments).values({
+            id: docId,
+            workspaceId: req.workspaceId,
+            sourceType: req.sourceType,
+            sourceId: req.sourceId ?? null,
+            sourceCollection: req.sourceCollection ?? null,
+            title: req.title ?? null,
+            mimeType: req.mimeType ?? 'text/plain',
+            rawContent: req.rawContent,
+            wordCount: estimateWordCount(req.rawContent),
+            tokenEstimate: estimateTokens(req.rawContent),
+            status: 'processing',
+            createdAt: now,
+            updatedAt: now,
+        });
+        try {
+            // 2. Parse
+            const parser = createParser(req.mimeType ?? 'text/plain');
+            const { text } = parser.parse(req.rawContent);
+            // 3. Split
+            const chunks = this.splitter.split(text, {
+                chunkSize: req.chunkSize ?? 512,
+                overlap: req.chunkOverlap ?? 64,
+            });
+            // 4. Embed and insert each chunk
+            for (const chunk of chunks) {
+                const embedding = await this.embeddingFn(chunk.content);
+                const chunkId = generateId('rchk');
+                await this.db.insert(ragChunks).values({
+                    id: chunkId,
+                    documentId: docId,
+                    workspaceId: req.workspaceId,
+                    content: chunk.content,
+                    tokenCount: chunk.tokenCount,
+                    chunkIndex: chunk.index,
+                    embedding,
+                    embeddingModel: 'nomic-embed-text',
+                    metadata: chunk.metadata ?? {},
+                    createdAt: now,
+                });
+            }
+            // 5. Mark as indexed
+            await this.db
+                .update(ragDocuments)
+                .set({ status: 'indexed', indexedAt: now, updatedAt: now })
+                .where(eq(ragDocuments.id, docId));
+            return { documentId: docId, chunkCount: chunks.length, status: 'indexed' };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            await this.db
+                .update(ragDocuments)
+                .set({ status: 'failed', errorMessage: message, updatedAt: new Date() })
+                .where(eq(ragDocuments.id, docId));
+            return { documentId: docId, chunkCount: 0, status: 'failed', error: message };
+        }
+    }
+    async ingestBatch(docs) {
+        const results = [];
+        const concurrencyLimit = 3;
+        for (let i = 0; i < docs.length; i += concurrencyLimit) {
+            const batch = docs.slice(i, i + concurrencyLimit);
+            const batchResults = await Promise.all(batch.map((doc) => this.ingest(doc)));
+            results.push(...batchResults);
+        }
+        return results;
+    }
+    async deleteDocument(documentId) {
+        // Chunks cascade-delete via FK
+        await this.db.delete(ragDocuments).where(eq(ragDocuments.id, documentId));
+    }
+    async deleteBySource(workspaceId, sourceCollection, sourceId) {
+        // Find and delete documents by source identity
+        const docs = await this.db
+            .select({ id: ragDocuments.id })
+            .from(ragDocuments)
+            .where(and(eq(ragDocuments.workspaceId, workspaceId), eq(ragDocuments.sourceCollection, sourceCollection), eq(ragDocuments.sourceId, sourceId)));
+        for (const doc of docs) {
+            await this.deleteDocument(doc.id);
+        }
+    }
+}

package/dist/ingestion/rag-vector-service.d.ts

@@ -0,0 +1,34 @@
+/**
+ * RAG Vector Service
+ *
+ * Semantic similarity search over rag_chunks using pgvector cosine distance.
+ * Modeled after VectorMemoryService — same Drizzle cosine-distance pattern.
+ */
+import type { RagChunk, RagDocument } from '@revealui/db/schema/rag';
+export interface RagSearchOptions {
+    workspaceId: string;
+    limit?: number;
+    threshold?: number;
+    sourceCollection?: string;
+}
+export interface RagSearchResult {
+    chunk: RagChunk;
+    document: Pick<RagDocument, 'id' | 'title' | 'sourceType' | 'sourceCollection' | 'sourceId' | 'createdAt'>;
+    similarity: number;
+}
+export declare class RagVectorService {
+    private _db;
+    private get db();
+    /**
+     * Search for RAG chunks similar to the query embedding.
+     *
+     * Uses cosine distance (<->) from pgvector.
+     * Validates that the embedding is 768-dimensional (nomic-embed-text).
+     */
+    searchSimilar(queryEmbedding: number[], options: RagSearchOptions): Promise<RagSearchResult[]>;
+    /**
+     * Get all chunks for a document ordered by chunk index.
+     */
+    getChunksByDocument(documentId: string): Promise<RagChunk[]>;
+}
+//# sourceMappingURL=rag-vector-service.d.ts.map

package/dist/ingestion/rag-vector-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag-vector-service.d.ts","sourceRoot":"","sources":["../../src/ingestion/rag-vector-service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAA;AAIpE,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,QAAQ,CAAA;IACf,QAAQ,EAAE,IAAI,CACZ,WAAW,EACX,IAAI,GAAG,OAAO,GAAG,YAAY,GAAG,kBAAkB,GAAG,UAAU,GAAG,WAAW,CAC9E,CAAA;IACD,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,GAAG,CAAkD;IAE7D,OAAO,KAAK,EAAE,GAKb;IAED;;;;;OAKG;IACG,aAAa,CACjB,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,eAAe,EAAE,CAAC;IA8E7B;;OAEG;IACG,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;CAOnE"}

package/dist/ingestion/rag-vector-service.js

@@ -0,0 +1,98 @@
+/**
+ * RAG Vector Service
+ *
+ * Semantic similarity search over rag_chunks using pgvector cosine distance.
+ * Modeled after VectorMemoryService — same Drizzle cosine-distance pattern.
+ */
+import { getVectorClient } from '@revealui/db/client';
+import { ragChunks, ragDocuments } from '@revealui/db/schema/rag';
+import { and, asc, eq, sql } from 'drizzle-orm';
+export class RagVectorService {
+    _db = null;
+    get db() {
+        if (!this._db) {
+            this._db = getVectorClient();
+        }
+        return this._db;
+    }
+    /**
+     * Search for RAG chunks similar to the query embedding.
+     *
+     * Uses cosine distance (<->) from pgvector.
+     * Validates that the embedding is 768-dimensional (nomic-embed-text).
+     */
+    async searchSimilar(queryEmbedding, options) {
+        if (queryEmbedding.length !== 768) {
+            throw new Error(`Invalid embedding dimension: expected 768, got ${queryEmbedding.length}. ` +
+                'RAG uses nomic-embed-text (768-dim). Ensure OLLAMA_BASE_URL is set.');
+        }
+        const conditions = [
+            sql `${ragChunks}.embedding IS NOT NULL`,
+            eq(ragChunks.workspaceId, options.workspaceId),
+        ];
+        if (options.sourceCollection) {
+            conditions.push(eq(ragDocuments.sourceCollection, options.sourceCollection));
+        }
+        const threshold = options.threshold ?? 0.6;
+        const limit = options.limit ?? 5;
+        const embeddingJson = JSON.stringify(queryEmbedding);
+        const rows = await this.db
+            .select({
+            chunkId: ragChunks.id,
+            documentId: ragChunks.documentId,
+            workspaceId: ragChunks.workspaceId,
+            content: ragChunks.content,
+            tokenCount: ragChunks.tokenCount,
+            chunkIndex: ragChunks.chunkIndex,
+            embedding: ragChunks.embedding,
+            embeddingModel: ragChunks.embeddingModel,
+            metadata: ragChunks.metadata,
+            chunkCreatedAt: ragChunks.createdAt,
+            docTitle: ragDocuments.title,
+            docSourceType: ragDocuments.sourceType,
+            docSourceCollection: ragDocuments.sourceCollection,
+            docSourceId: ragDocuments.sourceId,
+            docCreatedAt: ragDocuments.createdAt,
+            // 1 - (cosine_distance / 2) maps [0,2] distance to [0,1] similarity
+            similarity: sql `1 - (${ragChunks}.embedding <-> ${embeddingJson}::vector) / 2`,
+        })
+            .from(ragChunks)
+            .innerJoin(ragDocuments, eq(ragChunks.documentId, ragDocuments.id))
+            .where(and(...conditions, sql `1 - (${ragChunks}.embedding <-> ${embeddingJson}::vector) / 2 >= ${threshold}`))
+            .orderBy(sql `${ragChunks}.embedding <-> ${embeddingJson}::vector`)
+            .limit(limit);
+        return rows.map((row) => ({
+            chunk: {
+                id: row.chunkId,
+                documentId: row.documentId,
+                workspaceId: row.workspaceId,
+                content: row.content,
+                tokenCount: row.tokenCount,
+                chunkIndex: row.chunkIndex,
+                embedding: row.embedding,
+                embeddingModel: row.embeddingModel,
+                metadata: row.metadata,
+                createdAt: row.chunkCreatedAt,
+            },
+            document: {
+                id: row.documentId,
+                title: row.docTitle,
+                sourceType: row.docSourceType,
+                sourceCollection: row.docSourceCollection,
+                sourceId: row.docSourceId,
+                createdAt: row.docCreatedAt,
+            },
+            similarity: row.similarity ?? 0,
+        }));
+    }
+    /**
+     * Get all chunks for a document ordered by chunk index.
+     */
+    async getChunksByDocument(documentId) {
+        return this.db
+            .select()
+            .from(ragChunks)
+            .where(eq(ragChunks.documentId, documentId))
+            .orderBy(asc(ragChunks.chunkIndex));
+    }
+}

package/dist/ingestion/reranker.d.ts

@@ -0,0 +1,10 @@
+/**
+ * LLM-based Re-ranker
+ *
+ * Scores chunks 0–10 for relevance to the query using the configured LLMClient.
+ * No cross-encoder model required — uses the existing LLM.
+ */
+import type { LLMClient } from '../llm/client.js';
+import type { RagSearchResult } from './rag-vector-service.js';
+export declare function rerankChunks(query: string, chunks: RagSearchResult[], llmClient: LLMClient, topK?: number): Promise<RagSearchResult[]>;
+//# sourceMappingURL=reranker.d.ts.map

package/dist/ingestion/reranker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../../src/ingestion/reranker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AAI9D,wBAAsB,YAAY,CAChC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,eAAe,EAAE,EACzB,SAAS,EAAE,SAAS,EACpB,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC,CAqC5B"}