@juspay/neurolink 9.30.0 → 9.31.0

package/dist/client/rag/pipeline/contextAssembly.js

@@ -0,0 +1,341 @@
+/**
+ * Context Assembly Utilities
+ *
+ * Provides utilities for assembling, formatting, and optimizing context
+ * from retrieved chunks for LLM consumption.
+ *
+ * Features:
+ * - Context window management (token-aware truncation)
+ * - Citation formatting
+ * - Context deduplication
+ * - Relevance-based ordering
+ * - Context summarization
+ */
+import { logger } from "../../utils/logger.js";
+import { estimateTokens } from "../../utils/tokenEstimation.js";
+/**
+ * Assemble context from retrieved results
+ *
+ * Combines multiple chunks into a coherent context string
+ * suitable for LLM consumption.
+ *
+ * @param results - Retrieved chunks or query results
+ * @param options - Assembly options
+ * @returns Assembled context string
+ *
+ * @example
+ * ```typescript
+ * const context = assembleContext(results, {
+ *   maxTokens: 4000,
+ *   citationFormat: 'numbered',
+ *   deduplicate: true
+ * });
+ * ```
+ */
+export function assembleContext(results, options) {
+    const { maxChars, maxTokens = 4000, citationFormat = "none", separator = "\n\n---\n\n", includeMetadata = false, deduplicate = false, dedupeThreshold = 0.8, orderByRelevance = true, includeSectionHeaders = false, headerTemplate = "[{index}] Source: {source}", } = options || {};
+    if (results.length === 0) {
+        return "";
+    }
+    // Convert to unified format
+    let items = results.map((r, index) => ({
+        id: "id" in r ? r.id : `chunk-${index}`,
+        text: "text" in r ? r.text || "" : "",
+        score: "score" in r ? r.score || 0 : 0,
+        metadata: "metadata" in r ? r.metadata : {},
+        index,
+    }));
+    // Get text from metadata if not directly available
+    items = items.map((item) => ({
+        ...item,
+        text: item.text ||
+            item.metadata?.text ||
+            "",
+    }));
+    // Order by relevance if requested
+    if (orderByRelevance) {
+        items.sort((a, b) => b.score - a.score);
+    }
+    // Deduplicate if requested
+    if (deduplicate) {
+        // Ensure metadata is defined for deduplication
+        const itemsWithMetadata = items.map((item) => ({
+            ...item,
+            metadata: item.metadata || {},
+        }));
+        items = deduplicateChunks(itemsWithMetadata, dedupeThreshold);
+    }
+    // Calculate max characters
+    const effectiveMaxChars = maxChars || maxTokens * 4;
+    // Assemble context with token awareness
+    const parts = [];
+    let totalChars = 0;
+    for (const item of items) {
+        const header = includeSectionHeaders
+            ? formatHeader(headerTemplate, {
+                index: parts.length + 1,
+                source: item.metadata?.source || item.id,
+                score: item.score,
+            })
+            : "";
+        const metadata = includeMetadata ? formatMetadata(item.metadata) : "";
+        const citation = formatCitation(citationFormat, parts.length + 1, item.metadata);
+        const chunkText = [
+            header,
+            citation ? `${citation}\n` : "",
+            item.text,
+            metadata,
+        ]
+            .filter(Boolean)
+            .join("\n");
+        // Check if adding this chunk would exceed limit
+        const newTotalChars = totalChars + chunkText.length + separator.length;
+        if (newTotalChars > effectiveMaxChars) {
+            // Try to include partial chunk
+            const remainingChars = effectiveMaxChars - totalChars - separator.length - 50; // Buffer
+            if (remainingChars > 200) {
+                const truncatedText = truncateText(item.text, remainingChars);
+                parts.push([
+                    header,
+                    citation ? `${citation}\n` : "",
+                    truncatedText,
+                    "[truncated]",
+                ]
+                    .filter(Boolean)
+                    .join("\n"));
+            }
+            break;
+        }
+        parts.push(chunkText);
+        totalChars = newTotalChars;
+    }
+    return parts.join(separator);
+}
+/**
+ * Format context with inline citations
+ *
+ * @param results - Retrieved results
+ * @param options - Formatting options
+ * @returns Context with citations and citation list
+ */
+export function formatContextWithCitations(results, options) {
+    const citations = [];
+    const items = results.map((r, index) => {
+        const id = "id" in r ? r.id : `chunk-${index}`;
+        const metadata = "metadata" in r ? r.metadata : {};
+        const source = metadata?.source || id;
+        citations.push(`[${index + 1}] ${source}`);
+        return {
+            ...r,
+            citationMarker: `[${index + 1}]`,
+        };
+    });
+    const context = assembleContext(items, {
+        ...options,
+        citationFormat: "numbered",
+        includeSectionHeaders: true,
+        headerTemplate: "[{index}]",
+    });
+    return { context, citations };
+}
+/**
+ * Create a context window with detailed tracking
+ *
+ * @param results - Retrieved results
+ * @param options - Assembly options
+ * @returns Context window with metadata
+ */
+export function createContextWindow(results, options) {
+    const maxTokens = options?.maxTokens || 4000;
+    const maxChars = options?.maxChars || maxTokens * 4;
+    let text = "";
+    let chunkCount = 0;
+    let truncatedChunks = 0;
+    const citations = new Map();
+    const items = results.map((r, index) => ({
+        id: "id" in r ? r.id : `chunk-${index}`,
+        text: ("text" in r ? r.text : "") ||
+            r.metadata?.text ||
+            "",
+        metadata: "metadata" in r ? r.metadata : {},
+    }));
+    for (const item of items) {
+        const chunkText = item.text;
+        const newLength = text.length + chunkText.length + 10; // Buffer for separators
+        if (newLength > maxChars) {
+            // Try partial inclusion
+            const remaining = maxChars - text.length - 20;
+            if (remaining > 100) {
+                const truncated = truncateText(chunkText, remaining);
+                text += (text ? "\n\n" : "") + truncated + "...";
+                truncatedChunks++;
+                citations.set(item.id, `[${chunkCount + 1}] ${item.metadata?.source || item.id} (truncated)`);
+                chunkCount++;
+            }
+            else {
+                truncatedChunks++;
+            }
+            continue;
+        }
+        text += (text ? "\n\n" : "") + chunkText;
+        citations.set(item.id, `[${chunkCount + 1}] ${item.metadata?.source || item.id}`);
+        chunkCount++;
+    }
+    return {
+        text,
+        chunkCount,
+        charCount: text.length,
+        tokenCount: estimateTokens(text),
+        truncatedChunks,
+        citations,
+    };
+}
+/**
+ * Summarize context using LLM
+ *
+ * @param context - Context to summarize
+ * @param maxLength - Maximum summary length
+ * @param provider - LLM provider instance
+ * @returns Summarized context
+ */
+export async function summarizeContext(context, maxLength = 500, provider) {
+    if (!provider) {
+        // Simple truncation fallback
+        return truncateText(context, maxLength * 4);
+    }
+    try {
+        const result = await provider.generate({
+            prompt: `Summarize the following context in no more than ${maxLength} words, preserving the key information:\n\n${context}\n\nSummary:`,
+            maxTokens: Math.ceil(maxLength * 1.5),
+            temperature: 0.3,
+        });
+        return result?.content?.trim() || truncateText(context, maxLength * 4);
+    }
+    catch (error) {
+        logger.warn("[ContextAssembly] Summarization failed, using truncation", {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        return truncateText(context, maxLength * 4);
+    }
+}
+// ============================================================================
+// Helper Functions
+// ============================================================================
+/**
+ * Format section header using template
+ */
+function formatHeader(template, vars) {
+    return template
+        .replace("{index}", String(vars.index))
+        .replace("{source}", vars.source)
+        .replace("{score}", vars.score.toFixed(4));
+}
+/**
+ * Format citation based on style
+ */
+function formatCitation(format, index, metadata) {
+    switch (format) {
+        case "inline":
+            return `(Source: ${metadata?.source || `#${index}`})`;
+        case "footnote":
+            return `[^${index}]`;
+        case "numbered":
+            return `[${index}]`;
+        case "none":
+        default:
+            return "";
+    }
+}
+/**
+ * Format metadata for display
+ */
+function formatMetadata(metadata) {
+    if (!metadata) {
+        return "";
+    }
+    const relevant = ["source", "title", "author", "date", "page"];
+    const parts = [];
+    for (const key of relevant) {
+        if (metadata[key]) {
+            parts.push(`${key}: ${metadata[key]}`);
+        }
+    }
+    return parts.length > 0 ? `\n[${parts.join(" | ")}]` : "";
+}
+/**
+ * Truncate text at word boundary
+ */
+function truncateText(text, maxLength) {
+    if (text.length <= maxLength) {
+        return text;
+    }
+    // Find last space before maxLength
+    let truncateAt = text.lastIndexOf(" ", maxLength);
+    if (truncateAt === -1 || truncateAt < maxLength * 0.7) {
+        truncateAt = maxLength;
+    }
+    return text.slice(0, truncateAt).trim();
+}
+/**
+ * Deduplicate chunks based on text similarity
+ */
+function deduplicateChunks(items, threshold) {
+    const unique = [];
+    for (const item of items) {
+        // Check if this item is too similar to any already included
+        const isDuplicate = unique.some((existing) => textSimilarity(item.text, existing.text) > threshold);
+        if (!isDuplicate) {
+            unique.push(item);
+        }
+    }
+    return unique;
+}
+/**
+ * Simple text similarity using Jaccard index
+ */
+function textSimilarity(a, b) {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/));
+    const wordsB = new Set(b.toLowerCase().split(/\s+/));
+    const intersection = new Set([...wordsA].filter((x) => wordsB.has(x)));
+    const union = new Set([...wordsA, ...wordsB]);
+    return intersection.size / union.size;
+}
+/**
+ * Order chunks by document structure (if available)
+ */
+export function orderByDocumentStructure(chunks) {
+    // Group by document
+    const byDocument = new Map();
+    for (const chunk of chunks) {
+        const docId = chunk.metadata.documentId;
+        const group = byDocument.get(docId);
+        if (group) {
+            group.push(chunk);
+        }
+        else {
+            byDocument.set(docId, [chunk]);
+        }
+    }
+    // Sort each document's chunks by position
+    for (const docChunks of byDocument.values()) {
+        docChunks.sort((a, b) => (a.metadata.chunkIndex || 0) - (b.metadata.chunkIndex || 0));
+    }
+    // Flatten, keeping documents together
+    return [...byDocument.values()].flat();
+}
+/**
+ * Extract key sentences from chunks for summary
+ */
+export function extractKeySentences(text, count = 3) {
+    const sentences = text
+        .split(/[.!?]+/)
+        .map((s) => s.trim())
+        .filter((s) => s.length > 20);
+    // Simple scoring: longer sentences with more unique words
+    const scored = sentences.map((s) => ({
+        text: s,
+        score: s.length * new Set(s.toLowerCase().split(/\s+/)).size,
+    }));
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, count).map((s) => s.text);
+}

package/dist/client/rag/pipeline/index.js

@@ -0,0 +1,5 @@
+/**
+ * Pipeline Module Exports
+ */
+export { assembleContext, createContextWindow, extractKeySentences, formatContextWithCitations, orderByDocumentStructure, summarizeContext, } from "./contextAssembly.js";
+export { createRAGPipeline, RAGPipeline, } from "./RAGPipeline.js";

package/dist/client/rag/ragIntegration.js

@@ -0,0 +1,321 @@
+/**
+ * RAG Integration for generate() and stream()
+ *
+ * Provides automatic RAG pipeline setup when `rag` config is provided
+ * in GenerateOptions or StreamOptions. Handles file loading, chunking,
+ * embedding generation, vector storage, and tool creation internally
+ * so developers only need to pass `rag: { files: [...] }`.
+ */
+import { existsSync, readFileSync } from "fs";
+import { extname, resolve } from "path";
+import { z } from "zod";
+import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../observability/index.js";
+import { logger } from "../utils/logger.js";
+import { createChunker } from "./ChunkerFactory.js";
+import { createVectorQueryTool, InMemoryVectorStore, } from "./retrieval/vectorQueryTool.js";
+/**
+ * Maps file extensions to recommended chunking strategies
+ */
+const EXTENSION_TO_STRATEGY = {
+    ".md": "markdown",
+    ".mdx": "markdown",
+    ".html": "html",
+    ".htm": "html",
+    ".json": "json",
+    ".tex": "latex",
+    ".latex": "latex",
+    ".txt": "recursive",
+    ".csv": "recursive",
+    ".xml": "recursive",
+    ".yaml": "recursive",
+    ".yml": "recursive",
+    ".ts": "recursive",
+    ".js": "recursive",
+    ".py": "recursive",
+    ".java": "recursive",
+    ".go": "recursive",
+    ".rs": "recursive",
+    ".c": "recursive",
+    ".cpp": "recursive",
+    ".rb": "recursive",
+    ".php": "recursive",
+    ".swift": "recursive",
+    ".kt": "recursive",
+};
+/**
+ * Detect the best chunking strategy from file extension
+ */
+function detectStrategy(filePath) {
+    const ext = extname(filePath).toLowerCase();
+    return EXTENSION_TO_STRATEGY[ext] || "recursive";
+}
+/**
+ * Simple hash function for strings (FNV-1a variant).
+ * Maps a word to a bucket index deterministically.
+ */
+function hashWord(word, buckets) {
+    let hash = 2166136261;
+    for (let i = 0; i < word.length; i++) {
+        hash ^= word.charCodeAt(i);
+        hash = (hash * 16777619) >>> 0;
+    }
+    return hash % buckets;
+}
+/**
+ * Generate deterministic embeddings for chunks.
+ * Combines character-frequency (40%) with word-level hash features (60%)
+ * for better semantic discrimination than pure character frequency.
+ * When a real embedding provider is configured, it will be used instead.
+ */
+function generateSimpleEmbedding(text, dimension) {
+    const charEmbedding = new Array(dimension).fill(0);
+    const wordEmbedding = new Array(dimension).fill(0);
+    // Character-frequency features
+    for (let i = 0; i < text.length; i++) {
+        const charCode = text.charCodeAt(i);
+        const idx = charCode % dimension;
+        charEmbedding[idx] += 1;
+    }
+    // Word-level hash features (TF-IDF-like)
+    const words = text
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, "")
+        .split(/\s+/)
+        .filter((w) => w.length > 1);
+    for (const word of words) {
+        const idx = hashWord(word, dimension);
+        wordEmbedding[idx] += 1;
+    }
+    // Combine: 40% character, 60% word
+    const combined = new Array(dimension);
+    for (let i = 0; i < dimension; i++) {
+        combined[i] = 0.4 * charEmbedding[i] + 0.6 * wordEmbedding[i];
+    }
+    // Normalize to unit vector
+    const magnitude = Math.sqrt(combined.reduce((sum, v) => sum + v * v, 0));
+    if (magnitude > 0) {
+        for (let i = 0; i < dimension; i++) {
+            combined[i] /= magnitude;
+        }
+    }
+    return combined;
+}
+/**
+ * Diversify retrieval results via round-robin across source files.
+ * Ensures at least one chunk per source file appears in the top-K results,
+ * preventing any single file from dominating retrieval.
+ */
+function diversifyResults(results, topK) {
+    // Group by source file
+    const byFile = new Map();
+    for (const r of results) {
+        const source = r.metadata?.source || "unknown";
+        if (!byFile.has(source)) {
+            byFile.set(source, []);
+        }
+        const sourceGroup = byFile.get(source);
+        if (sourceGroup) {
+            sourceGroup.push(r);
+        }
+    }
+    // If only one source file, no diversification needed
+    if (byFile.size <= 1) {
+        return results.slice(0, topK);
+    }
+    // Round-robin selection from each source file group
+    const diversified = [];
+    const iterators = [...byFile.values()].map((arr) => ({ arr, idx: 0 }));
+    while (diversified.length < topK &&
+        iterators.some((it) => it.idx < it.arr.length)) {
+        for (const it of iterators) {
+            if (it.idx < it.arr.length && diversified.length < topK) {
+                diversified.push(it.arr[it.idx++]);
+            }
+        }
+    }
+    return diversified;
+}
+/**
+ * Prepare RAG tools from the provided configuration.
+ *
+ * This function:
+ * 1. Loads and reads all specified files
+ * 2. Chunks them using the configured (or auto-detected) strategy
+ * 3. Generates embeddings for each chunk
+ * 4. Stores them in an in-memory vector store
+ * 5. Creates a tool the AI model can use to search the documents
+ *
+ * @param ragConfig - RAG configuration from generate/stream options
+ * @param fallbackProvider - Provider to use for embeddings if not specified in ragConfig
+ * @returns Prepared RAG tool to inject into the tools record
+ */
+export async function prepareRAGTool(ragConfig, fallbackProvider) {
+    const span = SpanSerializer.createSpan(SpanType.RAG, "rag.prepare", {
+        "rag.operation": "prepare",
+        "rag.files_count": ragConfig.files?.length ?? 0,
+        "rag.strategy": ragConfig.strategy ?? "auto",
+        "rag.chunk_size": ragConfig.chunkSize ?? 1000,
+    });
+    const startTime = Date.now();
+    try {
+        const result = await _prepareRAGToolInner(ragConfig, fallbackProvider);
+        span.durationMs = Date.now() - startTime;
+        const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
+        endedSpan.attributes = {
+            ...endedSpan.attributes,
+            "rag.chunks_indexed": result.chunksIndexed,
+            "rag.files_loaded": result.filesLoaded,
+        };
+        getMetricsAggregator().recordSpan(endedSpan);
+        return result;
+    }
+    catch (error) {
+        span.durationMs = Date.now() - startTime;
+        const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR);
+        endedSpan.statusMessage =
+            error instanceof Error ? error.message : String(error);
+        getMetricsAggregator().recordSpan(endedSpan);
+        throw error;
+    }
+}
+async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
+    const { files, strategy: userStrategy, chunkSize = 1000, chunkOverlap = 200, topK: userTopK = 5, toolName = "search_knowledge_base", toolDescription = "REQUIRED: Search through pre-loaded local documents to find relevant information. Use this tool FIRST before any web search or other tools. This searches an indexed knowledge base of documents the user has provided.", embeddingProvider, embeddingModel, } = ragConfig;
+    if (!files || files.length === 0) {
+        throw new Error("RAG config requires at least one file path in 'files'");
+    }
+    // 1. Load files
+    const fileContents = [];
+    for (const filePath of files) {
+        const resolvedPath = resolve(filePath);
+        if (!existsSync(resolvedPath)) {
+            logger.warn(`[RAG] File not found, skipping: ${resolvedPath}`);
+            continue;
+        }
+        try {
+            const content = readFileSync(resolvedPath, "utf-8");
+            const strategy = userStrategy || detectStrategy(resolvedPath);
+            fileContents.push({ path: resolvedPath, content, strategy });
+        }
+        catch (error) {
+            logger.warn(`[RAG] Failed to read file: ${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    // Auto-increase topK for multi-file scenarios to ensure coverage
+    // (computed after loading so it reflects only files that actually exist)
+    const topK = fileContents.length > 1
+        ? Math.max(userTopK, fileContents.length * 3)
+        : userTopK;
+    if (fileContents.length === 0) {
+        throw new Error("RAG: No files could be loaded. Check that file paths exist and are readable.");
+    }
+    logger.info(`[RAG] Loaded ${fileContents.length} files for indexing`);
+    // 2. Chunk all files
+    const allChunks = [];
+    for (const { path, content, strategy } of fileContents) {
+        try {
+            const chunker = await createChunker(strategy, {
+                maxSize: chunkSize,
+                overlap: Math.min(chunkOverlap, Math.floor(chunkSize * 0.5)),
+            });
+            const chunks = await chunker.chunk(content, {
+                metadata: { source: path },
+            });
+            for (const chunk of chunks) {
+                allChunks.push({
+                    text: chunk.text,
+                    metadata: { ...chunk.metadata, source: path },
+                });
+            }
+        }
+        catch (error) {
+            logger.warn(`[RAG] Chunking failed for ${path}, using fallback: ${error instanceof Error ? error.message : String(error)}`);
+            // Fallback: treat entire file as one chunk
+            allChunks.push({
+                text: content.slice(0, chunkSize),
+                metadata: { source: path, fallback: true },
+            });
+        }
+    }
+    logger.info(`[RAG] Created ${allChunks.length} chunks from ${fileContents.length} files`);
+    // 3. Generate embeddings and store in vector store
+    const EMBEDDING_DIMENSION = 128;
+    const vectorStore = new InMemoryVectorStore();
+    const indexName = "rag-index";
+    const items = allChunks.map((chunk, i) => ({
+        id: `rag-chunk-${i}`,
+        vector: generateSimpleEmbedding(chunk.text, EMBEDDING_DIMENSION),
+        metadata: {
+            text: chunk.text,
+            ...chunk.metadata,
+        },
+    }));
+    await vectorStore.upsert(indexName, items);
+    logger.info(`[RAG] Indexed ${items.length} chunks in vector store`);
+    // 4. Create the search tool
+    // Determine embedding provider/model for the query tool
+    const provider = embeddingProvider || fallbackProvider || "vertex";
+    const model = embeddingModel || "gemini-2.5-flash";
+    const queryTool = createVectorQueryTool({
+        id: toolName,
+        description: toolDescription,
+        indexName,
+        embeddingModel: { provider, modelName: model },
+        topK,
+        includeSources: true,
+    }, vectorStore);
+    // Convert to Vercel AI SDK Tool format
+    const aiTool = {
+        description: queryTool.description,
+        inputSchema: z.object({
+            query: z
+                .string()
+                .describe("The search query to find relevant information"),
+        }),
+        execute: async ({ query }) => {
+            // For the in-memory store with simple embeddings,
+            // generate a query embedding using the same method
+            const queryEmbedding = generateSimpleEmbedding(query, EMBEDDING_DIMENSION);
+            // Fetch more candidates than needed so diversity can select across files
+            const fetchK = fileContents.length > 1 ? topK * 3 : topK;
+            const rawResults = await vectorStore.query({
+                indexName,
+                queryVector: queryEmbedding,
+                topK: fetchK,
+            });
+            // Apply source-file diversity for multi-file RAG
+            const results = fileContents.length > 1
+                ? diversifyResults(rawResults, topK)
+                : rawResults.slice(0, topK);
+            if (results.length === 0) {
+                return {
+                    relevantContext: "No relevant documents found for the query.",
+                    sources: [],
+                    totalResults: 0,
+                };
+            }
+            const relevantContext = results
+                .map((r, i) => `[${i + 1}] ${r.metadata?.text || r.text || ""}`)
+                .join("\n\n");
+            return {
+                relevantContext,
+                sources: results.map((r) => ({
+                    id: r.id,
+                    score: r.score,
+                    source: r.metadata?.source,
+                    text: (r.metadata?.text || r.text || "").slice(0, 200),
+                })),
+                totalResults: results.length,
+            };
+        },
+    };
+    return {
+        tool: aiTool,
+        toolName,
+        chunksIndexed: allChunks.length,
+        filesLoaded: fileContents.length,
+    };
+}
+/** @internal Exported for testing only */
+export { generateSimpleEmbedding as _generateSimpleEmbedding };
+/** @internal Exported for testing only */
+export { diversifyResults as _diversifyResults };