@juspay/neurolink 9.1.1 → 9.3.0

package/dist/lib/rag/pipeline/contextAssembly.js

@@ -0,0 +1,338 @@
+/**
+ * Context Assembly Utilities
+ *
+ * Provides utilities for assembling, formatting, and optimizing context
+ * from retrieved chunks for LLM consumption.
+ *
+ * Features:
+ * - Context window management (token-aware truncation)
+ * - Citation formatting
+ * - Context deduplication
+ * - Relevance-based ordering
+ * - Context summarization
+ */
+import { logger } from "../../utils/logger.js";
+/**
+ * Assemble context from retrieved results
+ *
+ * Combines multiple chunks into a coherent context string
+ * suitable for LLM consumption.
+ *
+ * @param results - Retrieved chunks or query results
+ * @param options - Assembly options
+ * @returns Assembled context string
+ *
+ * @example
+ * ```typescript
+ * const context = assembleContext(results, {
+ *   maxTokens: 4000,
+ *   citationFormat: 'numbered',
+ *   deduplicate: true
+ * });
+ * ```
+ */
+export function assembleContext(results, options) {
+    const { maxChars, maxTokens = 4000, citationFormat = "none", separator = "\n\n---\n\n", includeMetadata = false, deduplicate = false, dedupeThreshold = 0.8, orderByRelevance = true, includeSectionHeaders = false, headerTemplate = "[{index}] Source: {source}", } = options || {};
+    if (results.length === 0) {
+        return "";
+    }
+    // Convert to unified format
+    let items = results.map((r, index) => ({
+        id: "id" in r ? r.id : `chunk-${index}`,
+        text: "text" in r ? r.text || "" : "",
+        score: "score" in r ? r.score || 0 : 0,
+        metadata: "metadata" in r ? r.metadata : {},
+        index,
+    }));
+    // Get text from metadata if not directly available
+    items = items.map((item) => ({
+        ...item,
+        text: item.text ||
+            item.metadata?.text ||
+            "",
+    }));
+    // Order by relevance if requested
+    if (orderByRelevance) {
+        items.sort((a, b) => b.score - a.score);
+    }
+    // Deduplicate if requested
+    if (deduplicate) {
+        // Ensure metadata is defined for deduplication
+        const itemsWithMetadata = items.map((item) => ({
+            ...item,
+            metadata: item.metadata || {},
+        }));
+        items = deduplicateChunks(itemsWithMetadata, dedupeThreshold);
+    }
+    // Calculate max characters
+    const effectiveMaxChars = maxChars || maxTokens * 4;
+    // Assemble context with token awareness
+    const parts = [];
+    let totalChars = 0;
+    for (const item of items) {
+        const header = includeSectionHeaders
+            ? formatHeader(headerTemplate, {
+                index: parts.length + 1,
+                source: item.metadata?.source || item.id,
+                score: item.score,
+            })
+            : "";
+        const metadata = includeMetadata ? formatMetadata(item.metadata) : "";
+        const citation = formatCitation(citationFormat, parts.length + 1, item.metadata);
+        const chunkText = [
+            header,
+            citation ? `${citation}\n` : "",
+            item.text,
+            metadata,
+        ]
+            .filter(Boolean)
+            .join("\n");
+        // Check if adding this chunk would exceed limit
+        const newTotalChars = totalChars + chunkText.length + separator.length;
+        if (newTotalChars > effectiveMaxChars) {
+            // Try to include partial chunk
+            const remainingChars = effectiveMaxChars - totalChars - separator.length - 50; // Buffer
+            if (remainingChars > 200) {
+                const truncatedText = truncateText(item.text, remainingChars);
+                parts.push([
+                    header,
+                    citation ? `${citation}\n` : "",
+                    truncatedText,
+                    "[truncated]",
+                ]
+                    .filter(Boolean)
+                    .join("\n"));
+            }
+            break;
+        }
+        parts.push(chunkText);
+        totalChars = newTotalChars;
+    }
+    return parts.join(separator);
+}
+/**
+ * Format context with inline citations
+ *
+ * @param results - Retrieved results
+ * @param options - Formatting options
+ * @returns Context with citations and citation list
+ */
+export function formatContextWithCitations(results, options) {
+    const citations = [];
+    const items = results.map((r, index) => {
+        const id = "id" in r ? r.id : `chunk-${index}`;
+        const metadata = "metadata" in r ? r.metadata : {};
+        const source = metadata?.source || id;
+        citations.push(`[${index + 1}] ${source}`);
+        return {
+            ...r,
+            citationMarker: `[${index + 1}]`,
+        };
+    });
+    const context = assembleContext(items, {
+        ...options,
+        citationFormat: "numbered",
+        includeSectionHeaders: true,
+        headerTemplate: "[{index}]",
+    });
+    return { context, citations };
+}
+/**
+ * Create a context window with detailed tracking
+ *
+ * @param results - Retrieved results
+ * @param options - Assembly options
+ * @returns Context window with metadata
+ */
+export function createContextWindow(results, options) {
+    const maxTokens = options?.maxTokens || 4000;
+    const maxChars = options?.maxChars || maxTokens * 4;
+    let text = "";
+    let chunkCount = 0;
+    let truncatedChunks = 0;
+    const citations = new Map();
+    const items = results.map((r, index) => ({
+        id: "id" in r ? r.id : `chunk-${index}`,
+        text: ("text" in r ? r.text : "") ||
+            r.metadata?.text ||
+            "",
+        metadata: "metadata" in r ? r.metadata : {},
+    }));
+    for (const item of items) {
+        const chunkText = item.text;
+        const newLength = text.length + chunkText.length + 10; // Buffer for separators
+        if (newLength > maxChars) {
+            // Try partial inclusion
+            const remaining = maxChars - text.length - 20;
+            if (remaining > 100) {
+                const truncated = truncateText(chunkText, remaining);
+                text += (text ? "\n\n" : "") + truncated + "...";
+                truncatedChunks++;
+                citations.set(item.id, `[${chunkCount + 1}] ${item.metadata?.source || item.id} (truncated)`);
+                chunkCount++;
+            }
+            else {
+                truncatedChunks++;
+            }
+            continue;
+        }
+        text += (text ? "\n\n" : "") + chunkText;
+        citations.set(item.id, `[${chunkCount + 1}] ${item.metadata?.source || item.id}`);
+        chunkCount++;
+    }
+    return {
+        text,
+        chunkCount,
+        charCount: text.length,
+        tokenCount: Math.ceil(text.length / 4),
+        truncatedChunks,
+        citations,
+    };
+}
+/**
+ * Summarize context using LLM
+ *
+ * @param context - Context to summarize
+ * @param maxLength - Maximum summary length
+ * @param provider - LLM provider instance
+ * @returns Summarized context
+ */
+export async function summarizeContext(context, maxLength = 500, provider) {
+    if (!provider) {
+        // Simple truncation fallback
+        return truncateText(context, maxLength * 4);
+    }
+    try {
+        const result = await provider.generate({
+            prompt: `Summarize the following context in no more than ${maxLength} words, preserving the key information:\n\n${context}\n\nSummary:`,
+            maxTokens: Math.ceil(maxLength * 1.5),
+            temperature: 0.3,
+        });
+        return result?.content?.trim() || truncateText(context, maxLength * 4);
+    }
+    catch (error) {
+        logger.warn("[ContextAssembly] Summarization failed, using truncation", {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        return truncateText(context, maxLength * 4);
+    }
+}
+// ============================================================================
+// Helper Functions
+// ============================================================================
+/**
+ * Format section header using template
+ */
+function formatHeader(template, vars) {
+    return template
+        .replace("{index}", String(vars.index))
+        .replace("{source}", vars.source)
+        .replace("{score}", vars.score.toFixed(4));
+}
+/**
+ * Format citation based on style
+ */
+function formatCitation(format, index, metadata) {
+    switch (format) {
+        case "inline":
+            return `(Source: ${metadata?.source || `#${index}`})`;
+        case "footnote":
+            return `[^${index}]`;
+        case "numbered":
+            return `[${index}]`;
+        case "none":
+        default:
+            return "";
+    }
+}
+/**
+ * Format metadata for display
+ */
+function formatMetadata(metadata) {
+    if (!metadata) {
+        return "";
+    }
+    const relevant = ["source", "title", "author", "date", "page"];
+    const parts = [];
+    for (const key of relevant) {
+        if (metadata[key]) {
+            parts.push(`${key}: ${metadata[key]}`);
+        }
+    }
+    return parts.length > 0 ? `\n[${parts.join(" | ")}]` : "";
+}
+/**
+ * Truncate text at word boundary
+ */
+function truncateText(text, maxLength) {
+    if (text.length <= maxLength) {
+        return text;
+    }
+    // Find last space before maxLength
+    let truncateAt = text.lastIndexOf(" ", maxLength);
+    if (truncateAt === -1 || truncateAt < maxLength * 0.7) {
+        truncateAt = maxLength;
+    }
+    return text.slice(0, truncateAt).trim();
+}
+/**
+ * Deduplicate chunks based on text similarity
+ */
+function deduplicateChunks(items, threshold) {
+    const unique = [];
+    for (const item of items) {
+        // Check if this item is too similar to any already included
+        const isDuplicate = unique.some((existing) => textSimilarity(item.text, existing.text) > threshold);
+        if (!isDuplicate) {
+            unique.push(item);
+        }
+    }
+    return unique;
+}
+/**
+ * Simple text similarity using Jaccard index
+ */
+function textSimilarity(a, b) {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/));
+    const wordsB = new Set(b.toLowerCase().split(/\s+/));
+    const intersection = new Set([...wordsA].filter((x) => wordsB.has(x)));
+    const union = new Set([...wordsA, ...wordsB]);
+    return intersection.size / union.size;
+}
+/**
+ * Order chunks by document structure (if available)
+ */
+export function orderByDocumentStructure(chunks) {
+    // Group by document
+    const byDocument = new Map();
+    for (const chunk of chunks) {
+        const docId = chunk.metadata.documentId;
+        if (!byDocument.has(docId)) {
+            byDocument.set(docId, []);
+        }
+        byDocument.get(docId).push(chunk);
+    }
+    // Sort each document's chunks by position
+    for (const docChunks of byDocument.values()) {
+        docChunks.sort((a, b) => (a.metadata.chunkIndex || 0) - (b.metadata.chunkIndex || 0));
+    }
+    // Flatten, keeping documents together
+    return [...byDocument.values()].flat();
+}
+/**
+ * Extract key sentences from chunks for summary
+ */
+export function extractKeySentences(text, count = 3) {
+    const sentences = text
+        .split(/[.!?]+/)
+        .map((s) => s.trim())
+        .filter((s) => s.length > 20);
+    // Simple scoring: longer sentences with more unique words
+    const scored = sentences.map((s) => ({
+        text: s,
+        score: s.length * new Set(s.toLowerCase().split(/\s+/)).size,
+    }));
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, count).map((s) => s.text);
+}
+//# sourceMappingURL=contextAssembly.js.map

package/dist/lib/rag/pipeline/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Pipeline Module Exports
+ */
+export { assembleContext, type CitationFormat, type ContextAssemblyOptions, type ContextWindow, createContextWindow, extractKeySentences, formatContextWithCitations, orderByDocumentStructure, summarizeContext, } from "./contextAssembly.js";
+export { createRAGPipeline, type EmbeddingModelConfig, type GenerationModelConfig, type IngestOptions, type PipelineStats, type QueryOptions, RAGPipeline, type RAGPipelineConfig, type RAGResponse, } from "./RAGPipeline.js";

package/dist/lib/rag/pipeline/index.js

@@ -0,0 +1,6 @@
+/**
+ * Pipeline Module Exports
+ */
+export { assembleContext, createContextWindow, extractKeySentences, formatContextWithCitations, orderByDocumentStructure, summarizeContext, } from "./contextAssembly.js";
+export { createRAGPipeline, RAGPipeline, } from "./RAGPipeline.js";
+//# sourceMappingURL=index.js.map

package/dist/lib/rag/ragIntegration.d.ts

@@ -0,0 +1,38 @@
+/**
+ * RAG Integration for generate() and stream()
+ *
+ * Provides automatic RAG pipeline setup when `rag` config is provided
+ * in GenerateOptions or StreamOptions. Handles file loading, chunking,
+ * embedding generation, vector storage, and tool creation internally
+ * so developers only need to pass `rag: { files: [...] }`.
+ */
+import type { Tool } from "ai";
+import type { RAGConfig } from "./types.js";
+/**
+ * Result of preparing RAG for a generate/stream call
+ */
+export type RAGPreparedTool = {
+    /** The tool to inject into the tools Record */
+    tool: Tool;
+    /** Tool name (key for the tools Record) */
+    toolName: string;
+    /** Number of chunks indexed */
+    chunksIndexed: number;
+    /** Number of files loaded */
+    filesLoaded: number;
+};
+/**
+ * Prepare RAG tools from the provided configuration.
+ *
+ * This function:
+ * 1. Loads and reads all specified files
+ * 2. Chunks them using the configured (or auto-detected) strategy
+ * 3. Generates embeddings for each chunk
+ * 4. Stores them in an in-memory vector store
+ * 5. Creates a tool the AI model can use to search the documents
+ *
+ * @param ragConfig - RAG configuration from generate/stream options
+ * @param fallbackProvider - Provider to use for embeddings if not specified in ragConfig
+ * @returns Prepared RAG tool to inject into the tools record
+ */
+export declare function prepareRAGTool(ragConfig: RAGConfig, fallbackProvider?: string): Promise<RAGPreparedTool>;

package/dist/lib/rag/ragIntegration.js

@@ -0,0 +1,212 @@
+/**
+ * RAG Integration for generate() and stream()
+ *
+ * Provides automatic RAG pipeline setup when `rag` config is provided
+ * in GenerateOptions or StreamOptions. Handles file loading, chunking,
+ * embedding generation, vector storage, and tool creation internally
+ * so developers only need to pass `rag: { files: [...] }`.
+ */
+import { existsSync, readFileSync } from "fs";
+import { extname, resolve } from "path";
+import { z } from "zod";
+import { logger } from "../utils/logger.js";
+import { ChunkerRegistry } from "./chunking/index.js";
+import { createVectorQueryTool, InMemoryVectorStore, } from "./retrieval/vectorQueryTool.js";
+/**
+ * Maps file extensions to recommended chunking strategies
+ */
+const EXTENSION_TO_STRATEGY = {
+    ".md": "markdown",
+    ".mdx": "markdown",
+    ".html": "html",
+    ".htm": "html",
+    ".json": "json",
+    ".tex": "latex",
+    ".latex": "latex",
+    ".txt": "recursive",
+    ".csv": "recursive",
+    ".xml": "recursive",
+    ".yaml": "recursive",
+    ".yml": "recursive",
+    ".ts": "recursive",
+    ".js": "recursive",
+    ".py": "recursive",
+    ".java": "recursive",
+    ".go": "recursive",
+    ".rs": "recursive",
+    ".c": "recursive",
+    ".cpp": "recursive",
+    ".rb": "recursive",
+    ".php": "recursive",
+    ".swift": "recursive",
+    ".kt": "recursive",
+};
+/**
+ * Detect the best chunking strategy from file extension
+ */
+function detectStrategy(filePath) {
+    const ext = extname(filePath).toLowerCase();
+    return EXTENSION_TO_STRATEGY[ext] || "recursive";
+}
+/**
+ * Generate deterministic embeddings for chunks.
+ * Uses a simple hash-based approach for the in-memory vector store.
+ * When a real embedding provider is configured, it will be used instead.
+ */
+function generateSimpleEmbedding(text, dimension) {
+    const embedding = new Array(dimension).fill(0);
+    // Simple character-frequency based embedding
+    for (let i = 0; i < text.length; i++) {
+        const charCode = text.charCodeAt(i);
+        const idx = charCode % dimension;
+        embedding[idx] += 1;
+    }
+    // Normalize to unit vector
+    const magnitude = Math.sqrt(embedding.reduce((sum, v) => sum + v * v, 0));
+    if (magnitude > 0) {
+        for (let i = 0; i < dimension; i++) {
+            embedding[i] /= magnitude;
+        }
+    }
+    return embedding;
+}
+/**
+ * Prepare RAG tools from the provided configuration.
+ *
+ * This function:
+ * 1. Loads and reads all specified files
+ * 2. Chunks them using the configured (or auto-detected) strategy
+ * 3. Generates embeddings for each chunk
+ * 4. Stores them in an in-memory vector store
+ * 5. Creates a tool the AI model can use to search the documents
+ *
+ * @param ragConfig - RAG configuration from generate/stream options
+ * @param fallbackProvider - Provider to use for embeddings if not specified in ragConfig
+ * @returns Prepared RAG tool to inject into the tools record
+ */
+export async function prepareRAGTool(ragConfig, fallbackProvider) {
+    const { files, strategy: userStrategy, chunkSize = 1000, chunkOverlap = 200, topK = 5, toolName = "search_knowledge_base", toolDescription = "REQUIRED: Search through pre-loaded local documents to find relevant information. Use this tool FIRST before any web search or other tools. This searches an indexed knowledge base of documents the user has provided.", embeddingProvider, embeddingModel, } = ragConfig;
+    if (!files || files.length === 0) {
+        throw new Error("RAG config requires at least one file path in 'files'");
+    }
+    // 1. Load files
+    const fileContents = [];
+    for (const filePath of files) {
+        const resolvedPath = resolve(filePath);
+        if (!existsSync(resolvedPath)) {
+            logger.warn(`[RAG] File not found, skipping: ${resolvedPath}`);
+            continue;
+        }
+        try {
+            const content = readFileSync(resolvedPath, "utf-8");
+            const strategy = userStrategy || detectStrategy(resolvedPath);
+            fileContents.push({ path: resolvedPath, content, strategy });
+        }
+        catch (error) {
+            logger.warn(`[RAG] Failed to read file: ${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    if (fileContents.length === 0) {
+        throw new Error("RAG: No files could be loaded. Check that file paths exist and are readable.");
+    }
+    logger.info(`[RAG] Loaded ${fileContents.length} files for indexing`);
+    // 2. Chunk all files
+    const allChunks = [];
+    for (const { path, content, strategy } of fileContents) {
+        try {
+            const chunker = ChunkerRegistry.get(strategy);
+            const chunks = await chunker.chunk(content, {
+                maxSize: chunkSize,
+                overlap: chunkOverlap,
+                metadata: { source: path },
+            });
+            for (const chunk of chunks) {
+                allChunks.push({
+                    text: chunk.text,
+                    metadata: { ...chunk.metadata, source: path },
+                });
+            }
+        }
+        catch (error) {
+            logger.warn(`[RAG] Chunking failed for ${path}, using fallback: ${error instanceof Error ? error.message : String(error)}`);
+            // Fallback: treat entire file as one chunk
+            allChunks.push({
+                text: content.slice(0, chunkSize),
+                metadata: { source: path, fallback: true },
+            });
+        }
+    }
+    logger.info(`[RAG] Created ${allChunks.length} chunks from ${fileContents.length} files`);
+    // 3. Generate embeddings and store in vector store
+    const EMBEDDING_DIMENSION = 128;
+    const vectorStore = new InMemoryVectorStore();
+    const indexName = "rag-index";
+    const items = allChunks.map((chunk, i) => ({
+        id: `rag-chunk-${i}`,
+        vector: generateSimpleEmbedding(chunk.text, EMBEDDING_DIMENSION),
+        metadata: {
+            text: chunk.text,
+            ...chunk.metadata,
+        },
+    }));
+    await vectorStore.upsert(indexName, items);
+    logger.info(`[RAG] Indexed ${items.length} chunks in vector store`);
+    // 4. Create the search tool
+    // Determine embedding provider/model for the query tool
+    const provider = embeddingProvider || fallbackProvider || "vertex";
+    const model = embeddingModel || "gemini-2.5-flash";
+    const queryTool = createVectorQueryTool({
+        id: toolName,
+        description: toolDescription,
+        indexName,
+        embeddingModel: { provider, modelName: model },
+        topK,
+        includeSources: true,
+    }, vectorStore);
+    // Convert to Vercel AI SDK Tool format
+    const aiTool = {
+        description: queryTool.description,
+        parameters: z.object({
+            query: z
+                .string()
+                .describe("The search query to find relevant information"),
+        }),
+        execute: async ({ query }) => {
+            // For the in-memory store with simple embeddings,
+            // generate a query embedding using the same method
+            const queryEmbedding = generateSimpleEmbedding(query, EMBEDDING_DIMENSION);
+            const results = await vectorStore.query({
+                indexName,
+                queryVector: queryEmbedding,
+                topK,
+            });
+            if (results.length === 0) {
+                return {
+                    relevantContext: "No relevant documents found for the query.",
+                    sources: [],
+                    totalResults: 0,
+                };
+            }
+            const relevantContext = results
+                .map((r, i) => `[${i + 1}] ${r.metadata?.text || r.text || ""}`)
+                .join("\n\n");
+            return {
+                relevantContext,
+                sources: results.map((r) => ({
+                    id: r.id,
+                    score: r.score,
+                    source: r.metadata?.source,
+                    text: (r.metadata?.text || r.text || "").slice(0, 200),
+                })),
+                totalResults: results.length,
+            };
+        },
+    };
+    return {
+        tool: aiTool,
+        toolName,
+        chunksIndexed: allChunks.length,
+        filesLoaded: fileContents.length,
+    };
+}
+//# sourceMappingURL=ragIntegration.js.map