@juspay/neurolink 12.52.0 → 12.52.1

package/dist/rag/chunking/latexChunker.js

@@ -5,6 +5,31 @@
  * Best for academic papers, scientific documents, and mathematical content.
  */
 import { randomUUID } from "crypto";
+const BEGIN_DOCUMENT = "\\begin{document}";
+const END_DOCUMENT = "\\end{document}";
+/**
+ * Split a LaTeX source into the text before `\begin{document}` and the body up
+ * to the first `\end{document}` after it, or null when there is no such pair.
+ *
+ * This is what `/^([\s\S]*?)\\begin\{document\}([\s\S]*?)\\end\{document\}/`
+ * returned, without its cost: with no `\end{document}` in the text, that
+ * pattern rescanned to the end of the input from every `\begin{document}`,
+ * so N occurrences took O(N * length). If the first `\begin{document}` has no
+ * closing marker after it, a later one cannot either, so two `indexOf` calls
+ * decide the match and the work is linear.
+ */
+function splitDocumentBody(text) {
+    const begin = text.indexOf(BEGIN_DOCUMENT);
+    if (begin === -1) {
+        return null;
+    }
+    const bodyStart = begin + BEGIN_DOCUMENT.length;
+    const end = text.indexOf(END_DOCUMENT, bodyStart);
+    if (end === -1) {
+        return null;
+    }
+    return { preamble: text.slice(0, begin), body: text.slice(bodyStart, end) };
+}
 /**
  * LaTeX-aware chunker implementation
  * Splits based on LaTeX structure (sections, environments)
@@ -37,12 +62,12 @@ export class LaTeXChunker {
             return chunks;
         }
         // Extract preamble if present
-        const preambleMatch = text.match(/^([\s\S]*?)\\begin\{document\}([\s\S]*?)\\end\{document\}/);
+        const documentParts = splitDocumentBody(text);
         let preamble = "";
         let documentContent = text;
-        if (preambleMatch) {
-            preamble = preambleMatch[1].trim();
-            documentContent = preambleMatch[2];
+        if (documentParts) {
+            preamble = documentParts.preamble.trim();
+            documentContent = documentParts.body;
             // Add preamble as first chunk if requested
             if (includePreamble && preamble.length > 0) {
                 chunks.push({

package/dist/rag/ragIntegration.d.ts

@@ -25,6 +25,10 @@ declare function diversifyResults(results: VectorQueryResult[], topK: number): V
  * @param ragConfig - RAG configuration from generate/stream options
  * @param fallbackProvider - Provider to use for embeddings if not specified in ragConfig
  * @returns Prepared RAG tool to inject into the tools record
+ * @throws When `files` is empty, when no source can be loaded, and when the
+ * built-in hash embedding is given a chunk longer than 1,048,576 characters (it
+ * is the default, and the fallback when a configured embedding provider fails).
+ * The returned tool rejects a query over the same length the same way.
  */
 export declare function prepareRAGTool(ragConfig: RAGConfig, fallbackProvider?: string): Promise<RAGPreparedTool>;
 export {};

package/dist/rag/ragIntegration.js

@@ -4,6 +4,7 @@ import { z } from "zod";
 import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../observability/index.js";
 import { logger } from "../utils/logger.js";
 import { redactUrlForError } from "../utils/logSanitize.js";
+import { ErrorFactory } from "../utils/errorHandling.js";
 import { ImageProcessor } from "../utils/imageProcessor.js";
 import { createChunker } from "./ChunkerFactory.js";
 import { createVectorQueryTool, InMemoryVectorStore, } from "./retrieval/vectorQueryTool.js";
@@ -49,6 +50,22 @@ function detectStrategy(filePath) {
 }
 /** Embedding dimension used by {@link generateSimpleEmbedding}. */
 const EMBEDDING_DIMENSION = 128;
+/**
+ * Longest text, in UTF-16 code units, that {@link generateSimpleEmbedding}
+ * will hash: 1,048,576.
+ *
+ * The 128-dimension output does not bound the work. The function walks every
+ * character and every word of its input, and nothing upstream limits that
+ * input: a chunk is as large as the caller's `chunkSize` allows, and a search
+ * query is whatever the model sends. A default chunk is 1,000 characters, so
+ * this is about a thousand times that and no ordinary input reaches it. Longer
+ * text is refused with an error rather than truncated, because a clipped text
+ * would be embedded as a different document and rank as one.
+ *
+ * It applies only to the hash embedding. A configured embedding provider
+ * receives the text as before and applies its own limits.
+ */
+const MAX_HASH_EMBEDDING_TEXT_LENGTH = 1_048_576;
 /**
  * Simple hash function for strings (FNV-1a variant).
  * Maps a word to a bucket index deterministically.
@@ -68,6 +85,9 @@ function hashWord(word, buckets) {
  * When a real embedding provider is configured, it will be used instead.
  */
 function generateSimpleEmbedding(text, dimension) {
+    if (text.length > MAX_HASH_EMBEDDING_TEXT_LENGTH) {
+        throw ErrorFactory.invalidParameters("rag.hashEmbedding", new Error(`text is ${text.length} characters, over the ${MAX_HASH_EMBEDDING_TEXT_LENGTH} character limit of the built-in hash embedding. Lower chunkSize, shorten the query, or configure an embeddingProvider.`), { length: text.length, maxLength: MAX_HASH_EMBEDDING_TEXT_LENGTH });
+    }
     const charEmbedding = new Array(dimension).fill(0);
     const wordEmbedding = new Array(dimension).fill(0);
     // Character-frequency features
@@ -148,6 +168,10 @@ function diversifyResults(results, topK) {
  * @param ragConfig - RAG configuration from generate/stream options
  * @param fallbackProvider - Provider to use for embeddings if not specified in ragConfig
  * @returns Prepared RAG tool to inject into the tools record
+ * @throws When `files` is empty, when no source can be loaded, and when the
+ * built-in hash embedding is given a chunk longer than 1,048,576 characters (it
+ * is the default, and the fallback when a configured embedding provider fails).
+ * The returned tool rejects a query over the same length the same way.
  */
 export async function prepareRAGTool(ragConfig, fallbackProvider) {
     const span = SpanSerializer.createSpan(SpanType.RAG, "rag.prepare", {
@@ -356,6 +380,7 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
     const embedProviderName = embeddingProvider || fallbackProvider || "vertex";
     const embedModelName = embeddingModel || "gemini-2.5-flash";
     let embedFn = (text) => Promise.resolve(generateSimpleEmbedding(text, EMBEDDING_DIMENSION));
+    let embeddingFromProvider = false;
     if (wantProviderEmbeddings) {
         try {
             const { AIProviderFactory } = await import("../core/factory.js");
@@ -363,6 +388,7 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
             if (typeof embedderProvider.embed === "function") {
                 const providerEmbed = embedderProvider.embed.bind(embedderProvider);
                 embedFn = (text) => providerEmbed(text, embedModelName);
+                embeddingFromProvider = true;
             }
             else {
                 logger.warn(`[RAG] Embedding provider '${embedProviderName}' has no embed(); falling back to hash embeddings`);
@@ -387,6 +413,12 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
         imageVectors = allVectors.slice(allChunks.length);
     }
     catch (error) {
+        // The hash embedding refuses text over its length limit. With no provider
+        // in use there is nothing to fall back from, and retrying would only repeat
+        // that refusal after a log line claiming a provider failed.
+        if (!embeddingFromProvider) {
+            throw error;
+        }
         // One failed chunk must not leave a mixed-space index — flip the whole
         // index AND all queries back to the hash space together.
         logger.warn("[RAG] Provider embedding failed mid-index; falling back to hash embeddings for index and queries", { error: error instanceof Error ? error.message : String(error) });
@@ -456,7 +488,7 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
                 // would be scored against provider-space vectors of another
                 // dimension, which cosine reads as 0 for every chunk, so the tool
                 // would return arbitrary chunks as if they were relevant. A hash-built
-                // index never reaches this — its embedFn cannot reject.
+                // index only rejects here for a query over the hash length limit.
                 const queryEmbedding = await embedFn(query);
                 // Fetch more candidates than needed so diversity can select across files and images
                 const loadedSources = fileContents.length + imageChunks.length;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.52.0",
+  "version": "12.52.1",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -166,6 +166,7 @@
     "test:pdf-image-streaming": "pnpm exec tsx test/continuous-test-suite-pdf-image-streaming.ts",
     "test:rag": "pnpm exec tsx test/continuous-test-suite-rag.ts",
     "test:rag-html-markup": "pnpm exec tsx test/continuous-test-suite-rag-html-markup.ts",
+    "test:rag-bounds": "pnpm exec tsx test/continuous-test-suite-rag-bounds.ts",
     "test:skills": "pnpm exec tsx test/continuous-test-suite-skills.ts",
     "test:decide": "pnpm exec tsx test/continuous-test-suite-decide.ts",
     "test:decide-catalog": "pnpm exec tsx test/continuous-test-suite-decide-catalog.ts",
@@ -220,7 +221,7 @@
     "// CI tier — fast, no live AI calls, safe for every commit (test:unit; also see the separate provider-safety-net CI job, which runs build + test:providers-mocked + test:provider-structure + test:error-classifier-contract on every PR)": "",
     "test:tool-routing": "pnpm exec tsx test/continuous-test-suite-tool-routing.ts",
     "test:tool-routing-semantic": "pnpm exec tsx test/continuous-test-suite-tool-routing-semantic.ts",
-    "test:unit": "pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:spans && pnpm run test:tool-routing && pnpm run test:tool-routing-cli && pnpm run test:tool-dedup && pnpm run test:model-pool && pnpm run test:classifier-router && pnpm run test:tool-routing-semantic && pnpm run test:mcp-result-cache && pnpm run test:mcp-direct-name-repair && pnpm run test:mcp-breaker-resolved-errors && pnpm run test:archive:security && pnpm run test:office:security && pnpm run test:image-exif && pnpm run test:vector-chroma && pnpm run test:vector-pgvector && pnpm run test:vector-pinecone && pnpm run test:provider-wiring && pnpm run test:docs-mcp && pnpm run test:retired-model-defaults && pnpm run test:cli-fallback && pnpm run test:embedding-response-validation && pnpm run test:xiaomi-mimo-pilot",
+    "test:unit": "pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:spans && pnpm run test:tool-routing && pnpm run test:tool-routing-cli && pnpm run test:tool-dedup && pnpm run test:model-pool && pnpm run test:classifier-router && pnpm run test:tool-routing-semantic && pnpm run test:mcp-result-cache && pnpm run test:mcp-direct-name-repair && pnpm run test:mcp-breaker-resolved-errors && pnpm run test:archive:security && pnpm run test:office:security && pnpm run test:image-exif && pnpm run test:vector-chroma && pnpm run test:vector-pgvector && pnpm run test:vector-pinecone && pnpm run test:provider-wiring && pnpm run test:docs-mcp && pnpm run test:retired-model-defaults && pnpm run test:cli-fallback && pnpm run test:embedding-response-validation && pnpm run test:xiaomi-mimo-pilot && pnpm run test:rag-bounds",
     "// CI tier — live providers, runs only when API keys are present (test:credentials and test:dynamic make real provider calls when keys are set, so they live here, not in test:unit; test:matrix, a different suite covering the full provider capability matrix, runs nightly via .github/workflows/live-matrix.yml — test:providers itself is still only wired into test:live, not any GitHub Actions workflow)": "",
     "test:live": "pnpm run test:providers && pnpm run test:mcp:http && pnpm run test:mcp:sdk && pnpm run test:mcp:cli && pnpm run test:observability && pnpm run test:context && pnpm run test:memory && pnpm run test:tool-reliability && pnpm run test:evaluation && pnpm run test:autoresearch && pnpm run test:credentials && pnpm run test:dynamic && pnpm run test:model-not-found-retryable",
     "// CI tier — product output (image/video/TTS/PPT) — costs $$ per run (not wired into any GitHub Actions workflow as of this comment; run manually or add to live-matrix.yml if nightly coverage is needed)": "",