@juspay/neurolink 12.51.2 → 12.51.4

package/dist/rag/chunkers/HTMLChunker.js

@@ -3,6 +3,7 @@
  *
  * Splits HTML content by semantic tags.
  */
+import { removeHtmlMarkup } from "../../utils/htmlText.js";
 import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
 /**
  * HTML Chunker
@@ -28,10 +29,12 @@ export class HTMLChunker extends BaseChunker {
      * Strip HTML tags from content
      */
     stripHtml(html) {
-        return html
-            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
-            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
-            .replace(/<[^>]+>/g, " ")
+        return removeHtmlMarkup(html, {
+            tagReplacement: " ",
+            commentReplacement: " ",
+            lineBreakEndTags: [],
+            lineBreakVoidTags: [],
+        })
             .replace(/\s+/g, " ")
             .trim();
     }

package/dist/rag/chunking/htmlChunker.js

@@ -6,6 +6,7 @@
  */
 import { randomUUID } from "crypto";
 import { decodeHtmlTextEntitiesOnce } from "../../utils/htmlEntities.js";
+import { removeHtmlMarkup } from "../../utils/htmlText.js";
 /**
  * HTML-aware chunker implementation
  * Splits based on HTML structure (tags, elements)
@@ -171,16 +172,24 @@ export class HTMLChunker {
      * Extract plain text from HTML
      */
     extractText(html) {
-        return decodeHtmlTextEntitiesOnce(html
-            // Remove script and style elements
-            .replace(/<script[\s\S]*?<\/script>/gi, "")
-            .replace(/<style[\s\S]*?<\/style>/gi, "")
-            // Remove HTML comments
-            .replace(/<!--[\s\S]*?-->/g, "")
-            // Replace block elements with newlines
-            .replace(/<\/(p|div|br|h[1-6]|li|tr)>/gi, "\n")
-            // Remove remaining tags
-            .replace(/<[^>]+>/g, ""), { nbsp: " ", amp: "&", lt: "<", gt: ">", quot: '"', "#039": "'" })
+        return decodeHtmlTextEntitiesOnce(removeHtmlMarkup(html, {
+            tagReplacement: "",
+            commentReplacement: "",
+            lineBreakEndTags: [
+                "p",
+                "div",
+                "br",
+                "h1",
+                "h2",
+                "h3",
+                "h4",
+                "h5",
+                "h6",
+                "li",
+                "tr",
+            ],
+            lineBreakVoidTags: [],
+        }), { nbsp: " ", amp: "&", lt: "<", gt: ">", quot: '"', "#039": "'" })
             .replace(/\s+/g, " ")
             .trim();
     }

package/dist/rag/document/loaders.js

@@ -28,6 +28,7 @@ import { readFile } from "fs/promises";
 import { basename, extname } from "path";
 import { logger } from "../../utils/logger.js";
 import { decodeHtmlTextEntitiesOnce } from "../../utils/htmlEntities.js";
+import { removeHtmlMarkup } from "../../utils/htmlText.js";
 import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, dedupeColumnNames, sanitizeColumnName, } from "../../utils/csvProcessor.js";
 import { splitCSVFields } from "../../utils/csvUtils.js";
 import { MDocument } from "./MDocument.js";
@@ -429,17 +430,25 @@ export class WebLoader {
      * Convert HTML to plain text
      */
     htmlToText(html) {
-        return decodeHtmlTextEntitiesOnce(html
-            // Remove script and style elements
-            .replace(/<script[\s\S]*?<\/script>/gi, "")
-            .replace(/<style[\s\S]*?<\/style>/gi, "")
-            // Remove HTML comments
-            .replace(/<!--[\s\S]*?-->/g, "")
-            // Replace common block elements with newlines
-            .replace(/<\/(p|div|h[1-6]|br|li|tr|blockquote)>/gi, "\n")
-            .replace(/<(br|hr)\s*\/?>/gi, "\n")
-            // Remove remaining tags
-            .replace(/<[^>]+>/g, ""), {
+        return decodeHtmlTextEntitiesOnce(removeHtmlMarkup(html, {
+            tagReplacement: "",
+            commentReplacement: "",
+            lineBreakEndTags: [
+                "p",
+                "div",
+                "h1",
+                "h2",
+                "h3",
+                "h4",
+                "h5",
+                "h6",
+                "br",
+                "li",
+                "tr",
+                "blockquote",
+            ],
+            lineBreakVoidTags: ["br", "hr"],
+        }), {
             nbsp: " ",
             amp: "&",
             lt: "<",

package/dist/server/middleware/auth.js

@@ -135,13 +135,17 @@ function extractToken(ctx, type, headerName, customExtractor) {
     }
     switch (type) {
         case "bearer": {
-            // Extract token from "Bearer <token>"
-            const match = headerValue.match(/^Bearer\s+(.+)$/i);
+            // Extract token from "Bearer <token>". The token must open with a
+            // non-whitespace character. With `\s+(.+)` the whitespace run and the
+            // token could both claim the same tabs, so a header that failed to match
+            // (a line terminator after a long run of tabs) was retried at every
+            // split, which is quadratic in the header length.
+            const match = headerValue.match(/^Bearer\s+(\S.*)$/i);
             return match ? match[1] : null;
         }
         case "basic": {
-            // Extract credentials from "Basic <base64>"
-            const match = headerValue.match(/^Basic\s+(.+)$/i);
+            // Extract credentials from "Basic <base64>" (same shape as above).
+            const match = headerValue.match(/^Basic\s+(\S.*)$/i);
             return match ? match[1] : null;
         }
         case "api-key":

package/dist/server/voice/voiceServerApp.js

@@ -1,4 +1,5 @@
 import express from "express";
+import rateLimit from "express-rate-limit";
 import fs from "fs";
 import http from "http";
 import path from "path";
@@ -29,6 +30,24 @@ function resolvePublicPath() {
     }
     return compiled; // let express.static handle the 404
 }
+// Per-client request budget for the HTTP surface (the page and its static
+// assets). One page load is a handful of requests, so the default leaves normal
+// use untouched and only bounds a client that is hammering the file-serving
+// routes. Both values are operator settings, read once at startup.
+const DEFAULT_RATE_LIMIT_WINDOW_MS = 60_000;
+const DEFAULT_RATE_LIMIT_MAX = 300;
+function positiveIntegerFromEnv(name, fallback) {
+    const raw = process.env[name];
+    if (raw === undefined || raw.trim() === "") {
+        return fallback;
+    }
+    const parsed = Number(raw);
+    if (!Number.isSafeInteger(parsed) || parsed < 1) {
+        logger.warn(`[SERVER] Ignoring ${name}: expected a positive integer, using ${fallback}`);
+        return fallback;
+    }
+    return parsed;
+}
 export async function startVoiceServer(port = 3000) {
     const app = express();
     // NEW11: refuse to bind to non-loopback interfaces unless the operator
@@ -46,6 +65,22 @@ export async function startVoiceServer(port = 3000) {
     // set custom headers — see voiceWebSocketHandler.verifyClient. HTTP routes
     // intentionally reject `?token=` (would leak via Referer + access logs).
     const authToken = process.env.VOICE_SERVER_AUTH_TOKEN;
+    /* ---------- RATE LIMIT ---------- */
+    // Registered first so it also bounds failed-auth attempts and oversized
+    // bodies. /health stays exempt for the same reason the auth middleware skips
+    // it: a load balancer probes it on a fixed cadence with no credentials.
+    const rateLimitWindowMs = positiveIntegerFromEnv("VOICE_SERVER_RATE_LIMIT_WINDOW_MS", DEFAULT_RATE_LIMIT_WINDOW_MS);
+    const rateLimitMax = positiveIntegerFromEnv("VOICE_SERVER_RATE_LIMIT_MAX", DEFAULT_RATE_LIMIT_MAX);
+    app.use(rateLimit({
+        windowMs: rateLimitWindowMs,
+        limit: rateLimitMax,
+        standardHeaders: "draft-6",
+        legacyHeaders: false,
+        skip: (req) => req.path === "/health",
+        handler: (_req, res) => {
+            res.status(429).json({ error: "Too many requests" });
+        },
+    }));
     /* ---------- BODY LIMITS + AUTH ---------- */
     // NEW11: cap JSON / urlencoded body to 100kb. Express's default is 100kb
     // for json() but only when explicitly registered; without this any future
@@ -108,7 +143,7 @@ export async function startVoiceServer(port = 3000) {
             const exposure = allowPublic
                 ? `bound publicly on ${host}:${port} (VOICE_SERVER_ALLOW_PUBLIC=1)`
                 : `bound to loopback ${host}:${port} (set VOICE_SERVER_ALLOW_PUBLIC=1 to expose externally)`;
-            logger.info(`[SERVER] Voice server running — ${exposure}${authToken ? " (auth required)" : " (no auth — token via VOICE_SERVER_AUTH_TOKEN recommended)"}`);
+            logger.info(`[SERVER] Voice server running — ${exposure}${authToken ? " (auth required)" : " (no auth — token via VOICE_SERVER_AUTH_TOKEN recommended)"}; rate limit ${rateLimitMax} requests per ${rateLimitWindowMs}ms per client`);
             resolve();
         });
     });

package/dist/utils/htmlText.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Single-pass markup removal for turning HTML into indexable text.
+ *
+ * This is text conversion, not output sanitization. The result is plain text
+ * for retrieval and chunking; it is not meant to be written back into a page,
+ * and the caller's entity decoding can legitimately turn `<b>` into
+ * `<b>`.
+ *
+ * Why one pass: the previous implementation chained several global regex
+ * replacements (script, style, comment, block end, generic tag). Each removal
+ * could join the text on either side of it into a new tag for the next
+ * replacement (`<scr<!---->ipt>`), and the lazy `[\s\S]*?` bodies rescanned
+ * the rest of the input once per unclosed `<script`, which is quadratic.
+ * Here the input is read left to right exactly once. Every character of the
+ * output is copied from an unremoved stretch of the input, a removed stretch
+ * is never re-read, and every search either consumes what it scans or is
+ * answered in constant time from an index computed up front, so the work is
+ * linear in the input length.
+ *
+ * Invariant: the output never matches `<[^>]+>`. A `<` survives only when no
+ * `>` follows it in the input, or when the next character is the `>` itself
+ * (`<>`, which is not a tag). A removed span therefore cannot leave behind,
+ * or assemble, a tag.
+ */
+/**
+ * Remove HTML markup from `html` in one linear pass.
+ *
+ * - `<script>` / `<style>` elements are removed with their bodies. The end
+ *   tag may carry whitespace or attributes (`</script >`). An element with
+ *   no usable end tag loses only its start tag, as before.
+ * - `<!-- ... -->` is replaced with `commentReplacement`. An unterminated
+ *   comment is treated like any other tag and ends at the next `>`.
+ * - A closing tag named in `lineBreakEndTags` and a void tag named in
+ *   `lineBreakVoidTags` (`<br>`, `<hr />`) become a newline.
+ * - Any other `<...>` becomes `tagReplacement`. A `<` with no `>` after it,
+ *   and the empty `<>`, are literal text and are kept.
+ *
+ * Tag-name lists are lower case.
+ */
+export declare function removeHtmlMarkup(html: string, options: {
+    readonly tagReplacement: string;
+    readonly commentReplacement: string;
+    readonly lineBreakEndTags: readonly string[];
+    readonly lineBreakVoidTags: readonly string[];
+}): string;

package/dist/utils/htmlText.js

@@ -0,0 +1,148 @@
+/**
+ * Single-pass markup removal for turning HTML into indexable text.
+ *
+ * This is text conversion, not output sanitization. The result is plain text
+ * for retrieval and chunking; it is not meant to be written back into a page,
+ * and the caller's entity decoding can legitimately turn `<b>` into
+ * `<b>`.
+ *
+ * Why one pass: the previous implementation chained several global regex
+ * replacements (script, style, comment, block end, generic tag). Each removal
+ * could join the text on either side of it into a new tag for the next
+ * replacement (`<scr<!---->ipt>`), and the lazy `[\s\S]*?` bodies rescanned
+ * the rest of the input once per unclosed `<script`, which is quadratic.
+ * Here the input is read left to right exactly once. Every character of the
+ * output is copied from an unremoved stretch of the input, a removed stretch
+ * is never re-read, and every search either consumes what it scans or is
+ * answered in constant time from an index computed up front, so the work is
+ * linear in the input length.
+ *
+ * Invariant: the output never matches `<[^>]+>`. A `<` survives only when no
+ * `>` follows it in the input, or when the next character is the `>` itself
+ * (`<>`, which is not a tag). A removed span therefore cannot leave behind,
+ * or assemble, a tag.
+ */
+const SLASH = 0x2f;
+const GREATER_THAN = 0x3e;
+const WHITESPACE = /\s/;
+function isAsciiAlphanumeric(code) {
+    return ((code >= 0x30 && code <= 0x39) ||
+        (code >= 0x41 && code <= 0x5a) ||
+        (code >= 0x61 && code <= 0x7a));
+}
+// The characters after which an HTML tokenizer ends a tag name.
+function endsTagName(code) {
+    return (code === 0x09 ||
+        code === 0x0a ||
+        code === 0x0c ||
+        code === 0x0d ||
+        code === 0x20 ||
+        code === SLASH ||
+        code === GREATER_THAN);
+}
+function scanTagName(html, from) {
+    let end = from;
+    while (end < html.length && isAsciiAlphanumeric(html.charCodeAt(end))) {
+        end++;
+    }
+    return end;
+}
+/**
+ * Index of the next `</name` that is followed by whitespace, `/` or `>`
+ * (so `</script >` and `</SCRIPT foo>` count), or -1. Case-insensitive.
+ */
+function findEndTag(html, name, from) {
+    let at = html.indexOf("</", from);
+    while (at !== -1) {
+        const nameEnd = at + 2 + name.length;
+        if (html.slice(at + 2, nameEnd).toLowerCase() === name &&
+            endsTagName(html.charCodeAt(nameEnd))) {
+            return at;
+        }
+        at = html.indexOf("</", at + 2);
+    }
+    return -1;
+}
+/**
+ * Remove HTML markup from `html` in one linear pass.
+ *
+ * - `<script>` / `<style>` elements are removed with their bodies. The end
+ *   tag may carry whitespace or attributes (`</script >`). An element with
+ *   no usable end tag loses only its start tag, as before.
+ * - `<!-- ... -->` is replaced with `commentReplacement`. An unterminated
+ *   comment is treated like any other tag and ends at the next `>`.
+ * - A closing tag named in `lineBreakEndTags` and a void tag named in
+ *   `lineBreakVoidTags` (`<br>`, `<hr />`) become a newline.
+ * - Any other `<...>` becomes `tagReplacement`. A `<` with no `>` after it,
+ *   and the empty `<>`, are literal text and are kept.
+ *
+ * Tag-name lists are lower case.
+ */
+export function removeHtmlMarkup(html, options) {
+    const { tagReplacement, commentReplacement } = options;
+    const lineBreakEndTags = new Set(options.lineBreakEndTags);
+    const lineBreakVoidTags = new Set(options.lineBreakVoidTags);
+    // Answers "is there any `>` / `-->` after here" without a scan. Once an
+    // element finds no usable end tag, no later element of that name can.
+    const lastGreaterThan = html.lastIndexOf(">");
+    const lastCommentEnd = html.lastIndexOf("-->");
+    const withoutEndTag = new Set();
+    const markupAt = (at) => {
+        if (html.startsWith("<!--", at) && lastCommentEnd >= at + 4) {
+            return {
+                end: html.indexOf("-->", at + 4) + 3,
+                replacement: commentReplacement,
+            };
+        }
+        const closing = html.charCodeAt(at + 1) === SLASH;
+        const nameStart = closing ? at + 2 : at + 1;
+        const nameEnd = scanTagName(html, nameStart);
+        const name = html.slice(nameStart, nameEnd).toLowerCase();
+        if (!closing &&
+            (name === "script" || name === "style") &&
+            endsTagName(html.charCodeAt(nameEnd)) &&
+            !withoutEndTag.has(name)) {
+            const startTagEnd = html.indexOf(">", nameEnd);
+            const endTag = findEndTag(html, name, startTagEnd + 1);
+            if (endTag !== -1 && endTag < lastGreaterThan) {
+                return { end: html.indexOf(">", endTag) + 1, replacement: "" };
+            }
+            withoutEndTag.add(name);
+        }
+        if (closing &&
+            html.charCodeAt(nameEnd) === GREATER_THAN &&
+            lineBreakEndTags.has(name)) {
+            return { end: nameEnd + 1, replacement: "\n" };
+        }
+        if (!closing && lineBreakVoidTags.has(name)) {
+            let probe = nameEnd;
+            while (WHITESPACE.test(html.charAt(probe))) {
+                probe++;
+            }
+            if (html.charCodeAt(probe) === SLASH) {
+                probe++;
+            }
+            if (html.charCodeAt(probe) === GREATER_THAN) {
+                return { end: probe + 1, replacement: "\n" };
+            }
+        }
+        const end = html.indexOf(">", at + 1);
+        return end === at + 1
+            ? null
+            : { end: end + 1, replacement: tagReplacement };
+    };
+    let output = "";
+    let copied = 0;
+    let open = html.indexOf("<");
+    while (open !== -1 && open < lastGreaterThan) {
+        const markup = markupAt(open);
+        if (markup === null) {
+            open = html.indexOf("<", open + 1);
+            continue;
+        }
+        output += html.slice(copied, open) + markup.replacement;
+        copied = markup.end;
+        open = html.indexOf("<", copied);
+    }
+    return output + html.slice(copied);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.51.2",
+  "version": "12.51.4",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -164,11 +164,13 @@
     "test:embedding-response-validation": "pnpm exec tsx test/continuous-test-suite-embedding-response-validation.ts",
     "test:pdf-image-streaming": "pnpm exec tsx test/continuous-test-suite-pdf-image-streaming.ts",
     "test:rag": "pnpm exec tsx test/continuous-test-suite-rag.ts",
+    "test:rag-html-markup": "pnpm exec tsx test/continuous-test-suite-rag-html-markup.ts",
     "test:skills": "pnpm exec tsx test/continuous-test-suite-skills.ts",
     "test:decide": "pnpm exec tsx test/continuous-test-suite-decide.ts",
     "test:decide-catalog": "pnpm exec tsx test/continuous-test-suite-decide-catalog.ts",
     "test:agent-mode": "pnpm exec tsx test/continuous-test-suite-agent-mode.ts",
     "test:servers": "pnpm exec tsx test/continuous-test-suite-servers.ts",
+    "test:server-boundaries": "pnpm exec tsx test/continuous-test-suite-server-boundaries.ts",
     "test:websocket-agent-handler": "pnpm exec tsx test/continuous-test-suite-websocket-agent-handler.ts",
     "test:tool-reliability": "pnpm exec tsx test/continuous-test-suite-tool-reliability.ts",
     "test:video-abort": "pnpm exec tsx test/continuous-test-suite-video-abort.ts",