@juspay/neurolink 12.51.3 → 12.51.4

package/dist/rag/chunkers/HTMLChunker.js

@@ -3,6 +3,7 @@
  *
  * Splits HTML content by semantic tags.
  */
+import { removeHtmlMarkup } from "../../utils/htmlText.js";
 import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
 /**
  * HTML Chunker
@@ -28,10 +29,12 @@ export class HTMLChunker extends BaseChunker {
      * Strip HTML tags from content
      */
     stripHtml(html) {
-        return html
-            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
-            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
-            .replace(/<[^>]+>/g, " ")
+        return removeHtmlMarkup(html, {
+            tagReplacement: " ",
+            commentReplacement: " ",
+            lineBreakEndTags: [],
+            lineBreakVoidTags: [],
+        })
             .replace(/\s+/g, " ")
             .trim();
     }

package/dist/rag/chunking/htmlChunker.js

@@ -6,6 +6,7 @@
  */
 import { randomUUID } from "crypto";
 import { decodeHtmlTextEntitiesOnce } from "../../utils/htmlEntities.js";
+import { removeHtmlMarkup } from "../../utils/htmlText.js";
 /**
  * HTML-aware chunker implementation
  * Splits based on HTML structure (tags, elements)
@@ -171,16 +172,24 @@ export class HTMLChunker {
      * Extract plain text from HTML
      */
     extractText(html) {
-        return decodeHtmlTextEntitiesOnce(html
-            // Remove script and style elements
-            .replace(/<script[\s\S]*?<\/script>/gi, "")
-            .replace(/<style[\s\S]*?<\/style>/gi, "")
-            // Remove HTML comments
-            .replace(/<!--[\s\S]*?-->/g, "")
-            // Replace block elements with newlines
-            .replace(/<\/(p|div|br|h[1-6]|li|tr)>/gi, "\n")
-            // Remove remaining tags
-            .replace(/<[^>]+>/g, ""), { nbsp: " ", amp: "&", lt: "<", gt: ">", quot: '"', "#039": "'" })
+        return decodeHtmlTextEntitiesOnce(removeHtmlMarkup(html, {
+            tagReplacement: "",
+            commentReplacement: "",
+            lineBreakEndTags: [
+                "p",
+                "div",
+                "br",
+                "h1",
+                "h2",
+                "h3",
+                "h4",
+                "h5",
+                "h6",
+                "li",
+                "tr",
+            ],
+            lineBreakVoidTags: [],
+        }), { nbsp: " ", amp: "&", lt: "<", gt: ">", quot: '"', "#039": "'" })
             .replace(/\s+/g, " ")
             .trim();
     }

package/dist/rag/document/loaders.js

@@ -28,6 +28,7 @@ import { readFile } from "fs/promises";
 import { basename, extname } from "path";
 import { logger } from "../../utils/logger.js";
 import { decodeHtmlTextEntitiesOnce } from "../../utils/htmlEntities.js";
+import { removeHtmlMarkup } from "../../utils/htmlText.js";
 import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, dedupeColumnNames, sanitizeColumnName, } from "../../utils/csvProcessor.js";
 import { splitCSVFields } from "../../utils/csvUtils.js";
 import { MDocument } from "./MDocument.js";
@@ -429,17 +430,25 @@ export class WebLoader {
      * Convert HTML to plain text
      */
     htmlToText(html) {
-        return decodeHtmlTextEntitiesOnce(html
-            // Remove script and style elements
-            .replace(/<script[\s\S]*?<\/script>/gi, "")
-            .replace(/<style[\s\S]*?<\/style>/gi, "")
-            // Remove HTML comments
-            .replace(/<!--[\s\S]*?-->/g, "")
-            // Replace common block elements with newlines
-            .replace(/<\/(p|div|h[1-6]|br|li|tr|blockquote)>/gi, "\n")
-            .replace(/<(br|hr)\s*\/?>/gi, "\n")
-            // Remove remaining tags
-            .replace(/<[^>]+>/g, ""), {
+        return decodeHtmlTextEntitiesOnce(removeHtmlMarkup(html, {
+            tagReplacement: "",
+            commentReplacement: "",
+            lineBreakEndTags: [
+                "p",
+                "div",
+                "h1",
+                "h2",
+                "h3",
+                "h4",
+                "h5",
+                "h6",
+                "br",
+                "li",
+                "tr",
+                "blockquote",
+            ],
+            lineBreakVoidTags: ["br", "hr"],
+        }), {
             nbsp: " ",
             amp: "&",
             lt: "<",

package/dist/utils/htmlText.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Single-pass markup removal for turning HTML into indexable text.
+ *
+ * This is text conversion, not output sanitization. The result is plain text
+ * for retrieval and chunking; it is not meant to be written back into a page,
+ * and the caller's entity decoding can legitimately turn `<b>` into
+ * `<b>`.
+ *
+ * Why one pass: the previous implementation chained several global regex
+ * replacements (script, style, comment, block end, generic tag). Each removal
+ * could join the text on either side of it into a new tag for the next
+ * replacement (`<scr<!---->ipt>`), and the lazy `[\s\S]*?` bodies rescanned
+ * the rest of the input once per unclosed `<script`, which is quadratic.
+ * Here the input is read left to right exactly once. Every character of the
+ * output is copied from an unremoved stretch of the input, a removed stretch
+ * is never re-read, and every search either consumes what it scans or is
+ * answered in constant time from an index computed up front, so the work is
+ * linear in the input length.
+ *
+ * Invariant: the output never matches `<[^>]+>`. A `<` survives only when no
+ * `>` follows it in the input, or when the next character is the `>` itself
+ * (`<>`, which is not a tag). A removed span therefore cannot leave behind,
+ * or assemble, a tag.
+ */
+/**
+ * Remove HTML markup from `html` in one linear pass.
+ *
+ * - `<script>` / `<style>` elements are removed with their bodies. The end
+ *   tag may carry whitespace or attributes (`</script >`). An element with
+ *   no usable end tag loses only its start tag, as before.
+ * - `<!-- ... -->` is replaced with `commentReplacement`. An unterminated
+ *   comment is treated like any other tag and ends at the next `>`.
+ * - A closing tag named in `lineBreakEndTags` and a void tag named in
+ *   `lineBreakVoidTags` (`<br>`, `<hr />`) become a newline.
+ * - Any other `<...>` becomes `tagReplacement`. A `<` with no `>` after it,
+ *   and the empty `<>`, are literal text and are kept.
+ *
+ * Tag-name lists are lower case.
+ */
+export declare function removeHtmlMarkup(html: string, options: {
+    readonly tagReplacement: string;
+    readonly commentReplacement: string;
+    readonly lineBreakEndTags: readonly string[];
+    readonly lineBreakVoidTags: readonly string[];
+}): string;

package/dist/utils/htmlText.js

@@ -0,0 +1,148 @@
+/**
+ * Single-pass markup removal for turning HTML into indexable text.
+ *
+ * This is text conversion, not output sanitization. The result is plain text
+ * for retrieval and chunking; it is not meant to be written back into a page,
+ * and the caller's entity decoding can legitimately turn `<b>` into
+ * `<b>`.
+ *
+ * Why one pass: the previous implementation chained several global regex
+ * replacements (script, style, comment, block end, generic tag). Each removal
+ * could join the text on either side of it into a new tag for the next
+ * replacement (`<scr<!---->ipt>`), and the lazy `[\s\S]*?` bodies rescanned
+ * the rest of the input once per unclosed `<script`, which is quadratic.
+ * Here the input is read left to right exactly once. Every character of the
+ * output is copied from an unremoved stretch of the input, a removed stretch
+ * is never re-read, and every search either consumes what it scans or is
+ * answered in constant time from an index computed up front, so the work is
+ * linear in the input length.
+ *
+ * Invariant: the output never matches `<[^>]+>`. A `<` survives only when no
+ * `>` follows it in the input, or when the next character is the `>` itself
+ * (`<>`, which is not a tag). A removed span therefore cannot leave behind,
+ * or assemble, a tag.
+ */
+const SLASH = 0x2f;
+const GREATER_THAN = 0x3e;
+const WHITESPACE = /\s/;
+function isAsciiAlphanumeric(code) {
+    return ((code >= 0x30 && code <= 0x39) ||
+        (code >= 0x41 && code <= 0x5a) ||
+        (code >= 0x61 && code <= 0x7a));
+}
+// The characters after which an HTML tokenizer ends a tag name.
+function endsTagName(code) {
+    return (code === 0x09 ||
+        code === 0x0a ||
+        code === 0x0c ||
+        code === 0x0d ||
+        code === 0x20 ||
+        code === SLASH ||
+        code === GREATER_THAN);
+}
+function scanTagName(html, from) {
+    let end = from;
+    while (end < html.length && isAsciiAlphanumeric(html.charCodeAt(end))) {
+        end++;
+    }
+    return end;
+}
+/**
+ * Index of the next `</name` that is followed by whitespace, `/` or `>`
+ * (so `</script >` and `</SCRIPT foo>` count), or -1. Case-insensitive.
+ */
+function findEndTag(html, name, from) {
+    let at = html.indexOf("</", from);
+    while (at !== -1) {
+        const nameEnd = at + 2 + name.length;
+        if (html.slice(at + 2, nameEnd).toLowerCase() === name &&
+            endsTagName(html.charCodeAt(nameEnd))) {
+            return at;
+        }
+        at = html.indexOf("</", at + 2);
+    }
+    return -1;
+}
+/**
+ * Remove HTML markup from `html` in one linear pass.
+ *
+ * - `<script>` / `<style>` elements are removed with their bodies. The end
+ *   tag may carry whitespace or attributes (`</script >`). An element with
+ *   no usable end tag loses only its start tag, as before.
+ * - `<!-- ... -->` is replaced with `commentReplacement`. An unterminated
+ *   comment is treated like any other tag and ends at the next `>`.
+ * - A closing tag named in `lineBreakEndTags` and a void tag named in
+ *   `lineBreakVoidTags` (`<br>`, `<hr />`) become a newline.
+ * - Any other `<...>` becomes `tagReplacement`. A `<` with no `>` after it,
+ *   and the empty `<>`, are literal text and are kept.
+ *
+ * Tag-name lists are lower case.
+ */
+export function removeHtmlMarkup(html, options) {
+    const { tagReplacement, commentReplacement } = options;
+    const lineBreakEndTags = new Set(options.lineBreakEndTags);
+    const lineBreakVoidTags = new Set(options.lineBreakVoidTags);
+    // Answers "is there any `>` / `-->` after here" without a scan. Once an
+    // element finds no usable end tag, no later element of that name can.
+    const lastGreaterThan = html.lastIndexOf(">");
+    const lastCommentEnd = html.lastIndexOf("-->");
+    const withoutEndTag = new Set();
+    const markupAt = (at) => {
+        if (html.startsWith("<!--", at) && lastCommentEnd >= at + 4) {
+            return {
+                end: html.indexOf("-->", at + 4) + 3,
+                replacement: commentReplacement,
+            };
+        }
+        const closing = html.charCodeAt(at + 1) === SLASH;
+        const nameStart = closing ? at + 2 : at + 1;
+        const nameEnd = scanTagName(html, nameStart);
+        const name = html.slice(nameStart, nameEnd).toLowerCase();
+        if (!closing &&
+            (name === "script" || name === "style") &&
+            endsTagName(html.charCodeAt(nameEnd)) &&
+            !withoutEndTag.has(name)) {
+            const startTagEnd = html.indexOf(">", nameEnd);
+            const endTag = findEndTag(html, name, startTagEnd + 1);
+            if (endTag !== -1 && endTag < lastGreaterThan) {
+                return { end: html.indexOf(">", endTag) + 1, replacement: "" };
+            }
+            withoutEndTag.add(name);
+        }
+        if (closing &&
+            html.charCodeAt(nameEnd) === GREATER_THAN &&
+            lineBreakEndTags.has(name)) {
+            return { end: nameEnd + 1, replacement: "\n" };
+        }
+        if (!closing && lineBreakVoidTags.has(name)) {
+            let probe = nameEnd;
+            while (WHITESPACE.test(html.charAt(probe))) {
+                probe++;
+            }
+            if (html.charCodeAt(probe) === SLASH) {
+                probe++;
+            }
+            if (html.charCodeAt(probe) === GREATER_THAN) {
+                return { end: probe + 1, replacement: "\n" };
+            }
+        }
+        const end = html.indexOf(">", at + 1);
+        return end === at + 1
+            ? null
+            : { end: end + 1, replacement: tagReplacement };
+    };
+    let output = "";
+    let copied = 0;
+    let open = html.indexOf("<");
+    while (open !== -1 && open < lastGreaterThan) {
+        const markup = markupAt(open);
+        if (markup === null) {
+            open = html.indexOf("<", open + 1);
+            continue;
+        }
+        output += html.slice(copied, open) + markup.replacement;
+        copied = markup.end;
+        open = html.indexOf("<", copied);
+    }
+    return output + html.slice(copied);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.51.3",
+  "version": "12.51.4",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -164,6 +164,7 @@
     "test:embedding-response-validation": "pnpm exec tsx test/continuous-test-suite-embedding-response-validation.ts",
     "test:pdf-image-streaming": "pnpm exec tsx test/continuous-test-suite-pdf-image-streaming.ts",
     "test:rag": "pnpm exec tsx test/continuous-test-suite-rag.ts",
+    "test:rag-html-markup": "pnpm exec tsx test/continuous-test-suite-rag-html-markup.ts",
     "test:skills": "pnpm exec tsx test/continuous-test-suite-skills.ts",
     "test:decide": "pnpm exec tsx test/continuous-test-suite-decide.ts",
     "test:decide-catalog": "pnpm exec tsx test/continuous-test-suite-decide-catalog.ts",