@juspay/neurolink 9.94.2 → 9.94.4

package/dist/lib/utils/textEncoding.js

@@ -0,0 +1,141 @@
+/**
+ * Buffer → string decoding with encoding detection (#362).
+ *
+ * Order of resolution:
+ *   1. Explicit override (any iconv-lite label).
+ *   2. Deterministic BOM sniff (UTF-8 / UTF-16 LE / UTF-16 BE).
+ *   3. `chardet` statistical detection, mapped to an iconv-lite label.
+ *   4. Fallback to UTF-8 (preserves the historical hard-coded default).
+ *
+ * Decoding always goes through `iconv-lite`, and any residual BOM is stripped
+ * from the decoded string so it can't glue onto the first token.
+ */
+import chardet from "chardet";
+import iconv from "iconv-lite";
+import { logger } from "./logger.js";
+/** Strip a leading BOM (U+FEFF) from an already-decoded string. */
+export function stripBomString(text) {
+    return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+}
+/** Deterministic BOM detection from the raw bytes. Returns null when absent. */
+function sniffBom(buffer) {
+    if (buffer.length >= 3 &&
+        buffer[0] === 0xef &&
+        buffer[1] === 0xbb &&
+        buffer[2] === 0xbf) {
+        return "utf-8";
+    }
+    if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
+        return "utf-16le";
+    }
+    if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) {
+        return "utf-16be";
+    }
+    return null;
+}
+/**
+ * Map a chardet label to an iconv-lite-supported label. Returns null when the
+ * label is unknown/unsupported so the caller falls back to UTF-8.
+ */
+function toIconvLabel(detected) {
+    if (!detected) {
+        return null;
+    }
+    const normalized = detected.toLowerCase().replace(/[^a-z0-9]/g, "");
+    const map = {
+        utf8: "utf-8",
+        utf16le: "utf-16le",
+        utf16be: "utf-16be",
+        ascii: "utf-8", // ASCII is a strict UTF-8 subset
+        windows1252: "windows-1252",
+        iso88591: "latin1",
+        latin1: "latin1",
+        iso88592: "iso-8859-2",
+        windows1251: "windows-1251",
+        koi8r: "koi8-r",
+        gb18030: "gb18030",
+        big5: "big5",
+        shiftjis: "shift_jis",
+        euckr: "euc-kr",
+        eucjp: "euc-jp",
+    };
+    const label = map[normalized] ?? detected;
+    return iconv.encodingExists(label) ? label : null;
+}
+/**
+ * Decode a buffer to text, detecting the encoding unless overridden.
+ *
+ * @param isCompleteBuffer - Whether `buffer` is the entire source (default
+ *   `true`). Pass `false` when `buffer` is only a peeked prefix (e.g. a
+ *   streaming head-sniff) — the ASCII fast path still commits to "utf-8" for
+ *   the peeked bytes (unchanged behavior), but reports a lower confidence
+ *   since later bytes not yet seen could still be non-ASCII (#1199).
+ */
+export function decodeBuffer(buffer, override, isCompleteBuffer = true) {
+    // 1. Explicit override wins outright.
+    if (override && iconv.encodingExists(override)) {
+        return {
+            text: stripBomString(iconv.decode(buffer, override)),
+            encoding: override,
+            confidence: 100,
+        };
+    }
+    if (override && !iconv.encodingExists(override)) {
+        logger.warn(`[textEncoding] Unsupported encoding override "${override}"; falling back to detection.`);
+    }
+    // 2. BOM is authoritative when present.
+    const bom = sniffBom(buffer);
+    if (bom) {
+        return {
+            text: stripBomString(iconv.decode(buffer, bom)),
+            encoding: bom,
+            confidence: 100,
+        };
+    }
+    // 2b. Pure-ASCII fast path — chardet often reports ASCII as ISO-8859-1, but
+    // ASCII is a strict UTF-8 subset and decodes identically, so report "utf-8"
+    // to preserve the historical default for plain-ASCII files.
+    let ascii = true;
+    for (let i = 0; i < buffer.length; i++) {
+        if (buffer[i] >= 0x80) {
+            ascii = false;
+            break;
+        }
+    }
+    if (ascii) {
+        return {
+            text: stripBomString(iconv.decode(buffer, "utf-8")),
+            encoding: "utf-8",
+            // A partial peek being pure-ASCII only proves the peeked prefix is
+            // ASCII, not the whole file — a later chunk could still switch to a
+            // legacy encoding. Only report full certainty when `buffer` is known
+            // to be the complete source.
+            confidence: isCompleteBuffer ? 100 : 60,
+        };
+    }
+    // 3. Statistical detection.
+    let detected = null;
+    try {
+        detected = chardet.detect(buffer);
+    }
+    catch (error) {
+        logger.debug(`[textEncoding] chardet failed: ${String(error)}`);
+    }
+    const label = toIconvLabel(detected);
+    if (label) {
+        return {
+            text: stripBomString(iconv.decode(buffer, label)),
+            encoding: label,
+            // chardet gives a winner but not a numeric score via detect(); treat a
+            // confident non-UTF-8 hit as reasonably (not fully) certain.
+            confidence: label === "utf-8" ? 100 : 80,
+        };
+    }
+    // 4. Fallback: historical UTF-8 default.
+    return {
+        text: stripBomString(iconv.decode(buffer, "utf-8")),
+        encoding: "utf-8",
+        confidence: 0,
+    };
+}
+//# sourceMappingURL=textEncoding.js.map

package/dist/neurolink.js

@@ -3820,6 +3820,11 @@ Current user's request: ${currentInput}`;
             evaluationDomain: options.evaluationDomain,
             toolUsageContext: options.toolUsageContext,
             input: options.input,
+            // CSV processing options must survive the reconstruction into
+            // TextGenerationOptions — the message builder reads them (encoding,
+            // formatting, sanitization, parse timeout). Omitting them silently
+            // dropped every csvOptions field (incl. the CLI --csv-* flags).
+            csvOptions: options.csvOptions,
             region: options.region,
             tts: options.tts,
             stt: options.stt,

package/dist/rag/document/loaders.js

@@ -27,7 +27,7 @@ import { existsSync } from "fs";
 import { readFile } from "fs/promises";
 import { basename, extname } from "path";
 import { logger } from "../../utils/logger.js";
-import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, } from "../../utils/csvProcessor.js";
+import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, dedupeColumnNames, sanitizeColumnName, } from "../../utils/csvProcessor.js";
 import { splitCSVFields } from "../../utils/csvUtils.js";
 import { MDocument } from "./MDocument.js";
 /**
@@ -117,7 +117,7 @@ export class JSONLoader extends TextLoader {
 export class CSVLoader extends TextLoader {
     async load(source, options) {
         const content = await this.loadContent(source, options?.encoding);
-        const { delimiter, hasHeader = true, columns, outputFormat = "text", } = options || {};
+        const { delimiter, hasHeader = true, columns, outputFormat = "text", sanitizeColumnNames = false, columnNameCase = "snake_case", } = options || {};
         // #361: an explicit delimiter always wins; otherwise detect it from the
         // content + extension so a .tsv fed through RAG isn't collapsed into one
         // comma-column (CSVLoader.canHandle already accepts .tsv).
@@ -133,16 +133,32 @@ export class CSVLoader extends TextLoader {
         // escaping or quoted delimiters in the headerless-columns branch.
         const { dataLines: strippedLines } = stripMetadataLine(splitCsvLines(stripBom(content)));
         const lines = strippedLines.filter((line) => line.trim());
-        const headers = hasHeader
+        const rawHeaders = hasHeader
             ? splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((h) => h.trim())
             : columns ||
                 splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((_, i) => `col${i + 1}`);
+        // #378: apply the same opt-in header sanitization CSVProcessor offers, so
+        // the RAG ingestion path doesn't corrupt "Price ($)"-style headers as keys.
+        const headers = sanitizeColumnNames
+            ? dedupeColumnNames(rawHeaders.map((h, i) => sanitizeColumnName(h, columnNameCase, i)))
+            : rawHeaders;
         const dataLines = hasHeader ? lines.slice(1) : lines;
         const rows = dataLines.map((line) => splitCSVFields(line, effectiveDelimiter).map((field) => field.trim()));
         let formattedContent;
         switch (outputFormat) {
             case "json":
-                formattedContent = JSON.stringify(rows.map((row) => Object.fromEntries(headers.map((h, i) => [h, row[i]]))), null, 2);
+                formattedContent = JSON.stringify(rows.map((row) => {
+                    // Defense-in-depth against a caller-controlled header literally
+                    // named "__proto__"/"constructor"/"prototype" (#1199);
+                    // Object.fromEntries can't actually pollute Object.prototype here
+                    // (spec semantics), but a null-prototype target avoids relying on
+                    // that for future readers.
+                    const record = Object.create(null);
+                    headers.forEach((h, i) => {
+                        record[h] = row[i];
+                    });
+                    return record;
+                }), null, 2);
                 break;
             case "markdown":
                 formattedContent = this.toMarkdownTable(headers, rows);

package/dist/types/file.d.ts

@@ -80,6 +80,17 @@ export type FileProcessingResult = {
         hasHeaders?: boolean;
         /** Detected delimiter */
         detectedDelimiter?: string;
+        /** Detected (or overridden) character encoding used to decode the CSV (#362) */
+        detectedEncoding?: string;
+        /** Confidence (0-100) of the detected encoding (#362) */
+        encodingConfidence?: number;
+        /** Original→sanitized column-name mapping when sanitizeColumnNames is on (#378) */
+        columnNameMapping?: Array<{
+            original: string;
+            sanitized: string;
+        }>;
+        /** True when the parse hit its time budget and returned partial rows (#379) */
+        parseTimedOut?: boolean;
         version?: string;
         estimatedPages?: number | null;
         provider?: string;
@@ -125,6 +136,8 @@ export type CSVDataQualityWarning = {
  */
 export type CSVColumnMetadata = {
     name: string;
+    /** Original header text before sanitization, when sanitizeColumnNames rewrote it (#378) */
+    originalName?: string;
     index: number;
     detectedType: CSVColumnDataType;
     /** Confidence of type detection (0-100) */
@@ -146,6 +159,17 @@ export type CSVColumnMetadata = {
     /** Column name validation issues */
     nameIssues?: string[];
 };
+/** A parsed CSV row: string-keyed with string (or missing) cell values (#384). */
+export type CSVRow = Record<string, string | undefined>;
+/** Result of decoding a buffer with encoding detection (#362). */
+export type DecodedBuffer = {
+    /** Decoded text with any BOM removed. */
+    text: string;
+    /** iconv-lite label actually used to decode. */
+    encoding: string;
+    /** Detection confidence 0-100 (100 for BOM/override, 0 for the UTF-8 fallback). */
+    confidence: number;
+};
 /**
  * CSV processor options
  */
@@ -155,6 +179,25 @@ export type CSVProcessorOptions = {
     includeHeaders?: boolean;
     sampleDataFormat?: SampleDataFormat;
     extension?: string | null;
+    /**
+     * Character encoding override (#362). When omitted, the encoding is detected
+     * from a BOM then `chardet`, falling back to UTF-8. Accepts any label
+     * `iconv-lite` supports (e.g. "utf-8", "utf-16le", "windows-1252", "latin1").
+     */
+    encoding?: string;
+    /**
+     * Rewrite column headers into valid identifiers (#378). Opt-in; default false
+     * preserves the raw header strings as object keys.
+     */
+    sanitizeColumnNames?: boolean;
+    /** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
+    columnNameCase?: "camelCase" | "snake_case";
+    /**
+     * Wall-clock cap for the streaming parse in milliseconds (#379). On timeout the
+     * parse returns the rows collected so far and flags `metadata.parseTimedOut`,
+     * rather than hanging forever. Defaults: 30s for strings, 5min for files.
+     */
+    parseTimeoutMs?: number;
 };
 /**
  * PDF API types for different providers

package/dist/types/generate.d.ts

@@ -15,7 +15,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
 import type { MusicOptions, MusicResult } from "./music.js";
 import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
 import type { NeurolinkCredentials } from "./providers.js";
-import type { FileWithMetadata } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { Schema, Tool, ToolChoice } from "./tools.js";
 import type { StepResult, LanguageModel } from "./providers.js";
@@ -123,11 +123,7 @@ export type GenerateOptions = {
          */
         music?: MusicOptions;
     };
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     videoOptions?: {
         frames?: number;
         quality?: number;
@@ -1133,11 +1129,7 @@ export type TextGenerationOptions = {
     onError?: OnErrorCallback;
     expectedOutcome?: string;
     evaluationCriteria?: string[];
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     enableSummarization?: boolean;
     /**
      * Skip injecting tool schemas into the system prompt.

package/dist/types/rag.d.ts

@@ -500,6 +500,13 @@ export type CSVLoaderOptions = LoaderOptions & {
     columns?: string[];
     /** Output format */
     outputFormat?: "text" | "json" | "markdown";
+    /**
+     * Rewrite headers into valid identifiers (#378). Opt-in; default false keeps
+     * the raw header strings as JSON keys / table columns.
+     */
+    sanitizeColumnNames?: boolean;
+    /** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
+    columnNameCase?: "camelCase" | "snake_case";
 };
 /**
  * Abstract document loader type

package/dist/types/stream.d.ts

@@ -14,7 +14,7 @@ import type { AIModelProviderConfig, NeurolinkCredentials } from "./providers.js
 import type { TTSChunk, TTSOptions, TTSResult } from "./tts.js";
 import type { STTOptions, STTResult } from "./stt.js";
 import type { StandardRecord, ValidationSchema } from "./aliases.js";
-import type { FileWithMetadata } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { LanguageModel, StepResult } from "./providers.js";
 import type { Tool, ToolChoice } from "./tools.js";
@@ -205,11 +205,7 @@ export type StreamOptions = {
             enableProgress?: boolean;
         };
     };
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     videoOptions?: {
         frames?: number;
         quality?: number;

package/dist/utils/csvProcessor.d.ts

@@ -3,7 +3,33 @@
  * Converts CSV files to LLM-friendly text formats
  * Uses streaming for memory efficiency with large files
  */
-import type { FileProcessingResult, CSVProcessorOptions } from "../types/index.js";
+import type { FileProcessingResult, CSVProcessorOptions, CSVRow } from "../types/index.js";
+/**
+ * Rewrite a raw header into a valid identifier. Trims, replaces non-identifier
+ * runs with `_`, prefixes a `col_` when it starts with a digit or is empty,
+ * then applies the requested case style. Never returns an empty string.
+ */
+export declare function sanitizeColumnName(name: string, style?: "camelCase" | "snake_case", index?: number): string;
+/**
+ * Deduplicate a list of (already sanitized) names by suffixing `_2`, `_3`, … on
+ * collisions, preserving order. The suffixed candidate is bumped until it does
+ * not collide with ANY already-emitted name — so an existing `name_2` can't be
+ * silently overwritten by a generated `name_2` (which the naive counter did for
+ * `["name","name","name_2"]`).
+ */
+export declare function dedupeColumnNames(names: string[]): string[];
+/**
+ * Predicate form of the row-shape invariant: a non-null, non-array object whose
+ * every value is a string or undefined. csv-parser's default output always
+ * satisfies this; the guard is defense-in-depth at the parse boundary.
+ */
+export declare function isValidCsvRow(row: unknown): row is CSVRow;
+/**
+ * Assertion form used inside the streaming `data` handlers. Throws a
+ * `[CSVProcessor]`-prefixed error (so existing `includes("CSV")` catches still
+ * match) when a row violates the invariant.
+ */
+export declare function assertValidCsvRow(row: unknown, rowNumber: number): asserts row is CSVRow;
 /**
  * Detect if first line is CSV metadata (not actual data/headers)
  * Common patterns:
@@ -94,25 +120,57 @@ export declare class CSVProcessor {
      * @param maxRows - Maximum rows to parse (default: 1000)
      * @returns Array of row objects
      */
-    static parseCSVFile(filePath: string, maxRows?: number): Promise<unknown[]>;
+    static parseCSVFile(filePath: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
+    /**
+     * File-parse variant that also reports whether the parse timed out, used by
+     * `process()` to surface `metadata.parseTimedOut`.
+     */
+    static parseCSVFileWithMeta(filePath: string, maxRows?: number, timeoutMs?: number, encoding?: string): Promise<{
+        rows: CSVRow[];
+        timedOut: boolean;
+    }>;
+    /**
+     * Peek the head of a raw file stream to resolve encoding, metadata-line
+     * skipping, and the delimiter (#361), then return a decoded text stream
+     * that still delivers the whole file from byte 0 (via `unshift`) — a
+     * single disk read (#368/#362).
+     */
+    private static prepareFileSource;
     /**
      * Parse CSV string to array of row objects
      * Exposed for use by tools that need direct CSV parsing
      *
      * @param csvString - CSV data as string
      * @param maxRows - Maximum rows to parse (default: 1000)
+     * @param timeoutMs - Wall-clock parse cap; returns partial rows on timeout
      * @returns Array of row objects
      */
-    static parseCSVString(csvString: string, maxRows?: number, delimiter?: string): Promise<unknown[]>;
+    static parseCSVString(csvString: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
     /**
-     * Stream-parse already-split, metadata-stripped data lines into row
-     * objects with a known delimiter. Shared core for parseCSVString() and
-     * process(), both of which need this after already splitting the content
-     * and resolving the delimiter once — calling parseCSVString() directly
-     * from process() would re-split and re-run metadata/delimiter detection
-     * over the same content a second time.
+     * String-parse variant that also reports whether the parse timed out, used by
+     * `process()` to surface `metadata.parseTimedOut`.
+     */
+    static parseCSVStringWithMeta(csvString: string, maxRows?: number, timeoutMs?: number): Promise<{
+        rows: CSVRow[];
+        timedOut: boolean;
+    }>;
+    /**
+     * Shared streaming consumer for both parse methods. Guarantees:
+     *  - source, rawSource, and parser are all destroyed on ANY settle path
+     *    (#371) — `rawSource` matters for the file-parse path, where `source`
+     *    is a decode Transform piped FROM `rawSource`; `.pipe()` does not
+     *    propagate `.destroy()` upstream, so without this the underlying
+     *    `fs.createReadStream` fd would leak on abort/timeout/error,
+     *  - a wall-clock timeout returns partial rows instead of hanging (#379),
+     *  - every row is validated at the boundary (#384),
+     *  - reject messages carry columns/last-row-shape/format context (#375),
+     *  - the delimiter (#361, comma/tab/semicolon/pipe) detected by the caller
+     *    is honored, instead of csv-parser's hard-coded comma default.
+     *
+     * @param rawSource - The original file stream `source` was piped from, when
+     *   applicable (file-parse path only; `undefined` for string parsing).
      */
-    private static parseCSVLines;
+    private static streamParse;
     /**
      * Format parsed CSV data for LLM consumption
      * Only used for JSON and Markdown formats (raw format handled separately)