@juspay/neurolink 9.94.2 → 9.94.4

package/dist/cli/factories/commandFactory.d.ts

@@ -178,6 +178,12 @@ export declare class CLICommandFactory {
      * Build multimodal input from CLI arguments
      */
     private static buildGenerateMultimodalInput;
+    /**
+     * Build CSV processor options from CLI argv (#1199). Shared by executeGenerate
+     * and executeRealStream, which previously built byte-for-byte identical
+     * objects independently.
+     */
+    private static buildCsvOptionsFromArgv;
     /**
      * Build output configuration for generate request
      */

package/dist/cli/factories/commandFactory.js

@@ -198,6 +198,25 @@ export class CLICommandFactory {
                 "  • markdown: Formatted table (readable, best for small files <100 rows)\n" +
                 "  • json: Structured JSON array (best for programmatic use, higher tokens)",
         },
+        "csv-encoding": {
+            type: "string",
+            description: "Character encoding for CSV files (e.g. utf-8, utf-16le, windows-1252). Auto-detected when omitted (#362).",
+        },
+        "csv-sanitize-names": {
+            type: "boolean",
+            default: false,
+            description: "Rewrite CSV column headers into valid identifiers (e.g. 'Price ($)' → 'price') (#378).",
+        },
+        "csv-name-case": {
+            type: "string",
+            choices: ["snake_case", "camelCase"],
+            default: "snake_case",
+            description: "Case style for sanitized CSV column names (used with --csv-sanitize-names) (#378).",
+        },
+        "csv-parse-timeout-ms": {
+            type: "number",
+            description: "Wall-clock cap (ms) for CSV parsing; returns partial rows on timeout (#379).",
+        },
         model: {
             type: "string",
             description: "Specific model to use (e.g. gemini-2.5-pro, gemini-2.5-flash)",
@@ -2380,6 +2399,21 @@ export class CLICommandFactory {
             ...(files && { files }),
         };
     }
+    /**
+     * Build CSV processor options from CLI argv (#1199). Shared by executeGenerate
+     * and executeRealStream, which previously built byte-for-byte identical
+     * objects independently.
+     */
+    static buildCsvOptionsFromArgv(argv) {
+        return {
+            maxRows: argv.csvMaxRows,
+            formatStyle: argv.csvFormat,
+            encoding: argv.csvEncoding,
+            sanitizeColumnNames: argv.csvSanitizeNames,
+            columnNameCase: argv.csvNameCase,
+            parseTimeoutMs: argv.csvParseTimeoutMs,
+        };
+    }
     /**
      * Build output configuration for generate request
      */
@@ -2614,10 +2648,7 @@ export class CLICommandFactory {
             const inputAudioFormat = inferAudioFormatFromPath(inputAudioPath);
             const runGenerate = () => sdk.generate({
                 input: generateInput,
-                csvOptions: {
-                    maxRows: argv.csvMaxRows,
-                    formatStyle: argv.csvFormat,
-                },
+                csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
                 videoOptions: {
                     frames: argv.videoFrames,
                     quality: argv.videoQuality,
@@ -2869,10 +2900,7 @@ export class CLICommandFactory {
                 ...(videoFiles && { videoFiles }),
                 ...(files && { files }),
             },
-            csvOptions: {
-                maxRows: argv.csvMaxRows,
-                formatStyle: argv.csvFormat,
-            },
+            csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
             videoOptions: {
                 frames: argv.videoFrames,
                 quality: argv.videoQuality,

package/dist/lib/agent/directTools.js

@@ -357,7 +357,9 @@ export const directAgentTools = {
                 logger.debug(`[analyzeCSV] Resolved path: ${resolvedPath}`);
                 // Parse CSV using streaming from disk (memory efficient)
                 logger.info(`[analyzeCSV] Starting CSV parsing (max ${maxRows} rows)...`);
-                const rows = (await CSVProcessor.parseCSVFile(resolvedPath, maxRows));
+                // #384: parseCSVFile now returns validated Record<string, string |
+                // undefined>[] rows, so the previous unchecked cast is unnecessary.
+                const rows = await CSVProcessor.parseCSVFile(resolvedPath, maxRows);
                 logger.info(`[analyzeCSV] ✅ CSV parsing complete: ${rows.length} rows`);
                 if (rows.length === 0) {
                     logger.warn(`[analyzeCSV] No data rows found`);

package/dist/lib/neurolink.js

@@ -3820,6 +3820,11 @@ Current user's request: ${currentInput}`;
             evaluationDomain: options.evaluationDomain,
             toolUsageContext: options.toolUsageContext,
             input: options.input,
+            // CSV processing options must survive the reconstruction into
+            // TextGenerationOptions — the message builder reads them (encoding,
+            // formatting, sanitization, parse timeout). Omitting them silently
+            // dropped every csvOptions field (incl. the CLI --csv-* flags).
+            csvOptions: options.csvOptions,
             region: options.region,
             tts: options.tts,
             stt: options.stt,

package/dist/lib/rag/document/loaders.js

@@ -27,7 +27,7 @@ import { existsSync } from "fs";
 import { readFile } from "fs/promises";
 import { basename, extname } from "path";
 import { logger } from "../../utils/logger.js";
-import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, } from "../../utils/csvProcessor.js";
+import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, dedupeColumnNames, sanitizeColumnName, } from "../../utils/csvProcessor.js";
 import { splitCSVFields } from "../../utils/csvUtils.js";
 import { MDocument } from "./MDocument.js";
 /**
@@ -117,7 +117,7 @@ export class JSONLoader extends TextLoader {
 export class CSVLoader extends TextLoader {
     async load(source, options) {
         const content = await this.loadContent(source, options?.encoding);
-        const { delimiter, hasHeader = true, columns, outputFormat = "text", } = options || {};
+        const { delimiter, hasHeader = true, columns, outputFormat = "text", sanitizeColumnNames = false, columnNameCase = "snake_case", } = options || {};
         // #361: an explicit delimiter always wins; otherwise detect it from the
         // content + extension so a .tsv fed through RAG isn't collapsed into one
         // comma-column (CSVLoader.canHandle already accepts .tsv).
@@ -133,16 +133,32 @@ export class CSVLoader extends TextLoader {
         // escaping or quoted delimiters in the headerless-columns branch.
         const { dataLines: strippedLines } = stripMetadataLine(splitCsvLines(stripBom(content)));
         const lines = strippedLines.filter((line) => line.trim());
-        const headers = hasHeader
+        const rawHeaders = hasHeader
             ? splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((h) => h.trim())
             : columns ||
                 splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((_, i) => `col${i + 1}`);
+        // #378: apply the same opt-in header sanitization CSVProcessor offers, so
+        // the RAG ingestion path doesn't corrupt "Price ($)"-style headers as keys.
+        const headers = sanitizeColumnNames
+            ? dedupeColumnNames(rawHeaders.map((h, i) => sanitizeColumnName(h, columnNameCase, i)))
+            : rawHeaders;
         const dataLines = hasHeader ? lines.slice(1) : lines;
         const rows = dataLines.map((line) => splitCSVFields(line, effectiveDelimiter).map((field) => field.trim()));
         let formattedContent;
         switch (outputFormat) {
             case "json":
-                formattedContent = JSON.stringify(rows.map((row) => Object.fromEntries(headers.map((h, i) => [h, row[i]]))), null, 2);
+                formattedContent = JSON.stringify(rows.map((row) => {
+                    // Defense-in-depth against a caller-controlled header literally
+                    // named "__proto__"/"constructor"/"prototype" (#1199);
+                    // Object.fromEntries can't actually pollute Object.prototype here
+                    // (spec semantics), but a null-prototype target avoids relying on
+                    // that for future readers.
+                    const record = Object.create(null);
+                    headers.forEach((h, i) => {
+                        record[h] = row[i];
+                    });
+                    return record;
+                }), null, 2);
                 break;
             case "markdown":
                 formattedContent = this.toMarkdownTable(headers, rows);

package/dist/lib/types/file.d.ts

@@ -80,6 +80,17 @@ export type FileProcessingResult = {
         hasHeaders?: boolean;
         /** Detected delimiter */
         detectedDelimiter?: string;
+        /** Detected (or overridden) character encoding used to decode the CSV (#362) */
+        detectedEncoding?: string;
+        /** Confidence (0-100) of the detected encoding (#362) */
+        encodingConfidence?: number;
+        /** Original→sanitized column-name mapping when sanitizeColumnNames is on (#378) */
+        columnNameMapping?: Array<{
+            original: string;
+            sanitized: string;
+        }>;
+        /** True when the parse hit its time budget and returned partial rows (#379) */
+        parseTimedOut?: boolean;
         version?: string;
         estimatedPages?: number | null;
         provider?: string;
@@ -125,6 +136,8 @@ export type CSVDataQualityWarning = {
  */
 export type CSVColumnMetadata = {
     name: string;
+    /** Original header text before sanitization, when sanitizeColumnNames rewrote it (#378) */
+    originalName?: string;
     index: number;
     detectedType: CSVColumnDataType;
     /** Confidence of type detection (0-100) */
@@ -146,6 +159,17 @@ export type CSVColumnMetadata = {
     /** Column name validation issues */
     nameIssues?: string[];
 };
+/** A parsed CSV row: string-keyed with string (or missing) cell values (#384). */
+export type CSVRow = Record<string, string | undefined>;
+/** Result of decoding a buffer with encoding detection (#362). */
+export type DecodedBuffer = {
+    /** Decoded text with any BOM removed. */
+    text: string;
+    /** iconv-lite label actually used to decode. */
+    encoding: string;
+    /** Detection confidence 0-100 (100 for BOM/override, 0 for the UTF-8 fallback). */
+    confidence: number;
+};
 /**
  * CSV processor options
  */
@@ -155,6 +179,25 @@ export type CSVProcessorOptions = {
     includeHeaders?: boolean;
     sampleDataFormat?: SampleDataFormat;
     extension?: string | null;
+    /**
+     * Character encoding override (#362). When omitted, the encoding is detected
+     * from a BOM then `chardet`, falling back to UTF-8. Accepts any label
+     * `iconv-lite` supports (e.g. "utf-8", "utf-16le", "windows-1252", "latin1").
+     */
+    encoding?: string;
+    /**
+     * Rewrite column headers into valid identifiers (#378). Opt-in; default false
+     * preserves the raw header strings as object keys.
+     */
+    sanitizeColumnNames?: boolean;
+    /** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
+    columnNameCase?: "camelCase" | "snake_case";
+    /**
+     * Wall-clock cap for the streaming parse in milliseconds (#379). On timeout the
+     * parse returns the rows collected so far and flags `metadata.parseTimedOut`,
+     * rather than hanging forever. Defaults: 30s for strings, 5min for files.
+     */
+    parseTimeoutMs?: number;
 };
 /**
  * PDF API types for different providers

package/dist/lib/types/generate.d.ts

@@ -15,7 +15,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
 import type { MusicOptions, MusicResult } from "./music.js";
 import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
 import type { NeurolinkCredentials } from "./providers.js";
-import type { FileWithMetadata } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { Schema, Tool, ToolChoice } from "./tools.js";
 import type { StepResult, LanguageModel } from "./providers.js";
@@ -123,11 +123,7 @@ export type GenerateOptions = {
          */
         music?: MusicOptions;
     };
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     videoOptions?: {
         frames?: number;
         quality?: number;
@@ -1133,11 +1129,7 @@ export type TextGenerationOptions = {
     onError?: OnErrorCallback;
     expectedOutcome?: string;
     evaluationCriteria?: string[];
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     enableSummarization?: boolean;
     /**
      * Skip injecting tool schemas into the system prompt.

package/dist/lib/types/rag.d.ts

@@ -500,6 +500,13 @@ export type CSVLoaderOptions = LoaderOptions & {
     columns?: string[];
     /** Output format */
     outputFormat?: "text" | "json" | "markdown";
+    /**
+     * Rewrite headers into valid identifiers (#378). Opt-in; default false keeps
+     * the raw header strings as JSON keys / table columns.
+     */
+    sanitizeColumnNames?: boolean;
+    /** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
+    columnNameCase?: "camelCase" | "snake_case";
 };
 /**
  * Abstract document loader type

package/dist/lib/types/stream.d.ts

@@ -14,7 +14,7 @@ import type { AIModelProviderConfig, NeurolinkCredentials } from "./providers.js
 import type { TTSChunk, TTSOptions, TTSResult } from "./tts.js";
 import type { STTOptions, STTResult } from "./stt.js";
 import type { StandardRecord, ValidationSchema } from "./aliases.js";
-import type { FileWithMetadata } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { LanguageModel, StepResult } from "./providers.js";
 import type { Tool, ToolChoice } from "./tools.js";
@@ -205,11 +205,7 @@ export type StreamOptions = {
             enableProgress?: boolean;
         };
     };
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     videoOptions?: {
         frames?: number;
         quality?: number;

package/dist/lib/utils/csvProcessor.d.ts

@@ -3,7 +3,33 @@
  * Converts CSV files to LLM-friendly text formats
  * Uses streaming for memory efficiency with large files
  */
-import type { FileProcessingResult, CSVProcessorOptions } from "../types/index.js";
+import type { FileProcessingResult, CSVProcessorOptions, CSVRow } from "../types/index.js";
+/**
+ * Rewrite a raw header into a valid identifier. Trims, replaces non-identifier
+ * runs with `_`, prefixes a `col_` when it starts with a digit or is empty,
+ * then applies the requested case style. Never returns an empty string.
+ */
+export declare function sanitizeColumnName(name: string, style?: "camelCase" | "snake_case", index?: number): string;
+/**
+ * Deduplicate a list of (already sanitized) names by suffixing `_2`, `_3`, … on
+ * collisions, preserving order. The suffixed candidate is bumped until it does
+ * not collide with ANY already-emitted name — so an existing `name_2` can't be
+ * silently overwritten by a generated `name_2` (which the naive counter did for
+ * `["name","name","name_2"]`).
+ */
+export declare function dedupeColumnNames(names: string[]): string[];
+/**
+ * Predicate form of the row-shape invariant: a non-null, non-array object whose
+ * every value is a string or undefined. csv-parser's default output always
+ * satisfies this; the guard is defense-in-depth at the parse boundary.
+ */
+export declare function isValidCsvRow(row: unknown): row is CSVRow;
+/**
+ * Assertion form used inside the streaming `data` handlers. Throws a
+ * `[CSVProcessor]`-prefixed error (so existing `includes("CSV")` catches still
+ * match) when a row violates the invariant.
+ */
+export declare function assertValidCsvRow(row: unknown, rowNumber: number): asserts row is CSVRow;
 /**
  * Detect if first line is CSV metadata (not actual data/headers)
  * Common patterns:
@@ -94,25 +120,57 @@ export declare class CSVProcessor {
      * @param maxRows - Maximum rows to parse (default: 1000)
      * @returns Array of row objects
      */
-    static parseCSVFile(filePath: string, maxRows?: number): Promise<unknown[]>;
+    static parseCSVFile(filePath: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
+    /**
+     * File-parse variant that also reports whether the parse timed out, used by
+     * `process()` to surface `metadata.parseTimedOut`.
+     */
+    static parseCSVFileWithMeta(filePath: string, maxRows?: number, timeoutMs?: number, encoding?: string): Promise<{
+        rows: CSVRow[];
+        timedOut: boolean;
+    }>;
+    /**
+     * Peek the head of a raw file stream to resolve encoding, metadata-line
+     * skipping, and the delimiter (#361), then return a decoded text stream
+     * that still delivers the whole file from byte 0 (via `unshift`) — a
+     * single disk read (#368/#362).
+     */
+    private static prepareFileSource;
     /**
      * Parse CSV string to array of row objects
      * Exposed for use by tools that need direct CSV parsing
      *
      * @param csvString - CSV data as string
      * @param maxRows - Maximum rows to parse (default: 1000)
+     * @param timeoutMs - Wall-clock parse cap; returns partial rows on timeout
      * @returns Array of row objects
      */
-    static parseCSVString(csvString: string, maxRows?: number, delimiter?: string): Promise<unknown[]>;
+    static parseCSVString(csvString: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
     /**
-     * Stream-parse already-split, metadata-stripped data lines into row
-     * objects with a known delimiter. Shared core for parseCSVString() and
-     * process(), both of which need this after already splitting the content
-     * and resolving the delimiter once — calling parseCSVString() directly
-     * from process() would re-split and re-run metadata/delimiter detection
-     * over the same content a second time.
+     * String-parse variant that also reports whether the parse timed out, used by
+     * `process()` to surface `metadata.parseTimedOut`.
+     */
+    static parseCSVStringWithMeta(csvString: string, maxRows?: number, timeoutMs?: number): Promise<{
+        rows: CSVRow[];
+        timedOut: boolean;
+    }>;
+    /**
+     * Shared streaming consumer for both parse methods. Guarantees:
+     *  - source, rawSource, and parser are all destroyed on ANY settle path
+     *    (#371) — `rawSource` matters for the file-parse path, where `source`
+     *    is a decode Transform piped FROM `rawSource`; `.pipe()` does not
+     *    propagate `.destroy()` upstream, so without this the underlying
+     *    `fs.createReadStream` fd would leak on abort/timeout/error,
+     *  - a wall-clock timeout returns partial rows instead of hanging (#379),
+     *  - every row is validated at the boundary (#384),
+     *  - reject messages carry columns/last-row-shape/format context (#375),
+     *  - the delimiter (#361, comma/tab/semicolon/pipe) detected by the caller
+     *    is honored, instead of csv-parser's hard-coded comma default.
+     *
+     * @param rawSource - The original file stream `source` was piped from, when
+     *   applicable (file-parse path only; `undefined` for string parsing).
      */
-    private static parseCSVLines;
+    private static streamParse;
     /**
      * Format parsed CSV data for LLM consumption
      * Only used for JSON and Markdown formats (raw format handled separately)