@juspay/neurolink 9.94.1 → 9.94.3

package/dist/types/generate.d.ts

@@ -15,7 +15,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
 import type { MusicOptions, MusicResult } from "./music.js";
 import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
 import type { NeurolinkCredentials } from "./providers.js";
-import type { FileWithMetadata } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { Schema, Tool, ToolChoice } from "./tools.js";
 import type { StepResult, LanguageModel } from "./providers.js";
@@ -123,11 +123,7 @@ export type GenerateOptions = {
          */
         music?: MusicOptions;
     };
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     videoOptions?: {
         frames?: number;
         quality?: number;
@@ -1133,11 +1129,7 @@ export type TextGenerationOptions = {
     onError?: OnErrorCallback;
     expectedOutcome?: string;
     evaluationCriteria?: string[];
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     enableSummarization?: boolean;
     /**
      * Skip injecting tool schemas into the system prompt.

package/dist/types/rag.d.ts

@@ -500,6 +500,13 @@ export type CSVLoaderOptions = LoaderOptions & {
     columns?: string[];
     /** Output format */
     outputFormat?: "text" | "json" | "markdown";
+    /**
+     * Rewrite headers into valid identifiers (#378). Opt-in; default false keeps
+     * the raw header strings as JSON keys / table columns.
+     */
+    sanitizeColumnNames?: boolean;
+    /** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
+    columnNameCase?: "camelCase" | "snake_case";
 };
 /**
  * Abstract document loader type

package/dist/types/stream.d.ts

@@ -14,7 +14,7 @@ import type { AIModelProviderConfig, NeurolinkCredentials } from "./providers.js
 import type { TTSChunk, TTSOptions, TTSResult } from "./tts.js";
 import type { STTOptions, STTResult } from "./stt.js";
 import type { StandardRecord, ValidationSchema } from "./aliases.js";
-import type { FileWithMetadata } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { LanguageModel, StepResult } from "./providers.js";
 import type { Tool, ToolChoice } from "./tools.js";
@@ -205,11 +205,7 @@ export type StreamOptions = {
             enableProgress?: boolean;
         };
     };
-    csvOptions?: {
-        maxRows?: number;
-        formatStyle?: "raw" | "markdown" | "json";
-        includeHeaders?: boolean;
-    };
+    csvOptions?: CSVProcessorOptions;
     videoOptions?: {
         frames?: number;
         quality?: number;

package/dist/utils/csvProcessor.d.ts

@@ -3,7 +3,77 @@
  * Converts CSV files to LLM-friendly text formats
  * Uses streaming for memory efficiency with large files
  */
-import type { FileProcessingResult, CSVProcessorOptions } from "../types/index.js";
+import type { FileProcessingResult, CSVProcessorOptions, CSVRow } from "../types/index.js";
+/**
+ * Rewrite a raw header into a valid identifier. Trims, replaces non-identifier
+ * runs with `_`, prefixes a `col_` when it starts with a digit or is empty,
+ * then applies the requested case style. Never returns an empty string.
+ */
+export declare function sanitizeColumnName(name: string, style?: "camelCase" | "snake_case", index?: number): string;
+/**
+ * Deduplicate a list of (already sanitized) names by suffixing `_2`, `_3`, … on
+ * collisions, preserving order. The suffixed candidate is bumped until it does
+ * not collide with ANY already-emitted name — so an existing `name_2` can't be
+ * silently overwritten by a generated `name_2` (which the naive counter did for
+ * `["name","name","name_2"]`).
+ */
+export declare function dedupeColumnNames(names: string[]): string[];
+/**
+ * Predicate form of the row-shape invariant: a non-null, non-array object whose
+ * every value is a string or undefined. csv-parser's default output always
+ * satisfies this; the guard is defense-in-depth at the parse boundary.
+ */
+export declare function isValidCsvRow(row: unknown): row is CSVRow;
+/**
+ * Assertion form used inside the streaming `data` handlers. Throws a
+ * `[CSVProcessor]`-prefixed error (so existing `includes("CSV")` catches still
+ * match) when a row violates the invariant.
+ */
+export declare function assertValidCsvRow(row: unknown, rowNumber: number): asserts row is CSVRow;
+/**
+ * Detect if first line is CSV metadata (not actual data/headers)
+ * Common patterns:
+ * - Excel separator line: "SEP=,"
+ * - Lines with significantly different delimiter count than line 2
+ * - Lines that don't match CSV structure of subsequent lines
+ */
+/**
+ * Strip a leading UTF-8 BOM (U+FEFF) if present. Exported so every entry
+ * point that sniffs raw text (including the RAG CSVLoader) normalizes BOMs
+ * the same way before metadata/delimiter detection.
+ */
+export declare function stripBom(text: string): string;
+/**
+ * Strip a leading metadata line (e.g. Excel's `sep=;` preamble) from
+ * already-split lines, and read the delimiter it explicitly declares.
+ * Excel semantics: an explicit `sep=<char>` always wins over frequency-based
+ * detection; a metadata line without a recognized delimiter character is
+ * simply dropped from the sample without forcing a delimiter. This is the
+ * single place process(), detectDelimiter(), parseCSVFile(), and
+ * parseCSVString() all go through, so none of them detect a delimiter over
+ * the raw, un-stripped lines (which lets the preamble skew detection).
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) share it too.
+ */
+export declare function stripMetadataLine(lines: string[]): {
+    dataLines: string[];
+    hasMetadataLine: boolean;
+    explicitDelimiter?: string;
+};
+/**
+ * Split CSV text into logical rows for metadata detection and raw row limiting.
+ *
+ * Supports Unix (LF), Windows (CRLF), and classic Mac (CR) line endings, and is
+ * quote-aware: a line ending that appears *inside* a quoted field (RFC 4180) is
+ * kept as part of that field rather than treated as a row boundary. A doubled
+ * `""` inside quotes is the RFC-4180 escaped quote and does not toggle the quote
+ * state. For unquoted input this yields exactly the same result as a plain
+ * `split(/\r\n|\n|\r/)`.
+ *
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) get the same
+ * quote-aware row boundaries instead of a physical `content.split("\n")`,
+ * which would break rows on a newline embedded in a quoted field.
+ */
+export declare function splitCsvLines(csvString: string): string[];
 /**
  * CSV processor for converting CSV data to LLM-optimized formats
  *
@@ -35,9 +105,14 @@ export declare class CSVProcessor {
      */
     static process(content: Buffer, options?: CSVProcessorOptions): Promise<FileProcessingResult>;
     /**
-     * Parse CSV string into array of row objects using streaming
-     * Memory-efficient for large files
+     * Detect the delimiter of DSV content (comma/tab/semicolon/pipe), respecting
+     * RFC-4180 quoting. An optional extension hint (`tsv` → tab, `psv` → pipe)
+     * breaks ambiguity. Used by callers (e.g. the RAG CSV loader) that parse
+     * DSV content themselves and need the same detection (#361). A leading
+     * Excel `sep=` metadata line is stripped before sampling so it can't skew
+     * detection, and an explicit `sep=<char>` value wins outright.
      */
+    static detectDelimiter(csvString: string, extensionHint?: string): string;
     /**
      * Parse CSV file from disk using streaming (memory efficient)
      *
@@ -45,16 +120,57 @@ export declare class CSVProcessor {
      * @param maxRows - Maximum rows to parse (default: 1000)
      * @returns Array of row objects
      */
-    static parseCSVFile(filePath: string, maxRows?: number): Promise<unknown[]>;
+    static parseCSVFile(filePath: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
+    /**
+     * File-parse variant that also reports whether the parse timed out, used by
+     * `process()` to surface `metadata.parseTimedOut`.
+     */
+    static parseCSVFileWithMeta(filePath: string, maxRows?: number, timeoutMs?: number, encoding?: string): Promise<{
+        rows: CSVRow[];
+        timedOut: boolean;
+    }>;
+    /**
+     * Peek the head of a raw file stream to resolve encoding, metadata-line
+     * skipping, and the delimiter (#361), then return a decoded text stream
+     * that still delivers the whole file from byte 0 (via `unshift`) — a
+     * single disk read (#368/#362).
+     */
+    private static prepareFileSource;
     /**
      * Parse CSV string to array of row objects
      * Exposed for use by tools that need direct CSV parsing
      *
      * @param csvString - CSV data as string
      * @param maxRows - Maximum rows to parse (default: 1000)
+     * @param timeoutMs - Wall-clock parse cap; returns partial rows on timeout
      * @returns Array of row objects
      */
-    static parseCSVString(csvString: string, maxRows?: number): Promise<unknown[]>;
+    static parseCSVString(csvString: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
+    /**
+     * String-parse variant that also reports whether the parse timed out, used by
+     * `process()` to surface `metadata.parseTimedOut`.
+     */
+    static parseCSVStringWithMeta(csvString: string, maxRows?: number, timeoutMs?: number): Promise<{
+        rows: CSVRow[];
+        timedOut: boolean;
+    }>;
+    /**
+     * Shared streaming consumer for both parse methods. Guarantees:
+     *  - source, rawSource, and parser are all destroyed on ANY settle path
+     *    (#371) — `rawSource` matters for the file-parse path, where `source`
+     *    is a decode Transform piped FROM `rawSource`; `.pipe()` does not
+     *    propagate `.destroy()` upstream, so without this the underlying
+     *    `fs.createReadStream` fd would leak on abort/timeout/error,
+     *  - a wall-clock timeout returns partial rows instead of hanging (#379),
+     *  - every row is validated at the boundary (#384),
+     *  - reject messages carry columns/last-row-shape/format context (#375),
+     *  - the delimiter (#361, comma/tab/semicolon/pipe) detected by the caller
+     *    is honored, instead of csv-parser's hard-coded comma default.
+     *
+     * @param rawSource - The original file stream `source` was piped from, when
+     *   applicable (file-parse path only; `undefined` for string parsing).
+     */
+    private static streamParse;
     /**
      * Format parsed CSV data for LLM consumption
      * Only used for JSON and Markdown formats (raw format handled separately)