@juspay/neurolink 9.94.1 → 9.94.2

package/dist/lib/rag/document/loaders.d.ts

@@ -61,9 +61,20 @@ export declare class JSONLoader extends TextLoader {
 export declare class CSVLoader extends TextLoader {
     load(source: string, options?: CSVLoaderOptions): Promise<MDocument>;
     canHandle(source: string): boolean;
-    private parseCSVLine;
     private toMarkdownTable;
     private toTextTable;
+    /**
+     * Collapse embedded newlines to a single space so one logical CSV row
+     * renders as exactly one line in the text/markdown table output.
+     *
+     * The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
+     * preserves newlines embedded inside a quoted field verbatim — correct
+     * for the `json` output format (where `JSON.stringify` escapes them), but
+     * left as-is here it would split a single record across multiple
+     * physical table rows. Mirrors the same normalization `CSVProcessor`
+     * already applies in its own markdown formatter.
+     */
+    private collapseEmbeddedNewlines;
 }
 /**
  * PDF file loader

package/dist/lib/rag/document/loaders.js

@@ -27,6 +27,8 @@ import { existsSync } from "fs";
 import { readFile } from "fs/promises";
 import { basename, extname } from "path";
 import { logger } from "../../utils/logger.js";
+import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, } from "../../utils/csvProcessor.js";
+import { splitCSVFields } from "../../utils/csvUtils.js";
 import { MDocument } from "./MDocument.js";
 /**
  * Text file loader
@@ -115,13 +117,28 @@ export class JSONLoader extends TextLoader {
 export class CSVLoader extends TextLoader {
     async load(source, options) {
         const content = await this.loadContent(source, options?.encoding);
-        const { delimiter = ",", hasHeader = true, columns, outputFormat = "text", } = options || {};
-        const lines = content.split("\n").filter((line) => line.trim());
+        const { delimiter, hasHeader = true, columns, outputFormat = "text", } = options || {};
+        // #361: an explicit delimiter always wins; otherwise detect it from the
+        // content + extension so a .tsv fed through RAG isn't collapsed into one
+        // comma-column (CSVLoader.canHandle already accepts .tsv).
+        const effectiveDelimiter = delimiter ?? CSVProcessor.detectDelimiter(content, extname(source));
+        // Route header/row parsing through the same shared, quote-aware path
+        // detectDelimiter() above already uses internally: a quote-aware logical
+        // row split (splitCsvLines — a newline inside a quoted field stays part
+        // of that field instead of breaking the row) with a leading Excel `sep=`
+        // preamble line stripped (stripMetadataLine), so it can't be parsed as
+        // the header. Field-level splitting also goes through the shared
+        // RFC-4180-aware splitCSVFields() instead of the loader's own naive
+        // backslash-escape parser, which didn't handle doubled-quote (`""`)
+        // escaping or quoted delimiters in the headerless-columns branch.
+        const { dataLines: strippedLines } = stripMetadataLine(splitCsvLines(stripBom(content)));
+        const lines = strippedLines.filter((line) => line.trim());
         const headers = hasHeader
-            ? this.parseCSVLine(lines[0], delimiter)
-            : columns || lines[0]?.split(delimiter).map((_, i) => `col${i + 1}`);
+            ? splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((h) => h.trim())
+            : columns ||
+                splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((_, i) => `col${i + 1}`);
         const dataLines = hasHeader ? lines.slice(1) : lines;
-        const rows = dataLines.map((line) => this.parseCSVLine(line, delimiter));
+        const rows = dataLines.map((line) => splitCSVFields(line, effectiveDelimiter).map((field) => field.trim()));
         let formattedContent;
         switch (outputFormat) {
             case "json":
@@ -145,42 +162,39 @@ export class CSVLoader extends TextLoader {
         const ext = extname(source).toLowerCase();
         return ext === ".csv" || ext === ".tsv";
     }
-    parseCSVLine(line, delimiter) {
-        const result = [];
-        let current = "";
-        let inQuotes = false;
-        for (let i = 0; i < line.length; i++) {
-            const char = line[i];
-            if (char === '"' && (i === 0 || line[i - 1] !== "\\")) {
-                inQuotes = !inQuotes;
-            }
-            else if (char === delimiter && !inQuotes) {
-                result.push(current.trim());
-                current = "";
-            }
-            else {
-                current += char;
-            }
-        }
-        result.push(current.trim());
-        return result;
-    }
     toMarkdownTable(headers, rows) {
-        const headerRow = `| ${headers.join(" | ")} |`;
-        const separator = `| ${headers.map(() => "---").join(" | ")} |`;
-        const dataRows = rows.map((row) => `| ${row.join(" | ")} |`);
+        const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
+        const headerRow = `| ${safeHeaders.join(" | ")} |`;
+        const separator = `| ${safeHeaders.map(() => "---").join(" | ")} |`;
+        const dataRows = rows.map((row) => `| ${row.map((cell) => this.collapseEmbeddedNewlines(cell)).join(" | ")} |`);
         return [headerRow, separator, ...dataRows].join("\n");
     }
     toTextTable(headers, rows) {
-        const allRows = [headers, ...rows];
-        const colWidths = headers.map((_, i) => Math.max(...allRows.map((row) => (row[i] || "").length)));
+        const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
+        const safeRows = rows.map((row) => row.map((cell) => this.collapseEmbeddedNewlines(cell)));
+        const allRows = [safeHeaders, ...safeRows];
+        const colWidths = safeHeaders.map((_, i) => Math.max(...allRows.map((row) => (row[i] || "").length)));
         const formatRow = (row) => row.map((cell, i) => (cell || "").padEnd(colWidths[i])).join(" | ");
         return [
-            formatRow(headers),
+            formatRow(safeHeaders),
             colWidths.map((w) => "-".repeat(w)).join("-+-"),
-            ...rows.map(formatRow),
+            ...safeRows.map(formatRow),
         ].join("\n");
     }
+    /**
+     * Collapse embedded newlines to a single space so one logical CSV row
+     * renders as exactly one line in the text/markdown table output.
+     *
+     * The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
+     * preserves newlines embedded inside a quoted field verbatim — correct
+     * for the `json` output format (where `JSON.stringify` escapes them), but
+     * left as-is here it would split a single record across multiple
+     * physical table rows. Mirrors the same normalization `CSVProcessor`
+     * already applies in its own markdown formatter.
+     */
+    collapseEmbeddedNewlines(cell) {
+        return cell.replace(/\r\n|\n|\r/g, " ");
+    }
 }
 /**
  * PDF file loader

package/dist/lib/utils/csvProcessor.d.ts

@@ -4,6 +4,50 @@
  * Uses streaming for memory efficiency with large files
  */
 import type { FileProcessingResult, CSVProcessorOptions } from "../types/index.js";
+/**
+ * Detect if first line is CSV metadata (not actual data/headers)
+ * Common patterns:
+ * - Excel separator line: "SEP=,"
+ * - Lines with significantly different delimiter count than line 2
+ * - Lines that don't match CSV structure of subsequent lines
+ */
+/**
+ * Strip a leading UTF-8 BOM (U+FEFF) if present. Exported so every entry
+ * point that sniffs raw text (including the RAG CSVLoader) normalizes BOMs
+ * the same way before metadata/delimiter detection.
+ */
+export declare function stripBom(text: string): string;
+/**
+ * Strip a leading metadata line (e.g. Excel's `sep=;` preamble) from
+ * already-split lines, and read the delimiter it explicitly declares.
+ * Excel semantics: an explicit `sep=<char>` always wins over frequency-based
+ * detection; a metadata line without a recognized delimiter character is
+ * simply dropped from the sample without forcing a delimiter. This is the
+ * single place process(), detectDelimiter(), parseCSVFile(), and
+ * parseCSVString() all go through, so none of them detect a delimiter over
+ * the raw, un-stripped lines (which lets the preamble skew detection).
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) share it too.
+ */
+export declare function stripMetadataLine(lines: string[]): {
+    dataLines: string[];
+    hasMetadataLine: boolean;
+    explicitDelimiter?: string;
+};
+/**
+ * Split CSV text into logical rows for metadata detection and raw row limiting.
+ *
+ * Supports Unix (LF), Windows (CRLF), and classic Mac (CR) line endings, and is
+ * quote-aware: a line ending that appears *inside* a quoted field (RFC 4180) is
+ * kept as part of that field rather than treated as a row boundary. A doubled
+ * `""` inside quotes is the RFC-4180 escaped quote and does not toggle the quote
+ * state. For unquoted input this yields exactly the same result as a plain
+ * `split(/\r\n|\n|\r/)`.
+ *
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) get the same
+ * quote-aware row boundaries instead of a physical `content.split("\n")`,
+ * which would break rows on a newline embedded in a quoted field.
+ */
+export declare function splitCsvLines(csvString: string): string[];
 /**
  * CSV processor for converting CSV data to LLM-optimized formats
  *
@@ -35,9 +79,14 @@ export declare class CSVProcessor {
      */
     static process(content: Buffer, options?: CSVProcessorOptions): Promise<FileProcessingResult>;
     /**
-     * Parse CSV string into array of row objects using streaming
-     * Memory-efficient for large files
+     * Detect the delimiter of DSV content (comma/tab/semicolon/pipe), respecting
+     * RFC-4180 quoting. An optional extension hint (`tsv` → tab, `psv` → pipe)
+     * breaks ambiguity. Used by callers (e.g. the RAG CSV loader) that parse
+     * DSV content themselves and need the same detection (#361). A leading
+     * Excel `sep=` metadata line is stripped before sampling so it can't skew
+     * detection, and an explicit `sep=<char>` value wins outright.
      */
+    static detectDelimiter(csvString: string, extensionHint?: string): string;
     /**
      * Parse CSV file from disk using streaming (memory efficient)
      *
@@ -54,7 +103,16 @@ export declare class CSVProcessor {
      * @param maxRows - Maximum rows to parse (default: 1000)
      * @returns Array of row objects
      */
-    static parseCSVString(csvString: string, maxRows?: number): Promise<unknown[]>;
+    static parseCSVString(csvString: string, maxRows?: number, delimiter?: string): Promise<unknown[]>;
+    /**
+     * Stream-parse already-split, metadata-stripped data lines into row
+     * objects with a known delimiter. Shared core for parseCSVString() and
+     * process(), both of which need this after already splitting the content
+     * and resolving the delimiter once — calling parseCSVString() directly
+     * from process() would re-split and re-run metadata/delimiter detection
+     * over the same content a second time.
+     */
+    private static parseCSVLines;
     /**
      * Format parsed CSV data for LLM consumption
      * Only used for JSON and Markdown formats (raw format handled separately)

package/dist/lib/utils/csvProcessor.js

@@ -5,7 +5,9 @@
  */
 import csvParser from "csv-parser";
 import { Readable } from "stream";
+import { extname } from "path";
 import { logger } from "./logger.js";
+import { countCSVColumns, splitCSVFields, detectDelimiter as detectDelimiterUtil, CANDIDATE_DELIMITERS, } from "./csvUtils.js";
 // ============================================================================
 // Data Type Detection Patterns
 // ============================================================================
@@ -439,8 +441,12 @@ function detectHasHeaders(headerValues, dataRows) {
  * - Lines with significantly different delimiter count than line 2
  * - Lines that don't match CSV structure of subsequent lines
  */
-/** Strip a leading UTF-8 BOM (U+FEFF) if present. */
-function stripBom(text) {
+/**
+ * Strip a leading UTF-8 BOM (U+FEFF) if present. Exported so every entry
+ * point that sniffs raw text (including the RAG CSVLoader) normalizes BOMs
+ * the same way before metadata/delimiter detection.
+ */
+export function stripBom(text) {
     return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
 }
 /**
@@ -485,6 +491,31 @@ function isMetadataLine(lines) {
     }
     return false;
 }
+/**
+ * Strip a leading metadata line (e.g. Excel's `sep=;` preamble) from
+ * already-split lines, and read the delimiter it explicitly declares.
+ * Excel semantics: an explicit `sep=<char>` always wins over frequency-based
+ * detection; a metadata line without a recognized delimiter character is
+ * simply dropped from the sample without forcing a delimiter. This is the
+ * single place process(), detectDelimiter(), parseCSVFile(), and
+ * parseCSVString() all go through, so none of them detect a delimiter over
+ * the raw, un-stripped lines (which lets the preamble skew detection).
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) share it too.
+ */
+export function stripMetadataLine(lines) {
+    if (!isMetadataLine(lines)) {
+        return { dataLines: lines, hasMetadataLine: false };
+    }
+    const sepChar = lines[0]?.trim().match(/^sep=(.)/i)?.[1];
+    const explicitDelimiter = sepChar && CANDIDATE_DELIMITERS.includes(sepChar)
+        ? sepChar
+        : undefined;
+    return {
+        dataLines: lines.slice(1),
+        hasMetadataLine: true,
+        explicitDelimiter,
+    };
+}
 /**
  * Split CSV text into logical rows for metadata detection and raw row limiting.
  *
@@ -494,8 +525,12 @@ function isMetadataLine(lines) {
  * `""` inside quotes is the RFC-4180 escaped quote and does not toggle the quote
  * state. For unquoted input this yields exactly the same result as a plain
  * `split(/\r\n|\n|\r/)`.
+ *
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) get the same
+ * quote-aware row boundaries instead of a physical `content.split("\n")`,
+ * which would break rows on a newline embedded in a quoted field.
  */
-function splitCsvLines(csvString) {
+export function splitCsvLines(csvString) {
     const rows = [];
     let current = "";
     let inQuotes = false;
@@ -565,18 +600,22 @@ export class CSVProcessor {
             includeHeaders,
         });
         const csvString = content.toString("utf-8");
+        // #361: split once and strip a leading Excel `sep=` metadata line before
+        // detecting the delimiter (comma/tab/semicolon/pipe), so the preamble
+        // can't skew detection — then reuse dataLines below instead of
+        // re-splitting csvString for the raw-format row handling.
+        const lines = splitCsvLines(csvString);
+        const { dataLines, hasMetadataLine, explicitDelimiter } = stripMetadataLine(lines);
+        const delimiter = explicitDelimiter ??
+            detectDelimiterUtil(dataLines, extension ?? undefined);
         // For raw format, return CSV text with row limit (no parsing needed)
         // This preserves the CSV shape while normalizing line endings for row handling.
         if (formatStyle === "raw") {
-            const lines = splitCsvLines(csvString);
-            const hasMetadataLine = isMetadataLine(lines);
             if (hasMetadataLine) {
                 logger.debug("[CSVProcessor] Detected metadata line, skipping first line");
             }
-            // Skip metadata line if present, then take header + maxRows data rows
-            const csvLines = hasMetadataLine
-                ? lines.slice(1) // Skip metadata line
-                : lines;
+            // dataLines already has the metadata line (if any) stripped.
+            const csvLines = dataLines;
             const limitedLines = csvLines.slice(0, 1 + maxRows); // header + data rows
             const limitedCSV = limitedLines.join("\n");
             const rowCount = limitedLines
@@ -597,11 +636,16 @@ export class CSVProcessor {
             logger.info("[CSVProcessor] ✅ Processed CSV file", {
                 formatStyle: "raw",
                 rowCount,
-                columnCount: (limitedLines[0] || "").split(",").length,
+                columnCount: countCSVColumns(limitedLines[0] || "", delimiter),
                 truncated: wasTruncated,
             });
-            // Parse a sample for enhanced metadata analysis (raw format still benefits from column analysis)
-            const sampleForAnalysis = await this.parseCSVString(limitedCSV, Math.min(rowCount, 500));
+            // Parse a sample for enhanced metadata analysis (raw format still
+            // benefits from column analysis). Reuse the already-split, already
+            // metadata-stripped `dataLines` via the shared parseCSVLines() core
+            // instead of routing `limitedCSV` back through parseCSVString(), which
+            // would re-split/re-join it and re-run a redundant BOM/metadata pass.
+            const sampleRows = Math.min(rowCount, 500);
+            const sampleForAnalysis = await this.parseCSVLines(dataLines.slice(0, 1 + sampleRows), sampleRows, delimiter);
             const { columnMetadata, dataQualityWarnings, dataQualityScore } = analyzeColumns(sampleForAnalysis);
             // Log data quality summary
             if (dataQualityWarnings.length > 0) {
@@ -619,13 +663,13 @@ export class CSVProcessor {
                     size: content.length,
                     rowCount,
                     totalLines: limitedLines.length,
-                    columnCount: (limitedLines[0] || "").split(",").length,
+                    columnCount: countCSVColumns(limitedLines[0] || "", delimiter),
                     extension,
                     columnMetadata,
                     dataQualityWarnings,
                     dataQualityScore,
-                    hasHeaders: detectHasHeaders((limitedLines[0] || "").split(","), undefined),
-                    detectedDelimiter: ",",
+                    hasHeaders: detectHasHeaders(splitCSVFields(limitedLines[0] || "", delimiter), undefined),
+                    detectedDelimiter: delimiter,
                 },
             };
         }
@@ -634,7 +678,10 @@ export class CSVProcessor {
             formatStyle,
             maxRows,
         });
-        const rows = await this.parseCSVString(csvString, maxRows);
+        // dataLines was already split and metadata-stripped above (alongside
+        // `delimiter`) — reuse it instead of routing through parseCSVString(),
+        // which would re-split csvString and re-run metadata detection.
+        const rows = await this.parseCSVLines(dataLines, maxRows, delimiter);
         // Filter out empty rows (empty objects or rows with only whitespace values from blank lines)
         const nonEmptyRows = rows.filter((row) => {
             if (!row || typeof row !== "object") {
@@ -701,14 +748,22 @@ export class CSVProcessor {
                 dataQualityWarnings,
                 dataQualityScore,
                 hasHeaders: detectHasHeaders(columnNames, nonEmptyRows),
-                detectedDelimiter: ",",
+                detectedDelimiter: delimiter,
             },
         };
     }
     /**
-     * Parse CSV string into array of row objects using streaming
-     * Memory-efficient for large files
+     * Detect the delimiter of DSV content (comma/tab/semicolon/pipe), respecting
+     * RFC-4180 quoting. An optional extension hint (`tsv` → tab, `psv` → pipe)
+     * breaks ambiguity. Used by callers (e.g. the RAG CSV loader) that parse
+     * DSV content themselves and need the same detection (#361). A leading
+     * Excel `sep=` metadata line is stripped before sampling so it can't skew
+     * detection, and an explicit `sep=<char>` value wins outright.
      */
+    static detectDelimiter(csvString, extensionHint) {
+        const { dataLines, explicitDelimiter } = stripMetadataLine(splitCsvLines(stripBom(csvString)));
+        return explicitDelimiter ?? detectDelimiterUtil(dataLines, extensionHint);
+    }
     /**
      * Parse CSV file from disk using streaming (memory efficient)
      *
@@ -734,9 +789,13 @@ export class CSVProcessor {
             let buffer = "";
             lineReader.on("data", (chunk) => {
                 buffer += chunk.toString();
-                const splitBuffer = buffer.endsWith("\r")
-                    ? buffer.slice(0, -1)
-                    : buffer;
+                // Strip a leading UTF-8 BOM before the `sep=` check below, same as
+                // process(), parseCSVString(), and detectDelimiter() — otherwise an
+                // Excel export starting with a BOM + "sep=;" fails the /^sep=/i match.
+                const normalizedBuffer = stripBom(buffer);
+                const splitBuffer = normalizedBuffer.endsWith("\r")
+                    ? normalizedBuffer.slice(0, -1)
+                    : normalizedBuffer;
                 const lines = splitCsvLines(splitBuffer);
                 if (lines.length >= 2) {
                     firstLines.push(lines[0], lines[1]);
@@ -745,21 +804,32 @@ export class CSVProcessor {
                 }
             });
             lineReader.on("end", () => resolve());
-            // Metadata sniffing is best-effort — a read error here must not crash.
-            lineReader.on("error", () => resolve());
+            // Metadata sniffing is best-effort — a read error here must not crash
+            // (the main read stream below will surface a real failure properly),
+            // but silently dropping it loses the diagnostic entirely, so log it.
+            lineReader.on("error", (error) => {
+                logger.warn(`[CSVProcessor] Metadata sniffing read failed for ${filePath}, proceeding without it: ${error.message}`, { cause: error });
+                resolve();
+            });
         });
         await fileHandle.close();
-        const hasMetadataLine = isMetadataLine(firstLines);
+        const { dataLines, hasMetadataLine, explicitDelimiter } = stripMetadataLine(firstLines);
         const skipLines = hasMetadataLine ? 1 : 0;
+        // #361: pick the delimiter from the sniffed lines (metadata line
+        // stripped) + file extension so .tsv / semicolon / pipe files aren't
+        // collapsed into one column, and a `sep=` preamble can't skew detection.
+        const fileDelimiter = explicitDelimiter ?? detectDelimiterUtil(dataLines, extname(filePath));
         if (hasMetadataLine) {
             logger.debug("[CSVProcessor] Detected metadata line in file, will skip first line");
         }
         return new Promise((resolve, reject) => {
             const rows = [];
             let count = 0;
-            let lineCount = 0;
             const source = fs.createReadStream(filePath, { encoding: "utf-8" });
-            const parser = csvParser();
+            // Pass skipLines through to csv-parser itself so it skips the `sep=`
+            // metadata line BEFORE picking headers — otherwise csv-parser treats
+            // that line as the header row and the real header becomes a data row.
+            const parser = csvParser({ separator: fileDelimiter, skipLines });
             const abort = () => {
                 source.destroy();
                 parser.destroy();
@@ -773,10 +843,6 @@ export class CSVProcessor {
             source
                 .pipe(parser)
                 .on("data", (row) => {
-                lineCount++;
-                if (lineCount <= skipLines) {
-                    return;
-                }
                 rows.push(row);
                 count++;
                 if (count >= clampedMaxRows) {
@@ -803,32 +869,51 @@ export class CSVProcessor {
      * @param maxRows - Maximum rows to parse (default: 1000)
      * @returns Array of row objects
      */
-    static async parseCSVString(csvString, maxRows = 1000) {
+    static async parseCSVString(csvString, maxRows = 1000, delimiter) {
         if (typeof csvString !== "string" || csvString.trim().length === 0) {
             throw new Error("CSVProcessor.parseCSVString: csvString must be a non-empty string");
         }
         // Strip a leading UTF-8 BOM (common in Excel exports) so it does not glue
         // onto the first column name and break downstream key lookups.
         const normalized = stripBom(csvString);
-        const clampedMaxRows = Math.max(1, Math.min(10000, maxRows));
         logger.debug("[CSVProcessor] Starting string parsing", {
             inputLength: normalized.length,
-            maxRows: clampedMaxRows,
+            maxRows,
         });
-        // Detect and skip metadata line
+        // Detect and skip a leading metadata line before detecting the
+        // delimiter, so a `sep=` preamble can't skew detection (#361).
         const lines = splitCsvLines(normalized);
-        const hasMetadataLine = isMetadataLine(lines);
-        const csvData = hasMetadataLine
-            ? lines.slice(1).join("\n")
-            : lines.join("\n");
+        const { dataLines, hasMetadataLine, explicitDelimiter } = stripMetadataLine(lines);
+        // Caller-provided delimiter wins, then an explicit `sep=` declaration,
+        // then frequency-based detection over the post-metadata lines.
+        const effectiveDelimiter = delimiter ?? explicitDelimiter ?? detectDelimiterUtil(dataLines);
         if (hasMetadataLine) {
             logger.debug("[CSVProcessor] Detected metadata line in string, skipping");
         }
+        return this.parseCSVLines(dataLines, maxRows, effectiveDelimiter);
+    }
+    /**
+     * Stream-parse already-split, metadata-stripped data lines into row
+     * objects with a known delimiter. Shared core for parseCSVString() and
+     * process(), both of which need this after already splitting the content
+     * and resolving the delimiter once — calling parseCSVString() directly
+     * from process() would re-split and re-run metadata/delimiter detection
+     * over the same content a second time.
+     */
+    static parseCSVLines(dataLines, maxRows, delimiter) {
+        const clampedMaxRows = Math.max(1, Math.min(10000, maxRows));
+        const csvData = dataLines.join("\n");
+        if (csvData.trim().length === 0) {
+            // Shared core for both process() and parseCSVString() — use a generic
+            // message rather than naming one specific entry point, since either
+            // caller can land here with empty post-metadata data lines.
+            return Promise.reject(new Error("CSVProcessor: CSV data must be a non-empty string"));
+        }
         return new Promise((resolve, reject) => {
             const rows = [];
             let count = 0;
             const source = Readable.from([csvData]);
-            const parser = csvParser();
+            const parser = csvParser({ separator: delimiter });
             const abort = () => {
                 source.destroy();
                 parser.destroy();

package/dist/lib/utils/csvUtils.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Shared, RFC-4180-aware delimiter-separated-value helpers.
+ *
+ * Consolidates quote-aware field counting/splitting that was previously
+ * duplicated across csvProcessor.ts and fileDetector.ts (#359), and adds
+ * delimiter detection so TSV / semicolon / pipe files parse correctly (#361).
+ * Keeping these in one module prevents the same quoting edge case from having
+ * to be fixed in three places and drifting out of sync.
+ */
+/** Delimiter candidates, in tie-break preference order (comma first). */
+export declare const CANDIDATE_DELIMITERS: readonly [",", "\t", ";", "|"];
+/**
+ * Split a single line into fields, respecting RFC-4180 quoting: a delimiter
+ * inside a `"…"` quoted field does not split, and `""` is an escaped quote.
+ */
+export declare function splitCSVFields(line: string, delimiter?: string): string[];
+/** Count fields in a single line, respecting RFC-4180 quoting. */
+export declare function countCSVColumns(line: string, delimiter?: string): number;
+/**
+ * Detect the delimiter of DSV content by testing comma/tab/semicolon/pipe
+ * against up to 5 sampled non-empty lines and keeping the candidate with the
+ * most consistent (>1) column count. Comma wins ties and the "nothing splits"
+ * case, so plain CSVs are never reclassified. An optional extension hint
+ * (`tsv` → tab, `psv` → pipe) breaks ambiguity in the file's favor.
+ */
+export declare function detectDelimiter(lines: string[], extensionHint?: string): string;

package/dist/lib/utils/csvUtils.js

@@ -0,0 +1,85 @@
+/**
+ * Shared, RFC-4180-aware delimiter-separated-value helpers.
+ *
+ * Consolidates quote-aware field counting/splitting that was previously
+ * duplicated across csvProcessor.ts and fileDetector.ts (#359), and adds
+ * delimiter detection so TSV / semicolon / pipe files parse correctly (#361).
+ * Keeping these in one module prevents the same quoting edge case from having
+ * to be fixed in three places and drifting out of sync.
+ */
+/** Delimiter candidates, in tie-break preference order (comma first). */
+export const CANDIDATE_DELIMITERS = [",", "\t", ";", "|"];
+/**
+ * Split a single line into fields, respecting RFC-4180 quoting: a delimiter
+ * inside a `"…"` quoted field does not split, and `""` is an escaped quote.
+ */
+export function splitCSVFields(line, delimiter = ",") {
+    const fields = [];
+    let field = "";
+    let inQuotes = false;
+    for (let i = 0; i < line.length; i++) {
+        const ch = line[i];
+        if (ch === '"') {
+            if (inQuotes && line[i + 1] === '"') {
+                field += '"';
+                i++; // consume the escaped quote
+                continue;
+            }
+            inQuotes = !inQuotes;
+        }
+        else if (ch === delimiter && !inQuotes) {
+            fields.push(field);
+            field = "";
+        }
+        else {
+            field += ch;
+        }
+    }
+    fields.push(field);
+    return fields;
+}
+/** Count fields in a single line, respecting RFC-4180 quoting. */
+export function countCSVColumns(line, delimiter = ",") {
+    return splitCSVFields(line, delimiter).length;
+}
+/**
+ * Detect the delimiter of DSV content by testing comma/tab/semicolon/pipe
+ * against up to 5 sampled non-empty lines and keeping the candidate with the
+ * most consistent (>1) column count. Comma wins ties and the "nothing splits"
+ * case, so plain CSVs are never reclassified. An optional extension hint
+ * (`tsv` → tab, `psv` → pipe) breaks ambiguity in the file's favor.
+ */
+export function detectDelimiter(lines, extensionHint) {
+    const sample = lines.filter((l) => l.trim() !== "").slice(0, 5);
+    if (sample.length === 0) {
+        return ",";
+    }
+    const hint = extensionHint?.toLowerCase().replace(/^\./, "");
+    const hinted = hint === "tsv" ? "\t" : hint === "psv" ? "|" : undefined;
+    let best = ",";
+    let bestScore = -1;
+    for (const delim of CANDIDATE_DELIMITERS) {
+        const counts = sample.map((l) => countCSVColumns(l, delim));
+        const headerCols = counts[0];
+        if (headerCols <= 1) {
+            continue; // this delimiter doesn't split the header — not the delimiter
+        }
+        const consistent = counts.every((c) => c === headerCols);
+        // Prefer: a matching extension hint, then consistency, then more columns,
+        // with a small comma bias so comma stays the default on ties.
+        const score = (delim === hinted ? 1000 : 0) +
+            (consistent ? 100 : 0) +
+            headerCols +
+            (delim === "," ? 0.5 : 0);
+        if (score > bestScore) {
+            bestScore = score;
+            best = delim;
+        }
+    }
+    // Nothing split multi-column, but the extension says otherwise — honor it.
+    if (bestScore < 0 && hinted) {
+        return hinted;
+    }
+    return best;
+}
+//# sourceMappingURL=csvUtils.js.map

package/dist/rag/document/loaders.d.ts

@@ -61,9 +61,20 @@ export declare class JSONLoader extends TextLoader {
 export declare class CSVLoader extends TextLoader {
     load(source: string, options?: CSVLoaderOptions): Promise<MDocument>;
     canHandle(source: string): boolean;
-    private parseCSVLine;
     private toMarkdownTable;
     private toTextTable;
+    /**
+     * Collapse embedded newlines to a single space so one logical CSV row
+     * renders as exactly one line in the text/markdown table output.
+     *
+     * The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
+     * preserves newlines embedded inside a quoted field verbatim — correct
+     * for the `json` output format (where `JSON.stringify` escapes them), but
+     * left as-is here it would split a single record across multiple
+     * physical table rows. Mirrors the same normalization `CSVProcessor`
+     * already applies in its own markdown formatter.
+     */
+    private collapseEmbeddedNewlines;
 }
 /**
  * PDF file loader