@juspay/neurolink 9.94.1 → 9.94.3

package/dist/lib/utils/csvProcessor.js

@@ -4,8 +4,31 @@
  * Uses streaming for memory efficiency with large files
  */
 import csvParser from "csv-parser";
-import { Readable } from "stream";
+import iconv from "iconv-lite";
+import { Readable, Transform } from "stream";
+import { extname } from "path";
 import { logger } from "./logger.js";
+import { splitCSVFields, detectDelimiter as detectDelimiterUtil, CANDIDATE_DELIMITERS, } from "./csvUtils.js";
+import { isObject } from "./typeUtils.js";
+import { decodeBuffer } from "./textEncoding.js";
+import { ErrorFactory } from "./errorHandling.js";
+import { withTimeout, TimeoutError } from "./async/withTimeout.js";
+// ============================================================================
+// Parse Safety Limits (#371 / #379)
+// ============================================================================
+/**
+ * Upper bound on a single CSV row's byte size (#371). csv-parser defaults to an
+ * effectively-unbounded `maxRowBytes`; a pathological unbroken row would grow
+ * memory without limit and never fire the parser's own error path. 10 MB/row is
+ * generous for legitimate data (e.g. a large JSON blob in one cell) while
+ * bounding a single allocation and making the parser-error cleanup path
+ * reachable/testable.
+ */
+const CSV_MAX_ROW_BYTES = 10 * 1024 * 1024;
+/** Default wall-clock cap for `parseCSVString` (#379). */
+const DEFAULT_CSV_STRING_PARSE_TIMEOUT_MS = 30_000;
+/** Default wall-clock cap for `parseCSVFile` (#379). */
+const DEFAULT_CSV_FILE_PARSE_TIMEOUT_MS = 300_000;
 // ============================================================================
 // Data Type Detection Patterns
 // ============================================================================
@@ -66,6 +89,131 @@ function validateColumnName(name) {
     return issues;
 }
 // ============================================================================
+// Column Name Sanitization (#378)
+// ============================================================================
+/**
+ * Rewrite a raw header into a valid identifier. Trims, replaces non-identifier
+ * runs with `_`, prefixes a `col_` when it starts with a digit or is empty,
+ * then applies the requested case style. Never returns an empty string.
+ */
+export function sanitizeColumnName(name, style = "snake_case", index = 0) {
+    const trimmed = (name ?? "").trim();
+    // Split on runs of non-alphanumeric characters. Using split (linear) instead
+    // of a `^_+|_+$` trim regex avoids the polynomial-backtracking ReDoS CodeQL
+    // flags on long underscore/separator runs, and drops leading/trailing/empty
+    // segments naturally.
+    let parts = trimmed.split(/[^A-Za-z0-9]+/).filter((p) => p.length > 0);
+    let base = parts.join("_");
+    if (base === "") {
+        base = `col_${index + 1}`;
+        parts = [base];
+    }
+    else if (/^[0-9]/.test(base)) {
+        base = `col_${base}`;
+        parts = base.split("_").filter((p) => p.length > 0);
+    }
+    if (style === "camelCase") {
+        return parts
+            .map((p, i) => i === 0
+            ? p.charAt(0).toLowerCase() + p.slice(1)
+            : p.charAt(0).toUpperCase() + p.slice(1))
+            .join("");
+    }
+    return parts.map((p) => p.toLowerCase()).join("_");
+}
+/**
+ * Deduplicate a list of (already sanitized) names by suffixing `_2`, `_3`, … on
+ * collisions, preserving order. The suffixed candidate is bumped until it does
+ * not collide with ANY already-emitted name — so an existing `name_2` can't be
+ * silently overwritten by a generated `name_2` (which the naive counter did for
+ * `["name","name","name_2"]`).
+ */
+export function dedupeColumnNames(names) {
+    const used = new Set();
+    return names.map((name) => {
+        if (!used.has(name)) {
+            used.add(name);
+            return name;
+        }
+        let n = 2;
+        let candidate = `${name}_${n}`;
+        while (used.has(candidate)) {
+            n++;
+            candidate = `${name}_${n}`;
+        }
+        used.add(candidate);
+        return candidate;
+    });
+}
+/**
+ * Compute sanitized (and deduped) column names plus the original→sanitized
+ * mapping for the names that actually changed.
+ */
+function buildColumnNameMapping(headers, style) {
+    const sanitized = dedupeColumnNames(headers.map((h, i) => sanitizeColumnName(h, style, i)));
+    const mapping = headers
+        .map((original, i) => ({ original, sanitized: sanitized[i] }))
+        .filter((m) => m.original !== m.sanitized);
+    return { sanitized, mapping };
+}
+// ============================================================================
+// Row Validation (#384)
+// ============================================================================
+/**
+ * Predicate form of the row-shape invariant: a non-null, non-array object whose
+ * every value is a string or undefined. csv-parser's default output always
+ * satisfies this; the guard is defense-in-depth at the parse boundary.
+ */
+export function isValidCsvRow(row) {
+    if (!isObject(row) || Array.isArray(row)) {
+        return false;
+    }
+    return Object.values(row).every((v) => typeof v === "string" || v === undefined);
+}
+/**
+ * Assertion form used inside the streaming `data` handlers. Throws a
+ * `[CSVProcessor]`-prefixed error (so existing `includes("CSV")` catches still
+ * match) when a row violates the invariant.
+ */
+export function assertValidCsvRow(row, rowNumber) {
+    if (!isValidCsvRow(row)) {
+        throw ErrorFactory.csvRowInvalid(`[CSVProcessor] Invalid CSV row ${rowNumber}: expected a string-keyed object with string values, got ${Array.isArray(row) ? "array" : typeof row}`, rowNumber);
+    }
+}
+// ============================================================================
+// Parse Error Context (#375)
+// ============================================================================
+/**
+ * Build an enriched, consistent parse-failure message shared by all four
+ * source/parser reject sites so they can't re-diverge.
+ *
+ * Deliberately structural-only: `headerCount` and `lastRowColumnCount` report
+ * how many columns the header/last row had — counts, never the actual header
+ * or cell string values. Parsed header text is user-controlled CSV data (and,
+ * for headerless input, may literally be first-row cell values), so it must
+ * never be embedded in a thrown/logged message (#1199).
+ */
+function buildCsvParseErrorMessage(info) {
+    const parts = [
+        `[CSVProcessor] CSV ${info.stage === "read" ? "file read" : "parsing"} failed after ${info.count} row(s)`,
+    ];
+    if (info.location) {
+        parts.push(`(${info.location})`);
+    }
+    parts.push(`: ${info.cause}`);
+    parts.push(typeof info.headerCount === "number"
+        ? `| headerCount: ${info.headerCount}`
+        : "| headerCount: unknown (failed before headers were parsed)");
+    if (typeof info.bytesRead === "number") {
+        parts.push(`| bytesRead: ${info.bytesRead}`);
+    }
+    if (typeof info.lastRowColumnCount === "number") {
+        parts.push(`| lastRow columns: ${info.lastRowColumnCount}`);
+    }
+    parts.push("| Expected RFC 4180 CSV (comma-separated, double-quote escaped)");
+    return parts.join(" ");
+}
+// ============================================================================
 // Data Type Detection
 // ============================================================================
 /**
@@ -439,8 +587,12 @@ function detectHasHeaders(headerValues, dataRows) {
  * - Lines with significantly different delimiter count than line 2
  * - Lines that don't match CSV structure of subsequent lines
  */
-/** Strip a leading UTF-8 BOM (U+FEFF) if present. */
-function stripBom(text) {
+/**
+ * Strip a leading UTF-8 BOM (U+FEFF) if present. Exported so every entry
+ * point that sniffs raw text (including the RAG CSVLoader) normalizes BOMs
+ * the same way before metadata/delimiter detection.
+ */
+export function stripBom(text) {
     return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
 }
 /**
@@ -485,6 +637,31 @@ function isMetadataLine(lines) {
     }
     return false;
 }
+/**
+ * Strip a leading metadata line (e.g. Excel's `sep=;` preamble) from
+ * already-split lines, and read the delimiter it explicitly declares.
+ * Excel semantics: an explicit `sep=<char>` always wins over frequency-based
+ * detection; a metadata line without a recognized delimiter character is
+ * simply dropped from the sample without forcing a delimiter. This is the
+ * single place process(), detectDelimiter(), parseCSVFile(), and
+ * parseCSVString() all go through, so none of them detect a delimiter over
+ * the raw, un-stripped lines (which lets the preamble skew detection).
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) share it too.
+ */
+export function stripMetadataLine(lines) {
+    if (!isMetadataLine(lines)) {
+        return { dataLines: lines, hasMetadataLine: false };
+    }
+    const sepChar = lines[0]?.trim().match(/^sep=(.)/i)?.[1];
+    const explicitDelimiter = sepChar && CANDIDATE_DELIMITERS.includes(sepChar)
+        ? sepChar
+        : undefined;
+    return {
+        dataLines: lines.slice(1),
+        hasMetadataLine: true,
+        explicitDelimiter,
+    };
+}
 /**
  * Split CSV text into logical rows for metadata detection and raw row limiting.
  *
@@ -494,8 +671,12 @@ function isMetadataLine(lines) {
  * `""` inside quotes is the RFC-4180 escaped quote and does not toggle the quote
  * state. For unquoted input this yields exactly the same result as a plain
  * `split(/\r\n|\n|\r/)`.
+ *
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) get the same
+ * quote-aware row boundaries instead of a physical `content.split("\n")`,
+ * which would break rows on a newline embedded in a quoted field.
  */
-function splitCsvLines(csvString) {
+export function splitCsvLines(csvString) {
     const rows = [];
     let current = "";
     let inQuotes = false;
@@ -556,7 +737,7 @@ export class CSVProcessor {
      * @returns Formatted CSV data ready for LLM (JSON or Markdown)
      */
     static async process(content, options) {
-        const { maxRows: rawMaxRows = 1000, formatStyle = "raw", includeHeaders = true, sampleDataFormat = "json", extension = null, } = options || {};
+        const { maxRows: rawMaxRows = 1000, formatStyle = "raw", includeHeaders = true, sampleDataFormat = "json", extension = null, encoding, sanitizeColumnNames = false, columnNameCase = "snake_case", parseTimeoutMs = DEFAULT_CSV_STRING_PARSE_TIMEOUT_MS, } = options || {};
         const maxRows = Math.max(1, Math.min(10000, rawMaxRows));
         logger.debug("[CSVProcessor] Starting CSV processing", {
             contentSize: content.length,
@@ -564,21 +745,41 @@ export class CSVProcessor {
             maxRows,
             includeHeaders,
         });
-        const csvString = content.toString("utf-8");
+        // #362: detect the encoding (BOM → chardet → UTF-8 fallback) or honor the
+        // caller's override, instead of the previous hard-coded UTF-8 decode.
+        const { text: csvString, encoding: detectedEncoding, confidence: encodingConfidence, } = decodeBuffer(content, encoding);
+        // #361: split once and strip a leading Excel `sep=` metadata line before
+        // detecting the delimiter (comma/tab/semicolon/pipe), so the preamble
+        // can't skew detection — then reuse dataLines below instead of
+        // re-splitting csvString for the raw-format row handling.
+        const lines = splitCsvLines(csvString);
+        const { dataLines, hasMetadataLine, explicitDelimiter } = stripMetadataLine(lines);
+        const delimiter = explicitDelimiter ??
+            detectDelimiterUtil(dataLines, extension ?? undefined);
         // For raw format, return CSV text with row limit (no parsing needed)
         // This preserves the CSV shape while normalizing line endings for row handling.
         if (formatStyle === "raw") {
-            const lines = splitCsvLines(csvString);
-            const hasMetadataLine = isMetadataLine(lines);
             if (hasMetadataLine) {
                 logger.debug("[CSVProcessor] Detected metadata line, skipping first line");
             }
-            // Skip metadata line if present, then take header + maxRows data rows
-            const csvLines = hasMetadataLine
-                ? lines.slice(1) // Skip metadata line
-                : lines;
+            // dataLines already has the metadata line (if any) stripped.
+            const csvLines = dataLines;
             const limitedLines = csvLines.slice(0, 1 + maxRows); // header + data rows
+            // #378: opt-in — rewrite the literal header line with sanitized,
+            // deduped identifiers so the raw CSV text shown to the LLM has clean
+            // column names too. (Splits on commas to match the existing raw-branch
+            // columnCount logic.)
+            let rawColumnNameMapping;
+            if (sanitizeColumnNames && limitedLines.length > 0) {
+                const headerCols = splitCSVFields(limitedLines[0] || "", delimiter);
+                const { sanitized, mapping } = buildColumnNameMapping(headerCols, columnNameCase);
+                limitedLines[0] = sanitized.join(delimiter);
+                rawColumnNameMapping = mapping.length > 0 ? mapping : undefined;
+            }
             const limitedCSV = limitedLines.join("\n");
+            // #1199: quote-aware, delimiter-aware split (matches the
+            // sanitizeColumnNames branch above and #1192's detected delimiter).
+            const headerFields = splitCSVFields(limitedLines[0] || "", delimiter);
             const rowCount = limitedLines
                 .slice(1)
                 .filter((line) => line.trim() !== "").length;
@@ -597,11 +798,22 @@ export class CSVProcessor {
             logger.info("[CSVProcessor] ✅ Processed CSV file", {
                 formatStyle: "raw",
                 rowCount,
-                columnCount: (limitedLines[0] || "").split(",").length,
+                columnCount: headerFields.length,
                 truncated: wasTruncated,
             });
-            // Parse a sample for enhanced metadata analysis (raw format still benefits from column analysis)
-            const sampleForAnalysis = await this.parseCSVString(limitedCSV, Math.min(rowCount, 500));
+            // Parse a sample for enhanced metadata analysis (raw format still
+            // benefits from column analysis). Reuse the already-split, already
+            // metadata-stripped `dataLines` and the already-detected `delimiter`
+            // via the shared streamParse() core instead of routing `limitedCSV`
+            // back through parseCSVStringWithMeta(), which would re-split/re-join
+            // it and re-run a redundant BOM/metadata/delimiter-detection pass.
+            const sampleRows = Math.min(rowCount, 500);
+            const { rows: sampleForAnalysis, timedOut: rawTimedOut } = await this.streamParse(Readable.from([dataLines.slice(0, 1 + sampleRows).join("\n")]), undefined, {
+                maxRows: sampleRows,
+                skipLines: 0,
+                timeoutMs: parseTimeoutMs,
+                delimiter,
+            });
             const { columnMetadata, dataQualityWarnings, dataQualityScore } = analyzeColumns(sampleForAnalysis);
             // Log data quality summary
             if (dataQualityWarnings.length > 0) {
@@ -619,13 +831,19 @@ export class CSVProcessor {
                     size: content.length,
                     rowCount,
                     totalLines: limitedLines.length,
-                    columnCount: (limitedLines[0] || "").split(",").length,
+                    columnCount: headerFields.length,
                     extension,
                     columnMetadata,
                     dataQualityWarnings,
                     dataQualityScore,
-                    hasHeaders: detectHasHeaders((limitedLines[0] || "").split(","), undefined),
-                    detectedDelimiter: ",",
+                    hasHeaders: detectHasHeaders(headerFields, undefined),
+                    detectedDelimiter: delimiter,
+                    detectedEncoding,
+                    encodingConfidence,
+                    ...(rawColumnNameMapping
+                        ? { columnNameMapping: rawColumnNameMapping }
+                        : {}),
+                    ...(rawTimedOut ? { parseTimedOut: true } : {}),
                 },
             };
         }
@@ -634,9 +852,13 @@ export class CSVProcessor {
             formatStyle,
             maxRows,
         });
-        const rows = await this.parseCSVString(csvString, maxRows);
+        // dataLines was already split and metadata-stripped above (alongside
+        // `delimiter`) — reuse both via the shared streamParse() core instead of
+        // routing csvString back through parseCSVStringWithMeta(), which would
+        // re-split it and re-run metadata/delimiter detection a second time.
+        const { rows, timedOut: structuredTimedOut } = await this.streamParse(Readable.from([dataLines.join("\n")]), undefined, { maxRows, skipLines: 0, timeoutMs: parseTimeoutMs, delimiter });
         // Filter out empty rows (empty objects or rows with only whitespace values from blank lines)
-        const nonEmptyRows = rows.filter((row) => {
+        const filteredRows = rows.filter((row) => {
             if (!row || typeof row !== "object") {
                 return false;
             }
@@ -647,11 +869,34 @@ export class CSVProcessor {
             // Check if all values are empty or whitespace-only
             return !Object.values(row).every((val) => val === "" || (typeof val === "string" && val.trim() === ""));
         });
+        // #378: opt-in — remap each row's keys from original → sanitized identifiers.
+        let structuredColumnNameMapping;
+        const sanitizedToOriginal = new Map();
+        let nonEmptyRows = filteredRows;
+        if (sanitizeColumnNames && filteredRows.length > 0) {
+            const origHeaders = Object.keys(filteredRows[0]);
+            const { sanitized, mapping } = buildColumnNameMapping(origHeaders, columnNameCase);
+            origHeaders.forEach((original, i) => {
+                if (original !== sanitized[i]) {
+                    sanitizedToOriginal.set(sanitized[i], original);
+                }
+            });
+            nonEmptyRows = filteredRows.map((row) => {
+                // Defense-in-depth: a null-prototype target means an attacker-chosen
+                // header literally named "__proto__"/"constructor"/"prototype" can
+                // only ever create a harmless own property, never touch
+                // Object.prototype (#1199).
+                const out = Object.create(null);
+                origHeaders.forEach((h, i) => {
+                    out[sanitized[i]] = row[h];
+                });
+                return out;
+            });
+            structuredColumnNameMapping = mapping.length > 0 ? mapping : undefined;
+        }
         // Extract metadata from parsed results
         const rowCount = nonEmptyRows.length;
-        const columnNames = nonEmptyRows.length > 0
-            ? Object.keys(nonEmptyRows[0])
-            : [];
+        const columnNames = nonEmptyRows.length > 0 ? Object.keys(nonEmptyRows[0]) : [];
         const columnCount = columnNames.length;
         const hasEmptyColumns = columnNames.some((col) => !col || col.trim() === "");
         const sampleRows = nonEmptyRows.slice(0, 3);
@@ -666,6 +911,15 @@ export class CSVProcessor {
         }
         // Perform enhanced column analysis
         const { columnMetadata, dataQualityWarnings, dataQualityScore } = analyzeColumns(nonEmptyRows);
+        // #378: carry the pre-sanitization header on each renamed column.
+        if (sanitizedToOriginal.size > 0) {
+            for (const col of columnMetadata) {
+                const original = sanitizedToOriginal.get(col.name);
+                if (original) {
+                    col.originalName = original;
+                }
+            }
+        }
         // Log data quality summary
         if (dataQualityWarnings.length > 0) {
             logger.debug("[CSVProcessor] Data quality warnings detected", {
@@ -701,14 +955,28 @@ export class CSVProcessor {
                 dataQualityWarnings,
                 dataQualityScore,
                 hasHeaders: detectHasHeaders(columnNames, nonEmptyRows),
-                detectedDelimiter: ",",
+                detectedDelimiter: delimiter,
+                detectedEncoding,
+                encodingConfidence,
+                ...(structuredColumnNameMapping
+                    ? { columnNameMapping: structuredColumnNameMapping }
+                    : {}),
+                ...(structuredTimedOut ? { parseTimedOut: true } : {}),
             },
         };
     }
     /**
-     * Parse CSV string into array of row objects using streaming
-     * Memory-efficient for large files
+     * Detect the delimiter of DSV content (comma/tab/semicolon/pipe), respecting
+     * RFC-4180 quoting. An optional extension hint (`tsv` → tab, `psv` → pipe)
+     * breaks ambiguity. Used by callers (e.g. the RAG CSV loader) that parse
+     * DSV content themselves and need the same detection (#361). A leading
+     * Excel `sep=` metadata line is stripped before sampling so it can't skew
+     * detection, and an explicit `sep=<char>` value wins outright.
      */
+    static detectDelimiter(csvString, extensionHint) {
+        const { dataLines, explicitDelimiter } = stripMetadataLine(splitCsvLines(stripBom(csvString)));
+        return explicitDelimiter ?? detectDelimiterUtil(dataLines, extensionHint);
+    }
     /**
      * Parse CSV file from disk using streaming (memory efficient)
      *
@@ -716,83 +984,158 @@ export class CSVProcessor {
      * @param maxRows - Maximum rows to parse (default: 1000)
      * @returns Array of row objects
      */
-    static async parseCSVFile(filePath, maxRows = 1000) {
+    static async parseCSVFile(filePath, maxRows = 1000, timeoutMs = DEFAULT_CSV_FILE_PARSE_TIMEOUT_MS) {
+        const { rows } = await this.parseCSVFileWithMeta(filePath, maxRows, timeoutMs);
+        return rows;
+    }
+    /**
+     * File-parse variant that also reports whether the parse timed out, used by
+     * `process()` to surface `metadata.parseTimedOut`.
+     */
+    static async parseCSVFileWithMeta(filePath, maxRows = 1000, timeoutMs = DEFAULT_CSV_FILE_PARSE_TIMEOUT_MS, encoding) {
         if (typeof filePath !== "string" || filePath.trim().length === 0) {
-            throw new Error("CSVProcessor.parseCSVFile: filePath must be a non-empty string");
+            throw ErrorFactory.csvInvalidInput("CSVProcessor.parseCSVFile: filePath must be a non-empty string");
         }
         const clampedMaxRows = Math.max(1, Math.min(10000, maxRows));
         const fs = await import("fs");
+        // #1199: a single wall-clock deadline covers access() + prepareFileSource()
+        // + streamParse() together, not just the streamParse phase — otherwise a
+        // stalled disk/network filesystem during the open/peek steps defeats the
+        // #379 timeout guard entirely.
+        const startTime = Date.now();
+        const remaining = () => Math.max(0, timeoutMs - (Date.now() - startTime));
         logger.debug("[CSVProcessor] Starting file parsing", {
             filePath,
             maxRows: clampedMaxRows,
         });
-        // Read first 2 lines to detect metadata
-        const fileHandle = await fs.promises.open(filePath, "r");
-        const firstLines = [];
-        const lineReader = fileHandle.createReadStream({ encoding: "utf-8" });
-        await new Promise((resolve) => {
-            let buffer = "";
-            lineReader.on("data", (chunk) => {
-                buffer += chunk.toString();
-                const splitBuffer = buffer.endsWith("\r")
-                    ? buffer.slice(0, -1)
-                    : buffer;
-                const lines = splitCsvLines(splitBuffer);
-                if (lines.length >= 2) {
-                    firstLines.push(lines[0], lines[1]);
-                    lineReader.destroy();
-                    resolve();
-                }
-            });
-            lineReader.on("end", () => resolve());
-            // Metadata sniffing is best-effort — a read error here must not crash.
-            lineReader.on("error", () => resolve());
-        });
-        await fileHandle.close();
-        const hasMetadataLine = isMetadataLine(firstLines);
-        const skipLines = hasMetadataLine ? 1 : 0;
-        if (hasMetadataLine) {
-            logger.debug("[CSVProcessor] Detected metadata line in file, will skip first line");
+        // #375: a bad path (ENOENT/EACCES) should fail fast with clear context
+        // instead of a raw Node error surfacing from the stream. access() is a
+        // permission check, not a content read, so #368's single-read contract
+        // (one createReadStream, no second full pass) is preserved. Metadata-line
+        // and delimiter (#361) detection happen inside prepareFileSource() below,
+        // from the same peeked bytes used for encoding detection — not a second
+        // read of the file.
+        try {
+            await fs.promises.access(filePath, fs.constants.R_OK);
+        }
+        catch (error) {
+            throw ErrorFactory.csvFileAccessFailed(`[CSVProcessor] Failed to open CSV file (${filePath}): ${error instanceof Error ? error.message : String(error)}`, filePath, error instanceof Error ? error : undefined);
         }
+        // #368: a SINGLE createReadStream. We peek its leading bytes to sniff the
+        // encoding (#362) and detect a metadata line + delimiter (#361), then
+        // replay those bytes back onto the same stream — no second disk read.
+        const rawSource = fs.createReadStream(filePath);
+        let prepared;
+        try {
+            prepared = await withTimeout(this.prepareFileSource(rawSource, encoding, extname(filePath)), remaining(), "[CSVProcessor] Timed out preparing CSV file source");
+        }
+        catch (error) {
+            rawSource.destroy();
+            if (error instanceof TimeoutError) {
+                // Same contract as streamParse's own timeout path (#379): report a
+                // clean partial result instead of throwing, so the whole call never
+                // hangs OR surfaces a timeout as an exception.
+                logger.warn(`[CSVProcessor] Prepare phase exceeded the ${timeoutMs}ms parse budget for ${filePath}; returning an empty partial result`);
+                return { rows: [], timedOut: true };
+            }
+            throw ErrorFactory.csvParseFailed(buildCsvParseErrorMessage({
+                stage: "read",
+                count: 0,
+                location: filePath,
+                cause: error instanceof Error ? error.message : String(error),
+            }), { filePath }, error instanceof Error ? error : undefined);
+        }
+        return this.streamParse(prepared.source, rawSource, {
+            maxRows: clampedMaxRows,
+            skipLines: prepared.skipLines,
+            timeoutMs: remaining(),
+            location: filePath,
+            delimiter: prepared.delimiter,
+        });
+    }
+    /**
+     * Peek the head of a raw file stream to resolve encoding, metadata-line
+     * skipping, and the delimiter (#361), then return a decoded text stream
+     * that still delivers the whole file from byte 0 (via `unshift`) — a
+     * single disk read (#368/#362).
+     */
+    static prepareFileSource(rawSource, encodingOverride, extensionHint) {
         return new Promise((resolve, reject) => {
-            const rows = [];
-            let count = 0;
-            let lineCount = 0;
-            const source = fs.createReadStream(filePath, { encoding: "utf-8" });
-            const parser = csvParser();
-            const abort = () => {
-                source.destroy();
-                parser.destroy();
+            const PEEK_LIMIT = 64 * 1024;
+            const chunks = [];
+            let total = 0;
+            let settled = false;
+            const cleanup = () => {
+                rawSource.removeListener("data", onData);
+                rawSource.removeListener("end", onEnd);
+                rawSource.removeListener("error", onError);
             };
-            // .pipe() does not forward source errors (disk read/permission failures)
-            // to the parser, so listen on the source directly or it throws unhandled.
-            source.on("error", (error) => {
-                abort();
-                reject(new Error(`[CSVProcessor] CSV file read failed after ${count} row(s) (${filePath}): ${error.message}`));
-            });
-            source
-                .pipe(parser)
-                .on("data", (row) => {
-                lineCount++;
-                if (lineCount <= skipLines) {
+            // #361: resolve the metadata-line skip count and the delimiter
+            // together from the same peeked/decoded text — an explicit `sep=`
+            // line wins outright, otherwise fall back to frequency-based
+            // detection (+ the file extension as a tie-breaker) over the
+            // post-metadata lines.
+            const resolveMeta = (text) => {
+                const { dataLines, hasMetadataLine, explicitDelimiter } = stripMetadataLine(splitCsvLines(text));
+                const delimiter = explicitDelimiter ?? detectDelimiterUtil(dataLines, extensionHint);
+                return { hasMetadataLine, dataLines, delimiter };
+            };
+            const onData = (chunk) => {
+                chunks.push(chunk);
+                total += chunk.length;
+                const head = Buffer.concat(chunks);
+                // isCompleteBuffer=false: `head` is only a peek, not necessarily the
+                // whole file (#1199) — see the confidence note on decodeBuffer.
+                const { text, encoding } = decodeBuffer(head, encodingOverride, false);
+                if (splitCsvLines(text).length >= 2 || total >= PEEK_LIMIT) {
+                    if (settled) {
+                        return;
+                    }
+                    settled = true;
+                    cleanup();
+                    const { hasMetadataLine, delimiter } = resolveMeta(text);
+                    // Replay the consumed head bytes, then decode the full raw stream.
+                    // NOTE (#1282): the encoding is committed from the peeked head to keep
+                    // the parse streaming (which the parse-timeout guard #379 relies on).
+                    // A file whose head is pure ASCII but which switches to a legacy
+                    // encoding only later would be read as UTF-8; pass `encoding`
+                    // explicitly for such files.
+                    rawSource.pause();
+                    rawSource.unshift(head);
+                    const decodeStream = iconv.decodeStream(encoding);
+                    rawSource.on("error", (e) => decodeStream.destroy(e));
+                    resolve({
+                        source: rawSource.pipe(decodeStream),
+                        skipLines: hasMetadataLine ? 1 : 0,
+                        delimiter,
+                    });
+                }
+            };
+            const onEnd = () => {
+                if (settled) {
                     return;
                 }
-                rows.push(row);
-                count++;
-                if (count >= clampedMaxRows) {
-                    logger.debug(`[CSVProcessor] Reached row limit ${clampedMaxRows}, stopping parse`);
-                    abort();
-                    resolve(rows);
+                settled = true;
+                cleanup();
+                // Whole (small) file already buffered — decode it directly. unshift on
+                // an ended stream throws, so replay via an in-memory Readable instead.
+                const head = Buffer.concat(chunks);
+                const { text } = decodeBuffer(head, encodingOverride);
+                const { hasMetadataLine, dataLines, delimiter } = resolveMeta(text);
+                const data = hasMetadataLine ? dataLines.join("\n") : text;
+                resolve({ source: Readable.from([data]), skipLines: 0, delimiter });
+            };
+            const onError = (error) => {
+                if (settled) {
+                    return;
                 }
-            })
-                .on("end", () => {
-                logger.debug(`[CSVProcessor] File parsing complete: ${rows.length} rows parsed`);
-                resolve(rows);
-            })
-                .on("error", (error) => {
-                logger.error("[CSVProcessor] File parsing failed:", error);
-                reject(new Error(`[CSVProcessor] CSV parsing failed after ${count} row(s): ${error.message}`));
-            });
+                settled = true;
+                cleanup();
+                reject(error);
+            };
+            rawSource.on("data", onData);
+            rawSource.on("end", onEnd);
+            rawSource.on("error", onError);
         });
     }
     /**
@@ -801,11 +1144,20 @@ export class CSVProcessor {
      *
      * @param csvString - CSV data as string
      * @param maxRows - Maximum rows to parse (default: 1000)
+     * @param timeoutMs - Wall-clock parse cap; returns partial rows on timeout
      * @returns Array of row objects
      */
-    static async parseCSVString(csvString, maxRows = 1000) {
+    static async parseCSVString(csvString, maxRows = 1000, timeoutMs = DEFAULT_CSV_STRING_PARSE_TIMEOUT_MS) {
+        const { rows } = await this.parseCSVStringWithMeta(csvString, maxRows, timeoutMs);
+        return rows;
+    }
+    /**
+     * String-parse variant that also reports whether the parse timed out, used by
+     * `process()` to surface `metadata.parseTimedOut`.
+     */
+    static async parseCSVStringWithMeta(csvString, maxRows = 1000, timeoutMs = DEFAULT_CSV_STRING_PARSE_TIMEOUT_MS) {
         if (typeof csvString !== "string" || csvString.trim().length === 0) {
-            throw new Error("CSVProcessor.parseCSVString: csvString must be a non-empty string");
+            throw ErrorFactory.csvInvalidInput("CSVProcessor.parseCSVString: csvString must be a non-empty string");
         }
         // Strip a leading UTF-8 BOM (common in Excel exports) so it does not glue
         // onto the first column name and break downstream key lookups.
@@ -815,49 +1167,148 @@ export class CSVProcessor {
             inputLength: normalized.length,
             maxRows: clampedMaxRows,
         });
-        // Detect and skip metadata line
+        // Detect and skip a leading metadata line before detecting the
+        // delimiter, so a `sep=` preamble can't skew detection (#361). No
+        // caller currently needs to override the delimiter here (#1192's own
+        // tests rely on self-detection), so it's always auto-detected.
         const lines = splitCsvLines(normalized);
-        const hasMetadataLine = isMetadataLine(lines);
-        const csvData = hasMetadataLine
-            ? lines.slice(1).join("\n")
-            : lines.join("\n");
+        const { dataLines, hasMetadataLine, explicitDelimiter } = stripMetadataLine(lines);
+        const effectiveDelimiter = explicitDelimiter ?? detectDelimiterUtil(dataLines);
         if (hasMetadataLine) {
             logger.debug("[CSVProcessor] Detected metadata line in string, skipping");
         }
+        const csvData = dataLines.join("\n");
+        if (csvData.trim().length === 0) {
+            throw ErrorFactory.csvInvalidInput("CSVProcessor.parseCSVString: csvString must be a non-empty string");
+        }
+        return this.streamParse(Readable.from([csvData]), undefined, {
+            maxRows: clampedMaxRows,
+            skipLines: 0,
+            timeoutMs,
+            delimiter: effectiveDelimiter,
+        });
+    }
+    /**
+     * Shared streaming consumer for both parse methods. Guarantees:
+     *  - source, rawSource, and parser are all destroyed on ANY settle path
+     *    (#371) — `rawSource` matters for the file-parse path, where `source`
+     *    is a decode Transform piped FROM `rawSource`; `.pipe()` does not
+     *    propagate `.destroy()` upstream, so without this the underlying
+     *    `fs.createReadStream` fd would leak on abort/timeout/error,
+     *  - a wall-clock timeout returns partial rows instead of hanging (#379),
+     *  - every row is validated at the boundary (#384),
+     *  - reject messages carry columns/last-row-shape/format context (#375),
+     *  - the delimiter (#361, comma/tab/semicolon/pipe) detected by the caller
+     *    is honored, instead of csv-parser's hard-coded comma default.
+     *
+     * @param rawSource - The original file stream `source` was piped from, when
+     *   applicable (file-parse path only; `undefined` for string parsing).
+     */
+    static streamParse(source, rawSource, opts) {
         return new Promise((resolve, reject) => {
             const rows = [];
             let count = 0;
-            const source = Readable.from([csvData]);
-            const parser = csvParser();
+            let settled = false;
+            let capturedHeaders;
+            let lastRowColumnCount;
+            const startTime = Date.now();
+            // maxRowBytes bounds a single-row allocation (#371) — without it a
+            // pathological unbroken row grows memory unbounded and never errors.
+            // skipLines drops any pre-header metadata line (e.g. Excel's `sep=,`)
+            // BEFORE csv-parser reads the header, so the real header is used.
+            // separator (#361) defaults to comma when the caller has no better
+            // signal, matching csv-parser's own default.
+            const parser = csvParser({
+                separator: opts.delimiter ?? ",",
+                maxRowBytes: CSV_MAX_ROW_BYTES,
+                ...(opts.skipLines > 0 ? { skipLines: opts.skipLines } : {}),
+            });
             const abort = () => {
                 source.destroy();
+                rawSource?.destroy();
                 parser.destroy();
             };
-            // .pipe() does not forward source-stream errors to the destination, so a
-            // source failure must be listened for directly or it throws as an
-            // unhandled EventEmitter error and can crash the process.
+            const finish = (fn) => {
+                if (settled) {
+                    return;
+                }
+                settled = true;
+                clearTimeout(timer);
+                fn();
+            };
+            const timer = setTimeout(() => {
+                finish(() => {
+                    logger.warn(`[CSVProcessor] Parse timed out after ${Date.now() - startTime}ms with ${rows.length} partial row(s)`);
+                    abort();
+                    resolve({ rows, timedOut: true });
+                });
+            }, opts.timeoutMs);
+            if (typeof timer.unref === "function") {
+                timer.unref();
+            }
+            parser.on("headers", (h) => {
+                capturedHeaders = h;
+            });
+            // .pipe() does not forward source errors (disk/permission failures) to
+            // the destination, so listen on the source directly or it throws.
             source.on("error", (error) => {
-                abort();
-                reject(new Error(`[CSVProcessor] CSV source stream failed after ${count} row(s): ${error.message}`));
+                finish(() => {
+                    abort();
+                    reject(ErrorFactory.csvParseFailed(buildCsvParseErrorMessage({
+                        stage: "read",
+                        count,
+                        headerCount: capturedHeaders?.length,
+                        lastRowColumnCount,
+                        location: opts.location,
+                        cause: error.message,
+                    }), { location: opts.location, rowCount: count }, error));
+                });
             });
             source
                 .pipe(parser)
                 .on("data", (row) => {
+                try {
+                    assertValidCsvRow(row, count + 1);
+                }
+                catch (err) {
+                    finish(() => {
+                        abort();
+                        reject(err);
+                    });
+                    return;
+                }
+                lastRowColumnCount = Object.keys(row).length;
                 rows.push(row);
                 count++;
-                if (count >= clampedMaxRows) {
-                    logger.debug(`[CSVProcessor] Reached row limit ${clampedMaxRows}, stopping parse`);
-                    abort();
-                    resolve(rows);
+                if (count >= opts.maxRows) {
+                    logger.debug(`[CSVProcessor] Reached row limit ${opts.maxRows}, stopping parse`);
+                    finish(() => {
+                        abort();
+                        resolve({ rows, timedOut: false });
+                    });
                 }
             })
                 .on("end", () => {
-                logger.debug(`[CSVProcessor] String parsing complete: ${rows.length} rows parsed`);
-                resolve(rows);
+                finish(() => {
+                    logger.debug(`[CSVProcessor] Parsing complete: ${rows.length} rows parsed`);
+                    resolve({ rows, timedOut: false });
+                });
             })
                 .on("error", (error) => {
-                logger.error("[CSVProcessor] Parsing failed:", error);
-                reject(new Error(`[CSVProcessor] CSV parsing failed after ${count} row(s): ${error.message}`));
+                // #371: destroy the source too — .pipe() does NOT auto-destroy the
+                // upstream when the parser Transform errors, leaking the fd/stream.
+                finish(() => {
+                    abort();
+                    logger.error("[CSVProcessor] Parsing failed:", error);
+                    reject(ErrorFactory.csvParseFailed(buildCsvParseErrorMessage({
+                        stage: "parse",
+                        count,
+                        headerCount: capturedHeaders?.length,
+                        lastRowColumnCount,
+                        location: opts.location,
+                        cause: error.message,
+                    }), { location: opts.location, rowCount: count }, error));
+                });
             });
         });
     }
@@ -896,9 +1347,7 @@ export class CSVProcessor {
         rows.forEach((row) => {
             markdown +=
                 "| " +
-                    headers
-                        .map((h) => escapePipe(String(row[h] || "")))
-                        .join(" | ") +
+                    headers.map((h) => escapePipe(String(row[h] || ""))).join(" | ") +
                     " |\n";
         });
         return markdown;