@juspay/neurolink 9.94.1 → 9.94.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/agent/directTools.js +3 -1
  3. package/dist/browser/neurolink.min.js +404 -403
  4. package/dist/cli/factories/commandFactory.d.ts +6 -0
  5. package/dist/cli/factories/commandFactory.js +36 -8
  6. package/dist/lib/agent/directTools.js +3 -1
  7. package/dist/lib/neurolink.js +5 -0
  8. package/dist/lib/rag/document/loaders.d.ts +12 -1
  9. package/dist/lib/rag/document/loaders.js +64 -34
  10. package/dist/lib/types/file.d.ts +43 -0
  11. package/dist/lib/types/generate.d.ts +3 -11
  12. package/dist/lib/types/rag.d.ts +7 -0
  13. package/dist/lib/types/stream.d.ts +2 -6
  14. package/dist/lib/utils/csvProcessor.d.ts +121 -5
  15. package/dist/lib/utils/csvProcessor.js +563 -114
  16. package/dist/lib/utils/csvUtils.d.ts +26 -0
  17. package/dist/lib/utils/csvUtils.js +85 -0
  18. package/dist/lib/utils/errorHandling.d.ts +23 -0
  19. package/dist/lib/utils/errorHandling.js +65 -0
  20. package/dist/lib/utils/multimodalOptionsBuilder.d.ts +1 -5
  21. package/dist/lib/utils/textEncoding.d.ts +25 -0
  22. package/dist/lib/utils/textEncoding.js +141 -0
  23. package/dist/neurolink.js +5 -0
  24. package/dist/rag/document/loaders.d.ts +12 -1
  25. package/dist/rag/document/loaders.js +64 -34
  26. package/dist/types/file.d.ts +43 -0
  27. package/dist/types/generate.d.ts +3 -11
  28. package/dist/types/rag.d.ts +7 -0
  29. package/dist/types/stream.d.ts +2 -6
  30. package/dist/utils/csvProcessor.d.ts +121 -5
  31. package/dist/utils/csvProcessor.js +562 -113
  32. package/dist/utils/csvUtils.d.ts +26 -0
  33. package/dist/utils/csvUtils.js +84 -0
  34. package/dist/utils/errorHandling.d.ts +23 -0
  35. package/dist/utils/errorHandling.js +65 -0
  36. package/dist/utils/multimodalOptionsBuilder.d.ts +1 -5
  37. package/dist/utils/textEncoding.d.ts +25 -0
  38. package/dist/utils/textEncoding.js +140 -0
  39. package/package.json +3 -1
@@ -178,6 +178,12 @@ export declare class CLICommandFactory {
178
178
  * Build multimodal input from CLI arguments
179
179
  */
180
180
  private static buildGenerateMultimodalInput;
181
+ /**
182
+ * Build CSV processor options from CLI argv (#1199). Shared by executeGenerate
183
+ * and executeRealStream, which previously built byte-for-byte identical
184
+ * objects independently.
185
+ */
186
+ private static buildCsvOptionsFromArgv;
181
187
  /**
182
188
  * Build output configuration for generate request
183
189
  */
@@ -198,6 +198,25 @@ export class CLICommandFactory {
198
198
  " • markdown: Formatted table (readable, best for small files <100 rows)\n" +
199
199
  " • json: Structured JSON array (best for programmatic use, higher tokens)",
200
200
  },
201
+ "csv-encoding": {
202
+ type: "string",
203
+ description: "Character encoding for CSV files (e.g. utf-8, utf-16le, windows-1252). Auto-detected when omitted (#362).",
204
+ },
205
+ "csv-sanitize-names": {
206
+ type: "boolean",
207
+ default: false,
208
+ description: "Rewrite CSV column headers into valid identifiers (e.g. 'Price ($)' → 'price') (#378).",
209
+ },
210
+ "csv-name-case": {
211
+ type: "string",
212
+ choices: ["snake_case", "camelCase"],
213
+ default: "snake_case",
214
+ description: "Case style for sanitized CSV column names (used with --csv-sanitize-names) (#378).",
215
+ },
216
+ "csv-parse-timeout-ms": {
217
+ type: "number",
218
+ description: "Wall-clock cap (ms) for CSV parsing; returns partial rows on timeout (#379).",
219
+ },
201
220
  model: {
202
221
  type: "string",
203
222
  description: "Specific model to use (e.g. gemini-2.5-pro, gemini-2.5-flash)",
@@ -2380,6 +2399,21 @@ export class CLICommandFactory {
2380
2399
  ...(files && { files }),
2381
2400
  };
2382
2401
  }
2402
+ /**
2403
+ * Build CSV processor options from CLI argv (#1199). Shared by executeGenerate
2404
+ * and executeRealStream, which previously built byte-for-byte identical
2405
+ * objects independently.
2406
+ */
2407
+ static buildCsvOptionsFromArgv(argv) {
2408
+ return {
2409
+ maxRows: argv.csvMaxRows,
2410
+ formatStyle: argv.csvFormat,
2411
+ encoding: argv.csvEncoding,
2412
+ sanitizeColumnNames: argv.csvSanitizeNames,
2413
+ columnNameCase: argv.csvNameCase,
2414
+ parseTimeoutMs: argv.csvParseTimeoutMs,
2415
+ };
2416
+ }
2383
2417
  /**
2384
2418
  * Build output configuration for generate request
2385
2419
  */
@@ -2614,10 +2648,7 @@ export class CLICommandFactory {
2614
2648
  const inputAudioFormat = inferAudioFormatFromPath(inputAudioPath);
2615
2649
  const runGenerate = () => sdk.generate({
2616
2650
  input: generateInput,
2617
- csvOptions: {
2618
- maxRows: argv.csvMaxRows,
2619
- formatStyle: argv.csvFormat,
2620
- },
2651
+ csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
2621
2652
  videoOptions: {
2622
2653
  frames: argv.videoFrames,
2623
2654
  quality: argv.videoQuality,
@@ -2869,10 +2900,7 @@ export class CLICommandFactory {
2869
2900
  ...(videoFiles && { videoFiles }),
2870
2901
  ...(files && { files }),
2871
2902
  },
2872
- csvOptions: {
2873
- maxRows: argv.csvMaxRows,
2874
- formatStyle: argv.csvFormat,
2875
- },
2903
+ csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
2876
2904
  videoOptions: {
2877
2905
  frames: argv.videoFrames,
2878
2906
  quality: argv.videoQuality,
@@ -357,7 +357,9 @@ export const directAgentTools = {
357
357
  logger.debug(`[analyzeCSV] Resolved path: ${resolvedPath}`);
358
358
  // Parse CSV using streaming from disk (memory efficient)
359
359
  logger.info(`[analyzeCSV] Starting CSV parsing (max ${maxRows} rows)...`);
360
- const rows = (await CSVProcessor.parseCSVFile(resolvedPath, maxRows));
360
+ // #384: parseCSVFile now returns validated Record<string, string |
361
+ // undefined>[] rows, so the previous unchecked cast is unnecessary.
362
+ const rows = await CSVProcessor.parseCSVFile(resolvedPath, maxRows);
361
363
  logger.info(`[analyzeCSV] ✅ CSV parsing complete: ${rows.length} rows`);
362
364
  if (rows.length === 0) {
363
365
  logger.warn(`[analyzeCSV] No data rows found`);
@@ -3820,6 +3820,11 @@ Current user's request: ${currentInput}`;
3820
3820
  evaluationDomain: options.evaluationDomain,
3821
3821
  toolUsageContext: options.toolUsageContext,
3822
3822
  input: options.input,
3823
+ // CSV processing options must survive the reconstruction into
3824
+ // TextGenerationOptions — the message builder reads them (encoding,
3825
+ // formatting, sanitization, parse timeout). Omitting them silently
3826
+ // dropped every csvOptions field (incl. the CLI --csv-* flags).
3827
+ csvOptions: options.csvOptions,
3823
3828
  region: options.region,
3824
3829
  tts: options.tts,
3825
3830
  stt: options.stt,
@@ -61,9 +61,20 @@ export declare class JSONLoader extends TextLoader {
61
61
  export declare class CSVLoader extends TextLoader {
62
62
  load(source: string, options?: CSVLoaderOptions): Promise<MDocument>;
63
63
  canHandle(source: string): boolean;
64
- private parseCSVLine;
65
64
  private toMarkdownTable;
66
65
  private toTextTable;
66
+ /**
67
+ * Collapse embedded newlines to a single space so one logical CSV row
68
+ * renders as exactly one line in the text/markdown table output.
69
+ *
70
+ * The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
71
+ * preserves newlines embedded inside a quoted field verbatim — correct
72
+ * for the `json` output format (where `JSON.stringify` escapes them), but
73
+ * left as-is here it would split a single record across multiple
74
+ * physical table rows. Mirrors the same normalization `CSVProcessor`
75
+ * already applies in its own markdown formatter.
76
+ */
77
+ private collapseEmbeddedNewlines;
67
78
  }
68
79
  /**
69
80
  * PDF file loader
@@ -27,6 +27,8 @@ import { existsSync } from "fs";
27
27
  import { readFile } from "fs/promises";
28
28
  import { basename, extname } from "path";
29
29
  import { logger } from "../../utils/logger.js";
30
+ import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, dedupeColumnNames, sanitizeColumnName, } from "../../utils/csvProcessor.js";
31
+ import { splitCSVFields } from "../../utils/csvUtils.js";
30
32
  import { MDocument } from "./MDocument.js";
31
33
  /**
32
34
  * Text file loader
@@ -115,17 +117,48 @@ export class JSONLoader extends TextLoader {
115
117
  export class CSVLoader extends TextLoader {
116
118
  async load(source, options) {
117
119
  const content = await this.loadContent(source, options?.encoding);
118
- const { delimiter = ",", hasHeader = true, columns, outputFormat = "text", } = options || {};
119
- const lines = content.split("\n").filter((line) => line.trim());
120
- const headers = hasHeader
121
- ? this.parseCSVLine(lines[0], delimiter)
122
- : columns || lines[0]?.split(delimiter).map((_, i) => `col${i + 1}`);
120
+ const { delimiter, hasHeader = true, columns, outputFormat = "text", sanitizeColumnNames = false, columnNameCase = "snake_case", } = options || {};
121
+ // #361: an explicit delimiter always wins; otherwise detect it from the
122
+ // content + extension so a .tsv fed through RAG isn't collapsed into one
123
+ // comma-column (CSVLoader.canHandle already accepts .tsv).
124
+ const effectiveDelimiter = delimiter ?? CSVProcessor.detectDelimiter(content, extname(source));
125
+ // Route header/row parsing through the same shared, quote-aware path
126
+ // detectDelimiter() above already uses internally: a quote-aware logical
127
+ // row split (splitCsvLines — a newline inside a quoted field stays part
128
+ // of that field instead of breaking the row) with a leading Excel `sep=`
129
+ // preamble line stripped (stripMetadataLine), so it can't be parsed as
130
+ // the header. Field-level splitting also goes through the shared
131
+ // RFC-4180-aware splitCSVFields() instead of the loader's own naive
132
+ // backslash-escape parser, which didn't handle doubled-quote (`""`)
133
+ // escaping or quoted delimiters in the headerless-columns branch.
134
+ const { dataLines: strippedLines } = stripMetadataLine(splitCsvLines(stripBom(content)));
135
+ const lines = strippedLines.filter((line) => line.trim());
136
+ const rawHeaders = hasHeader
137
+ ? splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((h) => h.trim())
138
+ : columns ||
139
+ splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((_, i) => `col${i + 1}`);
140
+ // #378: apply the same opt-in header sanitization CSVProcessor offers, so
141
+ // the RAG ingestion path doesn't corrupt "Price ($)"-style headers as keys.
142
+ const headers = sanitizeColumnNames
143
+ ? dedupeColumnNames(rawHeaders.map((h, i) => sanitizeColumnName(h, columnNameCase, i)))
144
+ : rawHeaders;
123
145
  const dataLines = hasHeader ? lines.slice(1) : lines;
124
- const rows = dataLines.map((line) => this.parseCSVLine(line, delimiter));
146
+ const rows = dataLines.map((line) => splitCSVFields(line, effectiveDelimiter).map((field) => field.trim()));
125
147
  let formattedContent;
126
148
  switch (outputFormat) {
127
149
  case "json":
128
- formattedContent = JSON.stringify(rows.map((row) => Object.fromEntries(headers.map((h, i) => [h, row[i]]))), null, 2);
150
+ formattedContent = JSON.stringify(rows.map((row) => {
151
+ // Defense-in-depth against a caller-controlled header literally
152
+ // named "__proto__"/"constructor"/"prototype" (#1199);
153
+ // Object.fromEntries can't actually pollute Object.prototype here
154
+ // (spec semantics), but a null-prototype target avoids relying on
155
+ // that for future readers.
156
+ const record = Object.create(null);
157
+ headers.forEach((h, i) => {
158
+ record[h] = row[i];
159
+ });
160
+ return record;
161
+ }), null, 2);
129
162
  break;
130
163
  case "markdown":
131
164
  formattedContent = this.toMarkdownTable(headers, rows);
@@ -145,42 +178,39 @@ export class CSVLoader extends TextLoader {
145
178
  const ext = extname(source).toLowerCase();
146
179
  return ext === ".csv" || ext === ".tsv";
147
180
  }
148
- parseCSVLine(line, delimiter) {
149
- const result = [];
150
- let current = "";
151
- let inQuotes = false;
152
- for (let i = 0; i < line.length; i++) {
153
- const char = line[i];
154
- if (char === '"' && (i === 0 || line[i - 1] !== "\\")) {
155
- inQuotes = !inQuotes;
156
- }
157
- else if (char === delimiter && !inQuotes) {
158
- result.push(current.trim());
159
- current = "";
160
- }
161
- else {
162
- current += char;
163
- }
164
- }
165
- result.push(current.trim());
166
- return result;
167
- }
168
181
  toMarkdownTable(headers, rows) {
169
- const headerRow = `| ${headers.join(" | ")} |`;
170
- const separator = `| ${headers.map(() => "---").join(" | ")} |`;
171
- const dataRows = rows.map((row) => `| ${row.join(" | ")} |`);
182
+ const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
183
+ const headerRow = `| ${safeHeaders.join(" | ")} |`;
184
+ const separator = `| ${safeHeaders.map(() => "---").join(" | ")} |`;
185
+ const dataRows = rows.map((row) => `| ${row.map((cell) => this.collapseEmbeddedNewlines(cell)).join(" | ")} |`);
172
186
  return [headerRow, separator, ...dataRows].join("\n");
173
187
  }
174
188
  toTextTable(headers, rows) {
175
- const allRows = [headers, ...rows];
176
- const colWidths = headers.map((_, i) => Math.max(...allRows.map((row) => (row[i] || "").length)));
189
+ const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
190
+ const safeRows = rows.map((row) => row.map((cell) => this.collapseEmbeddedNewlines(cell)));
191
+ const allRows = [safeHeaders, ...safeRows];
192
+ const colWidths = safeHeaders.map((_, i) => Math.max(...allRows.map((row) => (row[i] || "").length)));
177
193
  const formatRow = (row) => row.map((cell, i) => (cell || "").padEnd(colWidths[i])).join(" | ");
178
194
  return [
179
- formatRow(headers),
195
+ formatRow(safeHeaders),
180
196
  colWidths.map((w) => "-".repeat(w)).join("-+-"),
181
- ...rows.map(formatRow),
197
+ ...safeRows.map(formatRow),
182
198
  ].join("\n");
183
199
  }
200
+ /**
201
+ * Collapse embedded newlines to a single space so one logical CSV row
202
+ * renders as exactly one line in the text/markdown table output.
203
+ *
204
+ * The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
205
+ * preserves newlines embedded inside a quoted field verbatim — correct
206
+ * for the `json` output format (where `JSON.stringify` escapes them), but
207
+ * left as-is here it would split a single record across multiple
208
+ * physical table rows. Mirrors the same normalization `CSVProcessor`
209
+ * already applies in its own markdown formatter.
210
+ */
211
+ collapseEmbeddedNewlines(cell) {
212
+ return cell.replace(/\r\n|\n|\r/g, " ");
213
+ }
184
214
  }
185
215
  /**
186
216
  * PDF file loader
@@ -80,6 +80,17 @@ export type FileProcessingResult = {
80
80
  hasHeaders?: boolean;
81
81
  /** Detected delimiter */
82
82
  detectedDelimiter?: string;
83
+ /** Detected (or overridden) character encoding used to decode the CSV (#362) */
84
+ detectedEncoding?: string;
85
+ /** Confidence (0-100) of the detected encoding (#362) */
86
+ encodingConfidence?: number;
87
+ /** Original→sanitized column-name mapping when sanitizeColumnNames is on (#378) */
88
+ columnNameMapping?: Array<{
89
+ original: string;
90
+ sanitized: string;
91
+ }>;
92
+ /** True when the parse hit its time budget and returned partial rows (#379) */
93
+ parseTimedOut?: boolean;
83
94
  version?: string;
84
95
  estimatedPages?: number | null;
85
96
  provider?: string;
@@ -125,6 +136,8 @@ export type CSVDataQualityWarning = {
125
136
  */
126
137
  export type CSVColumnMetadata = {
127
138
  name: string;
139
+ /** Original header text before sanitization, when sanitizeColumnNames rewrote it (#378) */
140
+ originalName?: string;
128
141
  index: number;
129
142
  detectedType: CSVColumnDataType;
130
143
  /** Confidence of type detection (0-100) */
@@ -146,6 +159,17 @@ export type CSVColumnMetadata = {
146
159
  /** Column name validation issues */
147
160
  nameIssues?: string[];
148
161
  };
162
+ /** A parsed CSV row: string-keyed with string (or missing) cell values (#384). */
163
+ export type CSVRow = Record<string, string | undefined>;
164
+ /** Result of decoding a buffer with encoding detection (#362). */
165
+ export type DecodedBuffer = {
166
+ /** Decoded text with any BOM removed. */
167
+ text: string;
168
+ /** iconv-lite label actually used to decode. */
169
+ encoding: string;
170
+ /** Detection confidence 0-100 (100 for BOM/override, 0 for the UTF-8 fallback). */
171
+ confidence: number;
172
+ };
149
173
  /**
150
174
  * CSV processor options
151
175
  */
@@ -155,6 +179,25 @@ export type CSVProcessorOptions = {
155
179
  includeHeaders?: boolean;
156
180
  sampleDataFormat?: SampleDataFormat;
157
181
  extension?: string | null;
182
+ /**
183
+ * Character encoding override (#362). When omitted, the encoding is detected
184
+ * from a BOM then `chardet`, falling back to UTF-8. Accepts any label
185
+ * `iconv-lite` supports (e.g. "utf-8", "utf-16le", "windows-1252", "latin1").
186
+ */
187
+ encoding?: string;
188
+ /**
189
+ * Rewrite column headers into valid identifiers (#378). Opt-in; default false
190
+ * preserves the raw header strings as object keys.
191
+ */
192
+ sanitizeColumnNames?: boolean;
193
+ /** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
194
+ columnNameCase?: "camelCase" | "snake_case";
195
+ /**
196
+ * Wall-clock cap for the streaming parse in milliseconds (#379). On timeout the
197
+ * parse returns the rows collected so far and flags `metadata.parseTimedOut`,
198
+ * rather than hanging forever. Defaults: 30s for strings, 5min for files.
199
+ */
200
+ parseTimeoutMs?: number;
158
201
  };
159
202
  /**
160
203
  * PDF API types for different providers
@@ -15,7 +15,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
15
15
  import type { MusicOptions, MusicResult } from "./music.js";
16
16
  import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
17
17
  import type { NeurolinkCredentials } from "./providers.js";
18
- import type { FileWithMetadata } from "./file.js";
18
+ import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
19
19
  import type { WorkflowConfig } from "./workflow.js";
20
20
  import type { Schema, Tool, ToolChoice } from "./tools.js";
21
21
  import type { StepResult, LanguageModel } from "./providers.js";
@@ -123,11 +123,7 @@ export type GenerateOptions = {
123
123
  */
124
124
  music?: MusicOptions;
125
125
  };
126
- csvOptions?: {
127
- maxRows?: number;
128
- formatStyle?: "raw" | "markdown" | "json";
129
- includeHeaders?: boolean;
130
- };
126
+ csvOptions?: CSVProcessorOptions;
131
127
  videoOptions?: {
132
128
  frames?: number;
133
129
  quality?: number;
@@ -1133,11 +1129,7 @@ export type TextGenerationOptions = {
1133
1129
  onError?: OnErrorCallback;
1134
1130
  expectedOutcome?: string;
1135
1131
  evaluationCriteria?: string[];
1136
- csvOptions?: {
1137
- maxRows?: number;
1138
- formatStyle?: "raw" | "markdown" | "json";
1139
- includeHeaders?: boolean;
1140
- };
1132
+ csvOptions?: CSVProcessorOptions;
1141
1133
  enableSummarization?: boolean;
1142
1134
  /**
1143
1135
  * Skip injecting tool schemas into the system prompt.
@@ -500,6 +500,13 @@ export type CSVLoaderOptions = LoaderOptions & {
500
500
  columns?: string[];
501
501
  /** Output format */
502
502
  outputFormat?: "text" | "json" | "markdown";
503
+ /**
504
+ * Rewrite headers into valid identifiers (#378). Opt-in; default false keeps
505
+ * the raw header strings as JSON keys / table columns.
506
+ */
507
+ sanitizeColumnNames?: boolean;
508
+ /** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
509
+ columnNameCase?: "camelCase" | "snake_case";
503
510
  };
504
511
  /**
505
512
  * Abstract document loader type
@@ -14,7 +14,7 @@ import type { AIModelProviderConfig, NeurolinkCredentials } from "./providers.js
14
14
  import type { TTSChunk, TTSOptions, TTSResult } from "./tts.js";
15
15
  import type { STTOptions, STTResult } from "./stt.js";
16
16
  import type { StandardRecord, ValidationSchema } from "./aliases.js";
17
- import type { FileWithMetadata } from "./file.js";
17
+ import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
18
18
  import type { WorkflowConfig } from "./workflow.js";
19
19
  import type { LanguageModel, StepResult } from "./providers.js";
20
20
  import type { Tool, ToolChoice } from "./tools.js";
@@ -205,11 +205,7 @@ export type StreamOptions = {
205
205
  enableProgress?: boolean;
206
206
  };
207
207
  };
208
- csvOptions?: {
209
- maxRows?: number;
210
- formatStyle?: "raw" | "markdown" | "json";
211
- includeHeaders?: boolean;
212
- };
208
+ csvOptions?: CSVProcessorOptions;
213
209
  videoOptions?: {
214
210
  frames?: number;
215
211
  quality?: number;
@@ -3,7 +3,77 @@
3
3
  * Converts CSV files to LLM-friendly text formats
4
4
  * Uses streaming for memory efficiency with large files
5
5
  */
6
- import type { FileProcessingResult, CSVProcessorOptions } from "../types/index.js";
6
+ import type { FileProcessingResult, CSVProcessorOptions, CSVRow } from "../types/index.js";
7
+ /**
8
+ * Rewrite a raw header into a valid identifier. Trims, replaces non-identifier
9
+ * runs with `_`, prefixes a `col_` when it starts with a digit or is empty,
10
+ * then applies the requested case style. Never returns an empty string.
11
+ */
12
+ export declare function sanitizeColumnName(name: string, style?: "camelCase" | "snake_case", index?: number): string;
13
+ /**
14
+ * Deduplicate a list of (already sanitized) names by suffixing `_2`, `_3`, … on
15
+ * collisions, preserving order. The suffixed candidate is bumped until it does
16
+ * not collide with ANY already-emitted name — so an existing `name_2` can't be
17
+ * silently overwritten by a generated `name_2` (which the naive counter did for
18
+ * `["name","name","name_2"]`).
19
+ */
20
+ export declare function dedupeColumnNames(names: string[]): string[];
21
+ /**
22
+ * Predicate form of the row-shape invariant: a non-null, non-array object whose
23
+ * every value is a string or undefined. csv-parser's default output always
24
+ * satisfies this; the guard is defense-in-depth at the parse boundary.
25
+ */
26
+ export declare function isValidCsvRow(row: unknown): row is CSVRow;
27
+ /**
28
+ * Assertion form used inside the streaming `data` handlers. Throws a
29
+ * `[CSVProcessor]`-prefixed error (so existing `includes("CSV")` catches still
30
+ * match) when a row violates the invariant.
31
+ */
32
+ export declare function assertValidCsvRow(row: unknown, rowNumber: number): asserts row is CSVRow;
33
+ /**
34
+ * Detect if first line is CSV metadata (not actual data/headers)
35
+ * Common patterns:
36
+ * - Excel separator line: "SEP=,"
37
+ * - Lines with significantly different delimiter count than line 2
38
+ * - Lines that don't match CSV structure of subsequent lines
39
+ */
40
+ /**
41
+ * Strip a leading UTF-8 BOM (U+FEFF) if present. Exported so every entry
42
+ * point that sniffs raw text (including the RAG CSVLoader) normalizes BOMs
43
+ * the same way before metadata/delimiter detection.
44
+ */
45
+ export declare function stripBom(text: string): string;
46
+ /**
47
+ * Strip a leading metadata line (e.g. Excel's `sep=;` preamble) from
48
+ * already-split lines, and read the delimiter it explicitly declares.
49
+ * Excel semantics: an explicit `sep=<char>` always wins over frequency-based
50
+ * detection; a metadata line without a recognized delimiter character is
51
+ * simply dropped from the sample without forcing a delimiter. This is the
52
+ * single place process(), detectDelimiter(), parseCSVFile(), and
53
+ * parseCSVString() all go through, so none of them detect a delimiter over
54
+ * the raw, un-stripped lines (which lets the preamble skew detection).
55
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) share it too.
56
+ */
57
+ export declare function stripMetadataLine(lines: string[]): {
58
+ dataLines: string[];
59
+ hasMetadataLine: boolean;
60
+ explicitDelimiter?: string;
61
+ };
62
+ /**
63
+ * Split CSV text into logical rows for metadata detection and raw row limiting.
64
+ *
65
+ * Supports Unix (LF), Windows (CRLF), and classic Mac (CR) line endings, and is
66
+ * quote-aware: a line ending that appears *inside* a quoted field (RFC 4180) is
67
+ * kept as part of that field rather than treated as a row boundary. A doubled
68
+ * `""` inside quotes is the RFC-4180 escaped quote and does not toggle the quote
69
+ * state. For unquoted input this yields exactly the same result as a plain
70
+ * `split(/\r\n|\n|\r/)`.
71
+ *
72
+ * Exported so non-CSVProcessor callers (the RAG CSVLoader) get the same
73
+ * quote-aware row boundaries instead of a physical `content.split("\n")`,
74
+ * which would break rows on a newline embedded in a quoted field.
75
+ */
76
+ export declare function splitCsvLines(csvString: string): string[];
7
77
  /**
8
78
  * CSV processor for converting CSV data to LLM-optimized formats
9
79
  *
@@ -35,9 +105,14 @@ export declare class CSVProcessor {
35
105
  */
36
106
  static process(content: Buffer, options?: CSVProcessorOptions): Promise<FileProcessingResult>;
37
107
  /**
38
- * Parse CSV string into array of row objects using streaming
39
- * Memory-efficient for large files
108
+ * Detect the delimiter of DSV content (comma/tab/semicolon/pipe), respecting
109
+ * RFC-4180 quoting. An optional extension hint (`tsv` → tab, `psv` → pipe)
110
+ * breaks ambiguity. Used by callers (e.g. the RAG CSV loader) that parse
111
+ * DSV content themselves and need the same detection (#361). A leading
112
+ * Excel `sep=` metadata line is stripped before sampling so it can't skew
113
+ * detection, and an explicit `sep=<char>` value wins outright.
40
114
  */
115
+ static detectDelimiter(csvString: string, extensionHint?: string): string;
41
116
  /**
42
117
  * Parse CSV file from disk using streaming (memory efficient)
43
118
  *
@@ -45,16 +120,57 @@ export declare class CSVProcessor {
45
120
  * @param maxRows - Maximum rows to parse (default: 1000)
46
121
  * @returns Array of row objects
47
122
  */
48
- static parseCSVFile(filePath: string, maxRows?: number): Promise<unknown[]>;
123
+ static parseCSVFile(filePath: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
124
+ /**
125
+ * File-parse variant that also reports whether the parse timed out, used by
126
+ * `process()` to surface `metadata.parseTimedOut`.
127
+ */
128
+ static parseCSVFileWithMeta(filePath: string, maxRows?: number, timeoutMs?: number, encoding?: string): Promise<{
129
+ rows: CSVRow[];
130
+ timedOut: boolean;
131
+ }>;
132
+ /**
133
+ * Peek the head of a raw file stream to resolve encoding, metadata-line
134
+ * skipping, and the delimiter (#361), then return a decoded text stream
135
+ * that still delivers the whole file from byte 0 (via `unshift`) — a
136
+ * single disk read (#368/#362).
137
+ */
138
+ private static prepareFileSource;
49
139
  /**
50
140
  * Parse CSV string to array of row objects
51
141
  * Exposed for use by tools that need direct CSV parsing
52
142
  *
53
143
  * @param csvString - CSV data as string
54
144
  * @param maxRows - Maximum rows to parse (default: 1000)
145
+ * @param timeoutMs - Wall-clock parse cap; returns partial rows on timeout
55
146
  * @returns Array of row objects
56
147
  */
57
- static parseCSVString(csvString: string, maxRows?: number): Promise<unknown[]>;
148
+ static parseCSVString(csvString: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
149
+ /**
150
+ * String-parse variant that also reports whether the parse timed out, used by
151
+ * `process()` to surface `metadata.parseTimedOut`.
152
+ */
153
+ static parseCSVStringWithMeta(csvString: string, maxRows?: number, timeoutMs?: number): Promise<{
154
+ rows: CSVRow[];
155
+ timedOut: boolean;
156
+ }>;
157
+ /**
158
+ * Shared streaming consumer for both parse methods. Guarantees:
159
+ * - source, rawSource, and parser are all destroyed on ANY settle path
160
+ * (#371) — `rawSource` matters for the file-parse path, where `source`
161
+ * is a decode Transform piped FROM `rawSource`; `.pipe()` does not
162
+ * propagate `.destroy()` upstream, so without this the underlying
163
+ * `fs.createReadStream` fd would leak on abort/timeout/error,
164
+ * - a wall-clock timeout returns partial rows instead of hanging (#379),
165
+ * - every row is validated at the boundary (#384),
166
+ * - reject messages carry columns/last-row-shape/format context (#375),
167
+ * - the delimiter (#361, comma/tab/semicolon/pipe) detected by the caller
168
+ * is honored, instead of csv-parser's hard-coded comma default.
169
+ *
170
+ * @param rawSource - The original file stream `source` was piped from, when
171
+ * applicable (file-parse path only; `undefined` for string parsing).
172
+ */
173
+ private static streamParse;
58
174
  /**
59
175
  * Format parsed CSV data for LLM consumption
60
176
  * Only used for JSON and Markdown formats (raw format handled separately)