@juspay/neurolink 9.94.1 → 9.94.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/agent/directTools.js +3 -1
- package/dist/browser/neurolink.min.js +404 -403
- package/dist/cli/factories/commandFactory.d.ts +6 -0
- package/dist/cli/factories/commandFactory.js +36 -8
- package/dist/lib/agent/directTools.js +3 -1
- package/dist/lib/neurolink.js +5 -0
- package/dist/lib/rag/document/loaders.d.ts +12 -1
- package/dist/lib/rag/document/loaders.js +64 -34
- package/dist/lib/types/file.d.ts +43 -0
- package/dist/lib/types/generate.d.ts +3 -11
- package/dist/lib/types/rag.d.ts +7 -0
- package/dist/lib/types/stream.d.ts +2 -6
- package/dist/lib/utils/csvProcessor.d.ts +121 -5
- package/dist/lib/utils/csvProcessor.js +563 -114
- package/dist/lib/utils/csvUtils.d.ts +26 -0
- package/dist/lib/utils/csvUtils.js +85 -0
- package/dist/lib/utils/errorHandling.d.ts +23 -0
- package/dist/lib/utils/errorHandling.js +65 -0
- package/dist/lib/utils/multimodalOptionsBuilder.d.ts +1 -5
- package/dist/lib/utils/textEncoding.d.ts +25 -0
- package/dist/lib/utils/textEncoding.js +141 -0
- package/dist/neurolink.js +5 -0
- package/dist/rag/document/loaders.d.ts +12 -1
- package/dist/rag/document/loaders.js +64 -34
- package/dist/types/file.d.ts +43 -0
- package/dist/types/generate.d.ts +3 -11
- package/dist/types/rag.d.ts +7 -0
- package/dist/types/stream.d.ts +2 -6
- package/dist/utils/csvProcessor.d.ts +121 -5
- package/dist/utils/csvProcessor.js +562 -113
- package/dist/utils/csvUtils.d.ts +26 -0
- package/dist/utils/csvUtils.js +84 -0
- package/dist/utils/errorHandling.d.ts +23 -0
- package/dist/utils/errorHandling.js +65 -0
- package/dist/utils/multimodalOptionsBuilder.d.ts +1 -5
- package/dist/utils/textEncoding.d.ts +25 -0
- package/dist/utils/textEncoding.js +140 -0
- package/package.json +3 -1
|
@@ -178,6 +178,12 @@ export declare class CLICommandFactory {
|
|
|
178
178
|
* Build multimodal input from CLI arguments
|
|
179
179
|
*/
|
|
180
180
|
private static buildGenerateMultimodalInput;
|
|
181
|
+
/**
|
|
182
|
+
* Build CSV processor options from CLI argv (#1199). Shared by executeGenerate
|
|
183
|
+
* and executeRealStream, which previously built byte-for-byte identical
|
|
184
|
+
* objects independently.
|
|
185
|
+
*/
|
|
186
|
+
private static buildCsvOptionsFromArgv;
|
|
181
187
|
/**
|
|
182
188
|
* Build output configuration for generate request
|
|
183
189
|
*/
|
|
@@ -198,6 +198,25 @@ export class CLICommandFactory {
|
|
|
198
198
|
" • markdown: Formatted table (readable, best for small files <100 rows)\n" +
|
|
199
199
|
" • json: Structured JSON array (best for programmatic use, higher tokens)",
|
|
200
200
|
},
|
|
201
|
+
"csv-encoding": {
|
|
202
|
+
type: "string",
|
|
203
|
+
description: "Character encoding for CSV files (e.g. utf-8, utf-16le, windows-1252). Auto-detected when omitted (#362).",
|
|
204
|
+
},
|
|
205
|
+
"csv-sanitize-names": {
|
|
206
|
+
type: "boolean",
|
|
207
|
+
default: false,
|
|
208
|
+
description: "Rewrite CSV column headers into valid identifiers (e.g. 'Price ($)' → 'price') (#378).",
|
|
209
|
+
},
|
|
210
|
+
"csv-name-case": {
|
|
211
|
+
type: "string",
|
|
212
|
+
choices: ["snake_case", "camelCase"],
|
|
213
|
+
default: "snake_case",
|
|
214
|
+
description: "Case style for sanitized CSV column names (used with --csv-sanitize-names) (#378).",
|
|
215
|
+
},
|
|
216
|
+
"csv-parse-timeout-ms": {
|
|
217
|
+
type: "number",
|
|
218
|
+
description: "Wall-clock cap (ms) for CSV parsing; returns partial rows on timeout (#379).",
|
|
219
|
+
},
|
|
201
220
|
model: {
|
|
202
221
|
type: "string",
|
|
203
222
|
description: "Specific model to use (e.g. gemini-2.5-pro, gemini-2.5-flash)",
|
|
@@ -2380,6 +2399,21 @@ export class CLICommandFactory {
|
|
|
2380
2399
|
...(files && { files }),
|
|
2381
2400
|
};
|
|
2382
2401
|
}
|
|
2402
|
+
/**
|
|
2403
|
+
* Build CSV processor options from CLI argv (#1199). Shared by executeGenerate
|
|
2404
|
+
* and executeRealStream, which previously built byte-for-byte identical
|
|
2405
|
+
* objects independently.
|
|
2406
|
+
*/
|
|
2407
|
+
static buildCsvOptionsFromArgv(argv) {
|
|
2408
|
+
return {
|
|
2409
|
+
maxRows: argv.csvMaxRows,
|
|
2410
|
+
formatStyle: argv.csvFormat,
|
|
2411
|
+
encoding: argv.csvEncoding,
|
|
2412
|
+
sanitizeColumnNames: argv.csvSanitizeNames,
|
|
2413
|
+
columnNameCase: argv.csvNameCase,
|
|
2414
|
+
parseTimeoutMs: argv.csvParseTimeoutMs,
|
|
2415
|
+
};
|
|
2416
|
+
}
|
|
2383
2417
|
/**
|
|
2384
2418
|
* Build output configuration for generate request
|
|
2385
2419
|
*/
|
|
@@ -2614,10 +2648,7 @@ export class CLICommandFactory {
|
|
|
2614
2648
|
const inputAudioFormat = inferAudioFormatFromPath(inputAudioPath);
|
|
2615
2649
|
const runGenerate = () => sdk.generate({
|
|
2616
2650
|
input: generateInput,
|
|
2617
|
-
csvOptions:
|
|
2618
|
-
maxRows: argv.csvMaxRows,
|
|
2619
|
-
formatStyle: argv.csvFormat,
|
|
2620
|
-
},
|
|
2651
|
+
csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
|
|
2621
2652
|
videoOptions: {
|
|
2622
2653
|
frames: argv.videoFrames,
|
|
2623
2654
|
quality: argv.videoQuality,
|
|
@@ -2869,10 +2900,7 @@ export class CLICommandFactory {
|
|
|
2869
2900
|
...(videoFiles && { videoFiles }),
|
|
2870
2901
|
...(files && { files }),
|
|
2871
2902
|
},
|
|
2872
|
-
csvOptions:
|
|
2873
|
-
maxRows: argv.csvMaxRows,
|
|
2874
|
-
formatStyle: argv.csvFormat,
|
|
2875
|
-
},
|
|
2903
|
+
csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
|
|
2876
2904
|
videoOptions: {
|
|
2877
2905
|
frames: argv.videoFrames,
|
|
2878
2906
|
quality: argv.videoQuality,
|
|
@@ -357,7 +357,9 @@ export const directAgentTools = {
|
|
|
357
357
|
logger.debug(`[analyzeCSV] Resolved path: ${resolvedPath}`);
|
|
358
358
|
// Parse CSV using streaming from disk (memory efficient)
|
|
359
359
|
logger.info(`[analyzeCSV] Starting CSV parsing (max ${maxRows} rows)...`);
|
|
360
|
-
|
|
360
|
+
// #384: parseCSVFile now returns validated Record<string, string |
|
|
361
|
+
// undefined>[] rows, so the previous unchecked cast is unnecessary.
|
|
362
|
+
const rows = await CSVProcessor.parseCSVFile(resolvedPath, maxRows);
|
|
361
363
|
logger.info(`[analyzeCSV] ✅ CSV parsing complete: ${rows.length} rows`);
|
|
362
364
|
if (rows.length === 0) {
|
|
363
365
|
logger.warn(`[analyzeCSV] No data rows found`);
|
package/dist/lib/neurolink.js
CHANGED
|
@@ -3820,6 +3820,11 @@ Current user's request: ${currentInput}`;
|
|
|
3820
3820
|
evaluationDomain: options.evaluationDomain,
|
|
3821
3821
|
toolUsageContext: options.toolUsageContext,
|
|
3822
3822
|
input: options.input,
|
|
3823
|
+
// CSV processing options must survive the reconstruction into
|
|
3824
|
+
// TextGenerationOptions — the message builder reads them (encoding,
|
|
3825
|
+
// formatting, sanitization, parse timeout). Omitting them silently
|
|
3826
|
+
// dropped every csvOptions field (incl. the CLI --csv-* flags).
|
|
3827
|
+
csvOptions: options.csvOptions,
|
|
3823
3828
|
region: options.region,
|
|
3824
3829
|
tts: options.tts,
|
|
3825
3830
|
stt: options.stt,
|
|
@@ -61,9 +61,20 @@ export declare class JSONLoader extends TextLoader {
|
|
|
61
61
|
export declare class CSVLoader extends TextLoader {
|
|
62
62
|
load(source: string, options?: CSVLoaderOptions): Promise<MDocument>;
|
|
63
63
|
canHandle(source: string): boolean;
|
|
64
|
-
private parseCSVLine;
|
|
65
64
|
private toMarkdownTable;
|
|
66
65
|
private toTextTable;
|
|
66
|
+
/**
|
|
67
|
+
* Collapse embedded newlines to a single space so one logical CSV row
|
|
68
|
+
* renders as exactly one line in the text/markdown table output.
|
|
69
|
+
*
|
|
70
|
+
* The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
|
|
71
|
+
* preserves newlines embedded inside a quoted field verbatim — correct
|
|
72
|
+
* for the `json` output format (where `JSON.stringify` escapes them), but
|
|
73
|
+
* left as-is here it would split a single record across multiple
|
|
74
|
+
* physical table rows. Mirrors the same normalization `CSVProcessor`
|
|
75
|
+
* already applies in its own markdown formatter.
|
|
76
|
+
*/
|
|
77
|
+
private collapseEmbeddedNewlines;
|
|
67
78
|
}
|
|
68
79
|
/**
|
|
69
80
|
* PDF file loader
|
|
@@ -27,6 +27,8 @@ import { existsSync } from "fs";
|
|
|
27
27
|
import { readFile } from "fs/promises";
|
|
28
28
|
import { basename, extname } from "path";
|
|
29
29
|
import { logger } from "../../utils/logger.js";
|
|
30
|
+
import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, dedupeColumnNames, sanitizeColumnName, } from "../../utils/csvProcessor.js";
|
|
31
|
+
import { splitCSVFields } from "../../utils/csvUtils.js";
|
|
30
32
|
import { MDocument } from "./MDocument.js";
|
|
31
33
|
/**
|
|
32
34
|
* Text file loader
|
|
@@ -115,17 +117,48 @@ export class JSONLoader extends TextLoader {
|
|
|
115
117
|
export class CSVLoader extends TextLoader {
|
|
116
118
|
async load(source, options) {
|
|
117
119
|
const content = await this.loadContent(source, options?.encoding);
|
|
118
|
-
const { delimiter
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
120
|
+
const { delimiter, hasHeader = true, columns, outputFormat = "text", sanitizeColumnNames = false, columnNameCase = "snake_case", } = options || {};
|
|
121
|
+
// #361: an explicit delimiter always wins; otherwise detect it from the
|
|
122
|
+
// content + extension so a .tsv fed through RAG isn't collapsed into one
|
|
123
|
+
// comma-column (CSVLoader.canHandle already accepts .tsv).
|
|
124
|
+
const effectiveDelimiter = delimiter ?? CSVProcessor.detectDelimiter(content, extname(source));
|
|
125
|
+
// Route header/row parsing through the same shared, quote-aware path
|
|
126
|
+
// detectDelimiter() above already uses internally: a quote-aware logical
|
|
127
|
+
// row split (splitCsvLines — a newline inside a quoted field stays part
|
|
128
|
+
// of that field instead of breaking the row) with a leading Excel `sep=`
|
|
129
|
+
// preamble line stripped (stripMetadataLine), so it can't be parsed as
|
|
130
|
+
// the header. Field-level splitting also goes through the shared
|
|
131
|
+
// RFC-4180-aware splitCSVFields() instead of the loader's own naive
|
|
132
|
+
// backslash-escape parser, which didn't handle doubled-quote (`""`)
|
|
133
|
+
// escaping or quoted delimiters in the headerless-columns branch.
|
|
134
|
+
const { dataLines: strippedLines } = stripMetadataLine(splitCsvLines(stripBom(content)));
|
|
135
|
+
const lines = strippedLines.filter((line) => line.trim());
|
|
136
|
+
const rawHeaders = hasHeader
|
|
137
|
+
? splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((h) => h.trim())
|
|
138
|
+
: columns ||
|
|
139
|
+
splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((_, i) => `col${i + 1}`);
|
|
140
|
+
// #378: apply the same opt-in header sanitization CSVProcessor offers, so
|
|
141
|
+
// the RAG ingestion path doesn't corrupt "Price ($)"-style headers as keys.
|
|
142
|
+
const headers = sanitizeColumnNames
|
|
143
|
+
? dedupeColumnNames(rawHeaders.map((h, i) => sanitizeColumnName(h, columnNameCase, i)))
|
|
144
|
+
: rawHeaders;
|
|
123
145
|
const dataLines = hasHeader ? lines.slice(1) : lines;
|
|
124
|
-
const rows = dataLines.map((line) =>
|
|
146
|
+
const rows = dataLines.map((line) => splitCSVFields(line, effectiveDelimiter).map((field) => field.trim()));
|
|
125
147
|
let formattedContent;
|
|
126
148
|
switch (outputFormat) {
|
|
127
149
|
case "json":
|
|
128
|
-
formattedContent = JSON.stringify(rows.map((row) =>
|
|
150
|
+
formattedContent = JSON.stringify(rows.map((row) => {
|
|
151
|
+
// Defense-in-depth against a caller-controlled header literally
|
|
152
|
+
// named "__proto__"/"constructor"/"prototype" (#1199);
|
|
153
|
+
// Object.fromEntries can't actually pollute Object.prototype here
|
|
154
|
+
// (spec semantics), but a null-prototype target avoids relying on
|
|
155
|
+
// that for future readers.
|
|
156
|
+
const record = Object.create(null);
|
|
157
|
+
headers.forEach((h, i) => {
|
|
158
|
+
record[h] = row[i];
|
|
159
|
+
});
|
|
160
|
+
return record;
|
|
161
|
+
}), null, 2);
|
|
129
162
|
break;
|
|
130
163
|
case "markdown":
|
|
131
164
|
formattedContent = this.toMarkdownTable(headers, rows);
|
|
@@ -145,42 +178,39 @@ export class CSVLoader extends TextLoader {
|
|
|
145
178
|
const ext = extname(source).toLowerCase();
|
|
146
179
|
return ext === ".csv" || ext === ".tsv";
|
|
147
180
|
}
|
|
148
|
-
parseCSVLine(line, delimiter) {
|
|
149
|
-
const result = [];
|
|
150
|
-
let current = "";
|
|
151
|
-
let inQuotes = false;
|
|
152
|
-
for (let i = 0; i < line.length; i++) {
|
|
153
|
-
const char = line[i];
|
|
154
|
-
if (char === '"' && (i === 0 || line[i - 1] !== "\\")) {
|
|
155
|
-
inQuotes = !inQuotes;
|
|
156
|
-
}
|
|
157
|
-
else if (char === delimiter && !inQuotes) {
|
|
158
|
-
result.push(current.trim());
|
|
159
|
-
current = "";
|
|
160
|
-
}
|
|
161
|
-
else {
|
|
162
|
-
current += char;
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
result.push(current.trim());
|
|
166
|
-
return result;
|
|
167
|
-
}
|
|
168
181
|
toMarkdownTable(headers, rows) {
|
|
169
|
-
const
|
|
170
|
-
const
|
|
171
|
-
const
|
|
182
|
+
const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
|
|
183
|
+
const headerRow = `| ${safeHeaders.join(" | ")} |`;
|
|
184
|
+
const separator = `| ${safeHeaders.map(() => "---").join(" | ")} |`;
|
|
185
|
+
const dataRows = rows.map((row) => `| ${row.map((cell) => this.collapseEmbeddedNewlines(cell)).join(" | ")} |`);
|
|
172
186
|
return [headerRow, separator, ...dataRows].join("\n");
|
|
173
187
|
}
|
|
174
188
|
toTextTable(headers, rows) {
|
|
175
|
-
const
|
|
176
|
-
const
|
|
189
|
+
const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
|
|
190
|
+
const safeRows = rows.map((row) => row.map((cell) => this.collapseEmbeddedNewlines(cell)));
|
|
191
|
+
const allRows = [safeHeaders, ...safeRows];
|
|
192
|
+
const colWidths = safeHeaders.map((_, i) => Math.max(...allRows.map((row) => (row[i] || "").length)));
|
|
177
193
|
const formatRow = (row) => row.map((cell, i) => (cell || "").padEnd(colWidths[i])).join(" | ");
|
|
178
194
|
return [
|
|
179
|
-
formatRow(
|
|
195
|
+
formatRow(safeHeaders),
|
|
180
196
|
colWidths.map((w) => "-".repeat(w)).join("-+-"),
|
|
181
|
-
...
|
|
197
|
+
...safeRows.map(formatRow),
|
|
182
198
|
].join("\n");
|
|
183
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* Collapse embedded newlines to a single space so one logical CSV row
|
|
202
|
+
* renders as exactly one line in the text/markdown table output.
|
|
203
|
+
*
|
|
204
|
+
* The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
|
|
205
|
+
* preserves newlines embedded inside a quoted field verbatim — correct
|
|
206
|
+
* for the `json` output format (where `JSON.stringify` escapes them), but
|
|
207
|
+
* left as-is here it would split a single record across multiple
|
|
208
|
+
* physical table rows. Mirrors the same normalization `CSVProcessor`
|
|
209
|
+
* already applies in its own markdown formatter.
|
|
210
|
+
*/
|
|
211
|
+
collapseEmbeddedNewlines(cell) {
|
|
212
|
+
return cell.replace(/\r\n|\n|\r/g, " ");
|
|
213
|
+
}
|
|
184
214
|
}
|
|
185
215
|
/**
|
|
186
216
|
* PDF file loader
|
package/dist/lib/types/file.d.ts
CHANGED
|
@@ -80,6 +80,17 @@ export type FileProcessingResult = {
|
|
|
80
80
|
hasHeaders?: boolean;
|
|
81
81
|
/** Detected delimiter */
|
|
82
82
|
detectedDelimiter?: string;
|
|
83
|
+
/** Detected (or overridden) character encoding used to decode the CSV (#362) */
|
|
84
|
+
detectedEncoding?: string;
|
|
85
|
+
/** Confidence (0-100) of the detected encoding (#362) */
|
|
86
|
+
encodingConfidence?: number;
|
|
87
|
+
/** Original→sanitized column-name mapping when sanitizeColumnNames is on (#378) */
|
|
88
|
+
columnNameMapping?: Array<{
|
|
89
|
+
original: string;
|
|
90
|
+
sanitized: string;
|
|
91
|
+
}>;
|
|
92
|
+
/** True when the parse hit its time budget and returned partial rows (#379) */
|
|
93
|
+
parseTimedOut?: boolean;
|
|
83
94
|
version?: string;
|
|
84
95
|
estimatedPages?: number | null;
|
|
85
96
|
provider?: string;
|
|
@@ -125,6 +136,8 @@ export type CSVDataQualityWarning = {
|
|
|
125
136
|
*/
|
|
126
137
|
export type CSVColumnMetadata = {
|
|
127
138
|
name: string;
|
|
139
|
+
/** Original header text before sanitization, when sanitizeColumnNames rewrote it (#378) */
|
|
140
|
+
originalName?: string;
|
|
128
141
|
index: number;
|
|
129
142
|
detectedType: CSVColumnDataType;
|
|
130
143
|
/** Confidence of type detection (0-100) */
|
|
@@ -146,6 +159,17 @@ export type CSVColumnMetadata = {
|
|
|
146
159
|
/** Column name validation issues */
|
|
147
160
|
nameIssues?: string[];
|
|
148
161
|
};
|
|
162
|
+
/** A parsed CSV row: string-keyed with string (or missing) cell values (#384). */
|
|
163
|
+
export type CSVRow = Record<string, string | undefined>;
|
|
164
|
+
/** Result of decoding a buffer with encoding detection (#362). */
|
|
165
|
+
export type DecodedBuffer = {
|
|
166
|
+
/** Decoded text with any BOM removed. */
|
|
167
|
+
text: string;
|
|
168
|
+
/** iconv-lite label actually used to decode. */
|
|
169
|
+
encoding: string;
|
|
170
|
+
/** Detection confidence 0-100 (100 for BOM/override, 0 for the UTF-8 fallback). */
|
|
171
|
+
confidence: number;
|
|
172
|
+
};
|
|
149
173
|
/**
|
|
150
174
|
* CSV processor options
|
|
151
175
|
*/
|
|
@@ -155,6 +179,25 @@ export type CSVProcessorOptions = {
|
|
|
155
179
|
includeHeaders?: boolean;
|
|
156
180
|
sampleDataFormat?: SampleDataFormat;
|
|
157
181
|
extension?: string | null;
|
|
182
|
+
/**
|
|
183
|
+
* Character encoding override (#362). When omitted, the encoding is detected
|
|
184
|
+
* from a BOM then `chardet`, falling back to UTF-8. Accepts any label
|
|
185
|
+
* `iconv-lite` supports (e.g. "utf-8", "utf-16le", "windows-1252", "latin1").
|
|
186
|
+
*/
|
|
187
|
+
encoding?: string;
|
|
188
|
+
/**
|
|
189
|
+
* Rewrite column headers into valid identifiers (#378). Opt-in; default false
|
|
190
|
+
* preserves the raw header strings as object keys.
|
|
191
|
+
*/
|
|
192
|
+
sanitizeColumnNames?: boolean;
|
|
193
|
+
/** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
|
|
194
|
+
columnNameCase?: "camelCase" | "snake_case";
|
|
195
|
+
/**
|
|
196
|
+
* Wall-clock cap for the streaming parse in milliseconds (#379). On timeout the
|
|
197
|
+
* parse returns the rows collected so far and flags `metadata.parseTimedOut`,
|
|
198
|
+
* rather than hanging forever. Defaults: 30s for strings, 5min for files.
|
|
199
|
+
*/
|
|
200
|
+
parseTimeoutMs?: number;
|
|
158
201
|
};
|
|
159
202
|
/**
|
|
160
203
|
* PDF API types for different providers
|
|
@@ -15,7 +15,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
|
|
|
15
15
|
import type { MusicOptions, MusicResult } from "./music.js";
|
|
16
16
|
import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
|
|
17
17
|
import type { NeurolinkCredentials } from "./providers.js";
|
|
18
|
-
import type { FileWithMetadata } from "./file.js";
|
|
18
|
+
import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
|
|
19
19
|
import type { WorkflowConfig } from "./workflow.js";
|
|
20
20
|
import type { Schema, Tool, ToolChoice } from "./tools.js";
|
|
21
21
|
import type { StepResult, LanguageModel } from "./providers.js";
|
|
@@ -123,11 +123,7 @@ export type GenerateOptions = {
|
|
|
123
123
|
*/
|
|
124
124
|
music?: MusicOptions;
|
|
125
125
|
};
|
|
126
|
-
csvOptions?:
|
|
127
|
-
maxRows?: number;
|
|
128
|
-
formatStyle?: "raw" | "markdown" | "json";
|
|
129
|
-
includeHeaders?: boolean;
|
|
130
|
-
};
|
|
126
|
+
csvOptions?: CSVProcessorOptions;
|
|
131
127
|
videoOptions?: {
|
|
132
128
|
frames?: number;
|
|
133
129
|
quality?: number;
|
|
@@ -1133,11 +1129,7 @@ export type TextGenerationOptions = {
|
|
|
1133
1129
|
onError?: OnErrorCallback;
|
|
1134
1130
|
expectedOutcome?: string;
|
|
1135
1131
|
evaluationCriteria?: string[];
|
|
1136
|
-
csvOptions?:
|
|
1137
|
-
maxRows?: number;
|
|
1138
|
-
formatStyle?: "raw" | "markdown" | "json";
|
|
1139
|
-
includeHeaders?: boolean;
|
|
1140
|
-
};
|
|
1132
|
+
csvOptions?: CSVProcessorOptions;
|
|
1141
1133
|
enableSummarization?: boolean;
|
|
1142
1134
|
/**
|
|
1143
1135
|
* Skip injecting tool schemas into the system prompt.
|
package/dist/lib/types/rag.d.ts
CHANGED
|
@@ -500,6 +500,13 @@ export type CSVLoaderOptions = LoaderOptions & {
|
|
|
500
500
|
columns?: string[];
|
|
501
501
|
/** Output format */
|
|
502
502
|
outputFormat?: "text" | "json" | "markdown";
|
|
503
|
+
/**
|
|
504
|
+
* Rewrite headers into valid identifiers (#378). Opt-in; default false keeps
|
|
505
|
+
* the raw header strings as JSON keys / table columns.
|
|
506
|
+
*/
|
|
507
|
+
sanitizeColumnNames?: boolean;
|
|
508
|
+
/** Case style used when `sanitizeColumnNames` is on (#378). Default "snake_case". */
|
|
509
|
+
columnNameCase?: "camelCase" | "snake_case";
|
|
503
510
|
};
|
|
504
511
|
/**
|
|
505
512
|
* Abstract document loader type
|
|
@@ -14,7 +14,7 @@ import type { AIModelProviderConfig, NeurolinkCredentials } from "./providers.js
|
|
|
14
14
|
import type { TTSChunk, TTSOptions, TTSResult } from "./tts.js";
|
|
15
15
|
import type { STTOptions, STTResult } from "./stt.js";
|
|
16
16
|
import type { StandardRecord, ValidationSchema } from "./aliases.js";
|
|
17
|
-
import type { FileWithMetadata } from "./file.js";
|
|
17
|
+
import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
|
|
18
18
|
import type { WorkflowConfig } from "./workflow.js";
|
|
19
19
|
import type { LanguageModel, StepResult } from "./providers.js";
|
|
20
20
|
import type { Tool, ToolChoice } from "./tools.js";
|
|
@@ -205,11 +205,7 @@ export type StreamOptions = {
|
|
|
205
205
|
enableProgress?: boolean;
|
|
206
206
|
};
|
|
207
207
|
};
|
|
208
|
-
csvOptions?:
|
|
209
|
-
maxRows?: number;
|
|
210
|
-
formatStyle?: "raw" | "markdown" | "json";
|
|
211
|
-
includeHeaders?: boolean;
|
|
212
|
-
};
|
|
208
|
+
csvOptions?: CSVProcessorOptions;
|
|
213
209
|
videoOptions?: {
|
|
214
210
|
frames?: number;
|
|
215
211
|
quality?: number;
|
|
@@ -3,7 +3,77 @@
|
|
|
3
3
|
* Converts CSV files to LLM-friendly text formats
|
|
4
4
|
* Uses streaming for memory efficiency with large files
|
|
5
5
|
*/
|
|
6
|
-
import type { FileProcessingResult, CSVProcessorOptions } from "../types/index.js";
|
|
6
|
+
import type { FileProcessingResult, CSVProcessorOptions, CSVRow } from "../types/index.js";
|
|
7
|
+
/**
|
|
8
|
+
* Rewrite a raw header into a valid identifier. Trims, replaces non-identifier
|
|
9
|
+
* runs with `_`, prefixes a `col_` when it starts with a digit or is empty,
|
|
10
|
+
* then applies the requested case style. Never returns an empty string.
|
|
11
|
+
*/
|
|
12
|
+
export declare function sanitizeColumnName(name: string, style?: "camelCase" | "snake_case", index?: number): string;
|
|
13
|
+
/**
|
|
14
|
+
* Deduplicate a list of (already sanitized) names by suffixing `_2`, `_3`, … on
|
|
15
|
+
* collisions, preserving order. The suffixed candidate is bumped until it does
|
|
16
|
+
* not collide with ANY already-emitted name — so an existing `name_2` can't be
|
|
17
|
+
* silently overwritten by a generated `name_2` (which the naive counter did for
|
|
18
|
+
* `["name","name","name_2"]`).
|
|
19
|
+
*/
|
|
20
|
+
export declare function dedupeColumnNames(names: string[]): string[];
|
|
21
|
+
/**
|
|
22
|
+
* Predicate form of the row-shape invariant: a non-null, non-array object whose
|
|
23
|
+
* every value is a string or undefined. csv-parser's default output always
|
|
24
|
+
* satisfies this; the guard is defense-in-depth at the parse boundary.
|
|
25
|
+
*/
|
|
26
|
+
export declare function isValidCsvRow(row: unknown): row is CSVRow;
|
|
27
|
+
/**
|
|
28
|
+
* Assertion form used inside the streaming `data` handlers. Throws a
|
|
29
|
+
* `[CSVProcessor]`-prefixed error (so existing `includes("CSV")` catches still
|
|
30
|
+
* match) when a row violates the invariant.
|
|
31
|
+
*/
|
|
32
|
+
export declare function assertValidCsvRow(row: unknown, rowNumber: number): asserts row is CSVRow;
|
|
33
|
+
/**
|
|
34
|
+
* Detect if first line is CSV metadata (not actual data/headers)
|
|
35
|
+
* Common patterns:
|
|
36
|
+
* - Excel separator line: "SEP=,"
|
|
37
|
+
* - Lines with significantly different delimiter count than line 2
|
|
38
|
+
* - Lines that don't match CSV structure of subsequent lines
|
|
39
|
+
*/
|
|
40
|
+
/**
|
|
41
|
+
* Strip a leading UTF-8 BOM (U+FEFF) if present. Exported so every entry
|
|
42
|
+
* point that sniffs raw text (including the RAG CSVLoader) normalizes BOMs
|
|
43
|
+
* the same way before metadata/delimiter detection.
|
|
44
|
+
*/
|
|
45
|
+
export declare function stripBom(text: string): string;
|
|
46
|
+
/**
|
|
47
|
+
* Strip a leading metadata line (e.g. Excel's `sep=;` preamble) from
|
|
48
|
+
* already-split lines, and read the delimiter it explicitly declares.
|
|
49
|
+
* Excel semantics: an explicit `sep=<char>` always wins over frequency-based
|
|
50
|
+
* detection; a metadata line without a recognized delimiter character is
|
|
51
|
+
* simply dropped from the sample without forcing a delimiter. This is the
|
|
52
|
+
* single place process(), detectDelimiter(), parseCSVFile(), and
|
|
53
|
+
* parseCSVString() all go through, so none of them detect a delimiter over
|
|
54
|
+
* the raw, un-stripped lines (which lets the preamble skew detection).
|
|
55
|
+
* Exported so non-CSVProcessor callers (the RAG CSVLoader) share it too.
|
|
56
|
+
*/
|
|
57
|
+
export declare function stripMetadataLine(lines: string[]): {
|
|
58
|
+
dataLines: string[];
|
|
59
|
+
hasMetadataLine: boolean;
|
|
60
|
+
explicitDelimiter?: string;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* Split CSV text into logical rows for metadata detection and raw row limiting.
|
|
64
|
+
*
|
|
65
|
+
* Supports Unix (LF), Windows (CRLF), and classic Mac (CR) line endings, and is
|
|
66
|
+
* quote-aware: a line ending that appears *inside* a quoted field (RFC 4180) is
|
|
67
|
+
* kept as part of that field rather than treated as a row boundary. A doubled
|
|
68
|
+
* `""` inside quotes is the RFC-4180 escaped quote and does not toggle the quote
|
|
69
|
+
* state. For unquoted input this yields exactly the same result as a plain
|
|
70
|
+
* `split(/\r\n|\n|\r/)`.
|
|
71
|
+
*
|
|
72
|
+
* Exported so non-CSVProcessor callers (the RAG CSVLoader) get the same
|
|
73
|
+
* quote-aware row boundaries instead of a physical `content.split("\n")`,
|
|
74
|
+
* which would break rows on a newline embedded in a quoted field.
|
|
75
|
+
*/
|
|
76
|
+
export declare function splitCsvLines(csvString: string): string[];
|
|
7
77
|
/**
|
|
8
78
|
* CSV processor for converting CSV data to LLM-optimized formats
|
|
9
79
|
*
|
|
@@ -35,9 +105,14 @@ export declare class CSVProcessor {
|
|
|
35
105
|
*/
|
|
36
106
|
static process(content: Buffer, options?: CSVProcessorOptions): Promise<FileProcessingResult>;
|
|
37
107
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
108
|
+
* Detect the delimiter of DSV content (comma/tab/semicolon/pipe), respecting
|
|
109
|
+
* RFC-4180 quoting. An optional extension hint (`tsv` → tab, `psv` → pipe)
|
|
110
|
+
* breaks ambiguity. Used by callers (e.g. the RAG CSV loader) that parse
|
|
111
|
+
* DSV content themselves and need the same detection (#361). A leading
|
|
112
|
+
* Excel `sep=` metadata line is stripped before sampling so it can't skew
|
|
113
|
+
* detection, and an explicit `sep=<char>` value wins outright.
|
|
40
114
|
*/
|
|
115
|
+
static detectDelimiter(csvString: string, extensionHint?: string): string;
|
|
41
116
|
/**
|
|
42
117
|
* Parse CSV file from disk using streaming (memory efficient)
|
|
43
118
|
*
|
|
@@ -45,16 +120,57 @@ export declare class CSVProcessor {
|
|
|
45
120
|
* @param maxRows - Maximum rows to parse (default: 1000)
|
|
46
121
|
* @returns Array of row objects
|
|
47
122
|
*/
|
|
48
|
-
static parseCSVFile(filePath: string, maxRows?: number): Promise<
|
|
123
|
+
static parseCSVFile(filePath: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
|
|
124
|
+
/**
|
|
125
|
+
* File-parse variant that also reports whether the parse timed out, used by
|
|
126
|
+
* `process()` to surface `metadata.parseTimedOut`.
|
|
127
|
+
*/
|
|
128
|
+
static parseCSVFileWithMeta(filePath: string, maxRows?: number, timeoutMs?: number, encoding?: string): Promise<{
|
|
129
|
+
rows: CSVRow[];
|
|
130
|
+
timedOut: boolean;
|
|
131
|
+
}>;
|
|
132
|
+
/**
|
|
133
|
+
* Peek the head of a raw file stream to resolve encoding, metadata-line
|
|
134
|
+
* skipping, and the delimiter (#361), then return a decoded text stream
|
|
135
|
+
* that still delivers the whole file from byte 0 (via `unshift`) — a
|
|
136
|
+
* single disk read (#368/#362).
|
|
137
|
+
*/
|
|
138
|
+
private static prepareFileSource;
|
|
49
139
|
/**
|
|
50
140
|
* Parse CSV string to array of row objects
|
|
51
141
|
* Exposed for use by tools that need direct CSV parsing
|
|
52
142
|
*
|
|
53
143
|
* @param csvString - CSV data as string
|
|
54
144
|
* @param maxRows - Maximum rows to parse (default: 1000)
|
|
145
|
+
* @param timeoutMs - Wall-clock parse cap; returns partial rows on timeout
|
|
55
146
|
* @returns Array of row objects
|
|
56
147
|
*/
|
|
57
|
-
static parseCSVString(csvString: string, maxRows?: number): Promise<
|
|
148
|
+
static parseCSVString(csvString: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
|
|
149
|
+
/**
|
|
150
|
+
* String-parse variant that also reports whether the parse timed out, used by
|
|
151
|
+
* `process()` to surface `metadata.parseTimedOut`.
|
|
152
|
+
*/
|
|
153
|
+
static parseCSVStringWithMeta(csvString: string, maxRows?: number, timeoutMs?: number): Promise<{
|
|
154
|
+
rows: CSVRow[];
|
|
155
|
+
timedOut: boolean;
|
|
156
|
+
}>;
|
|
157
|
+
/**
|
|
158
|
+
* Shared streaming consumer for both parse methods. Guarantees:
|
|
159
|
+
* - source, rawSource, and parser are all destroyed on ANY settle path
|
|
160
|
+
* (#371) — `rawSource` matters for the file-parse path, where `source`
|
|
161
|
+
* is a decode Transform piped FROM `rawSource`; `.pipe()` does not
|
|
162
|
+
* propagate `.destroy()` upstream, so without this the underlying
|
|
163
|
+
* `fs.createReadStream` fd would leak on abort/timeout/error,
|
|
164
|
+
* - a wall-clock timeout returns partial rows instead of hanging (#379),
|
|
165
|
+
* - every row is validated at the boundary (#384),
|
|
166
|
+
* - reject messages carry columns/last-row-shape/format context (#375),
|
|
167
|
+
* - the delimiter (#361, comma/tab/semicolon/pipe) detected by the caller
|
|
168
|
+
* is honored, instead of csv-parser's hard-coded comma default.
|
|
169
|
+
*
|
|
170
|
+
* @param rawSource - The original file stream `source` was piped from, when
|
|
171
|
+
* applicable (file-parse path only; `undefined` for string parsing).
|
|
172
|
+
*/
|
|
173
|
+
private static streamParse;
|
|
58
174
|
/**
|
|
59
175
|
* Format parsed CSV data for LLM consumption
|
|
60
176
|
* Only used for JSON and Markdown formats (raw format handled separately)
|