jtcsv 2.2.8 → 3.1.0

package/src/types/index.ts

@@ -0,0 +1,275 @@
+/**
+ * TypeScript интерфейсы для проекта jtcsv
+ */
+
+// Базовые типы
+export type AnyObject = Record<string, any>;
+export type AnyArray = any[];
+
+// JSON to CSV интерфейсы
+export interface JsonToCsvOptions {
+  /** CSV delimiter (default: ';') */
+  delimiter?: string;
+  /** Include headers row (default: true) */
+  includeHeaders?: boolean;
+  /** Rename column headers { oldKey: newKey } */
+  renameMap?: Record<string, string>;
+  /** Template for guaranteed column order */
+  template?: Record<string, any>;
+  /** Maximum number of records to process (optional, no limit by default) */
+  maxRecords?: number;
+  /** Prevent CSV injection attacks by escaping formulas (default: true) */
+  preventCsvInjection?: boolean;
+  /** Ensure RFC 4180 compliance (proper quoting, line endings) (default: true) */
+  rfc4180Compliant?: boolean;
+  /** Normalize excessive quotes in JSON string values before CSV export (default: true) */
+  normalizeQuotes?: boolean;
+  /** JSON schema for data validation and formatting */
+  schema?: Record<string, any>;
+  /** Whether to flatten nested objects into dot notation keys */
+  flatten?: boolean;
+  /** Separator for flattened keys (e.g., 'user.name' with '.') */
+  flattenSeparator?: string;
+  /** Maximum depth for flattening nested objects */
+  flattenMaxDepth?: number;
+  /** How to handle arrays ('stringify', 'join', 'expand') */
+  arrayHandling?: 'stringify' | 'join' | 'expand';
+  /** Warn when record count exceeds this threshold (default: 1000000) */
+  memoryWarningThreshold?: number;
+  /** Safety limit for in-memory conversion (default: 5000000). Set to Infinity to disable. */
+  memoryLimit?: number;
+}
+
+export interface SaveAsCsvOptions extends JsonToCsvOptions {
+  /** Validate file path security (default: true) */
+  validatePath?: boolean;
+}
+
+// CSV to JSON интерфейсы
+export interface CsvToJsonOptions {
+  /** CSV delimiter (default: auto-detected) */
+  delimiter?: string;
+  /** Auto-detect delimiter if not specified (default: true) */
+  autoDetect?: boolean;
+  /** Candidate delimiters for auto-detection (default: [';', ',', '\t', '|']) */
+  candidates?: string[];
+  /** Whether CSV has headers row (default: true) */
+  hasHeaders?: boolean;
+  /** Map for renaming column headers { newKey: oldKey } */
+  renameMap?: Record<string, string>;
+  /** Trim whitespace from values (default: true) */
+  trim?: boolean;
+  /** Parse numeric values (default: false) */
+  parseNumbers?: boolean;
+  /** Parse boolean values (default: false) */
+  parseBooleans?: boolean;
+  /** Maximum number of rows to process (optional, no limit by default) */
+  maxRows?: number;
+  /** Enable fast-path parsing (default: true) */
+  useFastPath?: boolean;
+  /** Fast-path output mode (default: 'objects') */
+  fastPathMode?: 'objects' | 'compact' | 'stream';
+  /** JSON schema for validation and formatting */
+  schema?: Record<string, any>;
+  /** Custom transform function for each row */
+  transform?: (row: Record<string, any>) => Record<string, any>;
+  /** Use delimiter cache for auto-detection (default: true) */
+  useCache?: boolean;
+  /** Custom delimiter cache instance */
+  cache?: any; // DelimiterCache type will be imported later
+  /** Hooks for custom processing */
+  hooks?: {
+    beforeConvert?: (csv: string, options: CsvToJsonOptions) => string;
+    perRow?: (row: Record<string, any>, index: number, context?: { options: CsvToJsonOptions }) => Record<string, any>;
+    afterConvert?: (result: any[], options: CsvToJsonOptions) => any[];
+    transformHooks?: import('../core/transform-hooks').TransformHooks;
+    onError?: (error: Error, csv: string, options: CsvToJsonOptions) => void;
+  };
+  /** Prevent CSV injection attacks by escaping formulas (default: true) */
+  preventCsvInjection?: boolean;
+  /** Ensure RFC 4180 compliance (proper quoting, line endings) (default: true) */
+  rfc4180Compliant?: boolean;
+  /** Warn about extra fields not in headers (default: false) */
+  warnExtraFields?: boolean;
+  /** Error recovery strategy for row-level errors (default: 'throw') */
+  onError?: 'skip' | 'warn' | 'throw';
+  /** Custom error handler for row-level errors */
+  errorHandler?: (error: Error, line: string, lineNumber: number) => void;
+  /** Warn when row count exceeds this threshold (default: 1000000) */
+  memoryWarningThreshold?: number;
+  /** Safety limit for in-memory conversion (default: 5000000). Set to Infinity to disable. */
+  memoryLimit?: number;
+  /** Attempt to repair shifted rows with trailing empty fields (default: true) */
+  repairRowShifts?: boolean;
+  /** Normalize excessive quotes in parsed fields (default: true) */
+  normalizeQuotes?: boolean;
+}
+
+// JSON save интерфейсы
+export interface SaveAsJsonOptions {
+  /** Format JSON with indentation (default: false) */
+  prettyPrint?: boolean;
+  /** Maximum file size in bytes (default: 10MB = 10485760) */
+  maxSize?: number;
+}
+
+// Streaming интерфейсы
+export interface JsonToCsvStreamOptions extends JsonToCsvOptions {
+  /** Custom transform function for each row */
+  transform?: (row: Record<string, any>) => Record<string, any>;
+  /** JSON schema for validation and formatting */
+  schema?: Record<string, any>;
+  /** Add UTF-8 BOM for Excel compatibility (default: true) */
+  addBOM?: boolean;
+}
+
+export interface CsvToJsonStreamOptions extends CsvToJsonOptions {
+  /** Custom transform function for each row */
+  transform?: (row: Record<string, any>) => Record<string, any>;
+  /** JSON schema for validation and formatting */
+  schema?: Record<string, any>;
+}
+
+// NDJSON интерфейсы
+export interface NdjsonOptions {
+  /** Buffer size for streaming (default: 64KB) */
+  bufferSize?: number;
+  /** Maximum line length (default: 10MB) */
+  maxLineLength?: number;
+  /** Error handler callback */
+  onError?: (error: Error, line: string, lineNumber: number) => void;
+  /** JSON stringify replacer function */
+  replacer?: (key: string, value: any) => any;
+  /** JSON stringify space (indentation) */
+  space?: number | string;
+  /** Filter function for rows */
+  filter?: (obj: Record<string, any>, index: number) => boolean;
+  /** Transform function for rows */
+  transform?: (obj: Record<string, any>, index: number) => any;
+}
+
+export interface NdjsonToCsvStreamOptions {
+  /** Delimiter for CSV output (default: ',') */
+  delimiter?: string;
+  /** Include headers row (default: true) */
+  includeHeaders?: boolean;
+  /** Rename column headers { oldKey: newKey } */
+  renameMap?: Record<string, string>;
+  /** Maximum number of records to process */
+  maxRecords?: number;
+  /** Prevent CSV injection attacks by escaping formulas (default: true) */
+  preventCsvInjection?: boolean;
+  /** Ensure RFC 4180 compliance (proper quoting, line endings) (default: true) */
+  rfc4180Compliant?: boolean;
+  /** JSON schema for validation and formatting */
+  schema?: Record<string, any>;
+  /** Custom transform function for each row */
+  transform?: (row: Record<string, any>) => Record<string, any>;
+  /** Add UTF-8 BOM for Excel compatibility (default: true) */
+  addBOM?: boolean;
+}
+
+// TSV интерфейсы
+export interface TsvOptions {
+  /** Whether TSV has headers row (default: true) */
+  hasHeaders?: boolean;
+  /** Trim whitespace from values (default: true) */
+  trim?: boolean;
+  /** Parse numeric values (default: false) */
+  parseNumbers?: boolean;
+  /** Parse boolean values (default: false) */
+  parseBooleans?: boolean;
+  /** Maximum number of rows to process */
+  maxRows?: number;
+  /** JSON schema for validation and formatting */
+  schema?: Record<string, any>;
+  /** Custom transform function for each row */
+  transform?: (row: Record<string, any>) => Record<string, any>;
+}
+
+export interface TsvValidationResult {
+  /** Whether the content is valid TSV */
+  isValid: boolean;
+  /** Number of rows detected */
+  rowCount: number;
+  /** Number of columns (if consistent) */
+  columnCount?: number;
+  /** Error message if invalid */
+  error?: string;
+  /** Line number where error occurred */
+  errorLine?: number;
+}
+
+export interface ValidateTsvOptions {
+  /** Maximum number of rows to check */
+  maxRows?: number;
+  /** Whether to check for consistent column count */
+  checkConsistency?: boolean;
+  /** Whether to validate TSV format strictly */
+  strict?: boolean;
+}
+
+// Worker интерфейсы для многопоточной обработки
+export interface WorkerTask<T = any, R = any> {
+  id: string;
+  data: T;
+  type: string;
+  options?: Record<string, any>;
+}
+
+export interface WorkerResult<R = any> {
+  id: string;
+  result: R;
+  error?: Error;
+  duration: number;
+}
+
+export interface WorkerPoolStats {
+  totalWorkers: number;
+  activeWorkers: number;
+  idleWorkers: number;
+  totalTasks: number;
+  completedTasks: number;
+  failedTasks: number;
+  averageTaskDuration: number;
+  memoryUsage?: NodeJS.MemoryUsage;
+}
+
+// Асинхронные интерфейсы
+export interface AsyncJsonToCsvOptions extends JsonToCsvOptions {
+  /** Use worker threads for processing (default: auto-detect based on data size) */
+  useWorkers?: boolean;
+  /** Number of worker threads to use (default: CPU cores - 1) */
+  workerCount?: number;
+  /** Size of data chunks for parallel processing */
+  chunkSize?: number;
+  /** Progress callback function */
+  onProgress?: (progress: { processed: number; total: number; percentage: number }) => void;
+}
+
+export interface AsyncCsvToJsonOptions extends CsvToJsonOptions {
+  /** Use worker threads for processing (default: auto-detect based on data size) */
+  useWorkers?: boolean;
+  /** Number of worker threads to use (default: CPU cores - 1) */
+  workerCount?: number;
+  /** Size of data chunks for parallel processing */
+  chunkSize?: number;
+  /** Progress callback function */
+  onProgress?: (progress: { processed: number; total: number; percentage: number }) => void;
+}
+
+// Утилитарные типы
+export type PreprocessOptions = {
+  flatten?: boolean;
+  flattenSeparator?: string;
+  flattenMaxDepth?: number;
+  arrayHandling?: 'stringify' | 'join' | 'expand';
+};
+
+export type DeepUnwrapOptions = {
+  maxDepth?: number;
+  preserveArrays?: boolean;
+};
+
+// Экспорт всех интерфейсов
+// Примечание: Типы ошибок экспортируются из errors.ts

package/src/utils/bom-utils.ts

@@ -0,0 +1,373 @@
+/**
+ * BOM (Byte Order Mark) Utilities for jtcsv
+ * 
+ * Provides functions to detect and strip BOM characters from UTF-8, UTF-16 LE/BE,
+ * and UTF-32 encoded strings/buffers.
+ * 
+ * @module bom-utils
+ */
+
+import { Transform } from 'stream';
+import * as fs from 'fs';
+import * as fsPromises from 'fs/promises';
+
+/**
+ * BOM signatures for different encodings
+ */
+export const BOM_SIGNATURES = {
+  'utf-8': Buffer.from([0xEF, 0xBB, 0xBF]),
+  'utf-16le': Buffer.from([0xFF, 0xFE]),
+  'utf-16be': Buffer.from([0xFE, 0xFF]),
+  'utf-32le': Buffer.from([0xFF, 0xFE, 0x00, 0x00]),
+  'utf-32be': Buffer.from([0x00, 0x00, 0xFE, 0xFF])
+} as const;
+
+export type Encoding = keyof typeof BOM_SIGNATURES;
+
+export interface BomDetectionResult {
+  encoding: Encoding;
+  bomLength: number;
+  hasBom: boolean;
+}
+
+export interface ReadFileWithBomResult {
+  data: Buffer | string;
+  encoding: string;
+  hadBom: boolean;
+  bomInfo: BomDetectionResult | null;
+}
+
+export interface NormalizeCsvInputOptions {
+  encoding?: string;
+  normalizeLineEndings?: boolean;
+}
+
+/**
+ * Detects if a buffer or string starts with a BOM
+ * 
+ * @param input - Input to check for BOM
+ * @returns Detection result or null if no BOM found
+ */
+export function detectBom(input: Buffer | string | null | undefined): BomDetectionResult | null {
+  if (!input) {
+    return null;
+  }
+  
+  let buffer: Buffer;
+  if (typeof input === 'string') {
+    buffer = Buffer.from(input, 'utf8');
+  } else if (Buffer.isBuffer(input)) {
+    buffer = input;
+  } else {
+    return null;
+  }
+  
+  // Check each BOM signature
+  for (const [encoding, signature] of Object.entries(BOM_SIGNATURES)) {
+    if (buffer.length >= signature.length) {
+      if (buffer.slice(0, signature.length).equals(signature)) {
+        return {
+          encoding: encoding as Encoding,
+          bomLength: signature.length,
+          hasBom: true
+        };
+      }
+    }
+  }
+  
+  return null;
+}
+
+/**
+ * Strips BOM from a buffer or string
+ * 
+ * @param input - Input to strip BOM from
+ * @returns Input without BOM
+ */
+export function stripBom(input: Buffer | string | null | undefined): Buffer | string {
+  if (!input) {
+    return input as any;
+  }
+  
+  const bomInfo = detectBom(input);
+  if (!bomInfo) {
+    return input;
+  }
+  
+  if (Buffer.isBuffer(input)) {
+    return input.slice(bomInfo.bomLength);
+  }
+  
+  if (typeof input === 'string') {
+    // Convert to buffer, strip BOM, then convert back to string
+    const buffer = Buffer.from(input, 'utf8');
+    const strippedBuffer = buffer.slice(bomInfo.bomLength);
+    
+    // Determine correct encoding for conversion
+    let encoding: BufferEncoding = 'utf8';
+    if (bomInfo.encoding === 'utf-16le') {
+      encoding = 'utf16le';
+    } else if (bomInfo.encoding === 'utf-16be') {
+      encoding = 'utf16le'; // Node.js uses utf16le for both LE and BE, conversion handled by Buffer
+    }
+    
+    return strippedBuffer.toString(encoding);
+  }
+  
+  return input;
+}
+
+/**
+ * Strips BOM from a string (optimized for strings)
+ * 
+ * @param str - String to strip BOM from
+ * @returns String without BOM
+ */
+export function stripBomFromString(str: string): string {
+  if (typeof str !== 'string') {
+    return str as any;
+  }
+  
+  // Check for UTF-8 BOM (most common)
+  if (str.charCodeAt(0) === 0xFEFF) {
+    return str.slice(1);
+  }
+  
+  // Check for UTF-8 BOM bytes as characters
+  if (str.length >= 3 && 
+      str.charCodeAt(0) === 0xEF && 
+      str.charCodeAt(1) === 0xBB && 
+      str.charCodeAt(2) === 0xBF) {
+    return str.slice(3);
+  }
+  
+  return str;
+}
+
+/**
+ * Creates a transform stream that strips BOM from incoming data
+ * 
+ * @returns Transform stream
+ */
+export function createBomStripStream(): Transform {
+  let bomStripped = false;
+  
+  return new Transform({
+    transform(chunk: Buffer, encoding: string, callback: (error?: Error | null, data?: Buffer) => void) {
+      if (!bomStripped) {
+        const bomInfo = detectBom(chunk);
+        if (bomInfo) {
+          // Strip BOM from first chunk
+          chunk = chunk.slice(bomInfo.bomLength);
+          bomStripped = true;
+        } else {
+          bomStripped = true; // No BOM found, but we've checked
+        }
+      }
+      
+      this.push(chunk);
+      callback();
+    }
+  });
+}
+
+/**
+ * Reads a file and automatically handles BOM
+ * 
+ * @param filePath - Path to file
+ * @param options - Read options
+ * @returns Promise with file data and BOM info
+ */
+export async function readFileWithBomHandling(
+  filePath: string, 
+  options: { encoding?: BufferEncoding } = {}
+): Promise<ReadFileWithBomResult> {
+  const buffer = await fsPromises.readFile(filePath);
+  
+  const bomInfo = detectBom(buffer);
+  const hadBom = !!bomInfo;
+  
+  let data: Buffer | string;
+  let encoding = options.encoding || 'utf8';
+  
+  if (bomInfo) {
+    // Strip BOM
+    data = buffer.slice(bomInfo.bomLength);
+    
+    // Use detected encoding if not specified
+    if (!options.encoding) {
+      // Convert our encoding names to Node.js BufferEncoding
+      if (bomInfo.encoding === 'utf-8') {
+        encoding = 'utf8';
+      } else if (bomInfo.encoding === 'utf-16le') {
+        encoding = 'utf16le';
+      } else if (bomInfo.encoding === 'utf-16be') {
+        encoding = 'utf16le'; // Node.js uses utf16le for both
+      } else {
+        encoding = 'utf8'; // fallback
+      }
+    }
+  } else {
+    data = buffer;
+  }
+  
+  // Convert to string if encoding is specified
+  if (options.encoding || bomInfo) {
+    data = data.toString(encoding);
+  }
+  
+  return {
+    data,
+    encoding,
+    hadBom,
+    bomInfo: bomInfo || null
+  };
+}
+
+/**
+ * Checks if a file has BOM (synchronous)
+ * 
+ * @param filePath - Path to file
+ * @returns BOM info or null
+ */
+export function fileHasBomSync(filePath: string): BomDetectionResult | null {
+  const fd = fs.openSync(filePath, 'r');
+  const buffer = Buffer.alloc(4);
+  const bytesRead = fs.readSync(fd, buffer, 0, 4, 0);
+  fs.closeSync(fd);
+  
+  if (bytesRead < 2) {
+    return null;
+  }
+  
+  return detectBom(buffer.slice(0, bytesRead));
+}
+
+/**
+ * Normalizes CSV input by stripping BOM and ensuring proper encoding
+ * 
+ * @param csvInput - CSV input
+ * @param options - Processing options
+ * @returns Normalized CSV string
+ */
+export function normalizeCsvInput(
+  csvInput: string | Buffer, 
+  options: NormalizeCsvInputOptions = {}
+): string {
+  if (!csvInput) {
+    return '';
+  }
+  
+  let normalized: string;
+  
+  if (Buffer.isBuffer(csvInput)) {
+    const bomInfo = detectBom(csvInput);
+    if (bomInfo) {
+      normalized = csvInput.slice(bomInfo.bomLength).toString(bomInfo.encoding as BufferEncoding);
+    } else {
+      normalized = csvInput.toString((options.encoding as BufferEncoding) || 'utf8');
+    }
+  } else if (typeof csvInput === 'string') {
+    normalized = stripBomFromString(csvInput);
+  } else {
+    throw new Error('CSV input must be a string or Buffer');
+  }
+  
+  // Ensure proper line endings
+  if (options.normalizeLineEndings !== false) {
+    normalized = normalized.replace(/\r\n|\r/g, '\n');
+  }
+  
+  return normalized;
+}
+
+/**
+ * Async version of normalizeCsvInput that can handle large files
+ * 
+ * @param csvInput - CSV input as string, Buffer, or file path
+ * @param options - Processing options
+ * @returns Promise with normalized CSV string
+ */
+export async function normalizeCsvInputAsync(
+  csvInput: string | Buffer | { filePath: string },
+  options: NormalizeCsvInputOptions = {}
+): Promise<string> {
+  if (typeof csvInput === 'object' && 'filePath' in csvInput) {
+    // Read file asynchronously
+    const result = await readFileWithBomHandling(csvInput.filePath, {
+      encoding: options.encoding as BufferEncoding || 'utf8'
+    });
+    let normalized = typeof result.data === 'string' ? result.data : result.data.toString();
+    
+    // Ensure proper line endings
+    if (options.normalizeLineEndings !== false) {
+      normalized = normalized.replace(/\r\n|\r/g, '\n');
+    }
+    
+    return normalized;
+  }
+  
+  // Handle string or Buffer input
+  return normalizeCsvInput(csvInput as string | Buffer, options);
+}
+
+/**
+ * Creates an async iterator that strips BOM from a stream
+ * 
+ * @param stream - Readable stream
+ * @returns Async iterator yielding chunks without BOM
+ */
+export async function* createBomStrippingIterator(
+  stream: NodeJS.ReadableStream
+): AsyncIterableIterator<Buffer> {
+  let bomStripped = false;
+  
+  for await (const chunk of stream) {
+    if (!bomStripped) {
+      const bomInfo = detectBom(chunk as Buffer);
+      if (bomInfo) {
+        // Strip BOM from first chunk
+        yield (chunk as Buffer).slice(bomInfo.bomLength);
+        bomStripped = true;
+        continue;
+      } else {
+        bomStripped = true;
+      }
+    }
+    
+    yield chunk as Buffer;
+  }
+}
+
+/**
+ * Detects BOM asynchronously for large files
+ * 
+ * @param filePath - Path to file
+ * @returns Promise with BOM info or null
+ */
+export async function detectBomAsync(filePath: string): Promise<BomDetectionResult | null> {
+  const fd = await fsPromises.open(filePath, 'r');
+  const buffer = Buffer.alloc(4);
+  const { bytesRead } = await fd.read(buffer, 0, 4, 0);
+  await fd.close();
+  
+  if (bytesRead < 2) {
+    return null;
+  }
+  
+  return detectBom(buffer.slice(0, bytesRead));
+}
+
+export default {
+  detectBom,
+  stripBom,
+  stripBomFromString,
+  createBomStripStream,
+  readFileWithBomHandling,
+  fileHasBomSync,
+  normalizeCsvInput,
+  normalizeCsvInputAsync,
+  createBomStrippingIterator,
+  detectBomAsync,
+  BOM_SIGNATURES
+};