jtcsv 2.2.8 → 3.1.0

package/src/browser/streams.ts

@@ -0,0 +1,403 @@
+import {
+  ValidationError,
+  ConfigurationError,
+  LimitError
+} from './errors-browser';
+import { csvToJsonIterator } from './csv-to-json-browser';
+
+import type { CsvToJsonOptions, JsonToCsvOptions } from '../types';
+
+const DEFAULT_MAX_CHUNK_SIZE = 64 * 1024;
+const PHONE_KEYS = new Set(['phone', 'phonenumber', 'phone_number', 'tel', 'telephone']);
+
+function isReadableStream(value: any): value is ReadableStream {
+  return value && typeof value.getReader === 'function';
+}
+
+function isAsyncIterable(value: any): value is AsyncIterable<any> {
+  return value && typeof value[Symbol.asyncIterator] === 'function';
+}
+
+function isIterable(value: any): value is Iterable<any> {
+  return value && typeof value[Symbol.iterator] === 'function';
+}
+
+function createReadableStreamFromIterator<T>(iterator: AsyncIterator<T>): ReadableStream<T> {
+  return new ReadableStream({
+    async pull(controller) {
+      try {
+        const { value, done } = await iterator.next();
+        if (done) {
+          controller.close();
+          return;
+        }
+        controller.enqueue(value);
+      } catch (error) {
+        controller.error(error);
+      }
+    },
+    cancel() {
+      if (iterator.return) {
+        iterator.return();
+      }
+    }
+  });
+}
+
+function detectInputFormat(input: any, options: any): 'json' | 'ndjson' | 'csv' | 'unknown' {
+  if (options && options.inputFormat) {
+    return options.inputFormat;
+  }
+
+  if (typeof input === 'string') {
+    const trimmed = input.trim();
+    if (trimmed === '') {
+      return 'unknown';
+    }
+
+    // Проверка на NDJSON (каждая строка - валидный JSON)
+    if (trimmed.includes('\n')) {
+      const lines = trimmed.split('\n').filter(line => line.trim() !== '');
+      if (lines.length > 0) {
+        try {
+          JSON.parse(lines[0]);
+          return 'ndjson';
+        } catch {
+          // Не NDJSON
+        }
+      }
+    }
+
+    // Проверка на JSON
+    try {
+      const parsed = JSON.parse(trimmed);
+      if (Array.isArray(parsed) || (parsed && typeof parsed === 'object')) {
+        return 'json';
+      }
+    } catch {
+      // Не JSON
+    }
+
+    // Проверка на CSV
+    if (trimmed.includes(',') || trimmed.includes(';') || trimmed.includes('\t')) {
+      return 'csv';
+    }
+  }
+
+  return 'unknown';
+}
+
+function normalizeQuotesInField(value: string): string {
+  // Не нормализуем кавычки в JSON-строках - это ломает структуру JSON
+  // Проверяем, выглядит ли значение как JSON (объект или массив)
+  if ((value.startsWith('{') && value.endsWith('}')) ||
+      (value.startsWith('[') && value.endsWith(']'))) {
+    return value; // Возвращаем как есть для JSON
+  }
+  
+  let normalized = value.replace(/"{2,}/g, '"');
+  // Убираем правило, которое ломает JSON: не заменяем "," на ","
+  // normalized = normalized.replace(/"\s*,\s*"/g, ',');
+  normalized = normalized.replace(/"\n/g, '\n').replace(/\n"/g, '\n');
+  if (normalized.length >= 2 && normalized.startsWith('"') && normalized.endsWith('"')) {
+    normalized = normalized.slice(1, -1);
+  }
+  return normalized;
+}
+
+function normalizePhoneValue(value: string): string {
+  const trimmed = value.trim();
+  if (trimmed === '') {
+    return trimmed;
+  }
+  return trimmed.replace(/["'\\]/g, '');
+}
+
+function normalizeValueForCsv(value: any, key: string | undefined, normalizeQuotes: boolean): any {
+  if (!normalizeQuotes || typeof value !== 'string') {
+    return value;
+  }
+  const base = normalizeQuotesInField(value);
+  if (key && PHONE_KEYS.has(String(key).toLowerCase())) {
+    return normalizePhoneValue(base);
+  }
+  return base;
+}
+
+async function* jsonToCsvChunkIterator(input: any, options: JsonToCsvOptions = {}): AsyncGenerator<string> {
+  const format = detectInputFormat(input, options);
+  
+  if (format === 'csv') {
+    throw new ValidationError('Input appears to be CSV, not JSON');
+  }
+
+  // Вспомогательная функция для создания асинхронного итератора
+  function toAsyncIterator<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterator<T> {
+    if (isAsyncIterable(iterable)) {
+      return iterable[Symbol.asyncIterator]();
+    }
+    
+    if (isIterable(iterable)) {
+      const syncIterator = iterable[Symbol.iterator]();
+      return {
+        next: () => Promise.resolve(syncIterator.next()),
+        return: syncIterator.return ? () => Promise.resolve(syncIterator.return!()) : undefined,
+        throw: syncIterator.throw ? (error: any) => Promise.resolve(syncIterator.throw!(error)) : undefined
+      };
+    }
+    
+    throw new ValidationError('Input is not iterable');
+  }
+
+  let iterator: AsyncIterator<any>;
+  
+  if (isAsyncIterable(input) || isIterable(input)) {
+    iterator = toAsyncIterator(input);
+  } else if (typeof input === 'string') {
+    const parsed = JSON.parse(input);
+    if (Array.isArray(parsed)) {
+      iterator = toAsyncIterator(parsed);
+    } else {
+      iterator = toAsyncIterator([parsed]);
+    }
+  } else if (Array.isArray(input)) {
+    iterator = toAsyncIterator(input);
+  } else {
+    iterator = toAsyncIterator([input]);
+  }
+
+  const delimiter = options.delimiter || ';';
+  const includeHeaders = options.includeHeaders !== false;
+  const preventInjection = options.preventCsvInjection !== false;
+  const normalizeQuotes = options.normalizeQuotes !== false;
+  const isPotentialFormula = (input: string): boolean => {
+    let idx = 0;
+    while (idx < input.length) {
+      const code = input.charCodeAt(idx);
+      if (code === 32 || code === 9 || code === 10 || code === 13 || code === 0xfeff) {
+        idx++;
+        continue;
+      }
+      break;
+    }
+    if (idx < input.length && (input[idx] === '"' || input[idx] === "'")) {
+      idx++;
+      while (idx < input.length) {
+        const code = input.charCodeAt(idx);
+        if (code === 32 || code === 9) {
+          idx++;
+          continue;
+        }
+        break;
+      }
+    }
+    if (idx >= input.length) {
+      return false;
+    }
+    const char = input[idx];
+    return char === '=' || char === '+' || char === '-' || char === '@';
+  };
+  
+  let isFirstChunk = true;
+  let headers: string[] = [];
+  
+  while (true) {
+    const { value, done } = await iterator.next();
+    if (done) break;
+
+    const item = value;
+    
+    if (isFirstChunk) {
+      // Извлечение заголовков из первого элемента
+      headers = Object.keys(item);
+      
+      if (includeHeaders) {
+        const headerLine = headers.map(header => {
+          const escaped = header.includes('"') ? `"${header.replace(/"/g, '""')}"` : header;
+          return preventInjection && isPotentialFormula(escaped) ? `'${escaped}` : escaped;
+        }).join(delimiter);
+        
+        yield headerLine + '\n';
+      }
+      
+      isFirstChunk = false;
+    }
+    
+    const row = headers.map(header => {
+      const value = item[header];
+      const normalized = normalizeValueForCsv(value, header, normalizeQuotes);
+      const strValue = normalized === null || normalized === undefined ? '' : String(normalized);
+      
+      if (strValue.includes('"') || strValue.includes('\n') || strValue.includes('\r') || strValue.includes(delimiter)) {
+        return `"${strValue.replace(/"/g, '""')}"`;
+      }
+      
+      if (preventInjection && isPotentialFormula(strValue)) {
+        return `'${strValue}`;
+      }
+      
+      return strValue;
+    }).join(delimiter);
+    
+    yield row + '\n';
+  }
+}
+
+async function* jsonToNdjsonChunkIterator(input: any, options: any = {}): AsyncGenerator<string> {
+  const format = detectInputFormat(input, options);
+  
+  // Вспомогательная функция для создания асинхронного итератора
+  function toAsyncIterator<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterator<T> {
+    if (isAsyncIterable(iterable)) {
+      return iterable[Symbol.asyncIterator]();
+    }
+    
+    if (isIterable(iterable)) {
+      const syncIterator = iterable[Symbol.iterator]();
+      return {
+        next: () => Promise.resolve(syncIterator.next()),
+        return: syncIterator.return ? () => Promise.resolve(syncIterator.return!()) : undefined,
+        throw: syncIterator.throw ? (error: any) => Promise.resolve(syncIterator.throw!(error)) : undefined
+      };
+    }
+    
+    throw new ValidationError('Input is not iterable');
+  }
+
+  let iterator: AsyncIterator<any>;
+  
+  if (isAsyncIterable(input) || isIterable(input)) {
+    iterator = toAsyncIterator(input);
+  } else if (typeof input === 'string') {
+    if (format === 'ndjson') {
+      const lines = input.split('\n').filter(line => line.trim() !== '');
+      iterator = toAsyncIterator(lines);
+    } else {
+      const parsed = JSON.parse(input);
+      if (Array.isArray(parsed)) {
+        iterator = toAsyncIterator(parsed);
+      } else {
+        iterator = toAsyncIterator([parsed]);
+      }
+    }
+  } else if (Array.isArray(input)) {
+    iterator = toAsyncIterator(input);
+  } else {
+    iterator = toAsyncIterator([input]);
+  }
+
+  while (true) {
+    const { value, done } = await iterator.next();
+    if (done) break;
+
+    let jsonStr: string;
+    
+    if (typeof value === 'string') {
+      try {
+        // Проверяем, является ли строка валидным JSON
+        JSON.parse(value);
+        jsonStr = value;
+      } catch {
+        // Если нет, сериализуем как JSON
+        jsonStr = JSON.stringify(value);
+      }
+    } else {
+      jsonStr = JSON.stringify(value);
+    }
+    
+    yield jsonStr + '\n';
+  }
+}
+
+async function* csvToJsonChunkIterator(input: any, options: CsvToJsonOptions = {}): AsyncGenerator<any> {
+  if (typeof input === 'string') {
+    // Используем csvToJsonIterator из csv-to-json-browser
+    yield* csvToJsonIterator(input, options);
+  } else if (input instanceof File || input instanceof Blob) {
+    const text = await input.text();
+    yield* csvToJsonIterator(text, options);
+  } else if (isReadableStream(input)) {
+    const reader = input.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    
+    try {
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        
+        buffer += decoder.decode(value, { stream: true });
+        
+        // Обработка буфера по строкам
+        const lines = buffer.split('\n');
+        buffer = lines.pop() || '';
+        
+        // TODO: Реализовать парсинг CSV из чанков
+        // Пока просто возвращаем сырые строки
+        for (const line of lines) {
+          if (line.trim()) {
+            yield { raw: line };
+          }
+        }
+      }
+      
+      // Обработка остатка буфера
+      if (buffer.trim()) {
+        yield { raw: buffer };
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  } else {
+    throw new ValidationError('Unsupported input type for CSV streaming');
+  }
+}
+
+export function jsonToCsvStream(input: any, options: JsonToCsvOptions = {}): ReadableStream<string> {
+  const iterator = jsonToCsvChunkIterator(input, options);
+  return createReadableStreamFromIterator(iterator);
+}
+
+export function jsonToNdjsonStream(input: any, options: any = {}): ReadableStream<string> {
+  const iterator = jsonToNdjsonChunkIterator(input, options);
+  return createReadableStreamFromIterator(iterator);
+}
+
+export function csvToJsonStream(input: any, options: CsvToJsonOptions = {}): ReadableStream<any> {
+  const iterator = csvToJsonChunkIterator(input, options);
+  return createReadableStreamFromIterator(iterator);
+}
+
+/**
+ * Асинхронная версия jsonToCsvStream
+ */
+export async function jsonToCsvStreamAsync(input: any, options: JsonToCsvOptions = {}): Promise<ReadableStream<string>> {
+  return jsonToCsvStream(input, options);
+}
+
+/**
+ * Асинхронная версия jsonToNdjsonStream
+ */
+export async function jsonToNdjsonStreamAsync(input: any, options: any = {}): Promise<ReadableStream<string>> {
+  return jsonToNdjsonStream(input, options);
+}
+
+/**
+ * Асинхронная версия csvToJsonStream
+ */
+export async function csvToJsonStreamAsync(input: any, options: CsvToJsonOptions = {}): Promise<ReadableStream<any>> {
+  return csvToJsonStream(input, options);
+}
+
+// Экспорт для Node.js совместимости
+if (typeof module !== 'undefined' && module.exports) {
+  module.exports = {
+    jsonToCsvStream,
+    jsonToCsvStreamAsync,
+    jsonToNdjsonStream,
+    jsonToNdjsonStreamAsync,
+    csvToJsonStream,
+    csvToJsonStreamAsync,
+    createReadableStreamFromIterator
+  };
+}

package/src/browser/workers/{csv-parser.worker.js → csv-parser.worker.ts}

@@ -2,8 +2,8 @@
 // Работает в отдельном потоке, не блокируя основной
 
 // Импорт функций парсинга (они будут bundled вместе с worker)
-import { csvToJson } from '../csv-to-json-browser.js';
-import { jsonToCsv } from '../json-to-csv-browser.js';
+import { csvToJson } from '../csv-to-json-browser';
+import { jsonToCsv } from '../json-to-csv-browser';
 
 const textDecoder = new TextDecoder('utf-8');
 
@@ -374,4 +374,4 @@ if (typeof module !== 'undefined' && module.exports) {
     getStats,
     clearCache
   };
-}
+}

package/src/browser/workers/{worker-pool.js → worker-pool.ts}

@@ -1,7 +1,7 @@
 // Worker Pool для параллельной обработки CSV
 // Использует Comlink для простой коммуникации с Web Workers
 
-import { ValidationError, ConfigurationError } from '../errors-browser.js';
+import { ValidationError, ConfigurationError } from '../errors-browser';
 
 // Проверка поддержки Web Workers
 const WORKERS_SUPPORTED = typeof Worker !== 'undefined';
@@ -37,8 +37,15 @@ function collectTransferables(args) {
   };
 
   args.forEach(collectFromValue);
-  return transferables.length ? transferables : null;
-}
+  return transferables.length ? transferables : null;
+}
+
+type WorkerWithMeta = Worker & {
+  id: string;
+  status: 'idle' | 'busy' | 'error';
+  lastUsed: number;
+  taskId: string | null;
+};
 
 /**
  * Опции для Worker Pool
@@ -72,8 +79,14 @@ function collectTransferables(args) {
 /**
  * Worker Pool для параллельной обработки CSV
  */
-export class WorkerPool {
-  /**
+export class WorkerPool {
+  workerScript: string;
+  options: any;
+  workers: WorkerWithMeta[];
+  taskQueue: any[];
+  activeTasks: Map<any, any>;
+  stats: any;
+  /**
    * Создает новый Worker Pool
    * @param {string} workerScript - URL скрипта worker
    * @param {WorkerPoolOptions} [options] - Опции pool
@@ -127,7 +140,7 @@ export class WorkerPool {
    */
   createWorker() {
     try {
-      const worker = new Worker(this.workerScript, { type: 'module' });
+      const worker = new Worker(this.workerScript, { type: 'module' }) as WorkerWithMeta;
       
       worker.id = `worker-${this.workers.length}`;
       worker.status = 'idle';
@@ -229,11 +242,11 @@ export class WorkerPool {
       
       // Завершение с ошибкой
       const workerError = new Error(errorData.message || 'Ошибка в worker');
-      if (errorData.code) {
-        workerError.code = errorData.code;
+      if ((errorData as any).code) {
+        (workerError as any).code = (errorData as any).code;
       }
-      if (errorData.details) {
-        workerError.details = errorData.details;
+      if ((errorData as any).details) {
+        (workerError as any).details = (errorData as any).details;
       }
       task.reject(workerError);
       this.activeTasks.delete(taskId);
@@ -363,9 +376,9 @@ export class WorkerPool {
    * @param {Array} args - Аргументы метода
    * @param {Object} [options] - Опции задачи
    * @param {Function} [onProgress] - Callback прогресса
-   * @returns {Promise<any>} Результат выполнения
+   * @returns {Promise<unknown>} Результат выполнения
    */
-  async exec(method, args = [], options = {}, onProgress = null) {
+  async exec(method, args = [], options: any = {}, onProgress = null) {
     return new Promise((resolve, reject) => {
       // Проверка размера очереди
       if (this.taskQueue.length >= this.options.maxQueueSize) {
@@ -457,10 +470,16 @@ export class WorkerPool {
  * @param {WorkerPoolOptions} [options] - Опции pool
  * @returns {WorkerPool} Worker Pool
  */
-export function createWorkerPool(options = {}) {
+export function createWorkerPool(options: any = {}): any {
   // Используем встроенный worker скрипт
-  const workerScript = new URL('./csv-parser.worker.js', import.meta.url).href;
-  return new WorkerPool(workerScript, options);
+  const baseUrl =
+    typeof document !== 'undefined'
+      ? document.baseURI
+      : (typeof self !== 'undefined' && (self as any).location
+          ? (self as any).location.href
+          : '');
+  const workerScript = new URL('./csv-parser.worker.js', baseUrl).href;
+  return new WorkerPool(workerScript, options);
 }
 
 /**
@@ -470,13 +489,14 @@ export function createWorkerPool(options = {}) {
  * @param {Function} [onProgress] - Callback прогресса
  * @returns {Promise<Array<Object>>} JSON данные
  */
-export async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
+export async function parseCSVWithWorker(csvInput, options: any = {}, onProgress = null) {
   // Создание pool если нужно
-  if (!parseCSVWithWorker.pool) {
-    parseCSVWithWorker.pool = createWorkerPool();
-  }
-  
-  const pool = parseCSVWithWorker.pool;
+  const poolHolder = parseCSVWithWorker as any;
+  if (!poolHolder.pool) {
+    poolHolder.pool = createWorkerPool();
+  }
+  
+  const pool = poolHolder.pool;
   
   // Подготовка CSV строки
   // ?????????? CSV ??????
@@ -508,20 +528,21 @@ export async function parseCSVWithWorker(csvInput, options = {}, onProgress = nu
  * Чтение файла как текст
  * @private
  */
-async function readFileAsArrayBuffer(file) {
+async function readFileAsArrayBuffer(file: File): Promise<ArrayBuffer> {
   return new Promise((resolve, reject) => {
     const reader = new FileReader();
-    reader.onload = (event) => resolve(event.target.result);
+    reader.onload = (event) =>
+      resolve((event.target as FileReader).result as ArrayBuffer);
     reader.onerror = (error) => reject(error);
     reader.readAsArrayBuffer(file);
   });
 }
 
 // Экспорт для Node.js совместимости
-if (typeof module !== 'undefined' && module.exports) {
-  module.exports = {
-    WorkerPool,
-    createWorkerPool,
-    parseCSVWithWorker
-  };
-}
+if (typeof module !== 'undefined' && module.exports) {
+  module.exports = {
+    WorkerPool,
+    createWorkerPool,
+    parseCSVWithWorker
+  };
+}