node-es-transformer 1.1.0 → 1.2.1

package/dist/node-es-transformer.esm.js

@@ -1,5 +1,9 @@
 import elasticsearch9 from 'es9';
 import elasticsearch8 from 'es8';
+import parquet from '@dsnp/parquetjs';
+import zlib from 'zlib';
+import { PARQUET_COMPRESSION_METHODS } from '@dsnp/parquetjs/dist/lib/compression.js';
+import * as arrow from 'apache-arrow';
 import fs from 'fs';
 import { parse } from 'csv-parse';
 import es from 'event-stream';
@@ -7,6 +11,7 @@ import { globSync } from 'glob';
 import split from 'split2';
 import { PassThrough } from 'stream';
 import cliProgress from 'cli-progress';
+import pino from 'pino';
 
 // In earlier versions this was used to set the number of docs to index in a
 // single bulk request. Since we switched to use the helpers.bulk() method from
@@ -27,9 +32,9 @@ function createMappingFactory({
   inferredIngestPipeline,
   mappingsOverride,
   indexMappingTotalFieldsLimit,
-  verbose,
   deleteIndex,
-  pipeline
+  pipeline,
+  logger
 }) {
   return async () => {
     let targetMappings = mappingsOverride ? undefined : mappings;
@@ -48,7 +53,10 @@ function createMappingFactory({
           }
         }
       } catch (err) {
-        console.log('Error reading source mapping', err);
+        logger.error({
+          err,
+          sourceIndexName
+        }, 'Error reading source mapping');
         return;
       }
     }
@@ -80,9 +88,14 @@ function createMappingFactory({
                 ...inferredIngestPipeline
               });
               defaultPipeline = inferredPipelineName;
-              if (verbose) console.log(`Created inferred ingest pipeline ${inferredPipelineName}`);
+              logger.info({
+                inferredPipelineName
+              }, 'Created inferred ingest pipeline');
             } catch (err) {
-              console.log('Error creating inferred ingest pipeline', err);
+              logger.error({
+                err,
+                inferredPipelineName
+              }, 'Error creating inferred ingest pipeline');
             }
           }
           const settings = {
@@ -95,22 +108,54 @@ function createMappingFactory({
               'index.number_of_replicas': 0
             } : {})
           };
-          const resp = await targetClient.indices.create({
+          const response = await targetClient.indices.create({
             index: targetIndexName,
             mappings: targetMappings,
             ...(Object.keys(settings).length > 0 ? {
               settings
             } : {})
           });
-          if (verbose) console.log('Created target mapping', resp);
+          logger.info({
+            targetIndexName,
+            response
+          }, 'Created target mapping');
         }
       } catch (err) {
-        console.log('Error creating target mapping', err);
+        logger.error({
+          err,
+          targetIndexName
+        }, 'Error creating target mapping');
       }
     }
   };
 }
 
+function registerZstdCompression() {
+  if (PARQUET_COMPRESSION_METHODS.ZSTD) {
+    return;
+  }
+  if (typeof zlib.zstdCompressSync !== 'function' || typeof zlib.zstdDecompressSync !== 'function') {
+    PARQUET_COMPRESSION_METHODS.ZSTD = {
+      deflate() {
+        throw new Error('ZSTD compression requires Node.js with zstd support.');
+      },
+      inflate() {
+        throw new Error('ZSTD compression requires Node.js with zstd support.');
+      }
+    };
+    return;
+  }
+  PARQUET_COMPRESSION_METHODS.ZSTD = {
+    deflate(value) {
+      return zlib.zstdCompressSync(value);
+    },
+    inflate(value) {
+      return zlib.zstdDecompressSync(value);
+    }
+  };
+}
+registerZstdCompression();
+
 function getCsvParserOptions(csvOptions = {}, skipHeader = false) {
   const options = {
     bom: true,
@@ -126,8 +171,36 @@ function getCsvParserOptions(csvOptions = {}, skipHeader = false) {
   return options;
 }
 
-function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}) {
-  function addParsedDoc(parsed, file, streamRef) {
+function createPauseWaiter$1(queueEmitter) {
+  let paused = false;
+  let waiters = [];
+  const onPause = () => {
+    paused = true;
+  };
+  const onResume = () => {
+    paused = false;
+    waiters.forEach(resolve => resolve());
+    waiters = [];
+  };
+  queueEmitter.on('pause', onPause);
+  queueEmitter.on('resume', onResume);
+  return {
+    async waitIfPaused() {
+      if (!paused) return;
+      await new Promise(resolve => {
+        waiters.push(resolve);
+      });
+    },
+    cleanup() {
+      queueEmitter.removeListener('pause', onPause);
+      queueEmitter.removeListener('resume', onResume);
+      waiters.forEach(resolve => resolve());
+      waiters = [];
+    }
+  };
+}
+function fileReaderFactory(indexer, fileName, transform, splitRegex, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}, logger) {
+  function addParsedDoc(parsed, file) {
     const context = {
       fileName: file
     };
@@ -135,7 +208,6 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, sk
 
     // if doc is null/undefined we'll skip indexing it
     if (doc === null || typeof doc === 'undefined') {
-      streamRef.resume();
       return;
     }
 
@@ -150,9 +222,101 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, sk
     }
     indexer.add(doc);
   }
-  function createNdjsonReader(file) {
+  async function processParquetFile(file) {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter$1(indexer.queueEmitter);
+    const reader = await parquet.ParquetReader.openFile(file);
+    try {
+      const cursor = reader.getCursor();
+      while (true) {
+        // eslint-disable-next-line no-await-in-loop
+        const row = await cursor.next();
+        if (row === null || typeof row === 'undefined') {
+          break;
+        }
+        addParsedDoc(row, file);
+        // eslint-disable-next-line no-await-in-loop
+        await waitIfPaused();
+      }
+      logger.info({
+        file
+      }, 'Read entire file');
+    } finally {
+      cleanup();
+      await reader.close();
+    }
+  }
+  async function processArrowFile(file) {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter$1(indexer.queueEmitter);
+    try {
+      const reader = await arrow.RecordBatchReader.from(fs.createReadStream(file));
+      for await (const recordBatch of reader) {
+        const {
+          fields
+        } = recordBatch.schema;
+        for (let rowIndex = 0; rowIndex < recordBatch.numRows; rowIndex++) {
+          const row = {};
+          fields.forEach(field => {
+            const vector = recordBatch.getChild(field.name);
+            row[field.name] = vector ? vector.get(rowIndex) : undefined;
+          });
+          addParsedDoc(row, file);
+          // eslint-disable-next-line no-await-in-loop
+          await waitIfPaused();
+        }
+      }
+      logger.info({
+        file
+      }, 'Read entire file');
+    } finally {
+      cleanup();
+    }
+  }
+  function processStreamFile(file, buildStream, errorMessage) {
+    return new Promise((resolve, reject) => {
+      let finished = false;
+      const s = buildStream();
+      const onPause = () => {
+        if (finished) return;
+        s.pause();
+      };
+      const onResume = () => {
+        if (finished) return;
+        s.resume();
+      };
+      function cleanup() {
+        indexer.queueEmitter.removeListener('pause', onPause);
+        indexer.queueEmitter.removeListener('resume', onResume);
+      }
+      indexer.queueEmitter.on('pause', onPause);
+      indexer.queueEmitter.on('resume', onResume);
+      s.on('end', () => {
+        finished = true;
+        cleanup();
+        logger.info({
+          file
+        }, 'Read entire file');
+        resolve();
+      });
+      s.on('error', err => {
+        finished = true;
+        cleanup();
+        logger.error({
+          err,
+          file
+        }, errorMessage);
+        reject(err);
+      });
+    });
+  }
+  function processNdjsonFile(file) {
     let skippedHeader = false;
-    const s = fs.createReadStream(file).pipe(split(splitRegex)).pipe(es.mapSync(line => {
+    return processStreamFile(file, () => fs.createReadStream(file).pipe(split(splitRegex)).pipe(es.mapSync(line => {
       try {
         // skip empty lines
         if (line === '') {
@@ -163,72 +327,115 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, sk
           return;
         }
         const parsed = JSON.parse(line);
-        addParsedDoc(parsed, file, s);
-      } catch (e) {
-        console.log('error', e);
+        addParsedDoc(parsed, file);
+      } catch (err) {
+        logger.error({
+          err,
+          file
+        }, 'Failed to process NDJSON line');
       }
     }).on('error', err => {
-      console.log('Error while reading file.', err);
-    }));
-    return s;
+      logger.error({
+        err,
+        file
+      }, 'Error while reading file');
+    })), 'Error while reading file');
   }
-  function createCsvReader(file) {
+  function processCsvFile(file) {
     const parserOptions = getCsvParserOptions(csvOptions, skipHeader);
-    const s = fs.createReadStream(file).pipe(parse(parserOptions)).pipe(es.mapSync(record => {
+    return processStreamFile(file, () => fs.createReadStream(file).pipe(parse(parserOptions)).pipe(es.mapSync(record => {
       try {
-        addParsedDoc(record, file, s);
-      } catch (e) {
-        console.log('error', e);
+        addParsedDoc(record, file);
+      } catch (err) {
+        logger.error({
+          err,
+          file
+        }, 'Failed to process CSV record');
       }
     }).on('error', err => {
-      console.log('Error while reading CSV file.', err);
-    }));
-    return s;
+      logger.error({
+        err,
+        file
+      }, 'Error while reading CSV file');
+    })), 'Error while reading CSV file');
   }
-  function startIndex(files) {
-    let finished = false;
+  async function processFile(file) {
+    if (sourceFormat === 'csv') {
+      await processCsvFile(file);
+      return;
+    }
+    if (sourceFormat === 'ndjson') {
+      await processNdjsonFile(file);
+      return;
+    }
+    if (sourceFormat === 'parquet') {
+      await processParquetFile(file);
+      return;
+    }
+    if (sourceFormat === 'arrow') {
+      await processArrowFile(file);
+      return;
+    }
+    throw Error(`Unsupported sourceFormat: ${sourceFormat}`);
+  }
+  async function startIndex(files) {
     if (files.length === 0) {
       indexer.finish();
       return;
     }
-    const file = files.shift();
-    const s = sourceFormat === 'csv' ? createCsvReader(file) : createNdjsonReader(file);
-    s.on('end', () => {
-      if (verbose) console.log('Read entire file: ', file);
-      if (files.length > 0) {
-        startIndex(files);
-        return;
+    try {
+      for (const file of files) {
+        // eslint-disable-next-line no-await-in-loop
+        await processFile(file);
       }
+    } catch (err) {
+      logger.error({
+        err,
+        files
+      }, 'Error while processing files');
+    } finally {
       indexer.finish();
-      finished = true;
-    });
-    indexer.queueEmitter.on('pause', () => {
-      if (finished) return;
-      s.pause();
-    });
-    indexer.queueEmitter.on('resume', () => {
-      if (finished) return;
-      s.resume();
-    });
+    }
   }
   return () => {
     try {
       const files = globSync(fileName);
       startIndex(files);
-    } catch (error) {
-      console.log('Error matching files:', error);
+    } catch (err) {
+      logger.error({
+        err,
+        fileName
+      }, 'Error matching files');
+      indexer.finish();
     }
   };
 }
 
 const EventEmitter = require('events');
 const parallelCalls = 5;
+const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
+const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
+function coerceBigInt(value) {
+  if (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) {
+    return Number(value);
+  }
+  return value.toString();
+}
+function safeStringify(doc) {
+  return JSON.stringify(doc, (_key, value) => {
+    if (typeof value === 'bigint') {
+      return coerceBigInt(value);
+    }
+    return value;
+  });
+}
 
 // a simple helper queue to bulk index documents
 function indexQueueFactory({
   targetClient: client,
   targetIndexName,
-  bufferSize = DEFAULT_BUFFER_SIZE
+  bufferSize = DEFAULT_BUFFER_SIZE,
+  logger
 }) {
   const queueEmitter = new EventEmitter();
   let docsPerSecond = 0;
@@ -261,8 +468,9 @@ function indexQueueFactory({
           try {
             yield JSON.parse(line); // Parse and yield the JSON object
           } catch (err) {
-            // Handle JSON parse errors if necessary
-            console.error('Failed to parse JSON:', err);
+            logger.error({
+              err
+            }, 'Failed to parse JSON from NDJSON stream');
           }
         }
       }
@@ -272,7 +480,9 @@ function indexQueueFactory({
         try {
           yield JSON.parse(buffer);
         } catch (err) {
-          console.error('Failed to parse final JSON:', err);
+          logger.error({
+            err
+          }, 'Failed to parse final JSON from NDJSON stream');
         }
       }
     } finally {
@@ -298,7 +508,7 @@ function indexQueueFactory({
         flushInterval: 1000,
         refreshOnCompletion: true,
         datasource: ndjsonStreamIterator(stream),
-        onDocument(doc) {
+        onDocument() {
           docsPerSecond++;
           return {
             index: {
@@ -307,9 +517,13 @@ function indexQueueFactory({
           };
         }
       });
-    } catch (error) {
-      console.error('Error during bulk indexing:', error);
-      throw error;
+    } catch (err) {
+      logger.error({
+        err,
+        targetIndexName
+      }, 'Error during bulk indexing');
+      queueEmitter.emit('error', err);
+      throw err;
     } finally {
       // Clean up interval
       clearInterval(interval);
@@ -338,7 +552,7 @@ function indexQueueFactory({
       if (finished) {
         throw new Error('Unexpected doc added after indexer should finish.');
       }
-      const canContinue = stream.write(`${JSON.stringify(doc)}\n`);
+      const canContinue = stream.write(`${safeStringify(doc)}\n`);
       if (!canContinue) {
         queueEmitter.emit('pause');
 
@@ -359,7 +573,7 @@ function indexQueueFactory({
 
 // create a new progress bar instance and use shades_classic theme
 const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
-function indexReaderFactory(indexer, sourceIndexName, transform, client, query, searchSize = DEFAULT_SEARCH_SIZE, populatedFields = false) {
+function indexReaderFactory(indexer, sourceIndexName, transform, client, query, searchSize = DEFAULT_SEARCH_SIZE, populatedFields = false, logger) {
   return async function indexReader() {
     let docsNum = 0;
     let scrollId;
@@ -378,8 +592,11 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query,
           maxRetries: 0
         });
         return Object.keys(response.fields);
-      } catch (e) {
-        console.log('error', e);
+      } catch (err) {
+        logger.error({
+          err,
+          sourceIndexName
+        }, 'Failed to fetch populated fields');
       }
     }
     function search(fields) {
@@ -423,8 +640,10 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query,
           return;
         }
         indexer.add(doc);
-      } catch (e) {
-        console.log('error', e);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process source index document');
       }
     }
     async function fetchNextResponse() {
@@ -495,17 +714,25 @@ async function inferMappingsFromSource({
   mappings,
   inferMappings,
   inferMappingsOptions,
-  verbose
+  logger
 }) {
   if (!inferMappings || typeof mappings !== 'undefined' || typeof fileName === 'undefined') {
     return emptyInferenceResult(mappings);
   }
+  if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv') {
+    logger.info({
+      sourceFormat
+    }, 'Skipping mapping inference. Inference is only supported for ndjson and csv.');
+    return emptyInferenceResult(mappings);
+  }
   if (typeof targetClient?.textStructure?.findStructure !== 'function' || sourceFormat === 'xml' || sourceFormat === 'semi_structured_text') {
     return emptyInferenceResult(mappings);
   }
   const files = globSync(fileName);
   if (files.length === 0) {
-    if (verbose) console.log(`No files matched for mapping inference: ${fileName}`);
+    logger.info({
+      fileName
+    }, 'No files matched for mapping inference');
     return emptyInferenceResult(mappings);
   }
   const {
@@ -514,7 +741,7 @@ async function inferMappingsFromSource({
   } = inferMappingsOptions || {};
   const sampleText = readSample(files[0], sampleBytes);
   if (!sampleText || sampleText.trim() === '') {
-    if (verbose) console.log('Skipping mapping inference because the sample text is empty.');
+    logger.info('Skipping mapping inference because the sample text is empty');
     return emptyInferenceResult(mappings);
   }
   const params = {
@@ -541,31 +768,98 @@ async function inferMappingsFromSource({
   }
   try {
     const response = await targetClient.textStructure.findStructure(params);
-    if (response?.mappings && verbose) {
-      console.log(`Inferred mappings via _text_structure/find_structure from ${files[0]}`);
+    if (response?.mappings) {
+      logger.info({
+        file: files[0]
+      }, 'Inferred mappings via _text_structure/find_structure');
     }
-    if (response?.ingest_pipeline && verbose) {
-      console.log('Inferred ingest pipeline via _text_structure/find_structure');
+    if (response?.ingest_pipeline) {
+      logger.info('Inferred ingest pipeline via _text_structure/find_structure');
     }
     return {
       mappings: response?.mappings || mappings,
       ingestPipeline: response?.ingest_pipeline
     };
-  } catch (error) {
-    if (verbose) {
-      console.log('Could not infer mappings via _text_structure/find_structure:', error.message);
-    }
+  } catch (err) {
+    logger.warn({
+      err
+    }, 'Could not infer mappings via _text_structure/find_structure');
     return emptyInferenceResult(mappings);
   }
 }
 
-function streamReaderFactory(indexer, stream, transform, splitRegex, verbose, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}) {
-  function addParsedDoc(parsed, streamRef) {
+const DEFAULT_LOG_LEVEL = 'info';
+function resolveLogLevel(verbose = true) {
+  if (typeof process.env.LOG_LEVEL === 'string' && process.env.LOG_LEVEL.trim() !== '') {
+    return process.env.LOG_LEVEL;
+  }
+  return verbose ? DEFAULT_LOG_LEVEL : 'error';
+}
+function createLogger({
+  logger,
+  verbose = true
+} = {}) {
+  if (logger && typeof logger === 'object') {
+    return logger;
+  }
+  return pino({
+    name: 'node-es-transformer',
+    level: resolveLogLevel(verbose),
+    timestamp: pino.stdTimeFunctions.isoTime,
+    serializers: {
+      err: pino.stdSerializers.err,
+      error: pino.stdSerializers.err
+    }
+  });
+}
+function createChildLogger(logger, bindings) {
+  if (!logger || typeof logger.child !== 'function') {
+    return logger;
+  }
+  return logger.child(bindings);
+}
+
+function createPauseWaiter(queueEmitter) {
+  let paused = false;
+  let waiters = [];
+  const onPause = () => {
+    paused = true;
+  };
+  const onResume = () => {
+    paused = false;
+    waiters.forEach(resolve => resolve());
+    waiters = [];
+  };
+  queueEmitter.on('pause', onPause);
+  queueEmitter.on('resume', onResume);
+  return {
+    async waitIfPaused() {
+      if (!paused) return;
+      await new Promise(resolve => {
+        waiters.push(resolve);
+      });
+    },
+    cleanup() {
+      queueEmitter.removeListener('pause', onPause);
+      queueEmitter.removeListener('resume', onResume);
+      waiters.forEach(resolve => resolve());
+      waiters = [];
+    }
+  };
+}
+async function readStreamToBuffer(stream) {
+  const chunks = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks);
+}
+function streamReaderFactory(indexer, stream, transform, splitRegex, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}, logger) {
+  function addParsedDoc(parsed) {
     const doc = typeof transform === 'function' ? transform(parsed) : parsed;
 
     // if doc is null/undefined we'll skip indexing it
     if (doc === null || typeof doc === 'undefined') {
-      streamRef.resume();
       return;
     }
 
@@ -580,50 +874,152 @@ function streamReaderFactory(indexer, stream, transform, splitRegex, verbose, sk
     }
     indexer.add(doc);
   }
-  function startIndex() {
-    let finished = false;
-    const s = sourceFormat === 'csv' ? stream.pipe(parse(getCsvParserOptions(csvOptions, skipHeader))).pipe(es.mapSync(record => {
+  async function processParquetStream() {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter(indexer.queueEmitter);
+    const parquetBuffer = await readStreamToBuffer(stream);
+    const reader = await parquet.ParquetReader.openBuffer(parquetBuffer);
+    try {
+      const cursor = reader.getCursor();
+      while (true) {
+        // eslint-disable-next-line no-await-in-loop
+        const row = await cursor.next();
+        if (row === null || typeof row === 'undefined') {
+          break;
+        }
+        addParsedDoc(row);
+        // eslint-disable-next-line no-await-in-loop
+        await waitIfPaused();
+      }
+      logger.info('Read entire stream');
+    } finally {
+      cleanup();
+      await reader.close();
+    }
+  }
+  async function processArrowStream() {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter(indexer.queueEmitter);
+    try {
+      const reader = await arrow.RecordBatchReader.from(stream);
+      for await (const recordBatch of reader) {
+        const {
+          fields
+        } = recordBatch.schema;
+        for (let rowIndex = 0; rowIndex < recordBatch.numRows; rowIndex++) {
+          const row = {};
+          fields.forEach(field => {
+            const vector = recordBatch.getChild(field.name);
+            row[field.name] = vector ? vector.get(rowIndex) : undefined;
+          });
+          addParsedDoc(row);
+          // eslint-disable-next-line no-await-in-loop
+          await waitIfPaused();
+        }
+      }
+      logger.info('Read entire stream');
+    } finally {
+      cleanup();
+    }
+  }
+  function processPipeline(buildPipeline, errorMessage) {
+    return new Promise((resolve, reject) => {
+      let finished = false;
+      const s = buildPipeline();
+      const onPause = () => {
+        if (finished) return;
+        s.pause();
+      };
+      const onResume = () => {
+        if (finished) return;
+        s.resume();
+      };
+      function cleanup() {
+        indexer.queueEmitter.removeListener('pause', onPause);
+        indexer.queueEmitter.removeListener('resume', onResume);
+      }
+      indexer.queueEmitter.on('pause', onPause);
+      indexer.queueEmitter.on('resume', onResume);
+      s.on('end', () => {
+        finished = true;
+        cleanup();
+        logger.info('Read entire stream');
+        resolve();
+      });
+      s.on('error', err => {
+        finished = true;
+        cleanup();
+        logger.error({
+          err
+        }, errorMessage);
+        reject(err);
+      });
+    });
+  }
+  function processCsvStream() {
+    return processPipeline(() => stream.pipe(parse(getCsvParserOptions(csvOptions, skipHeader))).pipe(es.mapSync(record => {
       try {
-        addParsedDoc(record, s);
-      } catch (e) {
-        console.log('error', e);
+        addParsedDoc(record);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process CSV stream record');
       }
     }).on('error', err => {
-      console.log('Error while reading CSV stream.', err);
-    })) : (() => {
-      let skippedHeader = false;
-      return stream.pipe(split(splitRegex)).pipe(es.mapSync(line => {
-        try {
-          // skip empty lines
-          if (line === '') {
-            return;
-          }
-          if (skipHeader && !skippedHeader) {
-            skippedHeader = true;
-            return;
-          }
-          const parsed = JSON.parse(line);
-          addParsedDoc(parsed, s);
-        } catch (e) {
-          console.log('error', e);
+      logger.error({
+        err
+      }, 'Error while reading CSV stream');
+    })), 'Error while reading CSV stream');
+  }
+  function processNdjsonStream() {
+    let skippedHeader = false;
+    return processPipeline(() => stream.pipe(split(splitRegex)).pipe(es.mapSync(line => {
+      try {
+        // skip empty lines
+        if (line === '') {
+          return;
         }
-      }).on('error', err => {
-        console.log('Error while reading stream.', err);
-      }));
-    })();
-    s.on('end', () => {
-      if (verbose) console.log('Read entire stream.');
+        if (skipHeader && !skippedHeader) {
+          skippedHeader = true;
+          return;
+        }
+        const parsed = JSON.parse(line);
+        addParsedDoc(parsed);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process NDJSON stream line');
+      }
+    }).on('error', err => {
+      logger.error({
+        err
+      }, 'Error while reading stream');
+    })), 'Error while reading stream');
+  }
+  async function startIndex() {
+    try {
+      if (sourceFormat === 'csv') {
+        await processCsvStream();
+      } else if (sourceFormat === 'ndjson') {
+        await processNdjsonStream();
+      } else if (sourceFormat === 'parquet') {
+        await processParquetStream();
+      } else if (sourceFormat === 'arrow') {
+        await processArrowStream();
+      } else {
+        throw Error(`Unsupported sourceFormat: ${sourceFormat}`);
+      }
+    } catch (err) {
+      logger.error({
+        err
+      }, 'Error while reading stream');
+    } finally {
       indexer.finish();
-      finished = true;
-    });
-    indexer.queueEmitter.on('pause', () => {
-      if (finished) return;
-      s.pause();
-    });
-    indexer.queueEmitter.on('resume', () => {
-      if (finished) return;
-      s.resume();
-    });
+    }
   }
   return () => {
     startIndex();
@@ -719,11 +1115,16 @@ async function transformer({
   query,
   skipHeader = false,
   transform,
-  verbose = true
+  verbose = true,
+  logger: loggerInput
 }) {
   if (typeof targetIndexName === 'undefined') {
     throw Error('targetIndexName must be specified.');
   }
+  const logger = createLogger({
+    logger: loggerInput,
+    verbose
+  });
   const defaultClientConfig = {
     node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200'
   };
@@ -740,7 +1141,9 @@ async function transformer({
     mappings,
     inferMappings,
     inferMappingsOptions,
-    verbose
+    logger: createChildLogger(logger, {
+      component: 'mapping-inference'
+    })
   });
   const createMapping = createMappingFactory({
     sourceClient,
@@ -751,17 +1154,23 @@ async function transformer({
     inferredIngestPipeline: inferenceResult.ingestPipeline,
     mappingsOverride,
     indexMappingTotalFieldsLimit,
-    verbose,
     deleteIndex,
-    pipeline
+    pipeline,
+    logger: createChildLogger(logger, {
+      component: 'create-mapping'
+    })
   });
   const indexer = indexQueueFactory({
     targetClient,
     targetIndexName,
-    bufferSize});
+    bufferSize,
+    logger: createChildLogger(logger, {
+      component: 'index-queue'
+    })
+  });
   function validateSourceFormat() {
-    if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv') {
-      throw Error(`Unsupported sourceFormat: ${sourceFormat}. Use "ndjson" or "csv".`);
+    if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv' && sourceFormat !== 'parquet' && sourceFormat !== 'arrow') {
+      throw Error(`Unsupported sourceFormat: ${sourceFormat}. Use "ndjson", "csv", "parquet", or "arrow".`);
     }
   }
   function getReader() {
@@ -773,18 +1182,27 @@ async function transformer({
     }
     if (typeof fileName !== 'undefined') {
       validateSourceFormat();
-      return fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, skipHeader, sourceFormat, csvOptions);
+      return fileReaderFactory(indexer, fileName, transform, splitRegex, skipHeader, sourceFormat, csvOptions, createChildLogger(logger, {
+        component: 'file-reader'
+      }));
     }
     if (typeof sourceIndexName !== 'undefined') {
-      return indexReaderFactory(indexer, sourceIndexName, transform, sourceClient, query, searchSize, populatedFields);
+      return indexReaderFactory(indexer, sourceIndexName, transform, sourceClient, query, searchSize, populatedFields, createChildLogger(logger, {
+        component: 'index-reader'
+      }));
     }
     if (typeof stream !== 'undefined') {
       validateSourceFormat();
-      return streamReaderFactory(indexer, stream, transform, splitRegex, verbose, skipHeader, sourceFormat, csvOptions);
+      return streamReaderFactory(indexer, stream, transform, splitRegex, skipHeader, sourceFormat, csvOptions, createChildLogger(logger, {
+        component: 'stream-reader'
+      }));
     }
     return null;
   }
   const reader = getReader();
+  if (typeof reader !== 'function') {
+    throw Error('One of fileName, sourceIndexName, or stream must be specified.');
+  }
   try {
     const indexExists = await targetClient.indices.exists({
       index: targetIndexName
@@ -801,8 +1219,11 @@ async function transformer({
     } else {
       reader();
     }
-  } catch (error) {
-    console.error('Error checking index existence:', error);
+  } catch (err) {
+    logger.error({
+      err,
+      targetIndexName
+    }, 'Error checking index existence');
   } finally {
     // targetClient.close();
   }