node-es-transformer 1.1.0 → 1.2.1

package/dist/node-es-transformer.cjs.js

@@ -2,6 +2,10 @@
 
 var elasticsearch9 = require('es9');
 var elasticsearch8 = require('es8');
+var parquet = require('@dsnp/parquetjs');
+var zlib = require('zlib');
+var compression_js = require('@dsnp/parquetjs/dist/lib/compression.js');
+var arrow = require('apache-arrow');
 var fs = require('fs');
 var csvParse = require('csv-parse');
 var es = require('event-stream');
@@ -9,6 +13,26 @@ var glob = require('glob');
 var split = require('split2');
 var stream = require('stream');
 var cliProgress = require('cli-progress');
+var pino = require('pino');
+
+function _interopNamespaceDefault(e) {
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
+var arrow__namespace = /*#__PURE__*/_interopNamespaceDefault(arrow);
 
 // In earlier versions this was used to set the number of docs to index in a
 // single bulk request. Since we switched to use the helpers.bulk() method from
@@ -29,9 +53,9 @@ function createMappingFactory({
   inferredIngestPipeline,
   mappingsOverride,
   indexMappingTotalFieldsLimit,
-  verbose,
   deleteIndex,
-  pipeline
+  pipeline,
+  logger
 }) {
   return async () => {
     let targetMappings = mappingsOverride ? undefined : mappings;
@@ -50,7 +74,10 @@ function createMappingFactory({
           }
         }
       } catch (err) {
-        console.log('Error reading source mapping', err);
+        logger.error({
+          err,
+          sourceIndexName
+        }, 'Error reading source mapping');
         return;
       }
     }
@@ -82,9 +109,14 @@ function createMappingFactory({
                 ...inferredIngestPipeline
               });
               defaultPipeline = inferredPipelineName;
-              if (verbose) console.log(`Created inferred ingest pipeline ${inferredPipelineName}`);
+              logger.info({
+                inferredPipelineName
+              }, 'Created inferred ingest pipeline');
             } catch (err) {
-              console.log('Error creating inferred ingest pipeline', err);
+              logger.error({
+                err,
+                inferredPipelineName
+              }, 'Error creating inferred ingest pipeline');
             }
           }
           const settings = {
@@ -97,22 +129,54 @@ function createMappingFactory({
               'index.number_of_replicas': 0
             } : {})
           };
-          const resp = await targetClient.indices.create({
+          const response = await targetClient.indices.create({
             index: targetIndexName,
             mappings: targetMappings,
             ...(Object.keys(settings).length > 0 ? {
               settings
             } : {})
           });
-          if (verbose) console.log('Created target mapping', resp);
+          logger.info({
+            targetIndexName,
+            response
+          }, 'Created target mapping');
         }
       } catch (err) {
-        console.log('Error creating target mapping', err);
+        logger.error({
+          err,
+          targetIndexName
+        }, 'Error creating target mapping');
       }
     }
   };
 }
 
+function registerZstdCompression() {
+  if (compression_js.PARQUET_COMPRESSION_METHODS.ZSTD) {
+    return;
+  }
+  if (typeof zlib.zstdCompressSync !== 'function' || typeof zlib.zstdDecompressSync !== 'function') {
+    compression_js.PARQUET_COMPRESSION_METHODS.ZSTD = {
+      deflate() {
+        throw new Error('ZSTD compression requires Node.js with zstd support.');
+      },
+      inflate() {
+        throw new Error('ZSTD compression requires Node.js with zstd support.');
+      }
+    };
+    return;
+  }
+  compression_js.PARQUET_COMPRESSION_METHODS.ZSTD = {
+    deflate(value) {
+      return zlib.zstdCompressSync(value);
+    },
+    inflate(value) {
+      return zlib.zstdDecompressSync(value);
+    }
+  };
+}
+registerZstdCompression();
+
 function getCsvParserOptions(csvOptions = {}, skipHeader = false) {
   const options = {
     bom: true,
@@ -128,8 +192,36 @@ function getCsvParserOptions(csvOptions = {}, skipHeader = false) {
   return options;
 }
 
-function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}) {
-  function addParsedDoc(parsed, file, streamRef) {
+function createPauseWaiter$1(queueEmitter) {
+  let paused = false;
+  let waiters = [];
+  const onPause = () => {
+    paused = true;
+  };
+  const onResume = () => {
+    paused = false;
+    waiters.forEach(resolve => resolve());
+    waiters = [];
+  };
+  queueEmitter.on('pause', onPause);
+  queueEmitter.on('resume', onResume);
+  return {
+    async waitIfPaused() {
+      if (!paused) return;
+      await new Promise(resolve => {
+        waiters.push(resolve);
+      });
+    },
+    cleanup() {
+      queueEmitter.removeListener('pause', onPause);
+      queueEmitter.removeListener('resume', onResume);
+      waiters.forEach(resolve => resolve());
+      waiters = [];
+    }
+  };
+}
+function fileReaderFactory(indexer, fileName, transform, splitRegex, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}, logger) {
+  function addParsedDoc(parsed, file) {
     const context = {
       fileName: file
     };
@@ -137,7 +229,6 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, sk
 
     // if doc is null/undefined we'll skip indexing it
     if (doc === null || typeof doc === 'undefined') {
-      streamRef.resume();
       return;
     }
 
@@ -152,9 +243,101 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, sk
     }
     indexer.add(doc);
   }
-  function createNdjsonReader(file) {
+  async function processParquetFile(file) {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter$1(indexer.queueEmitter);
+    const reader = await parquet.ParquetReader.openFile(file);
+    try {
+      const cursor = reader.getCursor();
+      while (true) {
+        // eslint-disable-next-line no-await-in-loop
+        const row = await cursor.next();
+        if (row === null || typeof row === 'undefined') {
+          break;
+        }
+        addParsedDoc(row, file);
+        // eslint-disable-next-line no-await-in-loop
+        await waitIfPaused();
+      }
+      logger.info({
+        file
+      }, 'Read entire file');
+    } finally {
+      cleanup();
+      await reader.close();
+    }
+  }
+  async function processArrowFile(file) {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter$1(indexer.queueEmitter);
+    try {
+      const reader = await arrow__namespace.RecordBatchReader.from(fs.createReadStream(file));
+      for await (const recordBatch of reader) {
+        const {
+          fields
+        } = recordBatch.schema;
+        for (let rowIndex = 0; rowIndex < recordBatch.numRows; rowIndex++) {
+          const row = {};
+          fields.forEach(field => {
+            const vector = recordBatch.getChild(field.name);
+            row[field.name] = vector ? vector.get(rowIndex) : undefined;
+          });
+          addParsedDoc(row, file);
+          // eslint-disable-next-line no-await-in-loop
+          await waitIfPaused();
+        }
+      }
+      logger.info({
+        file
+      }, 'Read entire file');
+    } finally {
+      cleanup();
+    }
+  }
+  function processStreamFile(file, buildStream, errorMessage) {
+    return new Promise((resolve, reject) => {
+      let finished = false;
+      const s = buildStream();
+      const onPause = () => {
+        if (finished) return;
+        s.pause();
+      };
+      const onResume = () => {
+        if (finished) return;
+        s.resume();
+      };
+      function cleanup() {
+        indexer.queueEmitter.removeListener('pause', onPause);
+        indexer.queueEmitter.removeListener('resume', onResume);
+      }
+      indexer.queueEmitter.on('pause', onPause);
+      indexer.queueEmitter.on('resume', onResume);
+      s.on('end', () => {
+        finished = true;
+        cleanup();
+        logger.info({
+          file
+        }, 'Read entire file');
+        resolve();
+      });
+      s.on('error', err => {
+        finished = true;
+        cleanup();
+        logger.error({
+          err,
+          file
+        }, errorMessage);
+        reject(err);
+      });
+    });
+  }
+  function processNdjsonFile(file) {
     let skippedHeader = false;
-    const s = fs.createReadStream(file).pipe(split(splitRegex)).pipe(es.mapSync(line => {
+    return processStreamFile(file, () => fs.createReadStream(file).pipe(split(splitRegex)).pipe(es.mapSync(line => {
       try {
         // skip empty lines
         if (line === '') {
@@ -165,72 +348,115 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, sk
           return;
         }
         const parsed = JSON.parse(line);
-        addParsedDoc(parsed, file, s);
-      } catch (e) {
-        console.log('error', e);
+        addParsedDoc(parsed, file);
+      } catch (err) {
+        logger.error({
+          err,
+          file
+        }, 'Failed to process NDJSON line');
       }
     }).on('error', err => {
-      console.log('Error while reading file.', err);
-    }));
-    return s;
+      logger.error({
+        err,
+        file
+      }, 'Error while reading file');
+    })), 'Error while reading file');
   }
-  function createCsvReader(file) {
+  function processCsvFile(file) {
     const parserOptions = getCsvParserOptions(csvOptions, skipHeader);
-    const s = fs.createReadStream(file).pipe(csvParse.parse(parserOptions)).pipe(es.mapSync(record => {
+    return processStreamFile(file, () => fs.createReadStream(file).pipe(csvParse.parse(parserOptions)).pipe(es.mapSync(record => {
       try {
-        addParsedDoc(record, file, s);
-      } catch (e) {
-        console.log('error', e);
+        addParsedDoc(record, file);
+      } catch (err) {
+        logger.error({
+          err,
+          file
+        }, 'Failed to process CSV record');
       }
     }).on('error', err => {
-      console.log('Error while reading CSV file.', err);
-    }));
-    return s;
+      logger.error({
+        err,
+        file
+      }, 'Error while reading CSV file');
+    })), 'Error while reading CSV file');
   }
-  function startIndex(files) {
-    let finished = false;
+  async function processFile(file) {
+    if (sourceFormat === 'csv') {
+      await processCsvFile(file);
+      return;
+    }
+    if (sourceFormat === 'ndjson') {
+      await processNdjsonFile(file);
+      return;
+    }
+    if (sourceFormat === 'parquet') {
+      await processParquetFile(file);
+      return;
+    }
+    if (sourceFormat === 'arrow') {
+      await processArrowFile(file);
+      return;
+    }
+    throw Error(`Unsupported sourceFormat: ${sourceFormat}`);
+  }
+  async function startIndex(files) {
     if (files.length === 0) {
       indexer.finish();
       return;
     }
-    const file = files.shift();
-    const s = sourceFormat === 'csv' ? createCsvReader(file) : createNdjsonReader(file);
-    s.on('end', () => {
-      if (verbose) console.log('Read entire file: ', file);
-      if (files.length > 0) {
-        startIndex(files);
-        return;
+    try {
+      for (const file of files) {
+        // eslint-disable-next-line no-await-in-loop
+        await processFile(file);
       }
+    } catch (err) {
+      logger.error({
+        err,
+        files
+      }, 'Error while processing files');
+    } finally {
       indexer.finish();
-      finished = true;
-    });
-    indexer.queueEmitter.on('pause', () => {
-      if (finished) return;
-      s.pause();
-    });
-    indexer.queueEmitter.on('resume', () => {
-      if (finished) return;
-      s.resume();
-    });
+    }
   }
   return () => {
     try {
       const files = glob.globSync(fileName);
       startIndex(files);
-    } catch (error) {
-      console.log('Error matching files:', error);
+    } catch (err) {
+      logger.error({
+        err,
+        fileName
+      }, 'Error matching files');
+      indexer.finish();
     }
   };
 }
 
 const EventEmitter = require('events');
 const parallelCalls = 5;
+const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
+const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
+function coerceBigInt(value) {
+  if (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) {
+    return Number(value);
+  }
+  return value.toString();
+}
+function safeStringify(doc) {
+  return JSON.stringify(doc, (_key, value) => {
+    if (typeof value === 'bigint') {
+      return coerceBigInt(value);
+    }
+    return value;
+  });
+}
 
 // a simple helper queue to bulk index documents
 function indexQueueFactory({
   targetClient: client,
   targetIndexName,
-  bufferSize = DEFAULT_BUFFER_SIZE
+  bufferSize = DEFAULT_BUFFER_SIZE,
+  logger
 }) {
   const queueEmitter = new EventEmitter();
   let docsPerSecond = 0;
@@ -263,8 +489,9 @@ function indexQueueFactory({
           try {
             yield JSON.parse(line); // Parse and yield the JSON object
           } catch (err) {
-            // Handle JSON parse errors if necessary
-            console.error('Failed to parse JSON:', err);
+            logger.error({
+              err
+            }, 'Failed to parse JSON from NDJSON stream');
           }
         }
       }
@@ -274,7 +501,9 @@ function indexQueueFactory({
         try {
           yield JSON.parse(buffer);
         } catch (err) {
-          console.error('Failed to parse final JSON:', err);
+          logger.error({
+            err
+          }, 'Failed to parse final JSON from NDJSON stream');
         }
       }
     } finally {
@@ -300,7 +529,7 @@ function indexQueueFactory({
         flushInterval: 1000,
         refreshOnCompletion: true,
         datasource: ndjsonStreamIterator(stream$1),
-        onDocument(doc) {
+        onDocument() {
           docsPerSecond++;
           return {
             index: {
@@ -309,9 +538,13 @@ function indexQueueFactory({
           };
         }
       });
-    } catch (error) {
-      console.error('Error during bulk indexing:', error);
-      throw error;
+    } catch (err) {
+      logger.error({
+        err,
+        targetIndexName
+      }, 'Error during bulk indexing');
+      queueEmitter.emit('error', err);
+      throw err;
     } finally {
       // Clean up interval
       clearInterval(interval);
@@ -340,7 +573,7 @@ function indexQueueFactory({
       if (finished) {
         throw new Error('Unexpected doc added after indexer should finish.');
       }
-      const canContinue = stream$1.write(`${JSON.stringify(doc)}\n`);
+      const canContinue = stream$1.write(`${safeStringify(doc)}\n`);
       if (!canContinue) {
         queueEmitter.emit('pause');
 
@@ -361,7 +594,7 @@ function indexQueueFactory({
 
 // create a new progress bar instance and use shades_classic theme
 const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
-function indexReaderFactory(indexer, sourceIndexName, transform, client, query, searchSize = DEFAULT_SEARCH_SIZE, populatedFields = false) {
+function indexReaderFactory(indexer, sourceIndexName, transform, client, query, searchSize = DEFAULT_SEARCH_SIZE, populatedFields = false, logger) {
   return async function indexReader() {
     let docsNum = 0;
     let scrollId;
@@ -380,8 +613,11 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query,
           maxRetries: 0
         });
         return Object.keys(response.fields);
-      } catch (e) {
-        console.log('error', e);
+      } catch (err) {
+        logger.error({
+          err,
+          sourceIndexName
+        }, 'Failed to fetch populated fields');
       }
     }
     function search(fields) {
@@ -425,8 +661,10 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query,
           return;
         }
         indexer.add(doc);
-      } catch (e) {
-        console.log('error', e);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process source index document');
       }
     }
     async function fetchNextResponse() {
@@ -497,17 +735,25 @@ async function inferMappingsFromSource({
   mappings,
   inferMappings,
   inferMappingsOptions,
-  verbose
+  logger
 }) {
   if (!inferMappings || typeof mappings !== 'undefined' || typeof fileName === 'undefined') {
     return emptyInferenceResult(mappings);
   }
+  if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv') {
+    logger.info({
+      sourceFormat
+    }, 'Skipping mapping inference. Inference is only supported for ndjson and csv.');
+    return emptyInferenceResult(mappings);
+  }
   if (typeof targetClient?.textStructure?.findStructure !== 'function' || sourceFormat === 'xml' || sourceFormat === 'semi_structured_text') {
     return emptyInferenceResult(mappings);
   }
   const files = glob.globSync(fileName);
   if (files.length === 0) {
-    if (verbose) console.log(`No files matched for mapping inference: ${fileName}`);
+    logger.info({
+      fileName
+    }, 'No files matched for mapping inference');
     return emptyInferenceResult(mappings);
   }
   const {
@@ -516,7 +762,7 @@ async function inferMappingsFromSource({
   } = inferMappingsOptions || {};
   const sampleText = readSample(files[0], sampleBytes);
   if (!sampleText || sampleText.trim() === '') {
-    if (verbose) console.log('Skipping mapping inference because the sample text is empty.');
+    logger.info('Skipping mapping inference because the sample text is empty');
     return emptyInferenceResult(mappings);
   }
   const params = {
@@ -543,31 +789,98 @@ async function inferMappingsFromSource({
   }
   try {
     const response = await targetClient.textStructure.findStructure(params);
-    if (response?.mappings && verbose) {
-      console.log(`Inferred mappings via _text_structure/find_structure from ${files[0]}`);
+    if (response?.mappings) {
+      logger.info({
+        file: files[0]
+      }, 'Inferred mappings via _text_structure/find_structure');
     }
-    if (response?.ingest_pipeline && verbose) {
-      console.log('Inferred ingest pipeline via _text_structure/find_structure');
+    if (response?.ingest_pipeline) {
+      logger.info('Inferred ingest pipeline via _text_structure/find_structure');
     }
     return {
       mappings: response?.mappings || mappings,
       ingestPipeline: response?.ingest_pipeline
     };
-  } catch (error) {
-    if (verbose) {
-      console.log('Could not infer mappings via _text_structure/find_structure:', error.message);
-    }
+  } catch (err) {
+    logger.warn({
+      err
+    }, 'Could not infer mappings via _text_structure/find_structure');
     return emptyInferenceResult(mappings);
   }
 }
 
-function streamReaderFactory(indexer, stream, transform, splitRegex, verbose, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}) {
-  function addParsedDoc(parsed, streamRef) {
+const DEFAULT_LOG_LEVEL = 'info';
+function resolveLogLevel(verbose = true) {
+  if (typeof process.env.LOG_LEVEL === 'string' && process.env.LOG_LEVEL.trim() !== '') {
+    return process.env.LOG_LEVEL;
+  }
+  return verbose ? DEFAULT_LOG_LEVEL : 'error';
+}
+function createLogger({
+  logger,
+  verbose = true
+} = {}) {
+  if (logger && typeof logger === 'object') {
+    return logger;
+  }
+  return pino({
+    name: 'node-es-transformer',
+    level: resolveLogLevel(verbose),
+    timestamp: pino.stdTimeFunctions.isoTime,
+    serializers: {
+      err: pino.stdSerializers.err,
+      error: pino.stdSerializers.err
+    }
+  });
+}
+function createChildLogger(logger, bindings) {
+  if (!logger || typeof logger.child !== 'function') {
+    return logger;
+  }
+  return logger.child(bindings);
+}
+
+function createPauseWaiter(queueEmitter) {
+  let paused = false;
+  let waiters = [];
+  const onPause = () => {
+    paused = true;
+  };
+  const onResume = () => {
+    paused = false;
+    waiters.forEach(resolve => resolve());
+    waiters = [];
+  };
+  queueEmitter.on('pause', onPause);
+  queueEmitter.on('resume', onResume);
+  return {
+    async waitIfPaused() {
+      if (!paused) return;
+      await new Promise(resolve => {
+        waiters.push(resolve);
+      });
+    },
+    cleanup() {
+      queueEmitter.removeListener('pause', onPause);
+      queueEmitter.removeListener('resume', onResume);
+      waiters.forEach(resolve => resolve());
+      waiters = [];
+    }
+  };
+}
+async function readStreamToBuffer(stream) {
+  const chunks = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks);
+}
+function streamReaderFactory(indexer, stream, transform, splitRegex, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}, logger) {
+  function addParsedDoc(parsed) {
     const doc = typeof transform === 'function' ? transform(parsed) : parsed;
 
     // if doc is null/undefined we'll skip indexing it
     if (doc === null || typeof doc === 'undefined') {
-      streamRef.resume();
       return;
     }
 
@@ -582,50 +895,152 @@ function streamReaderFactory(indexer, stream, transform, splitRegex, verbose, sk
     }
     indexer.add(doc);
   }
-  function startIndex() {
-    let finished = false;
-    const s = sourceFormat === 'csv' ? stream.pipe(csvParse.parse(getCsvParserOptions(csvOptions, skipHeader))).pipe(es.mapSync(record => {
+  async function processParquetStream() {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter(indexer.queueEmitter);
+    const parquetBuffer = await readStreamToBuffer(stream);
+    const reader = await parquet.ParquetReader.openBuffer(parquetBuffer);
+    try {
+      const cursor = reader.getCursor();
+      while (true) {
+        // eslint-disable-next-line no-await-in-loop
+        const row = await cursor.next();
+        if (row === null || typeof row === 'undefined') {
+          break;
+        }
+        addParsedDoc(row);
+        // eslint-disable-next-line no-await-in-loop
+        await waitIfPaused();
+      }
+      logger.info('Read entire stream');
+    } finally {
+      cleanup();
+      await reader.close();
+    }
+  }
+  async function processArrowStream() {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter(indexer.queueEmitter);
+    try {
+      const reader = await arrow__namespace.RecordBatchReader.from(stream);
+      for await (const recordBatch of reader) {
+        const {
+          fields
+        } = recordBatch.schema;
+        for (let rowIndex = 0; rowIndex < recordBatch.numRows; rowIndex++) {
+          const row = {};
+          fields.forEach(field => {
+            const vector = recordBatch.getChild(field.name);
+            row[field.name] = vector ? vector.get(rowIndex) : undefined;
+          });
+          addParsedDoc(row);
+          // eslint-disable-next-line no-await-in-loop
+          await waitIfPaused();
+        }
+      }
+      logger.info('Read entire stream');
+    } finally {
+      cleanup();
+    }
+  }
+  function processPipeline(buildPipeline, errorMessage) {
+    return new Promise((resolve, reject) => {
+      let finished = false;
+      const s = buildPipeline();
+      const onPause = () => {
+        if (finished) return;
+        s.pause();
+      };
+      const onResume = () => {
+        if (finished) return;
+        s.resume();
+      };
+      function cleanup() {
+        indexer.queueEmitter.removeListener('pause', onPause);
+        indexer.queueEmitter.removeListener('resume', onResume);
+      }
+      indexer.queueEmitter.on('pause', onPause);
+      indexer.queueEmitter.on('resume', onResume);
+      s.on('end', () => {
+        finished = true;
+        cleanup();
+        logger.info('Read entire stream');
+        resolve();
+      });
+      s.on('error', err => {
+        finished = true;
+        cleanup();
+        logger.error({
+          err
+        }, errorMessage);
+        reject(err);
+      });
+    });
+  }
+  function processCsvStream() {
+    return processPipeline(() => stream.pipe(csvParse.parse(getCsvParserOptions(csvOptions, skipHeader))).pipe(es.mapSync(record => {
       try {
-        addParsedDoc(record, s);
-      } catch (e) {
-        console.log('error', e);
+        addParsedDoc(record);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process CSV stream record');
       }
     }).on('error', err => {
-      console.log('Error while reading CSV stream.', err);
-    })) : (() => {
-      let skippedHeader = false;
-      return stream.pipe(split(splitRegex)).pipe(es.mapSync(line => {
-        try {
-          // skip empty lines
-          if (line === '') {
-            return;
-          }
-          if (skipHeader && !skippedHeader) {
-            skippedHeader = true;
-            return;
-          }
-          const parsed = JSON.parse(line);
-          addParsedDoc(parsed, s);
-        } catch (e) {
-          console.log('error', e);
+      logger.error({
+        err
+      }, 'Error while reading CSV stream');
+    })), 'Error while reading CSV stream');
+  }
+  function processNdjsonStream() {
+    let skippedHeader = false;
+    return processPipeline(() => stream.pipe(split(splitRegex)).pipe(es.mapSync(line => {
+      try {
+        // skip empty lines
+        if (line === '') {
+          return;
         }
-      }).on('error', err => {
-        console.log('Error while reading stream.', err);
-      }));
-    })();
-    s.on('end', () => {
-      if (verbose) console.log('Read entire stream.');
+        if (skipHeader && !skippedHeader) {
+          skippedHeader = true;
+          return;
+        }
+        const parsed = JSON.parse(line);
+        addParsedDoc(parsed);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process NDJSON stream line');
+      }
+    }).on('error', err => {
+      logger.error({
+        err
+      }, 'Error while reading stream');
+    })), 'Error while reading stream');
+  }
+  async function startIndex() {
+    try {
+      if (sourceFormat === 'csv') {
+        await processCsvStream();
+      } else if (sourceFormat === 'ndjson') {
+        await processNdjsonStream();
+      } else if (sourceFormat === 'parquet') {
+        await processParquetStream();
+      } else if (sourceFormat === 'arrow') {
+        await processArrowStream();
+      } else {
+        throw Error(`Unsupported sourceFormat: ${sourceFormat}`);
+      }
+    } catch (err) {
+      logger.error({
+        err
+      }, 'Error while reading stream');
+    } finally {
       indexer.finish();
-      finished = true;
-    });
-    indexer.queueEmitter.on('pause', () => {
-      if (finished) return;
-      s.pause();
-    });
-    indexer.queueEmitter.on('resume', () => {
-      if (finished) return;
-      s.resume();
-    });
+    }
   }
   return () => {
     startIndex();
@@ -721,11 +1136,16 @@ async function transformer({
   query,
   skipHeader = false,
   transform,
-  verbose = true
+  verbose = true,
+  logger: loggerInput
 }) {
   if (typeof targetIndexName === 'undefined') {
     throw Error('targetIndexName must be specified.');
   }
+  const logger = createLogger({
+    logger: loggerInput,
+    verbose
+  });
   const defaultClientConfig = {
     node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200'
   };
@@ -742,7 +1162,9 @@ async function transformer({
     mappings,
     inferMappings,
     inferMappingsOptions,
-    verbose
+    logger: createChildLogger(logger, {
+      component: 'mapping-inference'
+    })
   });
   const createMapping = createMappingFactory({
     sourceClient,
@@ -753,17 +1175,23 @@ async function transformer({
     inferredIngestPipeline: inferenceResult.ingestPipeline,
     mappingsOverride,
     indexMappingTotalFieldsLimit,
-    verbose,
     deleteIndex,
-    pipeline
+    pipeline,
+    logger: createChildLogger(logger, {
+      component: 'create-mapping'
+    })
   });
   const indexer = indexQueueFactory({
     targetClient,
     targetIndexName,
-    bufferSize});
+    bufferSize,
+    logger: createChildLogger(logger, {
+      component: 'index-queue'
+    })
+  });
   function validateSourceFormat() {
-    if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv') {
-      throw Error(`Unsupported sourceFormat: ${sourceFormat}. Use "ndjson" or "csv".`);
+    if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv' && sourceFormat !== 'parquet' && sourceFormat !== 'arrow') {
+      throw Error(`Unsupported sourceFormat: ${sourceFormat}. Use "ndjson", "csv", "parquet", or "arrow".`);
     }
   }
   function getReader() {
@@ -775,18 +1203,27 @@ async function transformer({
     }
     if (typeof fileName !== 'undefined') {
       validateSourceFormat();
-      return fileReaderFactory(indexer, fileName, transform, splitRegex, verbose, skipHeader, sourceFormat, csvOptions);
+      return fileReaderFactory(indexer, fileName, transform, splitRegex, skipHeader, sourceFormat, csvOptions, createChildLogger(logger, {
+        component: 'file-reader'
+      }));
     }
     if (typeof sourceIndexName !== 'undefined') {
-      return indexReaderFactory(indexer, sourceIndexName, transform, sourceClient, query, searchSize, populatedFields);
+      return indexReaderFactory(indexer, sourceIndexName, transform, sourceClient, query, searchSize, populatedFields, createChildLogger(logger, {
+        component: 'index-reader'
+      }));
     }
     if (typeof stream !== 'undefined') {
       validateSourceFormat();
-      return streamReaderFactory(indexer, stream, transform, splitRegex, verbose, skipHeader, sourceFormat, csvOptions);
+      return streamReaderFactory(indexer, stream, transform, splitRegex, skipHeader, sourceFormat, csvOptions, createChildLogger(logger, {
+        component: 'stream-reader'
+      }));
     }
     return null;
   }
   const reader = getReader();
+  if (typeof reader !== 'function') {
+    throw Error('One of fileName, sourceIndexName, or stream must be specified.');
+  }
   try {
     const indexExists = await targetClient.indices.exists({
       index: targetIndexName
@@ -803,8 +1240,11 @@ async function transformer({
     } else {
       reader();
     }
-  } catch (error) {
-    console.error('Error checking index existence:', error);
+  } catch (err) {
+    logger.error({
+      err,
+      targetIndexName
+    }, 'Error checking index existence');
   } finally {
     // targetClient.close();
   }