node-es-transformer 1.0.2 → 1.2.0

package/dist/node-es-transformer.cjs.js

@@ -2,12 +2,35 @@
 
 var elasticsearch9 = require('es9');
 var elasticsearch8 = require('es8');
+var parquet = require('@dsnp/parquetjs');
+var arrow = require('apache-arrow');
 var fs = require('fs');
+var csvParse = require('csv-parse');
 var es = require('event-stream');
 var glob = require('glob');
 var split = require('split2');
 var stream = require('stream');
 var cliProgress = require('cli-progress');
+var pino = require('pino');
+
+function _interopNamespaceDefault(e) {
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
+var arrow__namespace = /*#__PURE__*/_interopNamespaceDefault(arrow);
 
 // In earlier versions this was used to set the number of docs to index in a
 // single bulk request. Since we switched to use the helpers.bulk() method from
@@ -25,14 +48,16 @@ function createMappingFactory({
   targetClient,
   targetIndexName,
   mappings,
+  inferredIngestPipeline,
   mappingsOverride,
   indexMappingTotalFieldsLimit,
-  verbose,
   deleteIndex,
-  pipeline
+  pipeline,
+  logger
 }) {
   return async () => {
     let targetMappings = mappingsOverride ? undefined : mappings;
+    let defaultPipeline = pipeline;
     if (sourceClient && sourceIndexName && typeof targetMappings === 'undefined') {
       try {
         const mapping = await sourceClient.indices.getMapping({
@@ -47,7 +72,10 @@ function createMappingFactory({
           }
         }
       } catch (err) {
-        console.log('Error reading source mapping', err);
+        logger.error({
+          err,
+          sourceIndexName
+        }, 'Error reading source mapping');
         return;
       }
     }
@@ -71,93 +99,312 @@ function createMappingFactory({
           });
         }
         if (indexExists === false || deleteIndex === true) {
-          const resp = await targetClient.indices.create({
-            index: targetIndexName,
-            mappings: targetMappings,
-            ...(pipeline !== undefined ? {
-              settings: {
-                index: {
-                  default_pipeline: pipeline
-                }
-              }
+          if (typeof defaultPipeline === 'undefined' && typeof inferredIngestPipeline === 'object' && inferredIngestPipeline !== null && typeof targetClient?.ingest?.putPipeline === 'function') {
+            const inferredPipelineName = `${targetIndexName}-inferred-pipeline`;
+            try {
+              await targetClient.ingest.putPipeline({
+                id: inferredPipelineName,
+                ...inferredIngestPipeline
+              });
+              defaultPipeline = inferredPipelineName;
+              logger.info({
+                inferredPipelineName
+              }, 'Created inferred ingest pipeline');
+            } catch (err) {
+              logger.error({
+                err,
+                inferredPipelineName
+              }, 'Error creating inferred ingest pipeline');
+            }
+          }
+          const settings = {
+            ...(defaultPipeline !== undefined ? {
+              'index.default_pipeline': defaultPipeline
             } : {}),
             ...(indexMappingTotalFieldsLimit !== undefined ? {
-              settings: {
-                'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
-                'index.number_of_shards': 1,
-                'index.number_of_replicas': 0
-              }
+              'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
+              'index.number_of_shards': 1,
+              'index.number_of_replicas': 0
+            } : {})
+          };
+          const response = await targetClient.indices.create({
+            index: targetIndexName,
+            mappings: targetMappings,
+            ...(Object.keys(settings).length > 0 ? {
+              settings
             } : {})
           });
-          if (verbose) console.log('Created target mapping', resp);
+          logger.info({
+            targetIndexName,
+            response
+          }, 'Created target mapping');
         }
       } catch (err) {
-        console.log('Error creating target mapping', err);
+        logger.error({
+          err,
+          targetIndexName
+        }, 'Error creating target mapping');
       }
     }
   };
 }
 
-function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
-  function startIndex(files) {
-    let finished = false;
-    const file = files.shift();
-    const s = fs.createReadStream(file).pipe(split(splitRegex)).pipe(es.mapSync(line => {
+function getCsvParserOptions(csvOptions = {}, skipHeader = false) {
+  const options = {
+    bom: true,
+    columns: true,
+    trim: true,
+    skip_empty_lines: true,
+    ...csvOptions
+  };
+  const consumesHeader = options.columns === true || typeof options.columns === 'function';
+  if (skipHeader && !consumesHeader && typeof options.from_line === 'undefined') {
+    options.from_line = 2;
+  }
+  return options;
+}
+
+function createPauseWaiter$1(queueEmitter) {
+  let paused = false;
+  let waiters = [];
+  const onPause = () => {
+    paused = true;
+  };
+  const onResume = () => {
+    paused = false;
+    waiters.forEach(resolve => resolve());
+    waiters = [];
+  };
+  queueEmitter.on('pause', onPause);
+  queueEmitter.on('resume', onResume);
+  return {
+    async waitIfPaused() {
+      if (!paused) return;
+      await new Promise(resolve => {
+        waiters.push(resolve);
+      });
+    },
+    cleanup() {
+      queueEmitter.removeListener('pause', onPause);
+      queueEmitter.removeListener('resume', onResume);
+      waiters.forEach(resolve => resolve());
+      waiters = [];
+    }
+  };
+}
+function fileReaderFactory(indexer, fileName, transform, splitRegex, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}, logger) {
+  function addParsedDoc(parsed, file) {
+    const context = {
+      fileName: file
+    };
+    const doc = typeof transform === 'function' ? transform(parsed, context) : parsed;
+
+    // if doc is null/undefined we'll skip indexing it
+    if (doc === null || typeof doc === 'undefined') {
+      return;
+    }
+
+    // the transform callback may return an array of docs so we can emit
+    // multiple docs from a single line
+    if (Array.isArray(doc)) {
+      doc.forEach(d => {
+        if (d === null || typeof d === 'undefined') return;
+        indexer.add(d);
+      });
+      return;
+    }
+    indexer.add(doc);
+  }
+  async function processParquetFile(file) {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter$1(indexer.queueEmitter);
+    const reader = await parquet.ParquetReader.openFile(file);
+    try {
+      const cursor = reader.getCursor();
+      while (true) {
+        // eslint-disable-next-line no-await-in-loop
+        const row = await cursor.next();
+        if (row === null || typeof row === 'undefined') {
+          break;
+        }
+        addParsedDoc(row, file);
+        // eslint-disable-next-line no-await-in-loop
+        await waitIfPaused();
+      }
+      logger.info({
+        file
+      }, 'Read entire file');
+    } finally {
+      cleanup();
+      await reader.close();
+    }
+  }
+  async function processArrowFile(file) {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter$1(indexer.queueEmitter);
+    try {
+      const reader = await arrow__namespace.RecordBatchReader.from(fs.createReadStream(file));
+      for await (const recordBatch of reader) {
+        const {
+          fields
+        } = recordBatch.schema;
+        for (let rowIndex = 0; rowIndex < recordBatch.numRows; rowIndex++) {
+          const row = {};
+          fields.forEach(field => {
+            const vector = recordBatch.getChild(field.name);
+            row[field.name] = vector ? vector.get(rowIndex) : undefined;
+          });
+          addParsedDoc(row, file);
+          // eslint-disable-next-line no-await-in-loop
+          await waitIfPaused();
+        }
+      }
+      logger.info({
+        file
+      }, 'Read entire file');
+    } finally {
+      cleanup();
+    }
+  }
+  function processStreamFile(file, buildStream, errorMessage) {
+    return new Promise((resolve, reject) => {
+      let finished = false;
+      const s = buildStream();
+      const onPause = () => {
+        if (finished) return;
+        s.pause();
+      };
+      const onResume = () => {
+        if (finished) return;
+        s.resume();
+      };
+      function cleanup() {
+        indexer.queueEmitter.removeListener('pause', onPause);
+        indexer.queueEmitter.removeListener('resume', onResume);
+      }
+      indexer.queueEmitter.on('pause', onPause);
+      indexer.queueEmitter.on('resume', onResume);
+      s.on('end', () => {
+        finished = true;
+        cleanup();
+        logger.info({
+          file
+        }, 'Read entire file');
+        resolve();
+      });
+      s.on('error', err => {
+        finished = true;
+        cleanup();
+        logger.error({
+          err,
+          file
+        }, errorMessage);
+        reject(err);
+      });
+    });
+  }
+  function processNdjsonFile(file) {
+    let skippedHeader = false;
+    return processStreamFile(file, () => fs.createReadStream(file).pipe(split(splitRegex)).pipe(es.mapSync(line => {
       try {
         // skip empty lines
         if (line === '') {
           return;
         }
-        const doc = typeof transform === 'function' ? JSON.stringify(transform(JSON.parse(line))) : line;
-
-        // if doc is undefined we'll skip indexing it
-        if (typeof doc === 'undefined') {
-          s.resume();
-          return;
-        }
-
-        // the transform callback may return an array of docs so we can emit
-        // multiple docs from a single line
-        if (Array.isArray(doc)) {
-          doc.forEach(d => indexer.add(d));
+        if (skipHeader && !skippedHeader) {
+          skippedHeader = true;
           return;
         }
-        indexer.add(doc);
-      } catch (e) {
-        console.log('error', e);
+        const parsed = JSON.parse(line);
+        addParsedDoc(parsed, file);
+      } catch (err) {
+        logger.error({
+          err,
+          file
+        }, 'Failed to process NDJSON line');
       }
     }).on('error', err => {
-      console.log('Error while reading file.', err);
-    }).on('end', () => {
-      if (verbose) console.log('Read entire file: ', file);
-      if (files.length > 0) {
-        startIndex(files);
-        return;
+      logger.error({
+        err,
+        file
+      }, 'Error while reading file');
+    })), 'Error while reading file');
+  }
+  function processCsvFile(file) {
+    const parserOptions = getCsvParserOptions(csvOptions, skipHeader);
+    return processStreamFile(file, () => fs.createReadStream(file).pipe(csvParse.parse(parserOptions)).pipe(es.mapSync(record => {
+      try {
+        addParsedDoc(record, file);
+      } catch (err) {
+        logger.error({
+          err,
+          file
+        }, 'Failed to process CSV record');
+      }
+    }).on('error', err => {
+      logger.error({
+        err,
+        file
+      }, 'Error while reading CSV file');
+    })), 'Error while reading CSV file');
+  }
+  async function processFile(file) {
+    if (sourceFormat === 'csv') {
+      await processCsvFile(file);
+      return;
+    }
+    if (sourceFormat === 'ndjson') {
+      await processNdjsonFile(file);
+      return;
+    }
+    if (sourceFormat === 'parquet') {
+      await processParquetFile(file);
+      return;
+    }
+    if (sourceFormat === 'arrow') {
+      await processArrowFile(file);
+      return;
+    }
+    throw Error(`Unsupported sourceFormat: ${sourceFormat}`);
+  }
+  async function startIndex(files) {
+    if (files.length === 0) {
+      indexer.finish();
+      return;
+    }
+    try {
+      for (const file of files) {
+        // eslint-disable-next-line no-await-in-loop
+        await processFile(file);
       }
+    } catch (err) {
+      logger.error({
+        err,
+        files
+      }, 'Error while processing files');
+    } finally {
       indexer.finish();
-      finished = true;
-    }));
-    indexer.queueEmitter.on('pause', () => {
-      if (finished) return;
-      s.pause();
-    });
-    indexer.queueEmitter.on('resume', () => {
-      if (finished) return;
-      s.resume();
-    });
+    }
   }
   return () => {
     try {
       const files = glob.globSync(fileName);
       startIndex(files);
-    } catch (error) {
-      console.log('Error matching files:', error);
+    } catch (err) {
+      logger.error({
+        err,
+        fileName
+      }, 'Error matching files');
+      indexer.finish();
     }
   };
 }
 
 const EventEmitter = require('events');
-const queueEmitter = new EventEmitter();
 const parallelCalls = 5;
 
 // a simple helper queue to bulk index documents
@@ -165,21 +412,20 @@ function indexQueueFactory({
   targetClient: client,
   targetIndexName,
   bufferSize = DEFAULT_BUFFER_SIZE,
-  skipHeader = false
+  logger
 }) {
+  const queueEmitter = new EventEmitter();
   let docsPerSecond = 0;
   const flushBytes = bufferSize * 1024; // Convert KB to Bytes
   const highWaterMark = flushBytes * parallelCalls;
 
-  // Create a Readable stream
-  const stream$1 = new stream.Readable({
-    read() {},
-    // Implement read but we manage pushing manually
+  // Create a PassThrough stream (readable + writable) for proper backpressure
+  const stream$1 = new stream.PassThrough({
     highWaterMark // Buffer size for backpressure management
   });
   async function* ndjsonStreamIterator(readableStream) {
     let buffer = ''; // To hold the incomplete data
-    let skippedHeader = false;
+
     try {
       // Iterate over the stream using async iteration
       for await (const chunk of readableStream) {
@@ -193,16 +439,15 @@ function indexQueueFactory({
 
         // Yield each complete JSON object
         for (const line of lines) {
-          if (line.trim()) {
-            try {
-              if (!skipHeader || skipHeader && !skippedHeader) {
-                yield JSON.parse(line); // Parse and yield the JSON object
-                skippedHeader = true;
-              }
-            } catch (err) {
-              // Handle JSON parse errors if necessary
-              console.error('Failed to parse JSON:', err);
-            }
+          if (!line.trim()) {
+            continue;
+          }
+          try {
+            yield JSON.parse(line); // Parse and yield the JSON object
+          } catch (err) {
+            logger.error({
+              err
+            }, 'Failed to parse JSON from NDJSON stream');
           }
         }
       }
@@ -212,7 +457,9 @@ function indexQueueFactory({
         try {
           yield JSON.parse(buffer);
         } catch (err) {
-          console.error('Failed to parse final JSON:', err);
+          logger.error({
+            err
+          }, 'Failed to parse final JSON from NDJSON stream');
         }
       }
     } finally {
@@ -238,7 +485,7 @@ function indexQueueFactory({
         flushInterval: 1000,
         refreshOnCompletion: true,
         datasource: ndjsonStreamIterator(stream$1),
-        onDocument(doc) {
+        onDocument() {
           docsPerSecond++;
           return {
             index: {
@@ -247,9 +494,13 @@ function indexQueueFactory({
           };
         }
       });
-    } catch (error) {
-      console.error('Error during bulk indexing:', error);
-      throw error;
+    } catch (err) {
+      logger.error({
+        err,
+        targetIndexName
+      }, 'Error during bulk indexing');
+      queueEmitter.emit('error', err);
+      throw err;
     } finally {
       // Clean up interval
       clearInterval(interval);
@@ -278,7 +529,7 @@ function indexQueueFactory({
       if (finished) {
         throw new Error('Unexpected doc added after indexer should finish.');
       }
-      const canContinue = stream$1.push(`${JSON.stringify(doc)}\n`);
+      const canContinue = stream$1.write(`${JSON.stringify(doc)}\n`);
       if (!canContinue) {
         queueEmitter.emit('pause');
 
@@ -291,7 +542,7 @@ function indexQueueFactory({
     },
     finish: () => {
       finished = true;
-      stream$1.push(null);
+      stream$1.end();
     },
     queueEmitter
   };
@@ -299,7 +550,7 @@ function indexQueueFactory({
 
 // create a new progress bar instance and use shades_classic theme
 const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
-function indexReaderFactory(indexer, sourceIndexName, transform, client, query, searchSize = DEFAULT_SEARCH_SIZE, populatedFields = false) {
+function indexReaderFactory(indexer, sourceIndexName, transform, client, query, searchSize = DEFAULT_SEARCH_SIZE, populatedFields = false, logger) {
   return async function indexReader() {
     let docsNum = 0;
     let scrollId;
@@ -318,8 +569,11 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query,
           maxRetries: 0
         });
         return Object.keys(response.fields);
-      } catch (e) {
-        console.log('error', e);
+      } catch (err) {
+        logger.error({
+          err,
+          sourceIndexName
+        }, 'Failed to fetch populated fields');
       }
     }
     function search(fields) {
@@ -363,8 +617,10 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query,
           return;
         }
         indexer.add(doc);
-      } catch (e) {
-        console.log('error', e);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process source index document');
       }
     }
     async function fetchNextResponse() {
@@ -408,48 +664,339 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query,
   };
 }
 
-function streamReaderFactory(indexer, stream, transform, splitRegex, verbose) {
-  function startIndex() {
-    let finished = false;
-    const s = stream.pipe(split(splitRegex)).pipe(es.mapSync(line => {
+const DEFAULT_INFER_MAPPINGS_SAMPLE_BYTES = 100000;
+const DEFAULT_INFER_MAPPINGS_LINES_TO_SAMPLE = 1000;
+function readSample(filePath, sampleBytes) {
+  const fd = fs.openSync(filePath, 'r');
+  try {
+    const buffer = Buffer.alloc(sampleBytes);
+    const bytesRead = fs.readSync(fd, buffer, 0, sampleBytes, 0);
+    return buffer.subarray(0, bytesRead).toString('utf8');
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+function emptyInferenceResult(mappings) {
+  return {
+    mappings,
+    ingestPipeline: undefined
+  };
+}
+async function inferMappingsFromSource({
+  targetClient,
+  fileName,
+  sourceFormat,
+  csvOptions,
+  skipHeader,
+  mappings,
+  inferMappings,
+  inferMappingsOptions,
+  logger
+}) {
+  if (!inferMappings || typeof mappings !== 'undefined' || typeof fileName === 'undefined') {
+    return emptyInferenceResult(mappings);
+  }
+  if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv') {
+    logger.info({
+      sourceFormat
+    }, 'Skipping mapping inference. Inference is only supported for ndjson and csv.');
+    return emptyInferenceResult(mappings);
+  }
+  if (typeof targetClient?.textStructure?.findStructure !== 'function' || sourceFormat === 'xml' || sourceFormat === 'semi_structured_text') {
+    return emptyInferenceResult(mappings);
+  }
+  const files = glob.globSync(fileName);
+  if (files.length === 0) {
+    logger.info({
+      fileName
+    }, 'No files matched for mapping inference');
+    return emptyInferenceResult(mappings);
+  }
+  const {
+    sampleBytes = DEFAULT_INFER_MAPPINGS_SAMPLE_BYTES,
+    ...requestParams
+  } = inferMappingsOptions || {};
+  const sampleText = readSample(files[0], sampleBytes);
+  if (!sampleText || sampleText.trim() === '') {
+    logger.info('Skipping mapping inference because the sample text is empty');
+    return emptyInferenceResult(mappings);
+  }
+  const params = {
+    body: sampleText,
+    lines_to_sample: DEFAULT_INFER_MAPPINGS_LINES_TO_SAMPLE,
+    ...requestParams
+  };
+  if (typeof params.format === 'undefined') {
+    params.format = sourceFormat === 'csv' ? 'delimited' : 'ndjson';
+  }
+  if (sourceFormat === 'csv') {
+    if (typeof params.delimiter === 'undefined' && typeof csvOptions?.delimiter === 'string') {
+      params.delimiter = csvOptions.delimiter;
+    }
+    if (typeof params.quote === 'undefined' && typeof csvOptions?.quote === 'string') {
+      params.quote = csvOptions.quote;
+    }
+    if (typeof params.has_header_row === 'undefined' && typeof csvOptions?.columns === 'boolean') {
+      params.has_header_row = csvOptions.columns;
+    }
+    if (typeof params.has_header_row === 'undefined' && skipHeader) {
+      params.has_header_row = true;
+    }
+  }
+  try {
+    const response = await targetClient.textStructure.findStructure(params);
+    if (response?.mappings) {
+      logger.info({
+        file: files[0]
+      }, 'Inferred mappings via _text_structure/find_structure');
+    }
+    if (response?.ingest_pipeline) {
+      logger.info('Inferred ingest pipeline via _text_structure/find_structure');
+    }
+    return {
+      mappings: response?.mappings || mappings,
+      ingestPipeline: response?.ingest_pipeline
+    };
+  } catch (err) {
+    logger.warn({
+      err
+    }, 'Could not infer mappings via _text_structure/find_structure');
+    return emptyInferenceResult(mappings);
+  }
+}
+
+const DEFAULT_LOG_LEVEL = 'info';
+function resolveLogLevel(verbose = true) {
+  if (typeof process.env.LOG_LEVEL === 'string' && process.env.LOG_LEVEL.trim() !== '') {
+    return process.env.LOG_LEVEL;
+  }
+  return verbose ? DEFAULT_LOG_LEVEL : 'error';
+}
+function createLogger({
+  logger,
+  verbose = true
+} = {}) {
+  if (logger && typeof logger === 'object') {
+    return logger;
+  }
+  return pino({
+    name: 'node-es-transformer',
+    level: resolveLogLevel(verbose),
+    timestamp: pino.stdTimeFunctions.isoTime,
+    serializers: {
+      err: pino.stdSerializers.err,
+      error: pino.stdSerializers.err
+    }
+  });
+}
+function createChildLogger(logger, bindings) {
+  if (!logger || typeof logger.child !== 'function') {
+    return logger;
+  }
+  return logger.child(bindings);
+}
+
+function createPauseWaiter(queueEmitter) {
+  let paused = false;
+  let waiters = [];
+  const onPause = () => {
+    paused = true;
+  };
+  const onResume = () => {
+    paused = false;
+    waiters.forEach(resolve => resolve());
+    waiters = [];
+  };
+  queueEmitter.on('pause', onPause);
+  queueEmitter.on('resume', onResume);
+  return {
+    async waitIfPaused() {
+      if (!paused) return;
+      await new Promise(resolve => {
+        waiters.push(resolve);
+      });
+    },
+    cleanup() {
+      queueEmitter.removeListener('pause', onPause);
+      queueEmitter.removeListener('resume', onResume);
+      waiters.forEach(resolve => resolve());
+      waiters = [];
+    }
+  };
+}
+async function readStreamToBuffer(stream) {
+  const chunks = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks);
+}
+function streamReaderFactory(indexer, stream, transform, splitRegex, skipHeader = false, sourceFormat = 'ndjson', csvOptions = {}, logger) {
+  function addParsedDoc(parsed) {
+    const doc = typeof transform === 'function' ? transform(parsed) : parsed;
+
+    // if doc is null/undefined we'll skip indexing it
+    if (doc === null || typeof doc === 'undefined') {
+      return;
+    }
+
+    // the transform callback may return an array of docs so we can emit
+    // multiple docs from a single line
+    if (Array.isArray(doc)) {
+      doc.forEach(d => {
+        if (d === null || typeof d === 'undefined') return;
+        indexer.add(d);
+      });
+      return;
+    }
+    indexer.add(doc);
+  }
+  async function processParquetStream() {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter(indexer.queueEmitter);
+    const parquetBuffer = await readStreamToBuffer(stream);
+    const reader = await parquet.ParquetReader.openBuffer(parquetBuffer);
+    try {
+      const cursor = reader.getCursor();
+      while (true) {
+        // eslint-disable-next-line no-await-in-loop
+        const row = await cursor.next();
+        if (row === null || typeof row === 'undefined') {
+          break;
+        }
+        addParsedDoc(row);
+        // eslint-disable-next-line no-await-in-loop
+        await waitIfPaused();
+      }
+      logger.info('Read entire stream');
+    } finally {
+      cleanup();
+      await reader.close();
+    }
+  }
+  async function processArrowStream() {
+    const {
+      waitIfPaused,
+      cleanup
+    } = createPauseWaiter(indexer.queueEmitter);
+    try {
+      const reader = await arrow__namespace.RecordBatchReader.from(stream);
+      for await (const recordBatch of reader) {
+        const {
+          fields
+        } = recordBatch.schema;
+        for (let rowIndex = 0; rowIndex < recordBatch.numRows; rowIndex++) {
+          const row = {};
+          fields.forEach(field => {
+            const vector = recordBatch.getChild(field.name);
+            row[field.name] = vector ? vector.get(rowIndex) : undefined;
+          });
+          addParsedDoc(row);
+          // eslint-disable-next-line no-await-in-loop
+          await waitIfPaused();
+        }
+      }
+      logger.info('Read entire stream');
+    } finally {
+      cleanup();
+    }
+  }
+  function processPipeline(buildPipeline, errorMessage) {
+    return new Promise((resolve, reject) => {
+      let finished = false;
+      const s = buildPipeline();
+      const onPause = () => {
+        if (finished) return;
+        s.pause();
+      };
+      const onResume = () => {
+        if (finished) return;
+        s.resume();
+      };
+      function cleanup() {
+        indexer.queueEmitter.removeListener('pause', onPause);
+        indexer.queueEmitter.removeListener('resume', onResume);
+      }
+      indexer.queueEmitter.on('pause', onPause);
+      indexer.queueEmitter.on('resume', onResume);
+      s.on('end', () => {
+        finished = true;
+        cleanup();
+        logger.info('Read entire stream');
+        resolve();
+      });
+      s.on('error', err => {
+        finished = true;
+        cleanup();
+        logger.error({
+          err
+        }, errorMessage);
+        reject(err);
+      });
+    });
+  }
+  function processCsvStream() {
+    return processPipeline(() => stream.pipe(csvParse.parse(getCsvParserOptions(csvOptions, skipHeader))).pipe(es.mapSync(record => {
+      try {
+        addParsedDoc(record);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process CSV stream record');
+      }
+    }).on('error', err => {
+      logger.error({
+        err
+      }, 'Error while reading CSV stream');
+    })), 'Error while reading CSV stream');
+  }
+  function processNdjsonStream() {
+    let skippedHeader = false;
+    return processPipeline(() => stream.pipe(split(splitRegex)).pipe(es.mapSync(line => {
       try {
         // skip empty lines
         if (line === '') {
           return;
         }
-        const doc = typeof transform === 'function' ? JSON.stringify(transform(JSON.parse(line))) : line;
-
-        // if doc is undefined we'll skip indexing it
-        if (typeof doc === 'undefined') {
-          s.resume();
-          return;
-        }
-
-        // the transform callback may return an array of docs so we can emit
-        // multiple docs from a single line
-        if (Array.isArray(doc)) {
-          doc.forEach(d => indexer.add(d));
+        if (skipHeader && !skippedHeader) {
+          skippedHeader = true;
           return;
         }
-        indexer.add(doc);
-      } catch (e) {
-        console.log('error', e);
+        const parsed = JSON.parse(line);
+        addParsedDoc(parsed);
+      } catch (err) {
+        logger.error({
+          err
+        }, 'Failed to process NDJSON stream line');
       }
     }).on('error', err => {
-      console.log('Error while reading stream.', err);
-    }).on('end', () => {
-      if (verbose) console.log('Read entire stream.');
+      logger.error({
+        err
+      }, 'Error while reading stream');
+    })), 'Error while reading stream');
+  }
+  async function startIndex() {
+    try {
+      if (sourceFormat === 'csv') {
+        await processCsvStream();
+      } else if (sourceFormat === 'ndjson') {
+        await processNdjsonStream();
+      } else if (sourceFormat === 'parquet') {
+        await processParquetStream();
+      } else if (sourceFormat === 'arrow') {
+        await processArrowStream();
+      } else {
+        throw Error(`Unsupported sourceFormat: ${sourceFormat}`);
+      }
+    } catch (err) {
+      logger.error({
+        err
+      }, 'Error while reading stream');
+    } finally {
       indexer.finish();
-      finished = true;
-    }));
-    indexer.queueEmitter.on('pause', () => {
-      if (finished) return;
-      s.pause();
-    });
-    indexer.queueEmitter.on('resume', () => {
-      if (finished) return;
-      s.resume();
-    });
+    }
   }
   return () => {
     startIndex();
@@ -530,22 +1077,31 @@ async function transformer({
   searchSize = DEFAULT_SEARCH_SIZE,
   stream,
   fileName,
+  sourceFormat = 'ndjson',
+  csvOptions = {},
   splitRegex = /\n/,
   sourceIndexName,
   targetIndexName,
   mappings,
   mappingsOverride = false,
+  inferMappings = false,
+  inferMappingsOptions = {},
   indexMappingTotalFieldsLimit,
   pipeline,
   populatedFields = false,
   query,
   skipHeader = false,
   transform,
-  verbose = true
+  verbose = true,
+  logger: loggerInput
 }) {
   if (typeof targetIndexName === 'undefined') {
     throw Error('targetIndexName must be specified.');
   }
+  const logger = createLogger({
+    logger: loggerInput,
+    verbose
+  });
   const defaultClientConfig = {
     node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200'
   };
@@ -553,23 +1109,47 @@ async function transformer({
   // Support both old (config) and new (client instance) patterns
   const sourceClient = await getOrCreateClient(sourceClientInput || sourceClientConfig, defaultClientConfig, sourceClientVersion);
   const targetClient = await getOrCreateClient(targetClientInput || targetClientConfig || sourceClientInput || sourceClientConfig, defaultClientConfig, targetClientVersion);
+  const inferenceResult = await inferMappingsFromSource({
+    targetClient,
+    fileName,
+    sourceFormat,
+    csvOptions,
+    skipHeader,
+    mappings,
+    inferMappings,
+    inferMappingsOptions,
+    logger: createChildLogger(logger, {
+      component: 'mapping-inference'
+    })
+  });
   const createMapping = createMappingFactory({
     sourceClient,
     sourceIndexName,
     targetClient,
     targetIndexName,
-    mappings,
+    mappings: inferenceResult.mappings,
+    inferredIngestPipeline: inferenceResult.ingestPipeline,
     mappingsOverride,
     indexMappingTotalFieldsLimit,
-    verbose,
     deleteIndex,
-    pipeline
+    pipeline,
+    logger: createChildLogger(logger, {
+      component: 'create-mapping'
+    })
   });
   const indexer = indexQueueFactory({
     targetClient,
     targetIndexName,
     bufferSize,
-    skipHeader});
+    logger: createChildLogger(logger, {
+      component: 'index-queue'
+    })
+  });
+  function validateSourceFormat() {
+    if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv' && sourceFormat !== 'parquet' && sourceFormat !== 'arrow') {
+      throw Error(`Unsupported sourceFormat: ${sourceFormat}. Use "ndjson", "csv", "parquet", or "arrow".`);
+    }
+  }
   function getReader() {
     if (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') {
       throw Error('Only either one of fileName or sourceIndexName can be specified.');
@@ -578,17 +1158,28 @@ async function transformer({
       throw Error('Only one of fileName, sourceIndexName, or stream can be specified.');
     }
     if (typeof fileName !== 'undefined') {
-      return fileReaderFactory(indexer, fileName, transform, splitRegex, verbose);
+      validateSourceFormat();
+      return fileReaderFactory(indexer, fileName, transform, splitRegex, skipHeader, sourceFormat, csvOptions, createChildLogger(logger, {
+        component: 'file-reader'
+      }));
     }
     if (typeof sourceIndexName !== 'undefined') {
-      return indexReaderFactory(indexer, sourceIndexName, transform, sourceClient, query, searchSize, populatedFields);
+      return indexReaderFactory(indexer, sourceIndexName, transform, sourceClient, query, searchSize, populatedFields, createChildLogger(logger, {
+        component: 'index-reader'
+      }));
     }
     if (typeof stream !== 'undefined') {
-      return streamReaderFactory(indexer, stream, transform, splitRegex, verbose);
+      validateSourceFormat();
+      return streamReaderFactory(indexer, stream, transform, splitRegex, skipHeader, sourceFormat, csvOptions, createChildLogger(logger, {
+        component: 'stream-reader'
+      }));
     }
     return null;
   }
   const reader = getReader();
+  if (typeof reader !== 'function') {
+    throw Error('One of fileName, sourceIndexName, or stream must be specified.');
+  }
   try {
     const indexExists = await targetClient.indices.exists({
       index: targetIndexName
@@ -605,8 +1196,11 @@ async function transformer({
     } else {
       reader();
     }
-  } catch (error) {
-    console.error('Error checking index existence:', error);
+  } catch (err) {
+    logger.error({
+      err,
+      targetIndexName
+    }, 'Error checking index existence');
   } finally {
     // targetClient.close();
   }