node-es-transformer 1.0.2 → 1.2.0

package/README.md

@@ -8,7 +8,7 @@
 
 # node-es-transformer
 
-Stream-based library for ingesting and transforming large data files (CSV/JSON) into Elasticsearch indices.
+Stream-based library for ingesting and transforming large data files (NDJSON/CSV/Parquet/Arrow IPC) into Elasticsearch indices.
 
 ## Quick Start
 
@@ -36,7 +36,7 @@ See [Usage](#usage) for more examples.
 
 ## Why Use This?
 
-If you need to ingest large CSV/JSON files (GigaBytes) into Elasticsearch without running out of memory, this is the tool for you. Other solutions often run out of JS heap, hammer ES with too many requests, time out, or try to do everything in a single bulk request.
+If you need to ingest large NDJSON/CSV/Parquet/Arrow IPC files (GigaBytes) into Elasticsearch without running out of memory, this is the tool for you. Other solutions often run out of JS heap, hammer ES with too many requests, time out, or try to do everything in a single bulk request.
 
 **When to use this:**
 - Large file ingestion (20-30 GB tested)
@@ -96,7 +96,7 @@ yarn add node-es-transformer
 
 ## Usage
 
-### Read from a file
+### Read NDJSON from a file
 
 ```javascript
 const transformer = require('node-es-transformer');
@@ -129,6 +129,102 @@ transformer({
 });
 ```
 
+### Read CSV from a file
+
+```javascript
+const transformer = require('node-es-transformer');
+
+transformer({
+  fileName: 'users.csv',
+  sourceFormat: 'csv',
+  targetIndexName: 'users-index',
+  mappings: {
+    properties: {
+      id: { type: 'integer' },
+      first_name: { type: 'keyword' },
+      last_name: { type: 'keyword' },
+      full_name: { type: 'keyword' },
+    },
+  },
+  transform(row) {
+    return {
+      ...row,
+      id: Number(row.id),
+      full_name: `${row.first_name} ${row.last_name}`,
+    };
+  },
+});
+```
+
+### Read Parquet from a file
+
+```javascript
+const transformer = require('node-es-transformer');
+
+transformer({
+  fileName: 'users.parquet',
+  sourceFormat: 'parquet',
+  targetIndexName: 'users-index',
+  mappings: {
+    properties: {
+      id: { type: 'integer' },
+      first_name: { type: 'keyword' },
+      last_name: { type: 'keyword' },
+      full_name: { type: 'keyword' },
+    },
+  },
+  transform(row) {
+    return {
+      ...row,
+      id: Number(row.id),
+      full_name: `${row.first_name} ${row.last_name}`,
+    };
+  },
+});
+```
+
+### Read Arrow IPC from a file
+
+```javascript
+const transformer = require('node-es-transformer');
+
+transformer({
+  fileName: 'users.arrow',
+  sourceFormat: 'arrow',
+  targetIndexName: 'users-index',
+  mappings: {
+    properties: {
+      id: { type: 'integer' },
+      first_name: { type: 'keyword' },
+      last_name: { type: 'keyword' },
+    },
+  },
+  transform(row) {
+    return {
+      ...row,
+      id: Number(row.id),
+    };
+  },
+});
+```
+
+### Infer mappings from CSV sample
+
+```javascript
+const transformer = require('node-es-transformer');
+
+transformer({
+  fileName: 'users.csv',
+  sourceFormat: 'csv',
+  targetIndexName: 'users-index',
+  inferMappings: true,
+  inferMappingsOptions: {
+    sampleBytes: 200000,
+    lines_to_sample: 2000,
+  },
+});
+```
+
 ### Read from another index
 
 ```javascript
@@ -242,9 +338,13 @@ All options are passed to the main `transformer()` function.
 
 Choose **one** of these sources:
 
-- **`fileName`** (string): Source filename to ingest. Supports wildcards (e.g., `logs/*.json`).
+- **`fileName`** (string): Source filename to ingest. Supports wildcards (e.g., `logs/*.json`, `data/*.csv`, `data/*.parquet`, `data/*.arrow`).
 - **`sourceIndexName`** (string): Source Elasticsearch index to reindex from.
 - **`stream`** (Readable): Node.js readable stream to ingest from.
+- **`sourceFormat`** (`'ndjson' | 'csv' | 'parquet' | 'arrow'`): Format for file/stream sources. Default: `'ndjson'`.
+  - `arrow` expects Arrow IPC file/stream payloads.
+  - `parquet` stream sources are currently buffered in memory before row iteration (file sources remain streaming by row cursor).
+- **`csvOptions`** (object): CSV parser options (delimiter, quote, columns, etc.) used when `sourceFormat: 'csv'`.
 
 #### Client Configuration
 
@@ -259,10 +359,14 @@ Choose **one** of these sources:
 
 - **`mappings`** (object): Elasticsearch document mappings for target index. If reindexing and not provided, mappings are copied from source index.
 - **`mappingsOverride`** (boolean): When reindexing, apply `mappings` on top of source index mappings. Default: `false`.
+- **`inferMappings`** (boolean): Infer mappings for `fileName` sources via `/_text_structure/find_structure`. Supported for `sourceFormat: 'ndjson'` and `sourceFormat: 'csv'` only. Ignored when `mappings` is provided. If inference returns `ingest_pipeline`, it is created as `<targetIndexName>-inferred-pipeline` and applied as the index default pipeline (unless `pipeline` is explicitly set). Default: `false`.
+- **`inferMappingsOptions`** (object): Options for `/_text_structure/find_structure` (for example `sampleBytes`, `lines_to_sample`, `delimiter`, `quote`, `has_header_row`, `timeout`).
 - **`deleteIndex`** (boolean): Delete target index if it exists before starting. Default: `false`.
 - **`indexMappingTotalFieldsLimit`** (number): Field limit for target index (`index.mapping.total_fields.limit` setting).
 - **`pipeline`** (string): Elasticsearch ingest pipeline name to use during indexing.
 
+When `inferMappings` is enabled, the target cluster must allow `/_text_structure/find_structure` (cluster privilege: `monitor_text_structure`). If inferred ingest pipelines are used, the target cluster must also allow creating ingest pipelines (`_ingest/pipeline`).
+
 #### Performance Options
 
 - **`bufferSize`** (number): Buffer size threshold in KBytes for bulk indexing. Default: `5120` (5 MB).
@@ -276,9 +380,15 @@ Choose **one** of these sources:
   - Return array of documents to split one source into multiple targets
   - Return `null`/`undefined` to skip document
 - **`query`** (object): Elasticsearch [DSL query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html) to filter source documents.
-- **`splitRegex`** (RegExp): Line split regex for file/stream sources. Default: `/\n/`.
-- **`skipHeader`** (boolean): Skip first line of source file (e.g., CSV header). Default: `false`.
-- **`verbose`** (boolean): Enable logging and progress bars. Default: `true`.
+- **`splitRegex`** (RegExp): Line split regex for file/stream sources when `sourceFormat` is `'ndjson'`. Default: `/\n/`.
+- **`skipHeader`** (boolean): Header skipping for file/stream sources.
+  - NDJSON: skips the first non-empty line
+  - CSV: skips the first data line only when `csvOptions.columns` does not consume headers
+  - Parquet/Arrow: ignored
+  - Default: `false`
+  - Applies only to `fileName`/`stream` sources
+- **`verbose`** (boolean): Enable verbose logging and progress bars when using the built-in logger. Default: `true`.
+- **`logger`** (object): Optional custom Pino-compatible logger. If omitted, the library creates an internal Pino logger (`name: node-es-transformer`) and uses `LOG_LEVEL` (if set) or `info`/`error` based on `verbose`.
 
 ### Return Value
 
@@ -291,16 +401,19 @@ The `transformer()` function returns a Promise that resolves to an object with:
   - `'error'`: Error occurred
 
 ```javascript
+const pino = require('pino');
+const logger = pino({ name: 'my-app', level: process.env.LOG_LEVEL || 'info' });
+
 const result = await transformer({
   /* options */
 });
 
 result.events.on('complete', () => {
-  console.log('Ingestion complete!');
+  logger.info('Ingestion complete');
 });
 
 result.events.on('error', err => {
-  console.error('Error:', err);
+  logger.error({ err }, 'Ingestion failed');
 });
 ```
 
@@ -338,20 +451,23 @@ See [examples/typescript-example.ts](examples/typescript-example.ts) for more ex
 Always handle errors when using the library:
 
 ```javascript
+const pino = require('pino');
+const logger = pino({ name: 'my-app', level: process.env.LOG_LEVEL || 'info' });
+
 transformer({
   /* options */
 })
-  .then(() => console.log('Success'))
-  .catch(err => console.error('Error:', err));
+  .then(() => logger.info('Success'))
+  .catch(err => logger.error({ err }, 'Transformer failed'));
 
 // Or with async/await
 try {
   await transformer({
     /* options */
   });
-  console.log('Success');
+  logger.info('Success');
 } catch (err) {
-  console.error('Error:', err);
+  logger.error({ err }, 'Transformer failed');
 }
 ```