@jrmc/adonis-etl 1.0.0-alpha.3 → 1.0.2

package/README.md

@@ -78,11 +78,11 @@ This will create (with default configuration):
 - `app/etl/transforms/import_product_csv_to_db_transform.ts`
 - `app/etl/destinations/import_product_db_destination.ts`
 
-Or with custom directory configuration (`directories.etl: 'Opsone/jerem'`):
+Or with custom directory configuration (`directories.etl: 'etl'`):
 
-- `Opsone/jerem/sources/import_product_csv_source.ts`
-- `Opsone/jerem/transforms/import_product_csv_to_db_transform.ts`
-- `Opsone/jerem/destinations/import_product_db_destination.ts`
+- `etl/sources/import_product_csv_source.ts`
+- `etl/transforms/import_product_csv_to_db_transform.ts`
+- `etl/destinations/import_product_db_destination.ts`
 
 ## Generated Files
 
@@ -123,6 +123,74 @@ export default class ImportProductDbDestination implements Destination {
 }
 ```
 
+## Usage Examples
+
+The `sample/` folder contains two complete ETL implementation examples:
+
+### 1. Books Import (Source → Destination)
+
+This example shows a simple ETL process without transformation:
+
+**Command:** `node ace import:books`
+
+**Components:**
+- **Source**: `book_csv_source.ts` - Reads a CSV file of books (5M records) with batch processing (500 items)
+- **Destination**: `book_db_destination.ts` - Inserts data into database via `db.table().multiInsert()`
+
+**Features:**
+- Batch processing for performance optimization
+- CSV error handling (empty lines and errors ignored)
+- Optimized buffer (128KB) for large files
+
+### 2. Products Import (Source → Transform → Destination)
+
+This example shows a complete ETL process with data transformation:
+
+**Command:** `node ace import:products`
+
+**Components:**
+- **Source**: `product_csv_source.ts` - Reads a CSV file of products (500K records)
+- **Transform**: `product_csv_to_db_transform.ts` - Transforms CSV data (French column names → English)
+- **Destination**: `product_db_destination.ts` - Saves via Lucid model `Product.create()`
+
+**Features:**
+- Column name transformation (e.g., `Nom` → `name`, `Prix` → `price`)
+- AdonisJS model usage for persistence
+- Data processing logging
+
+### Example Files Structure
+
+```
+sample/
+├── commands/
+│   ├── import_books.ts      # Books import command
+│   └── import_products.ts   # Products import command
+├── etl/
+│   ├── sources/
+│   │   ├── book_csv_source.ts
+│   │   └── product_csv_source.ts
+│   ├── transforms/
+│   │   └── product_csv_to_db_transform.ts
+│   ├── destinations/
+│   │   ├── book_db_destination.ts
+│   │   └── product_db_destination.ts
+│   └── resources/
+│       ├── books.csv        # Sample data
+│       └── products.csv     # Sample data
+└── app/models/
+    ├── book.ts              # Book model
+    └── product.ts           # Product model
+```
+
+These examples demonstrate different possible approaches:
+- **Batch processing** vs **line-by-line processing**
+- **Direct database insertion** vs **Lucid model usage**
+- **With or without data transformation**
+
+## Performance Optimization
+
+For large-scale ETL operations, consider integrating with a job queue system (like BullMQ, or AdonisJS Queue package) to run ETL processes asynchronously, distribute workload across multiple workers, and improve reliability with automatic retry mechanisms.
+
 ## Dependencies
 
 This package requires:
@@ -190,11 +258,9 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Changelog
 
-### 1.0.0-alpha.1
+### 1.0.0
 - Initial release
 - Interactive ETL component generation
 - Support for Source, Transform, and Destination components
 - TypeScript support
-
-### 1.0.0-alpha.2
 - Custom directory configuration support via `directories.etl` in adonisrc.ts

package/build/commands/make/etl.js

@@ -19,17 +19,13 @@ export default class MakeEtl extends BaseCommand {
     static description = 'Create a new ETL files (source, transform, destination)';
     async run() {
         // Ask which ETL components to create
-        const components = await this.prompt.multiple('Which ETL components do you want to create?', [
-            'Source',
-            'Transform',
-            'Destination'
-        ], {
+        const components = await this.prompt.multiple('Which ETL components do you want to create?', ['Source', 'Transform', 'Destination'], {
             validate: (value) => {
                 if (!value || value.length === 0) {
                     return 'You must select at least one component';
                 }
                 return true;
-            }
+            },
         });
         // Ask for source details if source is selected
         let sourceDetails = null;
@@ -40,7 +36,7 @@ export default class MakeEtl extends BaseCommand {
                         return 'Source type is required';
                     }
                     return true;
-                }
+                },
             });
         }
         // Ask for destination details if destination is selected
@@ -52,7 +48,7 @@ export default class MakeEtl extends BaseCommand {
                         return 'Destination type is required';
                     }
                     return true;
-                }
+                },
             });
         }
         const codemods = await this.createCodemods();
@@ -61,30 +57,30 @@ export default class MakeEtl extends BaseCommand {
         for (const component of components) {
             if (component === 'Source') {
                 className = [
-                    string.snakeCase(this.name),
-                    string.snakeCase(sourceDetails),
-                    'source',
-                ].join('_');
+                    string.pascalCase(this.name),
+                    string.pascalCase(sourceDetails),
+                    'Source',
+                ].join('');
             }
             else if (component === 'Destination') {
                 className = [
-                    string.snakeCase(this.name),
-                    string.snakeCase(destinationDetails),
-                    'destination',
-                ].join('_');
+                    string.pascalCase(this.name),
+                    string.pascalCase(destinationDetails),
+                    'Destination',
+                ].join('');
             }
             else if (component === 'Transform') {
                 className = [
-                    string.snakeCase(this.name),
-                    string.snakeCase(sourceDetails),
-                    'to',
-                    string.snakeCase(destinationDetails),
-                    'transform',
-                ].join('_');
+                    string.pascalCase(this.name),
+                    string.pascalCase(sourceDetails),
+                    'To',
+                    string.pascalCase(destinationDetails),
+                    'Transform',
+                ].join('');
             }
             const stubPath = `make/etl/${component.toLowerCase()}s/main.ts.stub`;
             await codemods.makeUsingStub(stubsRoot, stubPath, {
-                className
+                className,
             });
         }
         this.logger.success(`ETL files created successfully for: ${this.name}`);

package/build/stubs/make/etl/destinations/main.ts.stub

@@ -4,7 +4,7 @@
     to: app.makePath(app.rcFile.directories['etl'] || 'app/etl/', 'destinations', resourceFileName)
   })
 }}}
-import { Destination } from '@jrmc/adonis-etl'
+import type { Destination } from '@jrmc/adonis-etl'
 
 export default class {{ className }} implements Destination {
   async write(row: unknown) {

package/build/stubs/make/etl/sources/main.ts.stub

@@ -4,7 +4,7 @@
     to: app.makePath(app.rcFile.directories['etl'] || 'app/etl/', 'sources', resourceFileName)
   })
 }}}
-import { Source } from '@jrmc/adonis-etl'
+import type { Source } from '@jrmc/adonis-etl'
 
 export default class {{ className }} implements Source {
   async *each() {

package/build/stubs/make/etl/transforms/main.ts.stub

@@ -4,7 +4,7 @@
     to: app.makePath(app.rcFile.directories['etl'] || 'app/etl/', 'transforms', resourceFileName)
   })
 }}}
-import { Transform } from '@jrmc/adonis-etl'
+import type { Transform } from '@jrmc/adonis-etl'
 
 export default class {{ className }} implements Transform {
   async process(row: unknown) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jrmc/adonis-etl",
-  "version": "1.0.0-alpha.3",
+  "version": "1.0.2",
   "keywords": [
     "adonisjs",
     "etl",
@@ -50,7 +50,7 @@
   "prettier": "@adonisjs/prettier-config",
   "publishConfig": {
     "access": "public",
-    "tag": "alpha"
+    "tag": "latest"
   },
   "volta": {
     "node": "22.17.0"