@forzalabs/remora 0.1.3-nasco.3 → 0.1.4-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.1.2-nasco',
+    cliVersion: '0.1.4-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/consumer-schema.json

@@ -464,6 +464,10 @@
                             "type": "string",
                             "description": "Cast the value to a specific type",
                             "enum": ["string", "number", "date", "boolean"]
+                        },
+                        "format": {
+                            "type": "string",
+                            "description": "Optional format for date parsing or string formatting (supports tokens: yyyy, mm, dd)"
                         }
                     },
                     "required": ["cast"],

package/drivers/DriverHelper.js

@@ -109,7 +109,7 @@ const DriverHelper = {
                         return keys.map(k => parsed[k]).join(delimiter);
                     }
                     catch (error) {
-                        Logger_1.default.log(`Failed parsing line in JSON - index: ${globalIndex}; line: ${line}; err: ${error === null || error === void 0 ? void 0 : error.name}`);
+                        Logger_1.default.log(`Failed parsing in JSON line -> file: ${fileKey}; index: ${globalIndex}; line: ${line}; err: ${error === null || error === void 0 ? void 0 : error.name}`);
                         throw error;
                     }
                 }

package/drivers/LocalDriver.js

@@ -143,6 +143,7 @@ class LocalSourceDriver {
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
                 Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
+                Affirm_1.default.hasItems(allFileKeys, `The file key "${fileKey}" doesn't have any matches in path "${this._path}".`);
                 // Get header line from the first file
                 const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
                 dataset.setFirstLine(headerLine);

package/drivers/S3Driver.js

@@ -216,6 +216,7 @@ class S3SourceDriver {
             if (fileKey.includes('%')) {
                 const allFileKeys = yield this.listFiles(fileKey);
                 Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
+                Affirm_1.default.hasItems(allFileKeys, `The file key "${fileKey}" doesn't have any matches in bucket "${this._bucketName}".`);
                 // Get header line from the first file
                 const firstFileCommand = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,

package/engines/consumer/PostProcessor.js

@@ -214,17 +214,19 @@ class PostProcessorClass {
                 return defaultValue;
             const fieldKey = alias !== null && alias !== void 0 ? alias : key;
             const fieldValue = record.getValue(fieldKey);
+            const fieldType = (_b = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string';
             if (Algo_1.default.hasVal(fieldValue) && !isNaN(fieldValue)) {
-                const fieldType = (_b = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string';
                 if (fieldType === 'number' && typeof fieldValue === 'string' && fieldValue.length === 0)
                     return (_c = field.cField.default) !== null && _c !== void 0 ? _c : fieldValue;
                 else
                     return fieldValue;
             }
-            else if ((!Algo_1.default.hasVal(fieldValue) || isNaN(fieldValue)) && Algo_1.default.hasVal(field.cField.default))
+            else if ((!Algo_1.default.hasVal(fieldValue) || (isNaN(fieldValue) && fieldType === 'number')) && Algo_1.default.hasVal(field.cField.default)) {
                 return field.cField.default;
-            else
+            }
+            else {
                 return fieldValue;
+            }
         };
         this.distinct = (dataset) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, 'Invalid dataset');

package/engines/dataset/ParallelDataset.js

@@ -20,9 +20,22 @@ const DatasetManager_1 = __importDefault(require("./DatasetManager"));
 const path_1 = __importDefault(require("path"));
 class ParallelDatasetClass {
     constructor() {
+        this.init = () => {
+            /**
+             * I need the init to be called after all the setup has been completed because I need the .env to be loaded
+             */
+            if (!this._filterPool || !this._projectionPool || !this._transformPool) {
+                const workerPath = this._getWorkerPath();
+                this._filterPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'FilterWorker.js'));
+                this._projectionPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'ProjectionWorker.js'));
+                this._transformPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'TransformWorker.js'));
+            }
+        };
         this._getWorkerPath = () => {
             // Get the current file's directory
             const currentDir = __dirname;
+            if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'development')
+                return path_1.default.resolve('./.build/workers');
             // Check if we're in a published npm package (no .build in path)
             if (!currentDir.includes('.build')) {
                 // We're in the published package, workers are relative to package root
@@ -47,6 +60,7 @@ class ParallelDatasetClass {
         this.filter = (dataset, filters) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, `Invalid dataset`);
             (0, Affirm_1.default)(filters, `Invalid filters`);
+            this.init();
             // Distribute the work of the filter among the various workers, trying to have them match the batch size
             const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
             dataset._startOperation('filter-parallel', { workerCount });
@@ -84,6 +98,7 @@ class ParallelDatasetClass {
         this.projection = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, `Invalid dataset`);
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            this.init();
             const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
             dataset._startOperation('projection-parallel', { workerCount });
             const threads = [];
@@ -118,6 +133,7 @@ class ParallelDatasetClass {
         this.transform = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, `Invalid dataset`);
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            this.init();
             const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
             dataset._startOperation('transform-parallel', { workerCount });
             const threads = [];
@@ -149,10 +165,6 @@ class ParallelDatasetClass {
             dataset._finishOperation('transform-parallel');
             return dataset;
         });
-        const workerPath = this._getWorkerPath();
-        this._filterPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'FilterWorker.js'));
-        this._projectionPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'ProjectionWorker.js'));
-        this._transformPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'TransformWorker.js'));
     }
 }
 const ParallelDataset = new ParallelDatasetClass();

package/engines/transform/TransformationEngine.js

@@ -73,9 +73,12 @@ class TransformationEngineClass {
             }
             // Single transformation
             if ('cast' in transformations) {
-                const casted = TypeCaster_1.default.cast(value, transformations.cast);
-                if (isNaN(casted) && transformations.cast === 'number')
+                const { cast, format } = transformations;
+                const casted = TypeCaster_1.default.cast(value, cast, format);
+                if (cast === 'number' && isNaN(casted))
                     throw new Error(`Cannot cast non-numeric value in field '${field.key}'`);
+                if (cast === 'date' && casted instanceof Date && isNaN(casted.getTime()))
+                    throw new Error(`Cannot cast value to date in field '${field.key}'`);
                 return casted;
             }
             if ('multiply' in transformations) {

package/engines/transform/TypeCaster.js

@@ -4,34 +4,45 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Algo_1 = __importDefault(require("../../core/Algo"));
+const dayjs_1 = __importDefault(require("dayjs"));
+const customParseFormat_1 = __importDefault(require("dayjs/plugin/customParseFormat"));
+const utc_1 = __importDefault(require("dayjs/plugin/utc"));
+dayjs_1.default.extend(customParseFormat_1.default);
+dayjs_1.default.extend(utc_1.default);
 class TypeCasterClass {
     /**
      * Casts the value to the requested type (only if needed)
+     * Optional format parameter currently supports:
+     *  - Parsing dates (type 'date'/'datetime') from string with tokens: yyyy, mm, dd
+     *  - Formatting dates when casting to string with same tokens
      */
-    cast(value, type) {
+    cast(value, type, format) {
         if (!Algo_1.default.hasVal(value))
             return value;
         switch (type) {
             case 'boolean': {
                 if (typeof value === 'boolean')
                     return value;
-                else
-                    return Boolean(value);
+                return Boolean(value);
             }
             case 'datetime':
-            case 'date':
-                return new Date(value);
+            case 'date': {
+                let dateValue = null;
+                if (format && typeof value === 'string')
+                    dateValue = dayjs_1.default.utc(value, format, true).toDate();
+                else
+                    dateValue = new Date(value);
+                return dateValue.toISOString();
+            }
             case 'number': {
                 if (typeof value === 'number')
                     return value;
-                else
-                    return Number(value);
+                return Number(value);
             }
             case 'string': {
                 if (typeof value === 'string')
                     return value;
-                else
-                    return String(value);
+                return String(value);
             }
         }
     }

package/engines/validation/Validator.js

@@ -154,9 +154,6 @@ class ValidatorClass {
                         else
                             trxToValidate.push(field.transform);
                         for (const trans of trxToValidate) {
-                            const trxKeys = Object.keys(trans);
-                            if (trxKeys.length !== 1)
-                                errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}.`);
                             if ('combine_fields' in trans) {
                                 const { combine_fields } = trans;
                                 if (!combine_fields.fields || combine_fields.fields.length === 0)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.1.3-nasco.3",
+    "version": "0.1.4-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -43,6 +43,7 @@
         "chalk": "^4.1.2",
         "commander": "^10.0.0",
         "cross-env": "^7.0.3",
+        "dayjs": "^1.11.13",
         "dotenv": "^16.0.3",
         "fast-xml-parser": "^5.2.3",
         "fs-extra": "^11.1.0",