@forzalabs/remora 0.0.27 → 0.0.29

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.27',
+    cliVersion: '0.0.29',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/producer-schema.json

@@ -68,9 +68,11 @@
                         "enum": [
                             "hash",
                             "mask",
-                            "crypt"
+                            "crypt",
+                            "random",
+                            "seeded-random"
                         ],
-                        "description": "Masking type to apply to this dimension"
+                        "description": "Masking type to apply to this dimension. 'hash' replaces with a hashed value. 'mask' replaces characters with a mask character. 'crypt' encrypts the value. 'random' replaces with a random value. 'seeded-random' replaces with a random value generated from a seed."
                     }
                 },
                 "required": [
@@ -132,13 +134,24 @@
                     "enum": [
                         "JSON",
                         "JSONL",
-                        "CSV"
+                        "CSV", 
+                        "TXT",
+                        "XLS",
+                        "XLSX"
                     ],
                     "description": "The type of file to read"
                 },
                 "delimiter": {
                     "type": "string",
-                    "description": "The column delimiter for the CSV file if different from the default (,)."
+                    "description": "The column delimiter for CSV or TXT files if different from the default (,)."
+                },
+                "hasHeaderRow": {
+                    "type": "boolean",
+                    "description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."
+                },
+                "sheetName": {
+                    "type": "string", 
+                    "description": "For Excel files (.xls/.xlsx), specifies the name of the sheet to read data from. If not specified, the first sheet will be used."
                 }
             },
             "additionalProperties": false

package/drivers/DriverFactory.js

@@ -1,37 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -45,10 +12,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const LocalDriver_1 = __importStar(require("./LocalDriver"));
+const LocalDriver_1 = require("./LocalDriver");
 const RedshiftDriver_1 = __importDefault(require("./RedshiftDriver"));
-const S3Driver_1 = __importDefault(require("./S3Driver"));
-const S3SourceDriver_1 = __importDefault(require("./S3SourceDriver"));
+const S3Driver_1 = require("./S3Driver");
 class DriverFactoryClass {
     constructor() {
         this.instantiateSource = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -60,12 +26,12 @@ class DriverFactoryClass {
                     return driver;
                 }
                 case 'aws-s3': {
-                    const driver = new S3SourceDriver_1.default();
+                    const driver = new S3Driver_1.S3SourceDriver();
                     yield driver.init(source);
                     return driver;
                 }
                 case 'local': {
-                    const driver = new LocalDriver_1.default();
+                    const driver = new LocalDriver_1.LocalSourceDriver();
                     yield driver.init(source);
                     return driver;
                 }
@@ -76,12 +42,12 @@ class DriverFactoryClass {
             switch (source.engine) {
                 // TODO: implement all the other engines
                 case 'aws-s3': {
-                    const driver = new S3Driver_1.default();
+                    const driver = new S3Driver_1.S3DestinationDriver();
                     yield driver.init(source);
                     return driver;
                 }
                 case 'local': {
-                    const driver = new LocalDriver_1.LocalDriverDestination();
+                    const driver = new LocalDriver_1.LocalDestinationDriver();
                     yield driver.init(source);
                     return driver;
                 }

package/drivers/LocalDriver.js

@@ -52,12 +52,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalDriverDestination = void 0;
+exports.LocalDestinationDriver = exports.LocalSourceDriver = void 0;
 const fs = __importStar(require("fs"));
 const path_1 = __importDefault(require("path"));
 const readline_1 = __importDefault(require("readline"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
-class LocalDriver {
+const Algo_1 = __importDefault(require("../core/Algo"));
+const xlsx_1 = __importDefault(require("xlsx"));
+class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             const fileURL = source.authentication['path'];
@@ -68,79 +70,109 @@ class LocalDriver {
             return this;
         });
         this.download = (request) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
             (0, Affirm_1.default)(this._path, `Invalid path`);
             (0, Affirm_1.default)(request, `Invalid download request`);
             (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
-            const fileUrl = path_1.default.join(this._path, request.fileKey);
-            const stream = fs.createReadStream(fileUrl);
-            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
-            const lines = [];
-            try {
-                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
-                    _c = reader_1_1.value;
-                    _d = false;
-                    const line = _c;
-                    lines.push(line);
-                }
+            (0, Affirm_1.default)(request.fileType, `Invalid file type for download request`);
+            const { fileKey, options } = request;
+            const fileUrl = path_1.default.join(this._path, fileKey);
+            switch (request.fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    return yield this._readLines(fileUrl);
+                case 'XLS':
+                case 'XLSX':
+                    return yield this._readExcelLines(fileUrl, options === null || options === void 0 ? void 0 : options.sheetName);
             }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
-                }
-                finally { if (e_1) throw e_1.error; }
+        });
+        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(request, 'Invalid read options');
+            (0, Affirm_1.default)(request.fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(request.fileType, `Invalid file type`);
+            (0, Affirm_1.default)(request.options, `Invalid request options`);
+            Affirm_1.default.hasValue(request.options.lineFrom, `Invalid request options line from`);
+            Affirm_1.default.hasValue(request.options.lineTo, `Invalid request options line to`);
+            const { fileKey, fileType, options: { lineFrom, lineTo, sheetName } } = request;
+            const fileUrl = path_1.default.join(this._path, fileKey);
+            switch (fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    return yield this._readLines(fileUrl, lineFrom, lineTo);
+                case 'XLS':
+                case 'XLSX':
+                    return yield this._readExcelLines(fileUrl, sheetName, lineFrom, lineTo);
             }
-            reader.close();
-            stream.close();
-            return lines;
         });
-        this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_2, _b, _c;
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(readOptions, 'Invalid read options');
-            (0, Affirm_1.default)(readOptions.fileKey, 'Invalid file key');
-            const fileUrl = path_1.default.join(this._path, readOptions.fileKey);
-            const { lineFrom, lineTo } = readOptions;
-            const stream = fs.createReadStream(fileUrl);
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            const fileKey = producer.settings.fileKey;
+            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
+            const fileUrl = path_1.default.join(this._path, fileKey);
+            return fs.existsSync(fileUrl);
+        });
+        this._readLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const stream = fs.createReadStream(fileUri);
             const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
             const lines = [];
             let lineCounter = 0;
             try {
-                for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
-                    _c = reader_2_1.value;
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
                     _d = false;
                     const line = _c;
-                    if (lineCounter >= lineFrom && lineCounter < lineTo) {
+                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                        if (lineCounter >= lineFrom && lineCounter < lineTo) {
+                            lines.push(line);
+                        }
+                        lineCounter++;
+                        if (lineCounter >= lineTo)
+                            break;
+                    }
+                    else {
                         lines.push(line);
                     }
-                    lineCounter++;
-                    if (lineCounter >= lineTo)
-                        break;
                 }
             }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
             finally {
                 try {
-                    if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
                 }
-                finally { if (e_2) throw e_2.error; }
+                finally { if (e_1) throw e_1.error; }
             }
             reader.close();
             stream.close();
             return lines;
         });
-        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(producer, `Invalid producer`);
-            const fileKey = producer.settings.fileKey;
-            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
-            const fileUrl = path_1.default.join(this._path, fileKey);
-            return fs.existsSync(fileUrl);
+        this._readExcelLines = (fileUri, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            const excel = xlsx_1.default.readFile(fileUri);
+            let targetSheetName = sheetName;
+            if (!targetSheetName) {
+                (0, Affirm_1.default)(excel.SheetNames.length > 0, 'The Excel file has no sheets.');
+                targetSheetName = excel.SheetNames[0];
+            }
+            else {
+                (0, Affirm_1.default)(excel.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
+            }
+            const sheet = excel.Sheets[targetSheetName];
+            const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
+            const lines = csv.split('\n');
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
+                return lines.slice(lineFrom, lineTo + 1);
+            else
+                return lines;
         });
     }
 }
-class LocalDriverDestination {
+exports.LocalSourceDriver = LocalSourceDriver;
+class LocalDestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(source, `Invalid source`);
@@ -206,5 +238,4 @@ class LocalDriverDestination {
         });
     }
 }
-exports.LocalDriverDestination = LocalDriverDestination;
-exports.default = LocalDriver;
+exports.LocalDestinationDriver = LocalDestinationDriver;

package/drivers/S3Driver.js

@@ -8,14 +8,25 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3SourceDriver = exports.S3DestinationDriver = void 0;
 const client_s3_1 = require("@aws-sdk/client-s3");
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
-class S3Driver {
+const readline_1 = __importDefault(require("readline"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+const xlsx_1 = __importDefault(require("xlsx"));
+class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             this._bucketName = source.authentication['bucket'];
@@ -99,4 +110,151 @@ class S3Driver {
         });
     }
 }
-exports.default = S3Driver;
+exports.S3DestinationDriver = S3DestinationDriver;
+class S3SourceDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            this._bucketName = source.authentication['bucket'];
+            const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
+            const config = {
+                region: source.authentication['region'],
+                credentials: {
+                    accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
+                    secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
+                    sessionToken: sessionToken ? sessionToken : undefined
+                }
+            };
+            this._client = new client_s3_1.S3Client(config);
+            // TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
+            return this;
+        });
+        this.download = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(request, `Invalid download request`);
+            (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
+            const { fileKey, fileType, options } = request;
+            const bucket = this._bucketName;
+            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
+                Bucket: bucket,
+                Key: fileKey
+            }));
+            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
+            const stream = response.Body;
+            switch (fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    return yield this._readLines(stream);
+                case 'XLS':
+                case 'XLSX':
+                    return yield this._readExcelLines(stream, options === null || options === void 0 ? void 0 : options.sheetName);
+            }
+        });
+        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(request, 'Invalid read request');
+            (0, Affirm_1.default)(request.options, 'Invalid read request options');
+            const { fileKey, fileType, options: { sheetName, lineFrom, lineTo } } = request;
+            const bucket = this._bucketName;
+            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
+                Bucket: bucket,
+                Key: fileKey
+            }));
+            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
+            const stream = response.Body;
+            switch (fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    return yield this._readLines(stream, lineFrom, lineTo);
+                case 'XLS':
+                case 'XLSX':
+                    return yield this._readExcelLines(stream, sheetName, lineFrom, lineTo);
+            }
+        });
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(producer, 'Invalid read producer');
+            const bucket = this._bucketName;
+            const fileKey = producer.settings.fileKey;
+            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
+            try {
+                yield this._client.send(new client_s3_1.HeadObjectCommand({ Bucket: bucket, Key: fileKey }));
+                return true;
+            }
+            catch (error) {
+                if (((_a = error.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode) === 404 || error.name === 'NotFound')
+                    return false;
+                throw error;
+            }
+        });
+        this._readLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            const lines = [];
+            let lineCounter = 0;
+            try {
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                        if (lineCounter >= lineFrom && lineCounter < lineTo) {
+                            lines.push(line);
+                        }
+                        lineCounter++;
+                        if (lineCounter >= lineTo)
+                            break;
+                    }
+                    else {
+                        lines.push(line);
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            reader.close();
+            return lines;
+        });
+        this._readExcelLines = (stream, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, stream_1, stream_1_1;
+            var _b, e_2, _c, _d;
+            (0, Affirm_1.default)(sheetName, `Invalid sheetname`);
+            const chunks = [];
+            try {
+                for (_a = true, stream_1 = __asyncValues(stream); stream_1_1 = yield stream_1.next(), _b = stream_1_1.done, !_b; _a = true) {
+                    _d = stream_1_1.value;
+                    _a = false;
+                    const chunk = _d;
+                    chunks.push(chunk);
+                }
+            }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            finally {
+                try {
+                    if (!_a && !_b && (_c = stream_1.return)) yield _c.call(stream_1);
+                }
+                finally { if (e_2) throw e_2.error; }
+            }
+            const buffer = Buffer.concat(chunks);
+            const excel = xlsx_1.default.read(buffer, { type: 'buffer' });
+            (0, Affirm_1.default)(excel.SheetNames.includes(sheetName), `The sheet "${sheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
+            const sheet = excel.Sheets[sheetName];
+            const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
+            const lines = csv.split('\n');
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
+                return lines.slice(lineFrom, lineTo + 1);
+            else
+                return lines;
+        });
+    }
+}
+exports.S3SourceDriver = S3SourceDriver;

package/engines/CryptoEngine.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const crypto_1 = __importDefault(require("crypto"));
 const Algo_1 = __importDefault(require("../core/Algo"));
+const RandomEngine_1 = __importDefault(require("./RandomEngine"));
 class CryptoEngineClass {
     constructor() {
         this.hashQuery = (maskType, fieldReference, fieldName) => {
@@ -24,7 +25,7 @@ class CryptoEngineClass {
         this.valueToHash = (value) => {
             return crypto_1.default.createHash('sha256').update(JSON.stringify(value)).digest('hex');
         };
-        this.hashValue = (maskType, value) => {
+        this.hashValue = (maskType, value, valueType) => {
             if (!Algo_1.default.hasVal(value))
                 return value;
             if (!Algo_1.default.hasVal(maskType))
@@ -32,6 +33,24 @@ class CryptoEngineClass {
             switch (maskType) {
                 case 'hash':
                     return this.valueToHash(value);
+                case 'random': {
+                    switch (valueType) {
+                        case 'datetime': return RandomEngine_1.default.rngDate();
+                        case 'number': return RandomEngine_1.default.rng();
+                        case 'string': return this.valueToHash(value);
+                        default:
+                            throw new Error('Not implemented yet');
+                    }
+                }
+                case 'seeded-random': {
+                    switch (valueType) {
+                        case 'datetime': return RandomEngine_1.default.sRngDate(value);
+                        case 'number': return RandomEngine_1.default.sRng(value);
+                        case 'string': return this.valueToHash(value);
+                        default:
+                            throw new Error('Not implemented yet');
+                    }
+                }
                 case 'crypt':
                     throw new Error('Not implemented yet');
                 case 'mask':

package/engines/ParseManager.js

@@ -18,9 +18,9 @@ class ParseManagerClass {
             (0, Affirm_1.default)(lines, 'Invalid csv lines');
             Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
             const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
-            const headerRow = lines[0];
-            const rows = lines.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
-            const headerColumns = this._extractHeader(headerRow, delimiterChar, producer, discover);
+            const { header, records } = this._getClassifiedRows(lines, delimiterChar, producer);
+            const headerColumns = this._extractHeader(header, delimiterChar, producer, discover);
+            const rows = records.map(x => x.split(delimiterChar).map(k => k.trim()));
             const result = [];
             for (const row of rows) {
                 const rowObject = {};
@@ -32,6 +32,23 @@ class ParseManagerClass {
             }
             return result;
         };
+        this._getClassifiedRows = (lines, delimiterChar, producer) => {
+            if (producer.settings.fileType === 'TXT' && !producer.settings.hasHeaderRow) {
+                // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
+                const source = Environment_1.default.getSource(producer.source);
+                const columns = FileCompiler_1.default.compileProducer(producer, source);
+                return {
+                    header: columns.map(x => x.nameInProducer).join(delimiterChar),
+                    records: lines
+                };
+            }
+            else {
+                return {
+                    header: lines[0],
+                    records: lines.slice(1)
+                };
+            }
+        };
         this._extractHeader = (headerLine, delimiter, producer, discover) => {
             var _a;
             (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);

package/engines/ProducerEngine.js

@@ -91,19 +91,24 @@ class ProducerEngineClass {
             (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
             const driver = yield DriverFactory_1.default.instantiateSource(source);
             (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+            const { settings: { fileKey, fileType, sheetName } } = producer;
             let lines = [];
             if (options.readmode === 'lines')
-                lines = yield driver.readLinesInRange({ fileKey: producer.settings.fileKey, lineFrom: options.lines.from, lineTo: options.lines.to });
+                lines = yield driver.readLinesInRange({ fileKey, fileType, options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName } });
             else
-                lines = yield driver.download({ fileKey: producer.settings.fileKey });
+                lines = yield driver.download({ fileKey, fileType, options: { sheetName } });
             switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
-                case 'CSV': {
+                case 'CSV':
+                case 'TXT':
+                    return { data: lines, dataType: 'lines-of-text' };
+                case 'XLS':
+                case 'XLSX':
                     return { data: lines, dataType: 'lines-of-text' };
-                }
                 case 'JSONL':
                 case 'JSON': {
-                    if (lines.length === 1)
+                    if (lines.length === 1) {
                         lines = lines[0].split('\n');
+                    }
                     const json = lines.map(x => JSON.parse(x));
                     return { data: json, dataType: 'array-of-json' };
                 }
@@ -112,7 +117,7 @@ class ProducerEngineClass {
             }
         });
         this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10, discover = false) {
-            var _a, _b, _c;
+            var _a;
             (0, Affirm_1.default)(producer, 'Invalid producer');
             (0, Affirm_1.default)(sampleSize > 0, 'Sample size must be greater than 0');
             const source = Environment_1.default.getSource(producer.source);
@@ -131,18 +136,29 @@ class ProducerEngineClass {
                 case 'local':
                 case 'aws-s3': {
                     const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
-                    if (((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) === 'CSV') {
-                        sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer, discover);
-                    }
-                    else if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'JSON' || ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) === 'JSONL') {
-                        // With JSON or JSONL the readFile function already parses the strings
-                        if (typeof fileData.data[0] === 'object')
+                    const fileType = (_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase();
+                    switch (fileType) {
+                        case 'CSV':
+                        case 'TXT':
+                        case 'XLS':
+                        case 'XLSX': {
+                            sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer, discover);
+                            break;
+                        }
+                        case 'JSON':
+                        case 'JSONL': {
+                            // With JSON or JSONL the readFile function already parses the strings
+                            if (typeof fileData.data[0] === 'object')
+                                sampleData = fileData.data;
+                            else
+                                sampleData = fileData.data.map(line => JSON.parse(line));
+                            sampleData = sampleData.slice(0, sampleSize);
+                            break;
+                        }
+                        default: {
                             sampleData = fileData.data;
-                        else
-                            sampleData = fileData.data.map(line => JSON.parse(line));
-                    }
-                    else {
-                        sampleData = fileData.data;
+                            break;
+                        }
                     }
                     break;
                 }

package/engines/RandomEngine.js

@@ -0,0 +1,35 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const seedrandom_1 = __importDefault(require("seedrandom"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+class RandomEngineClass {
+    constructor() {
+        this.rng = (min, max) => {
+            const rng = Math.random;
+            if (Algo_1.default.hasVal(min) || Algo_1.default.hasVal(max))
+                return Math.floor(rng() * (max - min + 1)) + min;
+            else
+                return rng();
+        };
+        this.sRng = (seed, min, max) => {
+            const rng = (0, seedrandom_1.default)(String(seed));
+            if (Algo_1.default.hasVal(min) || Algo_1.default.hasVal(max))
+                return Math.floor(rng() * (max - min + 1)) + min;
+            else
+                return rng();
+        };
+        this.rngDate = (min, max) => {
+            const randomNumber = this.rng(min, max);
+            return new Date(randomNumber).toJSON();
+        };
+        this.sRngDate = (seed, min, max) => {
+            const randomNumber = this.sRng(seed, min, max);
+            return new Date(randomNumber).toJSON();
+        };
+    }
+}
+const RandomEngine = new RandomEngineClass();
+exports.default = RandomEngine;

package/engines/consumer/PostProcessor.js

@@ -33,15 +33,16 @@ class PostProcessorClass {
                     const groups = Algo_1.default.groupBy(items, groupingRule.groupingKey);
                     const projections = [];
                     groups.forEach(gItems => {
-                        var _a;
+                        var _a, _b, _c;
                         const projected = {};
                         const first = gItems[0];
                         for (const field of allFields) {
                             const { key, alias, grouping } = field.cField;
                             const fieldKey = alias !== null && alias !== void 0 ? alias : key;
                             const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
+                            const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
                             if (!field.cField.grouping) {
-                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, first[fieldKey]);
+                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, first[fieldKey], fieldType);
                             }
                             else {
                                 const { subFields } = grouping;
@@ -64,7 +65,7 @@ class PostProcessorClass {
                             const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
                             const fieldValue = this._getFieldValue(x, field);
                             if (Algo_1.default.hasVal(maskType))
-                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, fieldValue);
+                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, fieldValue, fieldType);
                             else
                                 projected[fieldKey] = TypeCaster_1.default.cast(fieldValue, fieldType);
                         }
@@ -87,9 +88,10 @@ class PostProcessorClass {
             const columns = FileCompiler_1.default.compileProducer(producer, source);
             (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
             const unpackDimension = (item, dimension) => {
-                var _a;
+                var _a, _b, _c;
                 const { nameInProducer, aliasInProducer } = dimension;
                 const maskType = (_a = dimension.dimension.mask) !== null && _a !== void 0 ? _a : undefined;
+                const fieldType = (_c = (_b = dimension.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
                 const keys = aliasInProducer.split('.');
                 let prevValue = item;
                 for (const key of keys) {
@@ -114,7 +116,7 @@ class PostProcessorClass {
                             prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue[key];
                     }
                 }
-                prevValue = CryptoEngine_1.default.hashValue(maskType, prevValue);
+                prevValue = CryptoEngine_1.default.hashValue(maskType, prevValue, fieldType);
                 const res = { [nameInProducer]: prevValue };
                 return res;
             };

package/engines/execution/ExecutionPlanner.js

@@ -104,7 +104,8 @@ class ExecutionPlannerClas {
                         plan.push({ type: 'read-file-lines', producer, lines: { from: (_a = options.offset) !== null && _a !== void 0 ? _a : 0, to: options.limit ? (options.offset + options.limit) : undefined } });
                     else
                         plan.push({ type: 'read-file-whole', producer });
-                    if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'CSV')
+                    const fileType = (_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase();
+                    if (fileType === 'CSV' || fileType === 'TXT' || fileType === 'XLS' || fileType === 'XLSX')
                         plan.push({ type: 'csv-to-json', producer });
                     if (producer.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))
                         plan.push({ type: 'nested-field-unpacking', producer });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.27",
+    "version": "0.0.29",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -9,6 +9,7 @@
     },
     "scripts": {
         "sync": "cd ../dev_ops && npm run sync",
+        "tsc-check": "npx tsc --noemit",
         "init": "npx tsx ./src/index.ts init",
         "version": "npx tsx ./src/index.ts -v",
         "run": "npx tsx ./src/index.ts run",
@@ -53,6 +54,8 @@
         "ora": "^5.4.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
+        "seedrandom": "^3.0.5",
+        "xlsx": "^0.18.5",
         "zod": "^3.24.2"
     },
     "devDependencies": {