@forzalabs/remora 0.0.43-nasco.3 → 0.0.45-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.40-nasco',
+    cliVersion: '0.0.45-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/producer-schema.json

@@ -70,9 +70,14 @@
                             "mask",
                             "crypt",
                             "random",
-                            "seeded-random"
+                            "seeded-random",
+                            "none"
                         ],
-                        "description": "Masking type to apply to this dimension. 'hash' replaces with a hashed value. 'mask' replaces characters with a mask character. 'crypt' encrypts the value. 'random' replaces with a random value. 'seeded-random' replaces with a random value generated from a seed."
+                        "description": "Masking type to apply to this dimension. 'hash' replaces with a hashed value. 'mask' replaces characters with a mask character. 'crypt' encrypts the value. 'random' replaces with a random value. 'seeded-random' replaces with a random value generated from a seed. You can use environment variables by using the {your-env-var} notation",
+                        "examples": [
+                            "hash",
+                            "{REMORA_MASK_IN_DEV}"
+                        ]
                     }
                 },
                 "required": [

package/definitions/json_schemas/source-schema.json

@@ -29,7 +29,7 @@
         },
         "authentication": {
             "type": "object",
-            "description": "Authentication details for connecting to the data source",
+            "description": "Authentication details for connecting to the data source. You can use environment variables by using the {your-env-var} notation",
             "properties": {
                 "method": {
                     "type": "string",

package/engines/CryptoEngine.js

@@ -18,6 +18,8 @@ class CryptoEngineClass {
                     throw new Error('Not implemented yet');
                 case 'mask':
                     throw new Error('Not implemented yet');
+                case 'none':
+                    return `${fieldReference} AS "${fieldName}"`;
                 default:
                     throw new Error('This type is not ');
             }
@@ -55,6 +57,8 @@ class CryptoEngineClass {
                     throw new Error('Not implemented yet');
                 case 'mask':
                     throw new Error('Not implemented yet');
+                case 'none':
+                    return value;
                 default:
                     throw new Error(`This type doesn't exist`);
             }

package/engines/SecretManager.js

@@ -2,6 +2,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 class SecretManagerClass {
     constructor() {
+        /**
+         * If the value is a secret (or .env setting), replace it with the value inside the .env configuration file
+         * Starts with "{" and ends with "}".
+         * e.g. {AWS_ID}
+         */
         this.replaceSecret = (value) => {
             if (!value || value.length <= 2 || !value.startsWith('{') || !value.endsWith('}'))
                 return value;

package/engines/ai/DeveloperEngine.js

@@ -13,7 +13,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const ProducerEngine_1 = __importDefault(require("../ProducerEngine"));
+const ProducerEngine_1 = __importDefault(require("../producer/ProducerEngine"));
 const path_1 = __importDefault(require("path"));
 const promises_1 = __importDefault(require("fs/promises"));
 class DeveloperEngineClass {

package/engines/consumer/ConsumerEngine.js

@@ -19,6 +19,7 @@ const Helper_1 = __importDefault(require("../../helper/Helper"));
 const DeploymentPlanner_1 = __importDefault(require("../deployment/DeploymentPlanner"));
 const Environment_1 = __importDefault(require("../Environment"));
 const ExecutionEnvironment_1 = __importDefault(require("../execution/ExecutionEnvironment"));
+const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
 const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
 const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
 const UsageManager_1 = __importDefault(require("../UsageManager"));
@@ -145,14 +146,14 @@ class ConsumerEngineClass {
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
             const compiled = this.compile(consumer);
             const outDimensions = compiled.map(x => {
-                var _a, _b, _c, _d, _e, _f, _g, _h;
+                var _a, _b, _c, _d, _e, _f, _g;
                 return ({
                     name: (_a = x.consumerAlias) !== null && _a !== void 0 ? _a : x.consumerKey,
                     type: (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type,
                     classification: (_c = x.dimension) === null || _c === void 0 ? void 0 : _c.classification,
                     description: (_e = (_d = x.dimension) === null || _d === void 0 ? void 0 : _d.description) !== null && _e !== void 0 ? _e : (_f = x.measure) === null || _f === void 0 ? void 0 : _f.description,
-                    mask: (_g = x.dimension) === null || _g === void 0 ? void 0 : _g.mask,
-                    pk: (_h = x.dimension) === null || _h === void 0 ? void 0 : _h.pk
+                    mask: ProducerManager_1.default.getMask(x.dimension),
+                    pk: (_g = x.dimension) === null || _g === void 0 ? void 0 : _g.pk
                 });
             });
             return {

package/engines/consumer/PostProcessor.js

@@ -18,6 +18,7 @@ const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
 const DatasetRecord_1 = __importDefault(require("../dataset/DatasetRecord"));
 const Environment_1 = __importDefault(require("../Environment"));
 const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
+const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
 const TypeCaster_1 = __importDefault(require("../transform/TypeCaster"));
 const ConsumerManager_1 = __importDefault(require("./ConsumerManager"));
 class PostProcessorClass {
@@ -36,12 +37,12 @@ class PostProcessorClass {
             newDataset = this.renameDimensions(newDataset, consumer);
             newDataset = yield this.reorderDimensions(newDataset, consumer);
             newDataset = yield newDataset.map(record => {
-                var _a, _b, _c;
+                var _a, _b;
                 for (const field of fields) {
                     const { key, alias } = field.cField;
                     const fieldKey = alias !== null && alias !== void 0 ? alias : key;
-                    const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
-                    const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
+                    const maskType = ProducerManager_1.default.getMask(field.dimension);
+                    const fieldType = (_b = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string';
                     const fieldValue = this._getFieldValue(record, field);
                     if (Algo_1.default.hasVal(maskType))
                         record.setValue(fieldKey, CryptoEngine_1.default.hashValue(maskType, fieldValue, fieldType));
@@ -105,10 +106,10 @@ class PostProcessorClass {
             const columns = FileCompiler_1.default.compileProducer(producer, source);
             (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
             const unpackDimension = (item, dimension) => {
-                var _a, _b, _c;
+                var _a, _b;
                 const { nameInProducer, aliasInProducer } = dimension;
-                const maskType = (_a = dimension.dimension.mask) !== null && _a !== void 0 ? _a : undefined;
-                const fieldType = (_c = (_b = dimension.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
+                const maskType = ProducerManager_1.default.getMask(dimension.dimension);
+                const fieldType = (_b = (_a = dimension.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string';
                 const keys = aliasInProducer.split('.');
                 // Parse the JSON content from the DatasetRecord first
                 let prevValue;
@@ -128,7 +129,7 @@ class PostProcessorClass {
                         prevValue = recordObj;
                     }
                 }
-                catch (_d) {
+                catch (_c) {
                     // Fallback to using the record's values directly
                     const recordObj = {};
                     const dimensions = dataset.getDimensions();

package/engines/execution/ExecutionEnvironment.js

@@ -17,7 +17,7 @@ const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const PostProcessor_1 = __importDefault(require("../consumer/PostProcessor"));
 const FileExporter_1 = __importDefault(require("../file/FileExporter"));
-const ProducerEngine_1 = __importDefault(require("../ProducerEngine"));
+const ProducerEngine_1 = __importDefault(require("../producer/ProducerEngine"));
 const SQLBuilder_1 = __importDefault(require("../sql/SQLBuilder"));
 const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
 const ExecutionPlanner_1 = __importDefault(require("./ExecutionPlanner"));

package/engines/execution/RequestExecutor.js

@@ -21,7 +21,6 @@ class RequestExecutorClass {
         this.execute = (dataset, request) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, 'Invalid data');
             (0, Affirm_1.default)(request, 'Invalid request');
-            (0, Affirm_1.default)(Array.isArray(dataset), `Invalid data type: should be an array`);
             if (request.filters)
                 dataset = yield this.applyFilters(dataset, request.filters);
             if (request.order)

package/engines/producer/ProducerEngine.js

@@ -0,0 +1,160 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
+const DeploymentPlanner_1 = __importDefault(require("../deployment/DeploymentPlanner"));
+const Environment_1 = __importDefault(require("../Environment"));
+const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
+const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
+const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
+const DatasetManager_1 = __importDefault(require("../dataset/DatasetManager"));
+const Logger_1 = __importDefault(require("../../helper/Logger"));
+class ProducerEngineClass {
+    constructor() {
+        this.compile = (producer) => {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            switch (source.engine) {
+                case 'aws-redshift':
+                case 'postgres': {
+                    const sql = SQLCompiler_1.default.compileProducer(producer, source);
+                    (0, Affirm_1.default)(sql, `Invalid SQL from compilation for producer "${producer.name}"`);
+                    return sql;
+                }
+                case 'aws-s3': {
+                    const columns = FileCompiler_1.default.compileProducer(producer, source);
+                    (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
+                    break;
+                }
+                default: throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not implemented yet`);
+            }
+        };
+        this.deploy = (producer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            const driver = yield DriverFactory_1.default.instantiateSource(source);
+            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+            const plan = DeploymentPlanner_1.default.planProducer(producer);
+            for (const planStep of plan) {
+                switch (planStep.type) {
+                    case 'create-view': {
+                        const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
+                        (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${producer.name}" wanting to create a view)`);
+                        const sql = SQLCompiler_1.default.compileProducer(producer, source);
+                        const vSQL = `CREATE OR REPLACE VIEW "${internalSchema}"."${SQLUtils_1.default.viewName(producer.name)}" AS ${sql}`;
+                        yield driver.execute(vSQL);
+                        break;
+                    }
+                    default: throw new Error(`Invalid execution consumer plan step type "${planStep.type}"`);
+                }
+            }
+        });
+        this.readFile = (producer, options) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            (0, Affirm_1.default)(options, 'Invalid options');
+            if (options.readmode === 'lines')
+                (0, Affirm_1.default)(options.lines, 'Invalid lines');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            const driver = yield DriverFactory_1.default.instantiateSource(source);
+            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+            const { settings: { fileKey, fileType, sheetName, hasHeaderRow } } = producer;
+            let dataset = DatasetManager_1.default.create(producer);
+            let lines = [];
+            switch (options.readmode) {
+                case 'lines':
+                    lines = yield driver.readLinesInRange({ fileKey, fileType, options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName, hasHeaderRow } });
+                    break;
+                case 'all':
+                    lines = yield driver.readAll({ fileKey, fileType, options: { sheetName, hasHeaderRow } });
+                    break;
+                case 'download':
+                    dataset = yield driver.download(dataset);
+                    break;
+            }
+            switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
+                case 'CSV':
+                case 'TXT':
+                    return { data: lines, dataset, dataType: 'lines-of-text' };
+                case 'XLS':
+                case 'XLSX':
+                    return { data: lines, dataset, dataType: 'lines-of-text' };
+                case 'JSONL':
+                case 'JSON': {
+                    if (lines.length === 1) {
+                        // Attempt to handle cases where a single line might contain multiple JSON objects separated by newlines
+                        // Or if the entire file content is a single JSON array stringified.
+                        try {
+                            const parsedAsArray = JSON.parse(lines[0]);
+                            if (Array.isArray(parsedAsArray)) {
+                                return { data: parsedAsArray, dataset, dataType: 'array-of-json' };
+                            }
+                        }
+                        catch (error) {
+                            // If parsing as array fails, proceed to split by newline
+                            console.warn('Failed to parse single line as JSON array, splitting by newline:', error);
+                        }
+                        lines = lines[0].split('\\n');
+                    }
+                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
+                    return { data: json, dataset, dataType: 'array-of-json' };
+                }
+                case 'XML': {
+                    // The driver's _readXmlLines method now returns an array of JSON strings.
+                    // Each string needs to be parsed into a JSON object.
+                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
+                    return { data: json, dataset, dataType: 'array-of-json' };
+                }
+                default:
+                    throw new Error(`Invalid file type "${producer.settings.fileType}" for engine type "${source.engine}" for producer "${producer.name}": not supported`);
+            }
+        });
+        this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10, discover = false) {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            (0, Affirm_1.default)(sampleSize > 0, 'Sample size must be greater than 0');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            let dataset = DatasetManager_1.default.create(producer);
+            switch (source.engine) {
+                case 'aws-redshift': {
+                    const sql = `SELECT * FROM "${source.authentication['schema']}"."${producer.settings.sqlTable}" LIMIT ${sampleSize}`;
+                    (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for producer "${producer.name}"`);
+                    const driver = yield DriverFactory_1.default.instantiateSource(source);
+                    (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+                    const res = yield driver.query(sql);
+                    dataset = yield dataset.loadFromMemory(res.rows, producer, discover);
+                    break;
+                }
+                case 'local':
+                case 'aws-s3': {
+                    const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
+                    dataset = yield dataset.loadFromMemory(fileData.data, producer, discover);
+                    break;
+                }
+                default:
+                    throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not supported`);
+            }
+            const sampleData = [...yield dataset.readLines(sampleSize)];
+            dataset.destroy();
+            Logger_1.default.log(`Finished reading sample dataset:\n${dataset.printStats()}`);
+            return sampleData;
+        });
+    }
+}
+const ProducerEngine = new ProducerEngineClass();
+exports.default = ProducerEngine;

package/engines/producer/ProducerManager.js

@@ -0,0 +1,28 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const SecretManager_1 = __importDefault(require("../SecretManager"));
+class ProducerManagerClass {
+    constructor() {
+        this.getMask = (dimension) => {
+            if (!dimension || !dimension.mask)
+                return null;
+            const mask = SecretManager_1.default.replaceSecret(dimension.mask);
+            const allowedMaskValues = [
+                'hash',
+                'mask',
+                'crypt',
+                'random',
+                'seeded-random',
+                'none'
+            ];
+            (0, Affirm_1.default)(allowedMaskValues.includes(mask), `Mask value "${mask}" in dimension "${dimension.name}" is not a valid value.`);
+            return mask;
+        };
+    }
+}
+const ProducerManager = new ProducerManagerClass();
+exports.default = ProducerManager;

package/engines/sql/SQLCompiler.js

@@ -8,6 +8,7 @@ const Algo_1 = __importDefault(require("../../core/Algo"));
 const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
 const Environment_1 = __importDefault(require("../Environment"));
+const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
 const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
 class SQLCompilerClass {
     constructor() {
@@ -23,10 +24,10 @@ class SQLCompilerClass {
             (0, Affirm_1.default)(producer.settings.sqlTable, `Misconfiguration for producer "${producer.name}": SQL table name is missing. Required since no override to the SQL was used.`);
             const dimensions = producer.dimensions.map(x => {
                 var _a;
-                const maskType = x.mask;
+                const maskType = ProducerManager_1.default.getMask(x);
                 const columnReference = (_a = x.alias) !== null && _a !== void 0 ? _a : x.name;
                 const fieldReference = `"${producer.settings.sqlTable}"."${columnReference}"`;
-                if (x.mask && x.mask === 'hash')
+                if (maskType && maskType === 'hash')
                     return CryptoEngine_1.default.hashQuery(maskType, fieldReference, x.name);
                 else
                     return `${fieldReference} AS "${x.name}"`;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.43-nasco.3",
+    "version": "0.0.45-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,