@forzalabs/remora 0.2.2 → 0.2.4

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.2.2',
+    cliVersion: '0.2.4',
     backendVersion: 1,
     backendPort: 5088,
     workerVersion: 2,

package/definitions/json_schemas/consumer-schema.json

@@ -83,6 +83,10 @@
                     "union": {
                         "type": "boolean",
                         "description": "Merges the data from the various producers in a single dataset. They must have the same output dimensions. If true, then you can't set any joins on any producer, since all producers are merged in a single dataset."
+                    },
+                    "isOptional": {
+                        "type": "boolean",
+                        "description": "If true, if the data for this producer is not available when executing the consumer (e.g. the file is not present) the consumer will not fail, but just create a placeholder dataset and continue execution."
                     }
                 },
                 "required": [

package/engines/consumer/ConsumerManager.js

@@ -82,18 +82,34 @@ class ConsumerManagerClass {
             (0, Affirm_1.default)(field, 'Invalid consumer field');
             const expandedFields = [];
             if (field.cField.key === '*') {
-                const from = (_a = field.cField.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
-                availableColumns.filter(x => x.owner === from).forEach(col => {
-                    expandedFields.push({
+                // If the producers are "union" then they have the same dimensions, meaning that I can use the "*"
+                if (consumer.producers.length > 1 && consumer.producers.every(x => x.union)) {
+                    const first = consumer.producers[0];
+                    const firstColumns = availableColumns.filter(x => x.owner === first.name);
+                    expandedFields.push(...firstColumns.map(x => ({
                         cField: {
-                            key: col.nameInProducer,
-                            alias: col.nameInProducer,
-                            from: col.owner
+                            key: x.nameInProducer,
+                            alias: x.nameInProducer,
+                            from: x.owner
                         },
-                        dimension: col.dimension,
-                        measure: col.measure
+                        dimension: x.dimension,
+                        measure: x.measure
+                    })));
+                }
+                else {
+                    const from = (_a = field.cField.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
+                    availableColumns.filter(x => x.owner === from).forEach(col => {
+                        expandedFields.push({
+                            cField: {
+                                key: col.nameInProducer,
+                                alias: col.nameInProducer,
+                                from: col.owner
+                            },
+                            dimension: col.dimension,
+                            measure: col.measure
+                        });
                     });
-                });
+                }
             }
             else if (field.cField.grouping) {
                 expandedFields.push({

package/engines/consumer/PostProcessor.js

@@ -37,7 +37,7 @@ class PostProcessorClass {
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
             const dimensionsUpdates = DatasetManager_1.default.computeDimensionsUpdates(dataset, consumer);
             let updatedDimensions = null;
-            const newDataset = yield dataset.map(record => {
+            dataset = yield dataset.map(record => {
                 var _a, _b;
                 // First apply the updates to the dimensions of this record
                 if (dimensionsUpdates.length > 0) {
@@ -62,8 +62,16 @@ class PostProcessorClass {
                 }
                 return record;
             }, options);
-            newDataset.setDimensions(updatedDimensions);
-            return newDataset;
+            if (!updatedDimensions || updatedDimensions.length === 0) {
+                // This means that no updates were applied cause no records were present
+                // I need to force a fake update to get the new dimensions, since those might still have changed
+                const fakeRecord = new DatasetRecord_1.default('', dataset.getDimensions(), dataset.getDelimiter());
+                for (const update of dimensionsUpdates)
+                    fakeRecord.wholeUpdateDimension(update);
+                updatedDimensions = fakeRecord._dimensions;
+            }
+            dataset.setDimensions(updatedDimensions);
+            return dataset;
         });
         /**
          * Gets an array of objects (with potentially nested fields) and unpacks them to an array of objects with no nested fields

package/engines/dataset/Dataset.js

@@ -35,8 +35,9 @@ const Helper_1 = __importDefault(require("../../helper/Helper"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const Environment_1 = __importDefault(require("../Environment"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
+const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
 class Dataset {
-    constructor(name, file, batchSize, executionId) {
+    constructor(options) {
         var _a, _b;
         this.getPath = () => this._path;
         this.setPath = (path) => {
@@ -66,8 +67,25 @@ class Dataset {
         this.load = (source) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(source, 'Invalid source');
             this._startOperation('load', { source: source.engine });
-            const driver = yield DriverFactory_1.default.instantiateSource(source);
-            yield driver.download(this);
+            try {
+                const driver = yield DriverFactory_1.default.instantiateSource(source);
+                yield driver.download(this);
+            }
+            catch (error) {
+                if (this._file.isOptional) {
+                    Logger_1.default.log(`Error loading dataset "${this.name}", creating default configuration and mock data because "isOptional" is true.`);
+                    if (!this.getDimensions() || this.getDimensions().length === 0)
+                        this.setDimensions(ProducerManager_1.default.getColumns(this._baseProducer).map((x, i) => { var _a; return ({ index: i, key: x.nameInProducer, name: x.aliasInProducer, hidden: false, type: (_a = x.dimension) === null || _a === void 0 ? void 0 : _a.type }); }));
+                    if (!this.getFirstLine() || this.getFirstLine().length === 0) {
+                        if (this._file.hasHeaderRow)
+                            this.setFirstLine(this.getDimensions().map(x => x.key).join(this.getDelimiter()));
+                        else
+                            this.setFirstLine('');
+                    }
+                }
+                else
+                    throw error;
+            }
             this._size = this._computeSize();
             this._finishOperation('load');
             return this;
@@ -839,9 +857,11 @@ class Dataset {
             return searchInOperations(this._operations);
         };
         this._computeSize = () => fs_1.default.statSync(this._path).size / (1024 * 1024);
+        const { name, baseProducer, file, batchSize, executionId } = options;
         this.name = name;
         this._file = file;
         this._executionId = executionId;
+        this._baseProducer = baseProducer;
         this._batchSize = (_a = batchSize !== null && batchSize !== void 0 ? batchSize : parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
         this._dimensions = [];
         this._firstLine = '';

package/engines/dataset/DatasetManager.js

@@ -20,7 +20,7 @@ const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
 const ParseManager_1 = __importDefault(require("../parsing/ParseManager"));
 const Dataset_1 = __importDefault(require("./Dataset"));
 const promises_1 = require("stream/promises");
-const fs_1 = require("fs");
+const fs_1 = __importDefault(require("fs"));
 const DeveloperEngine_1 = __importDefault(require("../ai/DeveloperEngine"));
 const Constants_1 = __importDefault(require("../../Constants"));
 class DatasetManagerClass {
@@ -30,33 +30,41 @@ class DatasetManagerClass {
          * be isolated inside a sub-folder specific to that execution to avoid concurrency conflicts
          * when the same producer / consumer is executed multiple times in parallel.
          */
-        this.create = (producer, executionId) => {
+        this.create = (producer, options) => {
             var _a, _b;
             (0, Affirm_1.default)(producer, 'Invalid producer');
             const { name, settings: { delimiter, fileKey, fileType, hasHeaderRow, sheetName, httpApi } } = producer;
+            const executionId = options === null || options === void 0 ? void 0 : options.executionId;
+            const cProducer = options === null || options === void 0 ? void 0 : options.cProducer;
             // Check if any dimension has sourceFilename flag set to true
             const hasSourceFilenameDimension = (_b = (_a = producer.dimensions) === null || _a === void 0 ? void 0 : _a.some(d => d.sourceFilename === true)) !== null && _b !== void 0 ? _b : false;
-            const dataset = new Dataset_1.default(name, {
-                fileKey,
-                fileType,
-                hasHeaderRow,
-                sheetName,
-                delimiter,
-                httpApi,
-                includeSourceFilename: hasSourceFilenameDimension
-            }, undefined, executionId);
+            const dataset = new Dataset_1.default({
+                name,
+                baseProducer: producer,
+                file: {
+                    fileKey,
+                    fileType,
+                    hasHeaderRow,
+                    sheetName,
+                    delimiter,
+                    httpApi,
+                    includeSourceFilename: hasSourceFilenameDimension,
+                    isOptional: cProducer === null || cProducer === void 0 ? void 0 : cProducer.isOptional
+                },
+                executionId
+            });
             return dataset;
         };
         this.buildDimensions = (dataset_1, producer_1, ...args_1) => __awaiter(this, [dataset_1, producer_1, ...args_1], void 0, function* (dataset, producer, discover = false) {
             (0, Affirm_1.default)(dataset, `Invalid dataset`);
             (0, Affirm_1.default)(producer, `Invalid producer`);
             const firstLine = dataset.getFirstLine();
-            (0, Affirm_1.default)(firstLine, `The first line of the dataset was not set.`);
+            Affirm_1.default.hasValue(firstLine, `The first line of the dataset was not set.`);
             return this.buildDimensionsFromFirstLine(firstLine, dataset.getFile(), producer, discover);
         });
         this.buildDimensionsFromFirstLine = (firstLine_1, dsFile_1, producer_1, ...args_1) => __awaiter(this, [firstLine_1, dsFile_1, producer_1, ...args_1], void 0, function* (firstLine, dsFile, producer, discover = false) {
             var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
-            (0, Affirm_1.default)(firstLine, `Invalid first line`);
+            Affirm_1.default.hasValue(firstLine, `Invalid first line`);
             (0, Affirm_1.default)(dsFile, `Invalid dataset file`);
             (0, Affirm_1.default)(producer, `Invalid producer`);
             const file = dsFile;
@@ -83,6 +91,13 @@ class DatasetManagerClass {
                     const columns = FileCompiler_1.default.compileProducer(producer, source);
                     const firstObject = JSON.parse(firstLine);
                     const keys = Object.keys(firstObject);
+                    // const columnsWithDot = columns.filter(x => x.aliasInProducer.includes('.'))
+                    // if (columnsWithDot.length > 0) {
+                    //     console.log(columns, keys, 'PAPAPAPP')
+                    //     for (const colWithDot of columnsWithDot) {
+                    //         console.log(colWithDot)
+                    //     }
+                    // }
                     // If includeSourceFilename is enabled, the driver has added $source_filename column
                     // We need to add it to the keys list so dimensions can reference it
                     const includeSourceFilename = file.includeSourceFilename === true;
@@ -247,12 +262,16 @@ class DatasetManagerClass {
             const datasetPath = dataset.getPath();
             for (let i = 0; i < threadPaths.length; i++) {
                 const path = threadPaths[i];
-                const readStream = (0, fs_1.createReadStream)(path);
+                // If the thread skipped execution (maybe because no data needed to change), then the
+                // dataset file might not exist, in this case, just skip it
+                if (!fs_1.default.existsSync(path))
+                    continue;
+                const readStream = fs_1.default.createReadStream(path);
                 // For the first file, create a new write stream
                 // For subsequent files, append to the existing file
-                const writeStream = (0, fs_1.createWriteStream)(datasetPath, { flags: i === 0 ? 'w' : 'a' });
+                const writeStream = fs_1.default.createWriteStream(datasetPath, { flags: i === 0 ? 'w' : 'a' });
                 yield (0, promises_1.pipeline)(readStream, writeStream);
-                (0, fs_1.unlinkSync)(path);
+                fs_1.default.unlinkSync(path);
             }
             return dataset;
         });

package/engines/dataset/ParallelDataset.js

@@ -63,7 +63,7 @@ class ParallelDatasetClass {
             const datasetCount = dataset.getCount();
             const batchSize = (_a = parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
             const workerChunkSize = batchSize * Math.round(Constants_1.default.defaults.INDICATIVE_THREAD_LINE_COUNT / batchSize);
-            const workerCount = Math.min(Math.ceil(datasetCount / workerChunkSize), Constants_1.default.defaults.MAX_THREAD_COUNT);
+            const workerCount = Math.max(Math.min(Math.ceil(datasetCount / workerChunkSize), Constants_1.default.defaults.MAX_THREAD_COUNT), 1);
             const adjustedWorkerCount = Math.ceil(datasetCount / workerCount);
             return { workerCount, adjustedWorkerCount };
         };

package/engines/execution/ExecutionEnvironment.js

@@ -65,17 +65,17 @@ class ExecutionEnvironment {
                             (0, Affirm_1.default)(planStep.producer, `Invalid producer in execute-SQL step`);
                             const driver = yield DriverFactory_1.default.instantiateSource(planStep.source);
                             const queryData = (yield driver.query(this._envData.finalSQL)).rows;
-                            let dataset = DatasetManager_1.default.create(planStep.producer, this._executionId);
+                            let dataset = DatasetManager_1.default.create(planStep.producer, { cProducer: planStep.cProducer, executionId: this._executionId });
                             dataset = yield dataset.loadFromMemory(queryData, planStep.producer);
                             this._storeIntermidiate(planStep, dataset);
                             break;
                         }
                         case 'load-dataset': {
                             (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-lines step`);
-                            const { producer } = planStep;
+                            const { producer, cProducer } = planStep;
                             const source = Environment_1.default.getSource(producer.source);
                             (0, Affirm_1.default)(source, `Source "${producer.source}" of producer "${producer.name}" not found.`);
-                            let dataset = DatasetManager_1.default.create(producer, this._executionId);
+                            let dataset = DatasetManager_1.default.create(producer, { cProducer, executionId: this._executionId });
                             dataset = yield dataset.load(source);
                             this._storeIntermidiate(planStep, dataset);
                             break;

package/engines/execution/ExecutionPlanner.js

@@ -85,9 +85,9 @@ class ExecutionPlannerClas {
             const uniqEngineClasses = Algo_1.default.uniq(engineClasses);
             const plan = [];
             if (uniqEngineClasses.length === 1 && uniqEngineClasses[0] === 'sql')
-                plan.push(...this._planProducer(producers[0], options));
+                plan.push(...this._planProducer(producers[0], consumer.producers[0], options));
             else
-                plan.push(...(producers.flatMap(x => this._planProducer(x, options))));
+                plan.push(...(producers.flatMap((x, i) => this._planProducer(x, consumer.producers[i], options))));
             // I technically don't need this, but I keep it to merge all the datasets to a single one
             // so the other steps of the plan can work with a single dataset variable
             plan.push({ type: 'join-producers-data' });
@@ -95,7 +95,7 @@ class ExecutionPlannerClas {
                 plan.push({ type: 'apply-consumer-filters-on-JSON' });
             return plan;
         };
-        this._planProducer = (producer, options) => {
+        this._planProducer = (producer, cProducer, options) => {
             (0, Affirm_1.default)(producer, 'Invalid producer');
             const source = Environment_1.default.getSource(producer.source);
             (0, Affirm_1.default)(source, `Source "${producer.source}" of producer "${producer.name}" not found.`);
@@ -104,21 +104,21 @@ class ExecutionPlannerClas {
             switch (producerEngine) {
                 case 'postgres':
                 case 'aws-redshift': {
-                    plan.push({ type: 'compile-consumer-to-SQL', producer });
+                    plan.push({ type: 'compile-consumer-to-SQL', producer, cProducer });
                     if (Algo_1.default.hasVal(options))
-                        plan.push({ type: 'compile-execution-request-to-SQL', producer });
-                    plan.push({ type: 'execute-SQL', source: source, producer });
+                        plan.push({ type: 'compile-execution-request-to-SQL', producer, cProducer });
+                    plan.push({ type: 'execute-SQL', source: source, producer, cProducer });
                     break;
                 }
                 case 'local':
                 case 'aws-s3':
                 case 'delta-share':
                 case 'http-api': {
-                    plan.push({ type: 'load-dataset', producer });
-                    plan.push({ type: 'prepare-dataset', producer });
+                    plan.push({ type: 'load-dataset', producer, cProducer });
+                    plan.push({ type: 'prepare-dataset', producer, cProducer });
                     if (producer.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))
-                        plan.push({ type: 'nested-field-unpacking', producer });
-                    plan.push({ type: 'post-process-json', producer });
+                        plan.push({ type: 'nested-field-unpacking', producer, cProducer });
+                    plan.push({ type: 'post-process-json', producer, cProducer });
                     break;
                 }
                 default: throw new Error(`Engine "${producerEngine}" not supported`);

package/engines/parsing/ParseManager.js

@@ -4,10 +4,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Environment_1 = __importDefault(require("../Environment"));
-const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
 const CSVParser_1 = __importDefault(require("./CSVParser"));
 const Constants_1 = __importDefault(require("../../Constants"));
+const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
 class ParseManagerClass {
     constructor() {
         this._extractHeader = (headerLine, delimiter, producer, discover) => {
@@ -15,8 +14,7 @@ class ParseManagerClass {
             (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);
             (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
             (0, Affirm_1.default)(producer, 'Invalid producer');
-            const source = Environment_1.default.getSource(producer.source);
-            let columns = FileCompiler_1.default.compileProducer(producer, source);
+            let columns = ProducerManager_1.default.getColumns(producer);
             const headerColumns = CSVParser_1.default.parseRow(headerLine, delimiter).map(x => x.trim());
             // If I'm discovering the file, then it means that the dimensions are not set, so I use the ones that I get from the file directly
             if (discover)

package/engines/producer/ProducerManager.js

@@ -7,6 +7,21 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const SecretManager_1 = __importDefault(require("../SecretManager"));
 class ProducerManagerClass {
     constructor() {
+        this.getColumns = (producer) => {
+            var _a;
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            (0, Affirm_1.default)((_a = producer.settings.fileKey) !== null && _a !== void 0 ? _a : producer.settings.sqlTable, `Missing required file key in producer settings`);
+            (0, Affirm_1.default)(producer.settings.fileType, `Missing required file type in producer settings`);
+            const columns = producer.dimensions.map(x => ({
+                aliasInProducer: x.alias,
+                nameInProducer: x.name,
+                consumerAlias: null,
+                consumerKey: null,
+                owner: producer.name,
+                dimension: x
+            }));
+            return columns;
+        };
         this.getMask = (dimension) => {
             if (!dimension || !dimension.mask)
                 return null;

package/engines/transform/JoinEngine.js

@@ -97,9 +97,14 @@ class JoinEngineClass {
             const consumerShape = ConsumerEngine_1.default.getOutputShape(consumer);
             const consumerColumns = ConsumerEngine_1.default.compile(consumer);
             // Create a new dataset for the joined result
-            const resultDataset = new Dataset_1.default(`joined_${consumer.name}`, {
-                fileKey: 'temp',
-                fileType: 'CSV'
+            const resultDataset = new Dataset_1.default({
+                name: `joined_${consumer.name}`,
+                file: {
+                    fileKey: 'temp',
+                    fileType: 'CSV'
+                },
+                baseProducer: Environment_1.default.getProducer(consumer.producers[0].name),
+                executionId: producedData[0].dataset.getExecutionId()
             });
             // Get dimensions for the result dataset based on consumer columns
             const resultDimensions = consumerColumns.map((col, index) => {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.2.2",
+    "version": "0.2.4",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/FilterWorker.js

@@ -31,7 +31,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(datasetDimensions, `Invalid dataset dimensions`);
         (0, Affirm_1.default)(filter, `Invalid filter data`);
         (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
-        const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
+        const dataset = new Dataset_1.default({ name: datasetName, file: datasetFile, baseProducer: null, executionId });
         dataset
             .setDimensions(datasetDimensions)
             .setDelimiter(datasetDelimiter);

package/workers/ProjectionWorker.js

@@ -33,7 +33,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
         const consumer = Environment_1.default.getConsumer(projectionData.consumerName);
         (0, Affirm_1.default)(consumer, `Wrong consumer name sent to projection worker: "${projectionData.consumerName}" not found.`);
-        const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
+        const dataset = new Dataset_1.default({ name: datasetName, file: datasetFile, baseProducer: null, executionId });
         dataset
             .setDimensions(datasetDimensions)
             .setDelimiter(datasetDelimiter);

package/workers/TransformWorker.js

@@ -33,7 +33,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
         const consumer = Environment_1.default.getConsumer(transformData.consumerName);
         (0, Affirm_1.default)(consumer, `Wrong consumer name sent to projection worker: "${transformData.consumerName}" not found.`);
-        const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
+        const dataset = new Dataset_1.default({ name: datasetName, file: datasetFile, baseProducer: null, executionId });
         dataset
             .setDimensions(datasetDimensions)
             .setDelimiter(datasetDelimiter);