@forzalabs/remora 0.0.50-nasco.3 → 0.0.52-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.50-nasco',
+    cliVersion: '0.0.52-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/consumer-schema.json

@@ -79,6 +79,10 @@
                             "code"
                         ],
                         "additionalProperties": false
+                    },
+                    "union": {
+                        "type": "boolean",
+                        "description": "Merges the data from the various producers in a single dataset. They must have the same output dimensions. If true, then you can't set any joins on any producer, since all producers are merged in a single dataset."
                     }
                 },
                 "required": [
@@ -207,7 +211,7 @@
                         ],
                         "description": "The output format of the consumer"
                     },
-                    "accellerated": {
+                    "accelerated": {
                         "type": "boolean",
                         "description": "If true and supported, the consumer will be materialized to improve query performance"
                     },

package/definitions/json_schemas/producer-schema.json

@@ -28,7 +28,7 @@
                 "properties": {
                     "name": {
                         "type": "string",
-                        "description": "The name of the dimension"
+                        "description": "The name of the dimension. This is the output name of this dimension."
                     },
                     "description": {
                         "type": "string",
@@ -45,7 +45,7 @@
                     },
                     "alias": {
                         "type": "string",
-                        "description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name"
+                        "description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name."
                     },
                     "pk": {
                         "type": "boolean",

package/definitions/json_schemas/project-schema.json

@@ -1,8 +1,14 @@
 {
     "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Project Schema",
+    "description": "Schema for defining remora project configuration",
     "type": "object",
     "required": ["name", "version", "consumers", "producers", "sources", "schemas", "settings"],
     "properties": {
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
         "name": {
             "type": "string",
             "description": "Name of the remora project"
@@ -12,6 +18,10 @@
             "pattern": "^\\d+\\.\\d+\\.\\d+$",
             "description": "Version of the project in semver format"
         },
+        "description": {
+            "type": "string",
+            "description": "Optional description of the project"
+        },
         "consumers": {
             "type": "array",
             "items": {
@@ -53,11 +63,38 @@
                     "minimum": 1,
                     "description": "Maximum number of rows for SQL queries"
                 },
+                "STRING_MAX_CHARACTERS_LENGTH": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "description": "Maximum length for string fields"
+                },
+                "MAX_ITEMS_IN_MEMORY": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "description": "Maximum number of items to keep in memory"
+                },
                 "DEBUG_MODE": {
                     "type": "boolean",
-                    "description": "Enable logging of internal steps."
+                    "description": "Enable logging of internal steps"
                 }
             }
         }
-    }
+    },
+    "additionalProperties": false,
+    "examples": [
+        {
+            "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/project-schema.json",
+            "name": "analytics-project",
+            "version": "1.0.0",
+            "description": "Analytics data processing project",
+            "consumers": ["/consumers"],
+            "producers": ["/producers"],
+            "sources": ["/sources"],
+            "schemas": ["/schemas"],
+            "settings": {
+                "SQL_MAX_QUERY_ROWS": 10000,
+                "DEBUG_MODE": true
+            }
+        }
+    ]
 }

package/engines/consumer/ConsumerEngine.js

@@ -22,7 +22,7 @@ const ExecutionEnvironment_1 = __importDefault(require("../execution/ExecutionEn
 const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
 const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
 const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
-const UsageManager_1 = __importDefault(require("../UsageManager"));
+const UsageManager_1 = __importDefault(require("../usage/UsageManager"));
 const ConsumerManager_1 = __importDefault(require("./ConsumerManager"));
 class ConsumerEngineClass {
     constructor() {

package/engines/consumer/ConsumerManager.js

@@ -132,7 +132,12 @@ class ConsumerManagerClass {
             else {
                 const matches = columns.filter(x => x.nameInProducer === field.key);
                 (0, Affirm_1.default)(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map(x => x.name).join(', ')})`);
-                (0, Affirm_1.default)(matches.length === 1, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is ambiguos between the fields with same name from the producers: ${matches.map(x => x.owner).join(', ')}`);
+                if (matches.length === 1) {
+                    // Need to check if the producers have "union" if they do, I don't care about this check
+                    const cProd = consumer.producers.find(x => x.name === matches[0].owner);
+                    if (!cProd.union)
+                        (0, Affirm_1.default)(matches.length === 1, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is ambiguos between the fields with same name from the producers: ${matches.map(x => x.owner).join(', ')}`);
+                }
                 column = matches[0];
             }
             if (!column) {

package/engines/deployment/DeploymentPlanner.js

@@ -15,7 +15,7 @@ class DeploymentPlannerClass {
                 switch (output.format) {
                     // csv, json, parquet outputs do not need to generate anything at deploy
                     case 'SQL': {
-                        if (output.accellerated && !output.direct)
+                        if (output.accelerated && !output.direct)
                             plan.push({ type: 'create-materialized-view', output: output });
                         else if (!output.direct)
                             plan.push({ type: 'create-view', output: output });

package/engines/execution/ExecutionEnvironment.js

@@ -30,7 +30,7 @@ const Logger_1 = __importDefault(require("../../helper/Logger"));
 class ExecutionEnvironment {
     constructor(consumer) {
         this.run = (options) => __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b;
             (0, Affirm_1.default)(this._consumer, 'Invalid consumer');
             const plan = ExecutionPlanner_1.default.plan(this._consumer, options);
             (0, Affirm_1.default)(plan, `Invalid execution plan`);
@@ -153,7 +153,7 @@ class ExecutionEnvironment {
                 }
             }
             catch (error) {
-                const ds = (_a = this._resultingDataset) !== null && _a !== void 0 ? _a : this._getIntermidiate(currentStep);
+                const ds = (_a = this._resultingDataset) !== null && _a !== void 0 ? _a : (_b = this._producedData.at(-1)) === null || _b === void 0 ? void 0 : _b.dataset;
                 if (ds)
                     Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getSize()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
                 Logger_1.default.log(`\tFailed step: ${currentStep.type}->\n\t${error}`);

package/engines/sql/SQLCompiler.js

@@ -78,7 +78,7 @@ class SQLCompilerClass {
         };
         this.getConsumerReference = (consumer) => {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            if (consumer.outputs.some(x => x.format === 'SQL' && x.accellerated))
+            if (consumer.outputs.some(x => x.format === 'SQL' && x.accelerated))
                 return `SELECT * FROM "av_remora_${SQLUtils_1.default.sanitizeName(consumer.name)}"`;
             if (consumer.outputs.some(x => x.format === 'SQL' && !x.direct))
                 return `SELECT * FROM "v_remora_${SQLUtils_1.default.sanitizeName(consumer.name)}"`;

package/engines/transform/JoinEngine.js

@@ -92,6 +92,8 @@ class JoinEngineClass {
             (0, Affirm_1.default)(producedData, 'Invalid produced data');
             if (consumer.producers.length <= 1)
                 return this.findProducerData(consumer.producers[0].name, producedData);
+            if (consumer.producers.some(x => x.union))
+                return yield this.union(consumer, producedData);
             const consumerShape = ConsumerEngine_1.default.getOutputShape(consumer);
             const consumerColumns = ConsumerEngine_1.default.compile(consumer);
             // Create a new dataset for the joined result
@@ -132,6 +134,21 @@ class JoinEngineClass {
             }
             return resultDataset;
         });
+        this.union = (consumer, producedData) => __awaiter(this, void 0, void 0, function* () {
+            const getDimensionsKey = (ds) => ds.getDimensions().map(x => x.name.trim()).join(';').trim();
+            const mainDataset = producedData[0].dataset;
+            const mainDimKey = getDimensionsKey(mainDataset);
+            const otherProducedData = producedData.slice(1);
+            for (const prodData of otherProducedData) {
+                const prodDimKey = getDimensionsKey(prodData.dataset);
+                if (mainDimKey !== prodDimKey)
+                    throw new Error(`On consumer "${consumer.name}", can't union the dataset "${prodData.dataset['_name']}" (producer: ${prodData.producerKey}) because the dimensions are different from the main dataset "${mainDataset['_name']}" (producer: ${producedData[0].producerKey}). "${mainDimKey}" != "${prodDimKey}"`);
+                yield prodData.dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
+                    yield mainDataset.append(batch);
+                }));
+            }
+            return mainDataset;
+        });
         this.performStreamingJoin = (leftDataset, rightLookup, condition, relationship, consumerColumns, resultDataset) => __awaiter(this, void 0, void 0, function* () {
             const joinedRecords = [];
             const batchSize = leftDataset.getBatchSize();

package/engines/usage/DataframeManager.js

@@ -0,0 +1,55 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const Helper_1 = __importDefault(require("../../helper/Helper"));
+class DataframeManagerClass {
+    fill(points, from, to, onlyLastValue, maintainLastValue) {
+        const min = from !== null && from !== void 0 ? from : this.getMinDate(points);
+        const max = to !== null && to !== void 0 ? to : this.getMaxDate(points);
+        const orderPoints = points.length > 0 ? Algo_1.default.orderBy(points, 'x') : [];
+        const filledPoints = [];
+        const currentDate = new Date(min);
+        while (currentDate <= max) {
+            const monthKey = Helper_1.default.formatDateToYYYYMM(currentDate);
+            filledPoints.push({ x: monthKey, y: 0 });
+            currentDate.setMonth(currentDate.getMonth() + 1);
+        }
+        for (let i = 0; i < orderPoints.length; i++) {
+            const point = orderPoints[i];
+            const date = new Date(point.x);
+            const filledPoint = filledPoints.find(x => x.x === Helper_1.default.formatDateToYYYYMM(date));
+            if (filledPoint) {
+                if (!onlyLastValue)
+                    filledPoint.y += point.y;
+                else
+                    filledPoint.y = point.y;
+                if (maintainLastValue) {
+                    const index = filledPoints.findIndex(x => x.x === Helper_1.default.formatDateToYYYYMM(date));
+                    for (let k = index; k < filledPoints.length; k++) {
+                        const nextFilledPoint = filledPoints[k];
+                        nextFilledPoint.y = filledPoint.y;
+                    }
+                }
+            }
+        }
+        return filledPoints;
+    }
+    getMinDate(points) {
+        if (!points || points.length === 0) {
+            const currentDate = new Date();
+            return new Date(currentDate.getFullYear() - 1, currentDate.getMonth(), currentDate.getDate());
+        }
+        return points.reduce((min, point) => (new Date(point.x) < min ? new Date(point === null || point === void 0 ? void 0 : point.x) : min), new Date(points[0].x));
+    }
+    getMaxDate(points) {
+        if (!points || points.length === 0) {
+            return new Date();
+        }
+        return points.reduce((max, point) => (new Date(point.x) > max ? new Date(point.x) : max), new Date(points[0].x));
+    }
+}
+const DataframeManager = new DataframeManagerClass();
+exports.default = DataframeManager;

package/engines/usage/UsageManager.js

@@ -0,0 +1,61 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const DSTE_1 = __importDefault(require("../../core/dste/DSTE"));
+const DatabaseEngine_1 = __importDefault(require("../../database/DatabaseEngine"));
+const Helper_1 = __importDefault(require("../../helper/Helper"));
+const Settings_1 = __importDefault(require("../../helper/Settings"));
+class UsageManagerClass {
+    constructor() {
+        /**
+         * TODO: I need to group the usage stats into a bucket daily. When and how I do it is still a question...
+         */
+        this.getTodayBucketId = (consumer) => {
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            const now = DSTE_1.default.now();
+            return `${consumer.name}_${now.getUTCFullYear()}_${now.getUTCMonth()}_${now.getUTCDate()}`.toLowerCase();
+        };
+        this.startUsage = (consumer, user) => {
+            const newUsage = {
+                _id: Helper_1.default.uuid(),
+                consumer: consumer.name,
+                startedAt: DSTE_1.default.now(),
+                executedBy: { name: user.name, _id: user._id },
+                itemsCount: -1,
+                status: 'started',
+                _signature: ''
+            };
+            if (Helper_1.default.isDev())
+                return { usageId: newUsage._id, usage: Promise.resolve(newUsage) };
+            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, newUsage._id, newUsage);
+            return { usageId: newUsage._id, usage: updateRes };
+        };
+        this.endUsage = (usageId, itemsCount) => {
+            const update = {
+                itemsCount: itemsCount,
+                status: 'success',
+                finishedAt: DSTE_1.default.now()
+            };
+            if (Helper_1.default.isDev())
+                return { usageId: null, usage: Promise.resolve(update) };
+            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, usageId, update);
+            return { usageId: usageId, usage: updateRes };
+        };
+        this.failUsage = (usageId, error) => {
+            const update = {
+                status: 'failed',
+                error: error,
+                finishedAt: DSTE_1.default.now()
+            };
+            if (Helper_1.default.isDev())
+                return { usageId: null, usage: Promise.resolve(update) };
+            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, usageId, update);
+            return { usageId: usageId, usage: updateRes };
+        };
+    }
+}
+const UsageManager = new UsageManagerClass();
+exports.default = UsageManager;

package/engines/validation/Validator.js

@@ -90,6 +90,12 @@ class ValidatorClass {
                     errors.push('No producers found');
                 if (producers.some(x => !x))
                     errors.push(`Invalid producer found in consumer "${consumer.name}"`);
+                if (consumer.producers.length > 0) {
+                    const withJoins = consumer.producers.filter(x => (Algo_1.default.hasVal(x.joins) && x.joins.length > 0) || !x.union);
+                    const withUnions = consumer.producers.filter(x => x.union === true);
+                    if (withJoins.length > 0 && withUnions.length)
+                        errors.push(`Multiple producers in consumer have mixed "joins" and "union": you can either have multiple producers with "joins" or multiple producers with "union", but not both (joins: ${withJoins.map(x => x.name).join(', ')}; unions: ${withUnions.map(x => x.name).join(', ')})`);
+                }
                 // Validation on sources
                 const sources = producers.map(x => Environment_1.default.getSource(x.source));
                 if (sources.length === 0)
@@ -175,7 +181,7 @@ class ValidatorClass {
                 }
                 for (const output of consumer.outputs) {
                     const format = output.format.toUpperCase();
-                    if (format === 'SQL' && output.accellerated && output.direct)
+                    if (format === 'SQL' && output.accelerated && output.direct)
                         errors.push(`An output SQL cannot be both direct and accelerated (output: ${format})`);
                     if ((format === 'CSV' || format === 'JSON' || format === 'PARQUET')) {
                         if (!output.exportDestination)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.50-nasco.3",
+    "version": "0.0.52-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,