@forzalabs/remora 0.1.3-nasco.3 → 0.1.5-nasco.3

package/engines/transform/TypeCaster.js

@@ -4,34 +4,45 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Algo_1 = __importDefault(require("../../core/Algo"));
+const dayjs_1 = __importDefault(require("dayjs"));
+const customParseFormat_1 = __importDefault(require("dayjs/plugin/customParseFormat"));
+const utc_1 = __importDefault(require("dayjs/plugin/utc"));
+dayjs_1.default.extend(customParseFormat_1.default);
+dayjs_1.default.extend(utc_1.default);
 class TypeCasterClass {
     /**
      * Casts the value to the requested type (only if needed)
+     * Optional format parameter currently supports:
+     *  - Parsing dates (type 'date'/'datetime') from string with tokens: yyyy, mm, dd
+     *  - Formatting dates when casting to string with same tokens
      */
-    cast(value, type) {
+    cast(value, type, format) {
         if (!Algo_1.default.hasVal(value))
             return value;
         switch (type) {
             case 'boolean': {
                 if (typeof value === 'boolean')
                     return value;
-                else
-                    return Boolean(value);
+                return Boolean(value);
             }
             case 'datetime':
-            case 'date':
-                return new Date(value);
+            case 'date': {
+                let dateValue = null;
+                if (format && typeof value === 'string')
+                    dateValue = dayjs_1.default.utc(value, format, true).toDate();
+                else
+                    dateValue = new Date(value);
+                return dateValue.toISOString();
+            }
             case 'number': {
                 if (typeof value === 'number')
                     return value;
-                else
-                    return Number(value);
+                return Number(value);
             }
             case 'string': {
                 if (typeof value === 'string')
                     return value;
-                else
-                    return String(value);
+                return String(value);
             }
         }
     }

package/engines/usage/UsageDataManager.js

@@ -0,0 +1,110 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const DSTE_1 = __importDefault(require("../../core/dste/DSTE"));
+const DatabaseEngine_1 = __importDefault(require("../../database/DatabaseEngine"));
+const DataframeManager_1 = __importDefault(require("./DataframeManager"));
+class UsageDataManager {
+    getUsageDetails() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const now = DSTE_1.default.now();
+            const from = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
+            const prevMonthFrom = new Date(now.getTime() - 60 * 24 * 60 * 60 * 1000);
+            const yearAgo = new Date(now.getFullYear(), now.getMonth() - 11, 1);
+            const collection = 'usage';
+            // Aggregate status counts for current and previous month
+            const getStatusCounts = (start, end) => __awaiter(this, void 0, void 0, function* () {
+                const results = yield DatabaseEngine_1.default.aggregate(collection, [
+                    { $match: { startedAt: { $gte: start, $lte: end } } },
+                    { $group: { _id: '$status', count: { $sum: 1 } } }
+                ]);
+                let success = 0, failed = 0, total = 0;
+                results.forEach(r => {
+                    total += r.count;
+                    if (r._id === 'success')
+                        success = r.count;
+                    if (r._id === 'failed')
+                        failed = r.count;
+                });
+                return { total, success, failed };
+            });
+            const statusesRequests = yield getStatusCounts(from, now);
+            const prevStatusesRequests = yield getStatusCounts(prevMonthFrom, from);
+            // Monthly success and fails for last 12 months
+            const monthlySuccessPipeline = [
+                { $match: { status: 'success', startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { year: '$year', month: '$month' }, count: { $sum: 1 } } },
+                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
+                { $sort: { x: 1 } }
+            ];
+            const monthlyFailsPipeline = [
+                { $match: { status: 'failed', startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { year: '$year', month: '$month' }, count: { $sum: 1 } } },
+                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
+                { $sort: { x: 1 } }
+            ];
+            const rawMonthlySuccess = yield DatabaseEngine_1.default.aggregate(collection, monthlySuccessPipeline);
+            const rawMonthlyFails = yield DatabaseEngine_1.default.aggregate(collection, monthlyFailsPipeline);
+            // Top lines per month for last 12 months
+            const topLinesPipeline = [
+                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { year: '$year', month: '$month' }, itemsCount: { $max: '$itemsCount' } } },
+                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$itemsCount' } },
+                { $sort: { x: 1 } }
+            ];
+            const topLines = yield DatabaseEngine_1.default.aggregate(collection, topLinesPipeline);
+            // Top times per month for last 12 months
+            const topTimePipeline = [
+                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { durationMs: { $subtract: ['$finishedAt', '$startedAt'] }, year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { year: '$year', month: '$month' }, maxDuration: { $max: '$durationMs' } } },
+                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$maxDuration' } },
+                { $sort: { x: 1 } }
+            ];
+            const topTime = yield DatabaseEngine_1.default.aggregate(collection, topTimePipeline);
+            // Monthly consumers: for each consumer, per month count
+            const consumerPipeline = [
+                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { consumer: '$consumer', year: '$year', month: '$month' }, count: { $sum: 1 } } },
+                { $project: { _id: 0, consumer: '$_id.consumer', x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
+                { $sort: { consumer: 1, x: 1 } }
+            ];
+            const consumersData = yield DatabaseEngine_1.default.aggregate(collection, consumerPipeline);
+            // transform to consumer array
+            const consumerMap = {};
+            consumersData.forEach(r => {
+                consumerMap[r.consumer] = consumerMap[r.consumer] || [];
+                consumerMap[r.consumer].push({ x: r.x, y: r.y });
+            });
+            const consumers = Object.entries(consumerMap).map(([name, data]) => ({ name, data: DataframeManager_1.default.fill(data !== null && data !== void 0 ? data : [], yearAgo, now) }));
+            // Recent executions
+            const recentExecution = yield DatabaseEngine_1.default.query(collection, { startedAt: { $gte: from, $lte: now } }, { sort: { startedAt: -1 }, limit: 10 });
+            return {
+                statusesRequests,
+                prevStatusesRequests,
+                monthlySuccess: DataframeManager_1.default.fill(rawMonthlySuccess !== null && rawMonthlySuccess !== void 0 ? rawMonthlySuccess : [], yearAgo, now),
+                monthlyFails: DataframeManager_1.default.fill(rawMonthlyFails !== null && rawMonthlyFails !== void 0 ? rawMonthlyFails : [], yearAgo, now),
+                consumers: consumers,
+                topLine: DataframeManager_1.default.fill(topLines !== null && topLines !== void 0 ? topLines : [], yearAgo, now),
+                topTime: DataframeManager_1.default.fill(topTime !== null && topTime !== void 0 ? topTime : [], yearAgo, now),
+                recentExecution
+            };
+        });
+    }
+}
+exports.default = new UsageDataManager();

package/engines/validation/Validator.js

@@ -154,9 +154,6 @@ class ValidatorClass {
                         else
                             trxToValidate.push(field.transform);
                         for (const trans of trxToValidate) {
-                            const trxKeys = Object.keys(trans);
-                            if (trxKeys.length !== 1)
-                                errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}.`);
                             if ('combine_fields' in trans) {
                                 const { combine_fields } = trans;
                                 if (!combine_fields.fields || combine_fields.fields.length === 0)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.1.3-nasco.3",
+    "version": "0.1.5-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -43,9 +43,11 @@
         "chalk": "^4.1.2",
         "commander": "^10.0.0",
         "cross-env": "^7.0.3",
+        "dayjs": "^1.11.13",
         "dotenv": "^16.0.3",
         "fast-xml-parser": "^5.2.3",
         "fs-extra": "^11.1.0",
+        "hyparquet": "^1.17.4",
         "inquirer": "^8.2.5",
         "json-schema": "^0.4.0",
         "jsonwebtoken": "^9.0.2",

package/workers/FilterWorker.js

@@ -22,7 +22,7 @@ dotenv_1.default.configDotenv();
 const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
     Environment_1.default.load('./');
     try {
-        const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, filterData: filter } = workerData;
+        const { datasetName, fromLine, toLine, workerId, executionId, datasetFile, datasetDimensions, datasetDelimiter, filterData: filter } = workerData;
         Affirm_1.default.hasValue(fromLine, `Invalid from line`);
         Affirm_1.default.hasValue(toLine, `Invalid to line`);
         (0, Affirm_1.default)(datasetName, `Invalid dataset name`);
@@ -31,9 +31,9 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(datasetDimensions, `Invalid dataset dimensions`);
         (0, Affirm_1.default)(filter, `Invalid filter data`);
         (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
-        const dataset = new Dataset_1.default(datasetName, datasetFile);
+        const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
         dataset
-            .setDimensinons(datasetDimensions)
+            .setDimensions(datasetDimensions)
             .setDelimiter(datasetDelimiter);
         const outputPath = dataset['_tempPath'] + workerId;
         const rules = filter.rules.map(x => x.rule);

package/workers/ProjectionWorker.js

@@ -22,7 +22,7 @@ dotenv_1.default.configDotenv();
 const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
     Environment_1.default.load('./');
     try {
-        const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, projectionData } = workerData;
+        const { datasetName, fromLine, toLine, workerId, datasetFile, executionId, datasetDimensions, datasetDelimiter, projectionData } = workerData;
         Affirm_1.default.hasValue(fromLine, `Invalid from line`);
         Affirm_1.default.hasValue(toLine, `Invalid to line`);
         (0, Affirm_1.default)(datasetName, `Invalid dataset name`);
@@ -33,9 +33,9 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
         const consumer = Environment_1.default.getConsumer(projectionData.consumerName);
         (0, Affirm_1.default)(consumer, `Wrong consumer name sent to projection worker: "${projectionData.consumerName}" not found.`);
-        const dataset = new Dataset_1.default(datasetName, datasetFile);
+        const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
         dataset
-            .setDimensinons(datasetDimensions)
+            .setDimensions(datasetDimensions)
             .setDelimiter(datasetDelimiter);
         const outputPath = dataset['_tempPath'] + workerId;
         yield PostProcessor_1.default.doProjection(consumer, dataset, { outputPath, range: { fromLine, toLine } });

package/workers/TransformWorker.js

@@ -22,7 +22,7 @@ dotenv_1.default.configDotenv();
 const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
     Environment_1.default.load('./');
     try {
-        const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, transformData } = workerData;
+        const { datasetName, fromLine, toLine, workerId, executionId, datasetFile, datasetDimensions, datasetDelimiter, transformData } = workerData;
         Affirm_1.default.hasValue(fromLine, `Invalid from line`);
         Affirm_1.default.hasValue(toLine, `Invalid to line`);
         (0, Affirm_1.default)(datasetName, `Invalid dataset name`);
@@ -33,9 +33,9 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
         const consumer = Environment_1.default.getConsumer(transformData.consumerName);
         (0, Affirm_1.default)(consumer, `Wrong consumer name sent to projection worker: "${transformData.consumerName}" not found.`);
-        const dataset = new Dataset_1.default(datasetName, datasetFile);
+        const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
         dataset
-            .setDimensinons(datasetDimensions)
+            .setDimensions(datasetDimensions)
             .setDelimiter(datasetDelimiter);
         const outputPath = dataset['_tempPath'] + workerId;
         yield TransformationEngine_1.default.apply(consumer, dataset, { outputPath, range: { fromLine, toLine } });