@forzalabs/remora 1.0.1 → 1.0.2

package/drivers/S3SourceDriver.js

@@ -1,132 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const client_s3_1 = require("@aws-sdk/client-s3");
-const readline_1 = __importDefault(require("readline"));
-const Affirm_1 = __importDefault(require("../core/Affirm"));
-const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
-class S3SourceDriver {
-    constructor() {
-        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
-            this._bucketName = source.authentication['bucket'];
-            const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
-            const config = {
-                region: source.authentication['region'],
-                credentials: {
-                    accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
-                    secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
-                    sessionToken: sessionToken ? sessionToken : undefined
-                }
-            };
-            this._client = new client_s3_1.S3Client(config);
-            // TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
-            return this;
-        });
-        this.download = (request) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
-            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
-            (0, Affirm_1.default)(request, `Invalid download request`);
-            (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
-            const { fileKey } = request;
-            const bucket = this._bucketName;
-            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
-                Bucket: bucket,
-                Key: fileKey
-            }));
-            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
-            const stream = response.Body;
-            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
-            const lines = [];
-            try {
-                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
-                    _c = reader_1_1.value;
-                    _d = false;
-                    const line = _c;
-                    lines.push(line);
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            reader.close();
-            return lines;
-        });
-        this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_2, _b, _c;
-            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
-            (0, Affirm_1.default)(readOptions, 'Invalid read options');
-            const { fileKey, lineFrom, lineTo } = readOptions;
-            const bucket = this._bucketName;
-            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
-                Bucket: bucket,
-                Key: fileKey
-            }));
-            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
-            const stream = response.Body;
-            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
-            const lines = [];
-            let lineCounter = 0;
-            try {
-                for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
-                    _c = reader_2_1.value;
-                    _d = false;
-                    const line = _c;
-                    if (lineCounter >= lineFrom && lineCounter < lineTo)
-                        lines.push(line);
-                    if (lineCounter > lineTo)
-                        break;
-                    lineCounter++;
-                }
-            }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
-                }
-                finally { if (e_2) throw e_2.error; }
-            }
-            reader.close();
-            return lines;
-        });
-        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
-            (0, Affirm_1.default)(producer, 'Invalid read producer');
-            const bucket = this._bucketName;
-            const fileKey = producer.settings.fileKey;
-            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
-            try {
-                yield this._client.send(new client_s3_1.HeadObjectCommand({ Bucket: bucket, Key: fileKey }));
-                return true;
-            }
-            catch (error) {
-                if (((_a = error.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode) === 404 || error.name === 'NotFound')
-                    return false;
-                throw error;
-            }
-        });
-    }
-}
-exports.default = S3SourceDriver;

package/engines/DataframeManager.js

@@ -1,55 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Algo_1 = __importDefault(require("../core/Algo"));
-const Helper_1 = __importDefault(require("../helper/Helper"));
-class DataframeManagerClass {
-    fill(points, from, to, onlyLastValue, maintainLastValue) {
-        const min = from !== null && from !== void 0 ? from : this.getMinDate(points);
-        const max = to !== null && to !== void 0 ? to : this.getMaxDate(points);
-        const orderPoints = points.length > 0 ? Algo_1.default.orderBy(points, 'x') : [];
-        const filledPoints = [];
-        const currentDate = new Date(min);
-        while (currentDate <= max) {
-            const monthKey = Helper_1.default.formatDateToYYYYMM(currentDate);
-            filledPoints.push({ x: monthKey, y: 0 });
-            currentDate.setMonth(currentDate.getMonth() + 1);
-        }
-        for (let i = 0; i < orderPoints.length; i++) {
-            const point = orderPoints[i];
-            const date = new Date(point.x);
-            const filledPoint = filledPoints.find(x => x.x === Helper_1.default.formatDateToYYYYMM(date));
-            if (filledPoint) {
-                if (!onlyLastValue)
-                    filledPoint.y += point.y;
-                else
-                    filledPoint.y = point.y;
-                if (maintainLastValue) {
-                    const index = filledPoints.findIndex(x => x.x === Helper_1.default.formatDateToYYYYMM(date));
-                    for (let k = index; k < filledPoints.length; k++) {
-                        const nextFilledPoint = filledPoints[k];
-                        nextFilledPoint.y = filledPoint.y;
-                    }
-                }
-            }
-        }
-        return filledPoints;
-    }
-    getMinDate(points) {
-        if (!points || points.length === 0) {
-            const currentDate = new Date();
-            return new Date(currentDate.getFullYear() - 1, currentDate.getMonth(), currentDate.getDate());
-        }
-        return points.reduce((min, point) => (new Date(point.x) < min ? new Date(point === null || point === void 0 ? void 0 : point.x) : min), new Date(points[0].x));
-    }
-    getMaxDate(points) {
-        if (!points || points.length === 0) {
-            return new Date();
-        }
-        return points.reduce((max, point) => (new Date(point.x) > max ? new Date(point.x) : max), new Date(points[0].x));
-    }
-}
-const DataframeManager = new DataframeManagerClass();
-exports.default = DataframeManager;

package/engines/ParseManager.js

@@ -1,75 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../core/Affirm"));
-const Environment_1 = __importDefault(require("./Environment"));
-const FileCompiler_1 = __importDefault(require("./file/FileCompiler"));
-class ParseManagerClass {
-    constructor() {
-        this.csvToJson = (csv, producer) => {
-            (0, Affirm_1.default)(csv, 'Invalid csv content');
-            Affirm_1.default.hasValue(csv.length, 'Invalid csv content length');
-            return this.csvLinesToJson(csv, producer);
-        };
-        this.csvLinesToJson = (lines, producer, discover) => {
-            var _a;
-            (0, Affirm_1.default)(lines, 'Invalid csv lines');
-            Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
-            const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
-            const { header, records } = this._getClassifiedRows(lines, delimiterChar, producer);
-            const headerColumns = this._extractHeader(header, delimiterChar, producer, discover);
-            const rows = records.map(x => x.split(delimiterChar).map(k => k.trim()));
-            const result = [];
-            for (const row of rows) {
-                const rowObject = {};
-                for (let i = 0; i < headerColumns.length; i++) {
-                    const column = headerColumns[i];
-                    rowObject[column.saveAs] = row[column.index];
-                }
-                result.push(rowObject);
-            }
-            return result;
-        };
-        this._getClassifiedRows = (lines, delimiterChar, producer) => {
-            if (producer.settings.fileType === 'TXT' && !producer.settings.hasHeaderRow) {
-                // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
-                const source = Environment_1.default.getSource(producer.source);
-                const columns = FileCompiler_1.default.compileProducer(producer, source);
-                return {
-                    header: columns.map(x => x.nameInProducer).join(delimiterChar),
-                    records: lines
-                };
-            }
-            else {
-                return {
-                    header: lines[0],
-                    records: lines.slice(1)
-                };
-            }
-        };
-        this._extractHeader = (headerLine, delimiter, producer, discover) => {
-            var _a;
-            (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);
-            (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            const source = Environment_1.default.getSource(producer.source);
-            const columns = FileCompiler_1.default.compileProducer(producer, source);
-            const headerColumns = headerLine.split(delimiter).map(x => x.trim());
-            // If I'm discovering the file, then it means that the dimensions are not set, so I use the ones that I get from the file directly
-            if (discover)
-                columns.push(...headerColumns.map(x => ({ nameInProducer: x })));
-            const csvColumns = [];
-            for (const pColumn of columns) {
-                const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;
-                const csvColumnIndex = headerColumns.findIndex(x => x === columnKey);
-                (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
-                csvColumns.push({ index: csvColumnIndex, name: columnKey, saveAs: pColumn.nameInProducer });
-            }
-            return csvColumns;
-        };
-    }
-}
-const ParseManager = new ParseManagerClass();
-exports.default = ParseManager;

package/engines/ProducerEngine.js

@@ -1,160 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../core/Affirm"));
-const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
-const DeploymentPlanner_1 = __importDefault(require("./deployment/DeploymentPlanner"));
-const Environment_1 = __importDefault(require("./Environment"));
-const FileCompiler_1 = __importDefault(require("./file/FileCompiler"));
-const SQLCompiler_1 = __importDefault(require("./sql/SQLCompiler"));
-const SQLUtils_1 = __importDefault(require("./sql/SQLUtils"));
-const DatasetManager_1 = __importDefault(require("./dataset/DatasetManager"));
-const Logger_1 = __importDefault(require("../helper/Logger"));
-class ProducerEngineClass {
-    constructor() {
-        this.compile = (producer) => {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
-            switch (source.engine) {
-                case 'aws-redshift':
-                case 'postgres': {
-                    const sql = SQLCompiler_1.default.compileProducer(producer, source);
-                    (0, Affirm_1.default)(sql, `Invalid SQL from compilation for producer "${producer.name}"`);
-                    return sql;
-                }
-                case 'aws-s3': {
-                    const columns = FileCompiler_1.default.compileProducer(producer, source);
-                    (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
-                    break;
-                }
-                default: throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not implemented yet`);
-            }
-        };
-        this.deploy = (producer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
-            const driver = yield DriverFactory_1.default.instantiateSource(source);
-            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
-            const plan = DeploymentPlanner_1.default.planProducer(producer);
-            for (const planStep of plan) {
-                switch (planStep.type) {
-                    case 'create-view': {
-                        const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
-                        (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${producer.name}" wanting to create a view)`);
-                        const sql = SQLCompiler_1.default.compileProducer(producer, source);
-                        const vSQL = `CREATE OR REPLACE VIEW "${internalSchema}"."${SQLUtils_1.default.viewName(producer.name)}" AS ${sql}`;
-                        yield driver.execute(vSQL);
-                        break;
-                    }
-                    default: throw new Error(`Invalid execution consumer plan step type "${planStep.type}"`);
-                }
-            }
-        });
-        this.readFile = (producer, options) => __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            (0, Affirm_1.default)(options, 'Invalid options');
-            if (options.readmode === 'lines')
-                (0, Affirm_1.default)(options.lines, 'Invalid lines');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
-            const driver = yield DriverFactory_1.default.instantiateSource(source);
-            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
-            const { settings: { fileKey, fileType, sheetName, hasHeaderRow } } = producer;
-            let dataset = DatasetManager_1.default.create(producer);
-            let lines = [];
-            switch (options.readmode) {
-                case 'lines':
-                    lines = yield driver.readLinesInRange({ fileKey, fileType, options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName, hasHeaderRow } });
-                    break;
-                case 'all':
-                    lines = yield driver.readAll({ fileKey, fileType, options: { sheetName, hasHeaderRow } });
-                    break;
-                case 'download':
-                    dataset = yield driver.download(dataset);
-                    break;
-            }
-            switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
-                case 'CSV':
-                case 'TXT':
-                    return { data: lines, dataset, dataType: 'lines-of-text' };
-                case 'XLS':
-                case 'XLSX':
-                    return { data: lines, dataset, dataType: 'lines-of-text' };
-                case 'JSONL':
-                case 'JSON': {
-                    if (lines.length === 1) {
-                        // Attempt to handle cases where a single line might contain multiple JSON objects separated by newlines
-                        // Or if the entire file content is a single JSON array stringified.
-                        try {
-                            const parsedAsArray = JSON.parse(lines[0]);
-                            if (Array.isArray(parsedAsArray)) {
-                                return { data: parsedAsArray, dataset, dataType: 'array-of-json' };
-                            }
-                        }
-                        catch (error) {
-                            // If parsing as array fails, proceed to split by newline
-                            console.warn('Failed to parse single line as JSON array, splitting by newline:', error);
-                        }
-                        lines = lines[0].split('\\n');
-                    }
-                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
-                    return { data: json, dataset, dataType: 'array-of-json' };
-                }
-                case 'XML': {
-                    // The driver's _readXmlLines method now returns an array of JSON strings.
-                    // Each string needs to be parsed into a JSON object.
-                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
-                    return { data: json, dataset, dataType: 'array-of-json' };
-                }
-                default:
-                    throw new Error(`Invalid file type "${producer.settings.fileType}" for engine type "${source.engine}" for producer "${producer.name}": not supported`);
-            }
-        });
-        this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10, discover = false) {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            (0, Affirm_1.default)(sampleSize > 0, 'Sample size must be greater than 0');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
-            let dataset = DatasetManager_1.default.create(producer);
-            switch (source.engine) {
-                case 'aws-redshift': {
-                    const sql = `SELECT * FROM "${source.authentication['schema']}"."${producer.settings.sqlTable}" LIMIT ${sampleSize}`;
-                    (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for producer "${producer.name}"`);
-                    const driver = yield DriverFactory_1.default.instantiateSource(source);
-                    (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
-                    const res = yield driver.query(sql);
-                    dataset = yield dataset.loadFromMemory(res.rows, producer, discover);
-                    break;
-                }
-                case 'local':
-                case 'aws-s3': {
-                    const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
-                    dataset = yield dataset.loadFromMemory(fileData.data, producer, discover);
-                    break;
-                }
-                default:
-                    throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not supported`);
-            }
-            const sampleData = [...yield dataset.readLines(sampleSize)];
-            dataset.destroy();
-            Logger_1.default.log(`Finished reading sample dataset:\n${dataset.printStats()}`);
-            return sampleData;
-        });
-    }
-}
-const ProducerEngine = new ProducerEngineClass();
-exports.default = ProducerEngine;

package/engines/UsageDataManager.js

@@ -1,110 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const DSTE_1 = __importDefault(require("../core/dste/DSTE"));
-const DatabaseEngine_1 = __importDefault(require("../database/DatabaseEngine"));
-const DataframeManager_1 = __importDefault(require("./DataframeManager"));
-class UsageDataManager {
-    getUsageDetails() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const now = DSTE_1.default.now();
-            const from = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
-            const prevMonthFrom = new Date(now.getTime() - 60 * 24 * 60 * 60 * 1000);
-            const yearAgo = new Date(now.getFullYear(), now.getMonth() - 11, 1);
-            const collection = 'usage';
-            // Aggregate status counts for current and previous month
-            const getStatusCounts = (start, end) => __awaiter(this, void 0, void 0, function* () {
-                const results = yield DatabaseEngine_1.default.aggregate(collection, [
-                    { $match: { startedAt: { $gte: start, $lte: end } } },
-                    { $group: { _id: '$status', count: { $sum: 1 } } }
-                ]);
-                let success = 0, failed = 0, total = 0;
-                results.forEach((r) => {
-                    total += r.count;
-                    if (r._id === 'success')
-                        success = r.count;
-                    if (r._id === 'failed')
-                        failed = r.count;
-                });
-                return { total, success, failed };
-            });
-            const statusesRequests = yield getStatusCounts(from, now);
-            const prevStatusesRequests = yield getStatusCounts(prevMonthFrom, from);
-            // Monthly success and fails for last 12 months
-            const monthlySuccessPipeline = [
-                { $match: { status: 'success', startedAt: { $gte: yearAgo, $lte: now } } },
-                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
-                { $group: { _id: { year: '$year', month: '$month' }, count: { $sum: 1 } } },
-                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
-                { $sort: { x: 1 } }
-            ];
-            const monthlyFailsPipeline = [
-                { $match: { status: 'failed', startedAt: { $gte: yearAgo, $lte: now } } },
-                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
-                { $group: { _id: { year: '$year', month: '$month' }, count: { $sum: 1 } } },
-                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
-                { $sort: { x: 1 } }
-            ];
-            const rawMonthlySuccess = yield DatabaseEngine_1.default.aggregate(collection, monthlySuccessPipeline);
-            const rawMonthlyFails = yield DatabaseEngine_1.default.aggregate(collection, monthlyFailsPipeline);
-            // Top lines per month for last 12 months
-            const topLinesPipeline = [
-                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
-                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
-                { $group: { _id: { year: '$year', month: '$month' }, itemsCount: { $max: '$itemsCount' } } },
-                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$itemsCount' } },
-                { $sort: { x: 1 } }
-            ];
-            const topLines = yield DatabaseEngine_1.default.aggregate(collection, topLinesPipeline);
-            // Top times per month for last 12 months
-            const topTimePipeline = [
-                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
-                { $addFields: { durationMs: { $subtract: ['$finishedAt', '$startedAt'] }, year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
-                { $group: { _id: { year: '$year', month: '$month' }, maxDuration: { $max: '$durationMs' } } },
-                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$maxDuration' } },
-                { $sort: { x: 1 } }
-            ];
-            const topTime = yield DatabaseEngine_1.default.aggregate(collection, topTimePipeline);
-            // Monthly consumers: for each consumer, per month count
-            const consumerPipeline = [
-                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
-                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
-                { $group: { _id: { consumer: '$consumer', year: '$year', month: '$month' }, count: { $sum: 1 } } },
-                { $project: { _id: 0, consumer: '$_id.consumer', x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
-                { $sort: { consumer: 1, x: 1 } }
-            ];
-            const consumersData = yield DatabaseEngine_1.default.aggregate(collection, consumerPipeline);
-            // transform to consumer array
-            const consumerMap = {};
-            consumersData.forEach((r) => {
-                consumerMap[r.consumer] = consumerMap[r.consumer] || [];
-                consumerMap[r.consumer].push({ x: r.x, y: r.y });
-            });
-            const consumers = Object.entries(consumerMap).map(([name, data]) => ({ name, data: DataframeManager_1.default.fill(data !== null && data !== void 0 ? data : [], yearAgo, now) }));
-            // Recent executions
-            const recentExecution = yield DatabaseEngine_1.default.query(collection, { startedAt: { $gte: from, $lte: now } }, { sort: { startedAt: -1 }, limit: 10 });
-            return {
-                statusesRequests,
-                prevStatusesRequests,
-                monthlySuccess: DataframeManager_1.default.fill(rawMonthlySuccess !== null && rawMonthlySuccess !== void 0 ? rawMonthlySuccess : [], yearAgo, now),
-                monthlyFails: DataframeManager_1.default.fill(rawMonthlyFails !== null && rawMonthlyFails !== void 0 ? rawMonthlyFails : [], yearAgo, now),
-                consumers: consumers,
-                topLine: DataframeManager_1.default.fill(topLines !== null && topLines !== void 0 ? topLines : [], yearAgo, now),
-                topTime: DataframeManager_1.default.fill(topTime !== null && topTime !== void 0 ? topTime : [], yearAgo, now),
-                recentExecution
-            };
-        });
-    }
-}
-exports.default = new UsageDataManager();

package/engines/UsageManager.js

@@ -1,61 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../core/Affirm"));
-const DSTE_1 = __importDefault(require("../core/dste/DSTE"));
-const Helper_1 = __importDefault(require("../helper/Helper"));
-const DatabaseEngine_1 = __importDefault(require("../database/DatabaseEngine"));
-const Settings_1 = __importDefault(require("../helper/Settings"));
-class UsageManagerClass {
-    constructor() {
-        /**
-         * TODO: I need to group the usage stats into a bucket daily. When and how I do it is still a question...
-         */
-        this.getTodayBucketId = (consumer) => {
-            (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            const now = DSTE_1.default.now();
-            return `${consumer.name}_${now.getUTCFullYear()}_${now.getUTCMonth()}_${now.getUTCDate()}`.toLowerCase();
-        };
-        this.startUsage = (consumer, user) => {
-            const newUsage = {
-                _id: Helper_1.default.uuid(),
-                consumer: consumer.name,
-                startedAt: DSTE_1.default.now(),
-                executedBy: { name: user.name, _id: user._id },
-                itemsCount: -1,
-                status: 'started',
-                _signature: ''
-            };
-            if (Helper_1.default.isDev())
-                return { usageId: newUsage._id, usage: Promise.resolve(newUsage) };
-            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, newUsage._id, newUsage);
-            return { usageId: newUsage._id, usage: updateRes };
-        };
-        this.endUsage = (usageId, itemsCount) => {
-            const update = {
-                itemsCount: itemsCount,
-                status: 'success',
-                finishedAt: DSTE_1.default.now()
-            };
-            if (Helper_1.default.isDev())
-                return { usageId: null, usage: Promise.resolve(update) };
-            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, usageId, update);
-            return { usageId: usageId, usage: updateRes };
-        };
-        this.failUsage = (usageId, error) => {
-            const update = {
-                status: 'failed',
-                error: error,
-                finishedAt: DSTE_1.default.now()
-            };
-            if (Helper_1.default.isDev())
-                return { usageId: null, usage: Promise.resolve(update) };
-            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, usageId, update);
-            return { usageId: usageId, usage: updateRes };
-        };
-    }
-}
-const UsageManager = new UsageManagerClass();
-exports.default = UsageManager;