@forzalabs/remora 1.0.21 → 1.1.1

package/executors/ProducerExecutor.js

@@ -1,108 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../core/Affirm"));
-const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
-const Environment_1 = __importDefault(require("../engines/Environment"));
-const CSVParser_1 = __importDefault(require("../engines/parsing/CSVParser"));
-const ProducerManager_1 = __importDefault(require("../engines/producer/ProducerManager"));
-const Algo_1 = __importDefault(require("../core/Algo"));
-const LineParser_1 = __importDefault(require("../engines/parsing/LineParser"));
-const CryptoEngine_1 = __importDefault(require("../engines/CryptoEngine"));
-const path_1 = __importDefault(require("path"));
-class ProducerExecutorClass {
-    constructor() {
-        this.ready = (producer, scope) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `Invalid source ${producer.source} on producer ${producer.name}`);
-            const driver = yield DriverFactory_1.default.instantiateSource(source);
-            return yield driver.ready({ producer, scope });
-        });
-        this.processHeader = (line, producer) => {
-            const { settings: { fileType, hasHeaderRow, delimiter } } = producer;
-            switch (fileType) {
-                case 'PARQUET':
-                case 'XML':
-                case 'XLS':
-                case 'XLSX':
-                case 'CSV': {
-                    const parts = CSVParser_1.default.parseRow(line, delimiter);
-                    return parts;
-                }
-                case 'TXT': {
-                    if (hasHeaderRow) {
-                        const parts = CSVParser_1.default.parseRow(line, delimiter);
-                        return parts;
-                    }
-                    else {
-                        return producer.dimensions.map(x => { var _a; return (_a = x.alias) !== null && _a !== void 0 ? _a : x.name; });
-                    }
-                }
-                case 'JSON':
-                case 'JSONL': {
-                    const keys = Object.keys(JSON.parse(line));
-                    return keys;
-                }
-            }
-        };
-        this.reconcileHeader = (header, producer) => {
-            var _a;
-            const myHeader = [...header];
-            const producerDimensions = producer.dimensions;
-            if (producerDimensions.some(x => x.sourceFilename === true))
-                myHeader.push(...producerDimensions.filter(x => x.sourceFilename === true).map(x => x.name));
-            const dimensions = [];
-            for (const dimension of producerDimensions) {
-                const key = (_a = dimension.alias) !== null && _a !== void 0 ? _a : dimension.name;
-                const index = myHeader.findIndex(x => x === key);
-                if (index < 0)
-                    throw new Error(`The dimension "${dimension.name}" (with key "${key}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
-                dimensions.push({
-                    index,
-                    name: dimension.name,
-                    prodDimension: dimension
-                });
-            }
-            return dimensions;
-        };
-        this.processLine = (options) => {
-            var _a;
-            const { line, dimensions, index, producer, tracker, chunk } = options;
-            Affirm_1.default.hasValue(line, 'Invalid line');
-            Affirm_1.default.hasValue(index, 'Invalid index');
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            if (!line)
-                return null;
-            // 1. map the underlying aliases to the new names AND cast to the correct type
-            let counter = performance.now();
-            const record = LineParser_1.default.parse(line.trim(), producer, dimensions, tracker);
-            tracker.measure('process-line:line-parse', performance.now() - counter);
-            counter = performance.now();
-            for (const dimension of dimensions) {
-                // 2. apply source file name
-                if (dimension.prodDimension.sourceFilename === true)
-                    record[dimension.name] = path_1.default.basename(chunk.fileUri);
-                // 3. mask
-                const maskType = ProducerManager_1.default.getMask(dimension.prodDimension);
-                if (Algo_1.default.hasVal(maskType))
-                    record[dimension.name] = CryptoEngine_1.default.hashValue(maskType, (_a = record[dimension.name]) === null || _a === void 0 ? void 0 : _a.toString(), dimension.prodDimension.type);
-            }
-            tracker.measure('process-line:dimensions-filename-mask', performance.now() - counter);
-            return record;
-        };
-    }
-}
-const ProducerExecutor = new ProducerExecutorClass();
-exports.default = ProducerExecutor;

package/helper/Helper.js

@@ -1,149 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const fs = __importStar(require("fs"));
-const path = __importStar(require("path"));
-const uuid_1 = require("uuid");
-const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
-const Helper = {
-    isDev: () => ProcessENVManager_1.default.getEnvVariable('NODE_ENV') !== 'production',
-    uuid: () => (0, uuid_1.v4)(),
-    asError: (error) => error instanceof Error ? error : new Error(error),
-    readFile: (directory) => {
-        const functionFiles = [];
-        function read(directory) {
-            const files = fs.readdirSync(directory);
-            files.forEach(filename => {
-                const fullPath = path.join(directory, filename);
-                if (fs.statSync(fullPath).isDirectory()) {
-                    read(fullPath);
-                }
-                else if (filename.endsWith('.json')) {
-                    functionFiles.push({
-                        name: path.parse(filename).name,
-                        relativePath: directory
-                    });
-                }
-            });
-        }
-        read(directory);
-        return functionFiles;
-    },
-    formatDateToYYYYMM(date) {
-        if (!date)
-            return '';
-        const year = date.getFullYear();
-        const month = ('0' + (date.getMonth() + 1)).slice(-2);
-        return `${year}-${month}`;
-    },
-    formatDuration: (milliseconds) => {
-        if (!milliseconds || milliseconds < 0)
-            return '0ms';
-        if (milliseconds < 1000) {
-            return `${Math.round(milliseconds)}ms`;
-        }
-        const seconds = milliseconds / 1000;
-        if (seconds < 60) {
-            return `${seconds.toFixed(1)}s`;
-        }
-        const minutes = Math.floor(seconds / 60);
-        const remainingSeconds = seconds % 60;
-        if (minutes < 60) {
-            return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
-        }
-        const hours = Math.floor(minutes / 60);
-        const remainingMinutes = minutes % 60;
-        return `${hours}h ${remainingMinutes}m`;
-    },
-    matchPattern: (pattern, items) => {
-        let patternParts = [];
-        let hasWildcard = false;
-        let result = [...items];
-        if (pattern) {
-            if (pattern.includes('%')) {
-                hasWildcard = true;
-                const parts = pattern.split('%').filter(part => part.length > 0);
-                // Store all non-empty parts for pattern matching
-                patternParts = parts;
-            }
-        }
-        // Apply additional filtering for patterns with wildcards
-        if (hasWildcard && patternParts.length > 0) {
-            result = result.filter(key => {
-                // Create a function to check if the key matches the SQL-like pattern
-                const matchesPattern = (fileName, pattern) => {
-                    if (!pattern.includes('%')) {
-                        return fileName === pattern;
-                    }
-                    const parts = pattern.split('%');
-                    let currentIndex = 0;
-                    for (let i = 0; i < parts.length; i++) {
-                        const part = parts[i];
-                        if (part === '')
-                            continue; // Skip empty parts from consecutive %% or leading/trailing %
-                        if (i === 0 && !pattern.startsWith('%')) {
-                            // First part must be at the beginning
-                            if (!fileName.startsWith(part)) {
-                                return false;
-                            }
-                            currentIndex = part.length;
-                        }
-                        else if (i === parts.length - 1 && !pattern.endsWith('%')) {
-                            // Last part must be at the end
-                            if (!fileName.endsWith(part)) {
-                                return false;
-                            }
-                        }
-                        else {
-                            // Middle parts must exist somewhere after the current position
-                            const foundIndex = fileName.indexOf(part, currentIndex);
-                            if (foundIndex === -1) {
-                                return false;
-                            }
-                            currentIndex = foundIndex + part.length;
-                        }
-                    }
-                    return true;
-                };
-                return matchesPattern(key, pattern);
-            });
-        }
-        return result;
-    }
-};
-exports.default = Helper;

package/helper/Logger.js

@@ -1,84 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const chalk_1 = __importDefault(require("chalk"));
-class Logger {
-    constructor() {
-        this.setLevel = (level) => this._level = level;
-        this.log = (message, level) => {
-            const myLevel = level !== null && level !== void 0 ? level : this._level;
-            if (myLevel !== 'debug')
-                return;
-            console.log(chalk_1.default.cyanBright('DEBUG'), message);
-        };
-        this.info = (message) => {
-            console.info(message);
-        };
-        this.error = (error) => {
-            console.error(error);
-        };
-        this.formatObject = (obj, depth = 0) => {
-            if (obj === null || obj === undefined)
-                return chalk_1.default.gray(String(obj));
-            if (typeof obj !== 'object')
-                return String(obj);
-            if (Array.isArray(obj))
-                return this.formatList(obj);
-            try {
-                const keys = Object.keys(obj);
-                if (keys.length === 0)
-                    return chalk_1.default.gray('{}');
-                const indent = '  '.repeat(depth);
-                // Calculate column widths
-                const keyWidth = Math.max(...keys.map(key => key.length), 8) + 2;
-                const valueWidth = 40;
-                const rows = keys.map(key => {
-                    const value = obj[key];
-                    const formattedKey = chalk_1.default.yellow(key.padEnd(keyWidth));
-                    if (value === null || value === undefined) {
-                        const formattedValue = chalk_1.default.gray(String(value));
-                        return `${indent} ${formattedKey} : ${formattedValue}`;
-                    }
-                    if (typeof value === 'object' && !Array.isArray(value)) {
-                        const nestedObject = this.formatObject(value, depth + 1);
-                        const lines = nestedObject.split('\n');
-                        const firstLine = `${indent} ${formattedKey} : ${''.padEnd(valueWidth)}`;
-                        const nestedLines = lines.map(line => `${indent} ${''.padEnd(keyWidth)} : ${line.padEnd(valueWidth)}`);
-                        return [firstLine, ...nestedLines].join('\n');
-                    }
-                    if (Array.isArray(value)) {
-                        const arrayData = this.formatList(value);
-                        const nested = `${arrayData}`;
-                        return `${indent} ${formattedKey} : ${nested}`;
-                    }
-                    const formattedValue = String(value).padEnd(valueWidth);
-                    return `${indent} ${formattedKey} : ${formattedValue}`;
-                });
-                const result = rows.join('\n');
-                return result;
-            }
-            catch (error) {
-                return chalk_1.default.red('Error formatting object: ') + String(error) + '\n' + String(obj);
-            }
-        };
-        this.formatList = (list) => {
-            if (!Array.isArray(list)) {
-                return chalk_1.default.yellow('Warning: formatList called with non-array') + '\n' + this.formatObject(list);
-            }
-            const header = chalk_1.default.blue(`Array (${list.length} items):`);
-            if (list.length === 0)
-                return header;
-            const items = list.map((item, index) => {
-                const formattedItem = this.formatObject(item);
-                const indentedItem = formattedItem.split('\n').map(line => '      ' + line).join('\n');
-                return chalk_1.default.cyan(`  [${index}]:`) + '\n' + indentedItem;
-            });
-            return header + '\n' + items.join('\n');
-        };
-        this._level = 'prod';
-    }
-}
-const logger = new Logger();
-exports.default = logger;

package/helper/Runtime.js

@@ -1,20 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const node_v8_1 = __importDefault(require("node:v8"));
-class RuntimeClass {
-    constructor() {
-        this.getHeap = () => {
-            const { heap_size_limit, used_heap_size } = node_v8_1.default.getHeapStatistics();
-            return {
-                heapSizeMB: this._toMB(heap_size_limit),
-                usedHeapMB: this._toMB(used_heap_size)
-            };
-        };
-        this._toMB = (bytes) => Math.round(bytes / (1024 * 1024) * 100) / 100;
-    }
-}
-const Runtime = new RuntimeClass();
-exports.default = Runtime;

package/helper/Settings.js

@@ -1,13 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const SETTINGS = {
-    db: {
-        name: 'remora-db',
-        collections: {
-            usage: 'usage',
-            users: 'users',
-            apiKeys: 'apiKeys'
-        }
-    }
-};
-exports.default = SETTINGS;

package/licencing/LicenceManager.js

@@ -1,64 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const crypto_1 = __importDefault(require("crypto"));
-const PUBLICK_KEY = `-----BEGIN PUBLIC KEY-----
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7BWugM83YKGzTyZ6kJyy
-M01JoGYBQYn/9H9utQQyC/uugV4g9d7vv87I2yUfqiHtx7BQj0mOGctqnK7vuRcg
-py7fghTjsUj0MDhs8cEpc+47m0xdsU9MQQ8Xze8B3FcdvCc4xtT5nruTAUWMGIAM
-WLHG8lWxkHjTSZn4S0RdpRBB7fw4jyVkc+vXCTtVtYQwMWjcErNFrdCenvci4Vus
-nKlyfKS6ccLQphOc/I2w6zllFwD/l90vQeQ6kDO01aJLOmJPzvlU/TSP/lcxCYTA
-TgQJQxd5UdnTSDjNw+Uu/jzZ5mVlCbtYH1RoJ6cGNglzN8Q50mVk/iVyoZCZ94b4
-bwIDAQAB
------END PUBLIC KEY-----`;
-class LicenceManagerClass {
-    constructor() {
-        this.generate = (customer, expirationDays, features, privateKey) => {
-            const now = new Date();
-            const expirationDate = new Date(now.getTime() + expirationDays * 24 * 60 * 60 * 1000);
-            const licenceData = {
-                customer,
-                features,
-                issued: now.toISOString(),
-                expires: expirationDate.toISOString()
-            };
-            const licenceString = JSON.stringify(licenceData);
-            const sign = crypto_1.default.createSign('SHA256');
-            sign.update(licenceString);
-            sign.end();
-            const signature = sign.sign(privateKey, 'base64');
-            const licence = { data: licenceData, signature };
-            return Buffer.from(JSON.stringify(licence)).toString('base64');
-        };
-        this.validate = (licence) => {
-            try {
-                const licenceJSON = Buffer.from(licence, 'base64').toString();
-                const licenceData = JSON.parse(licenceJSON);
-                const { data, signature } = licenceData;
-                const now = new Date();
-                const expirationDate = new Date(data.expires);
-                if (now > expirationDate)
-                    return { valid: false, reason: 'License expired', expiryDate: expirationDate };
-                const verify = crypto_1.default.createVerify('SHA256');
-                verify.update(JSON.stringify(data));
-                const isSignatureValid = verify.verify(PUBLICK_KEY, signature, 'base64');
-                if (!isSignatureValid)
-                    return { valid: false, reason: 'Invalid license signature' };
-                return {
-                    valid: true,
-                    expiryDate: expirationDate,
-                    daysRemaining: Math.ceil((expirationDate.getTime() - now.getTime()) / (24 * 60 * 60 * 1000)),
-                    features: data.features,
-                    customer: data.customer
-                };
-            }
-            catch (error) {
-                return { valid: false, reason: 'License parsing error', details: error.message };
-            }
-        };
-    }
-}
-const LicenceManager = new LicenceManagerClass();
-exports.default = LicenceManager;

package/settings.js

@@ -1,12 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.REMORA_WORKER_USER = void 0;
-exports.REMORA_WORKER_USER = {
-    _id: '__remora_worker__',
-    auth: { oid: '', provider: 'internal' },
-    email: '',
-    name: 'Remora Worker',
-    roles: ['root'],
-    _signature: '',
-    lastLogin: new Date().toJSON()
-};