@forzalabs/remora 0.2.6 → 1.0.1

package/executors/ExecutorProgress.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+class ExecutorProgress {
+    constructor(isEnabled) {
+        this._isEnabled = false;
+        this._FPS = 2;
+        this._lastRenderTime = 0;
+        this._lastRenderedLines = -1;
+        this.register = (name) => {
+            this.workers[name] = 0;
+        };
+        this.update = (name, value) => {
+            this.workers[name] = value;
+            const now = Date.now();
+            const interval = 1000 / this._FPS;
+            if (now - this._lastRenderTime >= interval) {
+                this._lastRenderTime = now;
+                this.render();
+            }
+        };
+        this.complete = () => {
+            for (const key of Object.keys(this.workers)) {
+                this.workers[key] = 1;
+            }
+            this.render();
+        };
+        this.render = () => {
+            if (!this._isEnabled)
+                return;
+            if (this._lastRenderedLines > 0) {
+                for (let i = 0; i < this._lastRenderedLines; i++) {
+                    process.stdout.moveCursor(0, -1);
+                    process.stdout.clearLine(1);
+                }
+            }
+            this._lastRenderedLines = 0;
+            for (const key of Object.keys(this.workers)) {
+                const wrk = this.workers[key] * 100;
+                const percentage = Math.min(100, Math.max(0, wrk));
+                const barWidth = 30;
+                const filledWidth = Math.floor((percentage / 100) * barWidth);
+                const emptyWidth = barWidth - filledWidth;
+                const bar = '#'.repeat(filledWidth) + '-'.repeat(emptyWidth);
+                console.log(`Worker ${key.padStart(2, '0')}: [${bar}] ${percentage.toFixed(2)}%`);
+                this._lastRenderedLines++;
+            }
+        };
+        this._isEnabled = isEnabled;
+        this.workers = {};
+    }
+}
+exports.default = ExecutorProgress;

package/executors/OutputExecutor.js

@@ -0,0 +1,118 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Algo_1 = __importDefault(require("../core/Algo"));
+const DSTE_1 = __importDefault(require("../core/dste/DSTE"));
+const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const CSVParser_1 = __importDefault(require("../engines/parsing/CSVParser"));
+const ConsumerExecutor_1 = __importDefault(require("./ConsumerExecutor"));
+class OutputExecutorClass {
+    constructor() {
+        this._getInternalRecordFormat = (consumer) => {
+            const output = consumer.outputs[0];
+            const format = output.format === 'API'
+                ? 'JSON'
+                : output.format === 'PARQUET'
+                    ? 'CSV'
+                    : output.format;
+            return format;
+        };
+        this.outputRecord = (record, consumer, fields) => {
+            const format = this._getInternalRecordFormat(consumer);
+            switch (format) {
+                case 'CSV':
+                    return this.toCSV(record, fields, ',');
+                case 'JSON':
+                    return this.toJSON(record, fields);
+                default:
+                    throw new Error(`Export format ${format} not implemented yet.`);
+            }
+        };
+        this.toCSV = (record, fields, delimiter) => {
+            const myDelimtier = delimiter !== null && delimiter !== void 0 ? delimiter : ',';
+            // remove the not wanted dimension
+            const line = fields
+                .filter(x => !x.cField.hidden)
+                .map(x => { var _a, _b; return `"${Algo_1.default.replaceAll((_b = (_a = record[x.finalKey]) === null || _a === void 0 ? void 0 : _a.toString()) !== null && _b !== void 0 ? _b : '', '"', '""')}"`; })
+                .join(myDelimtier);
+            return line;
+        };
+        this.toJSON = (record, fields) => {
+            if (fields.some(x => x.cField.hidden)) {
+                // remove the un-wanted dimensions
+                for (const dim of fields) {
+                    if (dim.cField.hidden)
+                        delete record[dim.finalKey];
+                }
+                return JSON.stringify(record);
+            }
+            else {
+                return JSON.stringify(record);
+            }
+        };
+        this.exportResult = (consumer, executionId, fields) => __awaiter(this, void 0, void 0, function* () {
+            const internalFormat = this._getInternalRecordFormat(consumer);
+            for (const output of consumer.outputs) {
+                const destination = Environment_1.default.getSource(output.exportDestination);
+                const driver = yield DriverFactory_1.default.instantiateDestination(destination);
+                const currentPath = ConsumerExecutor_1.default._getWorkPath(consumer, executionId);
+                const destinationName = this._composeFileName(consumer, output, this._getExtension(output), executionId);
+                if (output.format === internalFormat) {
+                    return yield driver.move(currentPath, destinationName);
+                }
+                else {
+                    switch (output.format) {
+                        case 'CSV':
+                            return yield driver.transformAndMove(currentPath, line => {
+                                const parsed = JSON.parse(line);
+                                return Object.keys(parsed).map(x => `"${parsed[x]}"`).join(',');
+                            }, destinationName);
+                        case 'API':
+                        case 'JSON':
+                            return yield driver.transformAndMove(currentPath, line => {
+                                const parts = CSVParser_1.default.parseRow(line, ',');
+                                const value = {};
+                                for (const [index, field] of fields.entries())
+                                    value[field.finalKey] = parts[index];
+                                return JSON.stringify(value);
+                            }, destinationName);
+                        case 'PARQUET':
+                        default:
+                            throw new Error(`Export result to format ${output.format} not implemented yet.`);
+                    }
+                }
+            }
+        });
+        this._getExtension = (output) => {
+            return output.format === 'CSV'
+                ? 'csv'
+                : output.format === 'JSON'
+                    ? 'jsonl'
+                    : 'txt';
+        };
+        this._composeFileName = (consumer, output, extension, executionId) => {
+            if (output.exportName && output.exportName.trim().length > 0) {
+                // Ensure no extension duplication
+                const sanitized = output.exportName.replace(/\.[^.]+$/, '');
+                return `${sanitized}.${extension}`;
+            }
+            const baseTs = Algo_1.default.replaceAll(DSTE_1.default.now().toISOString().split('.')[0], ':', '-');
+            const suffix = executionId ? `_${executionId}` : '';
+            return `${consumer.name}_${baseTs}${suffix}.${extension}`;
+        };
+    }
+}
+const OutputExecutor = new OutputExecutorClass();
+exports.default = OutputExecutor;

package/executors/ProducerExecutor.js

@@ -0,0 +1,108 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const CSVParser_1 = __importDefault(require("../engines/parsing/CSVParser"));
+const ProducerManager_1 = __importDefault(require("../engines/producer/ProducerManager"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+const LineParser_1 = __importDefault(require("../engines/parsing/LineParser"));
+const CryptoEngine_1 = __importDefault(require("../engines/CryptoEngine"));
+class ProducerExecutorClass {
+    constructor() {
+        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `Invalid source ${producer.source} on producer ${producer.name}`);
+            const driver = yield DriverFactory_1.default.instantiateSource(source);
+            return yield driver.ready(producer);
+        });
+        this.processHeader = (line, producer) => {
+            const { settings: { fileType, hasHeaderRow, delimiter } } = producer;
+            switch (fileType) {
+                case 'PARQUET':
+                case 'XML':
+                case 'XLS':
+                case 'XLSX':
+                case 'CSV': {
+                    const parts = CSVParser_1.default.parseRow(line, delimiter);
+                    return parts;
+                }
+                case 'TXT': {
+                    if (hasHeaderRow) {
+                        const parts = CSVParser_1.default.parseRow(line, delimiter);
+                        return parts;
+                    }
+                    else {
+                        return producer.dimensions.map(x => { var _a; return (_a = x.alias) !== null && _a !== void 0 ? _a : x.name; });
+                    }
+                }
+                case 'JSON':
+                case 'JSONL': {
+                    const keys = Object.keys(JSON.parse(line));
+                    return keys;
+                }
+            }
+        };
+        this.reconcileHeader = (header, producer) => {
+            var _a;
+            const myHeader = [...header];
+            const producerDimensions = producer.dimensions;
+            if (producerDimensions.some(x => x.sourceFilename === true))
+                myHeader.push(...producerDimensions.filter(x => x.sourceFilename === true).map(x => x.name));
+            const dimensions = [];
+            for (const dimension of producerDimensions) {
+                const key = (_a = dimension.alias) !== null && _a !== void 0 ? _a : dimension.name;
+                const index = myHeader.findIndex(x => x === key);
+                if (index < 0)
+                    throw new Error(`The dimension "${dimension.name}" (with key "${key}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
+                dimensions.push({
+                    index,
+                    name: dimension.name,
+                    prodDimension: dimension
+                });
+            }
+            return dimensions;
+        };
+        this.processLine = (options) => {
+            var _a;
+            const { line, dimensions, index, producer, tracker } = options;
+            Affirm_1.default.hasValue(line, 'Invalid line');
+            Affirm_1.default.hasValue(index, 'Invalid index');
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            if (!line)
+                return null;
+            // 1. map the underlying aliases to the new names AND cast to the correct type
+            let counter = performance.now();
+            const record = LineParser_1.default.parse(line.trim(), producer, dimensions, tracker);
+            tracker.measure('process-line:line-parse', performance.now() - counter);
+            counter = performance.now();
+            for (const dimension of dimensions) {
+                // 2. apply source file name
+                // TODO: to replace with the actual full filename and not just the fileKey
+                if (dimension.prodDimension.sourceFilename === true)
+                    record[dimension.name] = producer.settings.fileKey;
+                // 3. mask
+                const maskType = ProducerManager_1.default.getMask(dimension.prodDimension);
+                if (Algo_1.default.hasVal(maskType))
+                    record[dimension.name] = CryptoEngine_1.default.hashValue(maskType, (_a = record[dimension.name]) === null || _a === void 0 ? void 0 : _a.toString(), dimension.prodDimension.type);
+            }
+            tracker.measure('process-line:dimensions-filename-mask', performance.now() - counter);
+            return record;
+        };
+    }
+}
+const ProducerExecutor = new ProducerExecutorClass();
+exports.default = ProducerExecutor;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.2.6",
+    "version": "1.0.1",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -11,7 +11,7 @@
         "sync": "cd ../dev_ops && npm run sync",
         "dev": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
         "dev:w": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
-        "tsc-check": "npx tsc --noemit",
+        "ts-check": "npx tsc --noemit",
         "init": "npx tsx ./src/index.ts init",
         "version": "npx tsx ./src/index.ts -v",
         "run": "npx tsx ./src/index.ts run",
@@ -23,7 +23,7 @@
         "copy-static-file": "npx tsx ./scripts/CopyStaticFile.js",
         "build": "npm i && npm run sync && tsc --outDir .build && npm run copy-static-file",
         "fast-build": "tsc --outDir .build",
-        "upload": "npm run build && cd .build && npm publish --access=public"
+        "upload": "npm run build && cd .build && npm login && npm publish --access=public"
     },
     "keywords": [
         "nextjs",

package/workers/ExecutorWorker.js

@@ -0,0 +1,48 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const workerpool_1 = __importDefault(require("workerpool"));
+const dotenv_1 = __importDefault(require("dotenv"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const Executor_1 = __importDefault(require("../executors/Executor"));
+dotenv_1.default.configDotenv();
+const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
+    Environment_1.default.load('./');
+    try {
+        const { workerId, chunk, consumer, producer, prodDimensions, options } = workerData;
+        (0, Affirm_1.default)(workerId, `Invalid worker id`);
+        (0, Affirm_1.default)(consumer, `Invalid consumer`);
+        (0, Affirm_1.default)(producer, `Invalid producer`);
+        (0, Affirm_1.default)(chunk, `Invalid chunk`);
+        const executor = new Executor_1.default();
+        const res = yield executor.run({
+            consumer,
+            producer,
+            prodDimensions,
+            workerId,
+            chunk,
+            options,
+            reportWork: packet => workerpool_1.default.workerEmit(packet)
+        });
+        return res;
+    }
+    catch (error) {
+        console.error(error);
+        return null;
+    }
+});
+workerpool_1.default.worker({
+    executor: run
+});