@forzalabs/remora 0.0.23 → 0.0.25

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.23',
+    cliVersion: '0.0.25',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/actions/run.js

@@ -26,9 +26,16 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
         console.log();
         const spinner = (0, ora_1.default)(chalk_1.default.blue('Running consumer(s)...\n')).start();
         const user = UserManager_1.default.getUser();
-        const consumersToExecute = (consumerName && consumerName.length > 0)
-            ? [Environment_1.default.getConsumer(consumerName)]
-            : Environment_1.default._env.consumers;
+        const consumersToExecute = [];
+        if (consumerName && consumerName.length > 0) {
+            const cons = Environment_1.default.getConsumer(consumerName);
+            if (!cons)
+                throw new Error(`Consumer with name "${consumerName}" was not found.`);
+            consumersToExecute.push(cons);
+        }
+        else {
+            consumersToExecute.push(...Environment_1.default._env.consumers);
+        }
         const results = [];
         for (let i = 0; i < consumersToExecute.length; i++) {
             const consumer = consumersToExecute[i];

package/engines/ParseManager.js

@@ -11,16 +11,16 @@ class ParseManagerClass {
         this.csvToJson = (csv, producer) => {
             (0, Affirm_1.default)(csv, 'Invalid csv content');
             Affirm_1.default.hasValue(csv.length, 'Invalid csv content length');
-            const fileRows = csv.split('\n');
-            return this.csvLinesToJson(fileRows, producer);
+            return this.csvLinesToJson(csv, producer);
         };
-        this.csvLinesToJson = (lines, producer) => {
+        this.csvLinesToJson = (lines, producer, discover) => {
             var _a;
             (0, Affirm_1.default)(lines, 'Invalid csv lines');
             Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
             const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
+            const headerRow = lines[0];
             const rows = lines.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
-            const headerColumns = this._extractHeader(lines[0], delimiterChar, producer);
+            const headerColumns = this._extractHeader(headerRow, delimiterChar, producer, discover);
             const result = [];
             for (const row of rows) {
                 const rowObject = {};
@@ -32,14 +32,17 @@ class ParseManagerClass {
             }
             return result;
         };
-        this._extractHeader = (headerLine, delimiter, producer) => {
+        this._extractHeader = (headerLine, delimiter, producer, discover) => {
             var _a;
-            (0, Affirm_1.default)(headerLine, 'Invalid CSV header line');
+            (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);
             (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
             (0, Affirm_1.default)(producer, 'Invalid producer');
             const source = Environment_1.default.getSource(producer.source);
             const columns = FileCompiler_1.default.compileProducer(producer, source);
             const headerColumns = headerLine.split(delimiter).map(x => x.trim());
+            // If I'm discovering the file, then it means that the dimensions are not set, so I use the ones that I get from the file directly
+            if (discover)
+                columns.push(...headerColumns.map(x => ({ nameInProducer: x })));
             const csvColumns = [];
             for (const pColumn of columns) {
                 const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;

package/engines/ProducerEngine.js

@@ -111,7 +111,7 @@ class ProducerEngineClass {
                     throw new Error(`Invalid file type "${producer.settings.fileType}" for engine type "${source.engine}" for producer "${producer.name}": not supported`);
             }
         });
-        this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10) {
+        this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10, discover = false) {
             var _a, _b, _c;
             (0, Affirm_1.default)(producer, 'Invalid producer');
             (0, Affirm_1.default)(sampleSize > 0, 'Sample size must be greater than 0');
@@ -132,7 +132,7 @@ class ProducerEngineClass {
                 case 'aws-s3': {
                     const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
                     if (((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) === 'CSV') {
-                        sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer);
+                        sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer, discover);
                     }
                     else if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'JSON' || ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) === 'JSONL') {
                         // With JSON or JSONL the readFile function already parses the strings

package/engines/ai/DeveloperEngine.js

@@ -21,7 +21,7 @@ class DeveloperEngineClass {
         this.discover = (producer) => __awaiter(this, void 0, void 0, function* () {
             var _a;
             (0, Affirm_1.default)(producer, 'Invalid producer');
-            const sampleData = yield ProducerEngine_1.default.readSampleData(producer);
+            const sampleData = yield ProducerEngine_1.default.readSampleData(producer, 10, true);
             (0, Affirm_1.default)(sampleData, 'Discover process failed: no result found');
             const typeDefinitions = this.extractFieldTypes(sampleData);
             const mappedProducer = {

package/engines/consumer/PostProcessor.js

@@ -145,10 +145,16 @@ class PostProcessorClass {
             return unpackedData;
         };
         this._getFieldValue = (record, field) => {
+            var _a, _b, _c;
             const fieldValue = record[field.cField.key];
-            if (Algo_1.default.hasVal(fieldValue))
-                return fieldValue;
-            else if (!Algo_1.default.hasVal(fieldValue) && Algo_1.default.hasVal(field.cField.default))
+            if (Algo_1.default.hasVal(fieldValue) && !isNaN(fieldValue)) {
+                const fieldType = (_b = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string';
+                if (fieldType === 'number' && typeof fieldValue === 'string' && fieldValue.length === 0)
+                    return (_c = field.cField.default) !== null && _c !== void 0 ? _c : fieldValue;
+                else
+                    return fieldValue;
+            }
+            else if ((!Algo_1.default.hasVal(fieldValue) || isNaN(fieldValue)) && Algo_1.default.hasVal(field.cField.default))
                 return field.cField.default;
             else
                 return fieldValue;

package/engines/execution/ExecutionEnvironment.js

@@ -24,6 +24,7 @@ const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
 const ExecutionPlanner_1 = __importDefault(require("./ExecutionPlanner"));
 const RequestExecutor_1 = __importDefault(require("./RequestExecutor"));
 const TransformationEngine_1 = __importDefault(require("../transform/TransformationEngine"));
+const JoinEngine_1 = __importDefault(require("../transform/JoinEngine"));
 class ExecutionEnvironment {
     constructor(consumer) {
         this.run = (options) => __awaiter(this, void 0, void 0, function* () {
@@ -37,28 +38,27 @@ class ExecutionEnvironment {
                 switch (planStep.type) {
                     case 'compile-consumer-to-SQL': {
                         const sql = SQLCompiler_1.default.getConsumerReference(this._consumer);
-                        this._data.consumerSQL = sql;
-                        this._data.finalSQL = sql;
+                        this._envData.consumerSQL = sql;
+                        this._envData.finalSQL = sql;
                         break;
                     }
                     case 'compile-execution-request-to-SQL': {
                         const sql = SQLBuilder_1.default.buildConsumerQuery(options);
-                        this._data.executionRequestSQL = sql;
-                        this._data.finalSQL = `WITH consumer AS (${this._data.consumerSQL})\nSELECT * FROM consumer${this._data.executionRequestSQL}`;
+                        this._envData.executionRequestSQL = sql;
+                        this._envData.finalSQL = `WITH consumer AS (${this._envData.consumerSQL})\nSELECT * FROM consumer${this._envData.executionRequestSQL}`;
                         break;
                     }
                     case 'execute-SQL': {
                         (0, Affirm_1.default)(planStep.source, `Invalid source in execute-SQL step`);
                         const driver = yield DriverFactory_1.default.instantiateSource(planStep.source);
-                        this._fetchedData = (yield driver.query(this._data.finalSQL)).rows;
-                        this._fetchedDataType = 'array-of-json';
+                        const queryData = (yield driver.query(this._envData.finalSQL)).rows;
+                        this._storeIntermidiate(planStep, queryData);
                         break;
                     }
                     case 'read-file-whole': {
                         (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-whole step`);
                         const fileData = yield ProducerEngine_1.default.readFile(planStep.producer, { readmode: 'all' });
-                        this._fetchedData = fileData.data;
-                        this._fetchedDataType = fileData.dataType;
+                        this._storeIntermidiate(planStep, fileData.data);
                         break;
                     }
                     case 'read-file-lines': {
@@ -66,56 +66,85 @@ class ExecutionEnvironment {
                         (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-lines step`);
                         const { producer, lines: { from, to } } = planStep;
                         const fileData = yield ProducerEngine_1.default.readFile(producer, { readmode: 'lines', lines: { from, to } });
-                        this._fetchedData = fileData.data;
-                        this._fetchedDataType = fileData.dataType;
+                        this._storeIntermidiate(planStep, fileData.data);
                         break;
                     }
                     case 'nested-field-unpacking': {
                         (0, Affirm_1.default)(planStep.producer, `Invalid producer in nested-field-unpacking step`);
-                        this._fetchedData = PostProcessor_1.default.unpack(this._fetchedData, planStep.producer);
+                        const unpackedData = PostProcessor_1.default.unpack(this._resultingData, planStep.producer);
+                        this._storeIntermidiate(planStep, unpackedData);
                         break;
                     }
                     case 'post-process-json': {
-                        this._fetchedData = PostProcessor_1.default.doProjection(this._consumer, this._fetchedData);
+                        // this._fetchedData = PostProcessor.doProjection(this._consumer, this._fetchedData)
+                        const myProdData = this._getIntermidiate(planStep);
+                        const processedData = PostProcessor_1.default.doProjection(this._consumer, myProdData);
+                        this._storeIntermidiate(planStep, processedData);
                         break;
                     }
                     case 'csv-to-json': {
-                        (0, Affirm_1.default)(this._fetchedData, 'Invalid data');
-                        (0, Affirm_1.default)(Array.isArray(this._fetchedData), 'Invalid data type, must be an array');
+                        (0, Affirm_1.default)(this._resultingData, 'Invalid data');
+                        (0, Affirm_1.default)(Array.isArray(this._resultingData), 'Invalid data type, must be an array');
                         (0, Affirm_1.default)(planStep.producer, `Invalid producer in csv-to-json step`);
-                        const csv = this._fetchedData[0];
-                        this._fetchedData = ParseManager_1.default.csvToJson(csv, planStep.producer);
+                        const csv = this._getIntermidiate(planStep);
+                        // this._fetchedData = ParseManager.csvToJson(csv, planStep.producer)
+                        const jsonData = ParseManager_1.default.csvToJson(csv, planStep.producer);
+                        this._storeIntermidiate(planStep, jsonData);
                         break;
                     }
                     case 'export-file': {
                         (0, Affirm_1.default)(planStep.output, `Invalid output in export-file step`);
-                        const res = yield FileExporter_1.default.export(this._consumer, planStep.output, this._fetchedData);
+                        const res = yield FileExporter_1.default.export(this._consumer, planStep.output, this._resultingData);
                         result.fileUri = res;
                         break;
                     }
                     case 'apply-execution-request-to-result': {
-                        this._fetchedData = RequestExecutor_1.default.execute(this._fetchedData, options);
+                        this._resultingData = RequestExecutor_1.default.execute(this._resultingData, options);
                         break;
                     }
                     case 'apply-consumer-filters-on-JSON': {
-                        this._fetchedData = RequestExecutor_1.default._applyFilters(this._fetchedData, this._consumer.filters.map(x => x.rule));
+                        this._resultingData = RequestExecutor_1.default._applyFilters(this._resultingData, this._consumer.filters.map(x => x.rule));
                         break;
                     }
                     case 'apply-transformations': {
-                        this._fetchedData = TransformationEngine_1.default.apply(this._consumer, this._fetchedData);
+                        this._resultingData = TransformationEngine_1.default.apply(this._consumer, this._resultingData);
+                        break;
+                    }
+                    case 'join-producers-data': {
+                        const joinedData = JoinEngine_1.default.join(this._consumer, this._producedData);
+                        this._resultingData = joinedData;
                         break;
                     }
                     default: throw new Error(`Invalid execution plan step type "${planStep.type}"`);
                 }
             }
-            result.data = this._fetchedData;
+            result.data = this._resultingData;
             result._elapsedMS = performance.now() - start;
             return result;
         });
+        this._storeIntermidiate = (step, data) => {
+            var _a, _b;
+            (0, Affirm_1.default)(step, 'Invalid step');
+            const key = (_b = (_a = step.producer) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : '_default_';
+            let pData = this._producedData.find(x => x.producerKey === key);
+            if (!pData) {
+                pData = { producerKey: key, data: [] };
+                this._producedData.push(pData);
+            }
+            pData.data = data;
+        };
+        this._getIntermidiate = (step) => {
+            var _a, _b;
+            (0, Affirm_1.default)(step, 'Invalid step');
+            const key = (_b = (_a = step.producer) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : '_default_';
+            const produced = this._producedData.find(x => x.producerKey === key);
+            (0, Affirm_1.default)(produced, `No produced dataset found for step "${step.type}" of producer "${key}".`);
+            return produced.data;
+        };
         this._consumer = consumer;
-        this._data = { consumerSQL: null, executionRequestSQL: null, finalSQL: null };
-        this._fetchedData = [];
-        this._fetchedDataType = null;
+        this._envData = { consumerSQL: null, executionRequestSQL: null, finalSQL: null };
+        this._producedData = [];
+        this._resultingData = [];
     }
 }
 exports.default = ExecutionEnvironment;

package/engines/execution/ExecutionPlanner.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
+const Environment_1 = __importDefault(require("../Environment"));
 class ExecutionPlannerClas {
     constructor() {
         this.getEngineClass = (engine) => {
@@ -18,39 +19,9 @@ class ExecutionPlannerClas {
             }
         };
         this.plan = (consumer, options) => {
-            var _a, _b;
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            const [source, producer] = ConsumerManager_1.default.getSource(consumer);
-            const producerEngine = source.engine;
-            const plan = [];
-            switch (producerEngine) {
-                case 'postgres':
-                case 'aws-redshift': {
-                    plan.push({ type: 'compile-consumer-to-SQL' });
-                    if (Algo_1.default.hasVal(options))
-                        plan.push({ type: 'compile-execution-request-to-SQL' });
-                    plan.push({ type: 'execute-SQL', source: source });
-                    break;
-                }
-                case 'local':
-                case 'aws-s3': {
-                    const prod = producer;
-                    if (Algo_1.default.hasVal(options) && (options.limit || options.offset))
-                        plan.push({ type: 'read-file-lines', producer: prod, lines: { from: (_a = options.offset) !== null && _a !== void 0 ? _a : 0, to: options.limit ? (options.offset + options.limit) : undefined } });
-                    else
-                        plan.push({ type: 'read-file-whole', producer: prod });
-                    if (((_b = prod.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'CSV') {
-                        plan.push({ type: 'csv-to-json', producer: prod });
-                    }
-                    if (prod.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))
-                        plan.push({ type: 'nested-field-unpacking', producer: prod });
-                    plan.push({ type: 'post-process-json' });
-                    if (consumer.filters && consumer.filters.length > 0)
-                        plan.push({ type: 'apply-consumer-filters-on-JSON' });
-                    break;
-                }
-                default: throw new Error(`Engine "${producerEngine}" not supported`);
-            }
+            const producersPlan = this._planProducers(consumer, options);
+            const plan = [...producersPlan];
             // At this point I have the data loaded in memory
             // TODO: can I handle streaming data? (e.g. a file that is too big to fit in memory)
             // TODO: how to handle pagination of SQL results?
@@ -58,7 +29,8 @@ class ExecutionPlannerClas {
             // TODO: transformations can also be applied directly to the producer... how???
             if (consumer.fields.some(x => Algo_1.default.hasVal(x.transform)))
                 plan.push({ type: 'apply-transformations' });
-            const engineClass = this.getEngineClass(producerEngine);
+            const [source] = ConsumerManager_1.default.getSource(consumer);
+            const engineClass = this.getEngineClass(source.engine);
             for (const output of consumer.outputs) {
                 switch (output.format.toUpperCase()) {
                     case 'JSON': {
@@ -90,6 +62,59 @@ class ExecutionPlannerClas {
             }
             return plan;
         };
+        this._planProducers = (consumer, options) => {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
+            (0, Affirm_1.default)(producers, `Invalid producers on consumer "${consumer.name}"`);
+            (0, Affirm_1.default)(producers.every(x => Algo_1.default.hasVal(x)), `One or more producers of consumer "${consumer.name}" not found.`);
+            const sources = producers.map(x => Environment_1.default.getSource(x.source));
+            (0, Affirm_1.default)(sources, `Invalid sources on consumer "${consumer.name}"`);
+            (0, Affirm_1.default)(sources.every(x => Algo_1.default.hasVal(x)), `One or more sources of consumer "${consumer.name}" not found.`);
+            const engineClasses = sources.map(x => this.getEngineClass(x.engine));
+            const uniqEngineClasses = Algo_1.default.uniq(engineClasses);
+            const plan = [];
+            if (uniqEngineClasses.length === 1 && uniqEngineClasses[0] === 'sql')
+                plan.push(...this._planProducer(producers[0], options));
+            else
+                plan.push(...(producers.flatMap(x => this._planProducer(x, options))));
+            plan.push({ type: 'join-producers-data' });
+            if (consumer.filters && consumer.filters.length > 0)
+                plan.push({ type: 'apply-consumer-filters-on-JSON' });
+            return plan;
+        };
+        this._planProducer = (producer, options) => {
+            var _a, _b;
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `Source "${producer.source}" of producer "${producer.name}" not found.`);
+            const plan = [];
+            const producerEngine = source.engine;
+            switch (producerEngine) {
+                case 'postgres':
+                case 'aws-redshift': {
+                    plan.push({ type: 'compile-consumer-to-SQL', producer });
+                    if (Algo_1.default.hasVal(options))
+                        plan.push({ type: 'compile-execution-request-to-SQL', producer });
+                    plan.push({ type: 'execute-SQL', source: source, producer });
+                    break;
+                }
+                case 'local':
+                case 'aws-s3': {
+                    if (Algo_1.default.hasVal(options) && (options.limit || options.offset))
+                        plan.push({ type: 'read-file-lines', producer, lines: { from: (_a = options.offset) !== null && _a !== void 0 ? _a : 0, to: options.limit ? (options.offset + options.limit) : undefined } });
+                    else
+                        plan.push({ type: 'read-file-whole', producer });
+                    if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'CSV')
+                        plan.push({ type: 'csv-to-json', producer });
+                    if (producer.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))
+                        plan.push({ type: 'nested-field-unpacking', producer });
+                    plan.push({ type: 'post-process-json', producer });
+                    break;
+                }
+                default: throw new Error(`Engine "${producerEngine}" not supported`);
+            }
+            return plan;
+        };
     }
 }
 const ExecutionPlanner = new ExecutionPlannerClas();

package/engines/transform/JoinEngine.js

@@ -0,0 +1,149 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Environment_1 = __importDefault(require("../Environment"));
+const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
+class JoinEngineClass {
+    constructor() {
+        this.validateFieldInProducer = (fieldName, producerName) => {
+            var _a, _b, _c, _d;
+            const producer = Environment_1.default.getProducer(producerName);
+            if (!producer) {
+                throw new Error(`Producer ${producerName} not found`);
+            }
+            // Check dimensions
+            const hasDimension = producer.dimensions.some(d => d.name === fieldName);
+            // Check measures
+            const hasMeasure = (_b = (_a = producer.measures) === null || _a === void 0 ? void 0 : _a.some(m => m.name === fieldName)) !== null && _b !== void 0 ? _b : false;
+            if (!hasDimension && !hasMeasure) {
+                throw new Error(`Field '${fieldName}' not found in producer '${producerName}'. Available fields: ${producer.dimensions.map(d => d.name).concat((_d = (_c = producer.measures) === null || _c === void 0 ? void 0 : _c.map(m => m.name)) !== null && _d !== void 0 ? _d : []).join(', ')}`);
+            }
+        };
+        this.validateFieldInConsumer = (fieldName, consumerShape) => {
+            const hasField = consumerShape.dimensions.find(x => x.name === fieldName);
+            if (!hasField)
+                throw new Error(`Field '${fieldName}' not found in consumer '${consumerShape.name}'. Your join condition must be on fields that are present in the consumer.`);
+        };
+        this.parseJoinCondition = (sql, producer) => {
+            // Extract field names from SQL condition like ${P.id} = ${orders.user_id}
+            const regex = /\${([^}]+)}/g;
+            const matches = Array.from(sql.matchAll(regex));
+            if (matches.length !== 2) {
+                throw new Error(`Invalid join condition: ${sql}. Expected format: \${P.field} = \${producer.field}`);
+            }
+            const [left, right] = matches.map(m => m[1]);
+            const [leftProducer, leftField] = left.split('.');
+            const [rightProducer, rightField] = right.split('.');
+            if (!leftField || !rightField) {
+                throw new Error(`Invalid join condition: ${sql}. Both sides must specify a field name after the dot.`);
+            }
+            // Replace P with actual producer name
+            const actualLeftProducer = leftProducer === 'P' ? producer.name : leftProducer;
+            const actualRightProducer = rightProducer === 'P' ? producer.name : rightProducer;
+            // Validate both fields exist in their respective producers
+            this.validateFieldInProducer(leftField, actualLeftProducer);
+            this.validateFieldInProducer(rightField, actualRightProducer);
+            return {
+                leftProducer: actualLeftProducer,
+                leftField: leftField,
+                rightProducer: actualRightProducer,
+                rightField: rightField
+            };
+        };
+        this.findProducerData = (producerName, producedData) => {
+            const data = producedData.find(pd => pd.producerKey === producerName);
+            if (!data) {
+                throw new Error(`No data found for producer: ${producerName}`);
+            }
+            return data.data;
+        };
+        this.createLookupMap = (data, key) => {
+            var _a;
+            const map = new Map();
+            for (const item of data) {
+                const row = item;
+                const keyValue = (_a = row[key]) === null || _a === void 0 ? void 0 : _a.toString();
+                if (keyValue === undefined)
+                    continue;
+                const existing = map.get(keyValue);
+                if (existing)
+                    existing.push(row);
+                else
+                    map.set(keyValue, [row]);
+            }
+            return map;
+        };
+        this.join = (consumer, producedData) => {
+            var _a;
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            (0, Affirm_1.default)(producedData, 'Invalid produced data');
+            if (consumer.producers.length <= 1) {
+                return this.findProducerData(consumer.producers[0].name, producedData);
+            }
+            // Start with the first producer's data
+            // let result = this.findProducerData(consumer.producers[0].name, producedData)
+            let result = [];
+            const consumerShape = ConsumerEngine_1.default.getOutputShape(consumer);
+            const consumerColumns = ConsumerEngine_1.default.compile(consumer);
+            // Iterate through each producer and its joins
+            for (let i = 0; i < consumer.producers.length; i++) {
+                const producer = consumer.producers[i];
+                if (!producer.joins)
+                    continue;
+                for (const join of producer.joins) {
+                    const otherProducer = consumer.producers.find(p => p.name === join.otherName);
+                    if (!otherProducer) {
+                        throw new Error(`Producer ${join.otherName} not found`);
+                    }
+                    const condition = this.parseJoinCondition(join.sql, producer);
+                    this.validateFieldInConsumer(condition.leftField, consumerShape);
+                    this.validateFieldInConsumer(condition.rightField, consumerShape);
+                    const leftData = this.findProducerData(condition.leftProducer, producedData);
+                    const rightData = this.findProducerData(condition.rightProducer, producedData);
+                    // Create lookup map for the right dataset
+                    const rightLookup = this.createLookupMap(rightData, condition.rightField);
+                    // Perform the join based on relationship type
+                    const joinedResult = [];
+                    for (const item of leftData) {
+                        const leftRow = item;
+                        const leftValue = (_a = leftRow[condition.leftField]) === null || _a === void 0 ? void 0 : _a.toString();
+                        if (leftValue === undefined)
+                            continue;
+                        const rightRows = rightLookup.get(leftValue) || [];
+                        if (rightRows.length === 0) {
+                            if (join.relationship !== 'one-to-many') {
+                                // For one-to-one and many-to-one, keep rows even without matches
+                                joinedResult.push(leftRow);
+                            }
+                            continue;
+                        }
+                        // Merge rows based on relationship type and field ownership
+                        for (const rightRow of rightRows) {
+                            const mergedRow = {};
+                            // For each field in the compiled consumer, get it from its owner
+                            for (const column of consumerColumns) {
+                                const fieldName = column.consumerAlias;
+                                // Get the data from the owner producer
+                                if (column.owner === condition.leftProducer) {
+                                    mergedRow[fieldName] = leftRow[fieldName];
+                                }
+                                else if (column.owner === condition.rightProducer) {
+                                    mergedRow[fieldName] = rightRow[fieldName];
+                                }
+                                // If neither has the field, it will be undefined
+                            }
+                            joinedResult.push(mergedRow);
+                        }
+                    }
+                    result = joinedResult;
+                }
+            }
+            return result;
+        };
+    }
+}
+const JoinEngine = new JoinEngineClass();
+exports.default = JoinEngine;

package/engines/transform/TypeCaster.js

@@ -1,10 +1,16 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+const Algo_1 = __importDefault(require("../../core/Algo"));
 class TypeCasterClass {
     /**
      * Casts the value to the requested type (only if needed)
      */
     cast(value, type) {
+        if (!Algo_1.default.hasVal(value))
+            return value;
         switch (type) {
             case 'boolean': {
                 if (typeof value === 'boolean')

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.23",
+    "version": "0.0.25",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -17,6 +17,7 @@
         "deploy": "npx tsx ./src/index.ts deploy",
         "debug": "npx tsx ./src/index.ts debug",
         "automap": "npx tsx ./src/index.ts automap",
+        "create-producer": "npx tsx ./src/index.ts create-producer",
         "copy-static-file": "npx tsx ./scripts/CopyStaticFile.js",
         "build": "npm i && npm run sync && tsc --outDir .build && npm run copy-static-file",
         "upload": "npm run build && cd .build && npm publish --access=public"