@forzalabs/remora 1.2.7 → 1.2.8

package/index.js

@@ -18770,7 +18770,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.2.7",
+  cliVersion: "1.2.8",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -19851,6 +19851,7 @@ var XLSParserClass = class {
   }
 };
 var XLSParser = new XLSParserClass();
+var XLSParser_default = XLSParser;
 
 // ../../packages/parsing/src/XMLParser.ts
 var import_fast_xml_parser = require("fast-xml-parser");
@@ -22864,8 +22865,10 @@ var ProducerEngineClass = class {
       for (const dimension of producerDimensions) {
         const key = dimension.alias ?? dimension.name;
         const index = myHeader.findIndex((x) => x === key);
-        if (index < 0)
+        if (index < 0) {
+          Logger_default.error(`Unable to find dimension "${dimension.name}" in: ${myHeader.join(" | ")}`);
           throw new Error(`The dimension "${dimension.name}" (with key "${key}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
+        }
         dimensions.push({
           index,
           name: dimension.name,
@@ -25430,7 +25433,9 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
         _progress.update({ phase: "Preparing source data", progress: 0 });
-        const sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
+        let sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
+        _progress.update({ phase: "Preparing source data", progress: 0.8 });
+        sourceFilesByProducer = await this.decodeSourceFiles(sourceFilesByProducer);
         _progress.update({ phase: "Preparing source data", progress: 1 });
         tracker.measure("ready-producers", performance.now() - counter);
         if (sourceFilesByProducer.length === 1 && sourceFilesByProducer[0].response?.files.length === 0)
@@ -25642,6 +25647,9 @@ var ExecutorOrchestratorClass = class {
       }
       return fileSize;
     };
+    /**
+     * Grabs the remote datasets and creates an internal .dataset to work on
+     */
     this.readySourceFiles = async (consumer, scope) => {
       const results = [];
       for (let i = 0; i < consumer.producers.length; i++) {
@@ -25654,6 +25662,95 @@ var ExecutorOrchestratorClass = class {
       }
       return results;
     };
+    this.decodeSourceFiles = async (readyResults) => {
+      const csvSafeValue = (value) => {
+        if (!Algo_default.hasVal(value))
+          return "";
+        if (typeof value === "object")
+          return JSON.stringify(value);
+        return String(value);
+      };
+      const normalizeXmlRows = (raw) => {
+        const rows = Array.isArray(raw) ? raw : [raw];
+        const normalized = [];
+        for (const row of rows) {
+          if (Array.isArray(row)) {
+            normalized.push({ value: JSON.stringify(row) });
+            continue;
+          }
+          if (typeof row === "object" && row !== null) {
+            normalized.push(row);
+            continue;
+          }
+          normalized.push({ value: row });
+        }
+        return normalized;
+      };
+      const decodedResults = await Promise.all(readyResults.map(async (readyResult) => {
+        const { prod, response } = readyResult;
+        const fileType = prod.settings.fileType?.toUpperCase();
+        const decodedFiles = [];
+        let decodedCount = 0;
+        for (const file of response.files) {
+          const inferredType = fileType ?? import_path21.default.extname(file.fullUri).replace(".", "").toUpperCase();
+          const decodedPath = `${file.fullUri}.decoded.csv`;
+          if (inferredType === "XLS" || inferredType === "XLSX") {
+            const xlsCsvStream = await XLSParser_default.getStreamXls(file.fullUri, {
+              sheetName: prod.settings.sheetName,
+              startRow: prod.settings.startRow,
+              startColumn: prod.settings.startColumn
+            });
+            await (0, import_promises11.pipeline)(
+              xlsCsvStream,
+              import_fs14.default.createWriteStream(decodedPath)
+            );
+            const fileStats = await import_promises10.default.stat(decodedPath);
+            decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
+            decodedCount++;
+            continue;
+          }
+          if (inferredType === "XML") {
+            const fileContent = await import_promises10.default.readFile(file.fullUri, "utf-8");
+            const jsonData = XMLParser_default.xmlToJson(fileContent);
+            const records = normalizeXmlRows(jsonData);
+            if (records.length === 0) {
+              await import_promises10.default.writeFile(decodedPath, "", "utf-8");
+            } else {
+              const columns = [];
+              for (const record of records) {
+                for (const key of Object.keys(record)) {
+                  if (!columns.includes(key))
+                    columns.push(key);
+                }
+              }
+              if (columns.length === 0)
+                columns.push("value");
+              const lines = [CSVParser_default.stringifyRow(columns)];
+              for (const record of records) {
+                const row = columns.map((column) => csvSafeValue(record[column]));
+                lines.push(CSVParser_default.stringifyRow(row));
+              }
+              await import_promises10.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
+            }
+            const fileStats = await import_promises10.default.stat(decodedPath);
+            decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
+            decodedCount++;
+            continue;
+          }
+          decodedFiles.push(file);
+        }
+        if (decodedCount > 0)
+          Logger_default.log(`Producer "${prod.name}": decoded ${decodedCount} encoded file(s) to CSV`);
+        return {
+          ...readyResult,
+          response: {
+            ...response,
+            files: decodedFiles
+          }
+        };
+      }));
+      return decodedResults;
+    };
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.2.7",
+    "version": "1.2.8",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -18764,7 +18764,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.2.7",
+  cliVersion: "1.2.8",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -20195,6 +20195,7 @@ var XLSParserClass = class {
   }
 };
 var XLSParser = new XLSParserClass();
+var XLSParser_default = XLSParser;
 
 // ../../packages/parsing/src/XMLParser.ts
 var import_fast_xml_parser = require("fast-xml-parser");
@@ -22189,8 +22190,10 @@ var ProducerEngineClass = class {
       for (const dimension of producerDimensions) {
         const key = dimension.alias ?? dimension.name;
         const index = myHeader.findIndex((x) => x === key);
-        if (index < 0)
+        if (index < 0) {
+          Logger_default.error(`Unable to find dimension "${dimension.name}" in: ${myHeader.join(" | ")}`);
           throw new Error(`The dimension "${dimension.name}" (with key "${key}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
+        }
         dimensions.push({
           index,
           name: dimension.name,
@@ -25198,7 +25201,9 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
         _progress.update({ phase: "Preparing source data", progress: 0 });
-        const sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
+        let sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
+        _progress.update({ phase: "Preparing source data", progress: 0.8 });
+        sourceFilesByProducer = await this.decodeSourceFiles(sourceFilesByProducer);
         _progress.update({ phase: "Preparing source data", progress: 1 });
         tracker.measure("ready-producers", performance.now() - counter);
         if (sourceFilesByProducer.length === 1 && sourceFilesByProducer[0].response?.files.length === 0)
@@ -25410,6 +25415,9 @@ var ExecutorOrchestratorClass = class {
       }
       return fileSize;
     };
+    /**
+     * Grabs the remote datasets and creates an internal .dataset to work on
+     */
     this.readySourceFiles = async (consumer, scope) => {
       const results = [];
       for (let i = 0; i < consumer.producers.length; i++) {
@@ -25422,6 +25430,95 @@ var ExecutorOrchestratorClass = class {
       }
       return results;
     };
+    this.decodeSourceFiles = async (readyResults) => {
+      const csvSafeValue = (value) => {
+        if (!Algo_default.hasVal(value))
+          return "";
+        if (typeof value === "object")
+          return JSON.stringify(value);
+        return String(value);
+      };
+      const normalizeXmlRows = (raw) => {
+        const rows = Array.isArray(raw) ? raw : [raw];
+        const normalized = [];
+        for (const row of rows) {
+          if (Array.isArray(row)) {
+            normalized.push({ value: JSON.stringify(row) });
+            continue;
+          }
+          if (typeof row === "object" && row !== null) {
+            normalized.push(row);
+            continue;
+          }
+          normalized.push({ value: row });
+        }
+        return normalized;
+      };
+      const decodedResults = await Promise.all(readyResults.map(async (readyResult) => {
+        const { prod, response } = readyResult;
+        const fileType = prod.settings.fileType?.toUpperCase();
+        const decodedFiles = [];
+        let decodedCount = 0;
+        for (const file of response.files) {
+          const inferredType = fileType ?? import_path18.default.extname(file.fullUri).replace(".", "").toUpperCase();
+          const decodedPath = `${file.fullUri}.decoded.csv`;
+          if (inferredType === "XLS" || inferredType === "XLSX") {
+            const xlsCsvStream = await XLSParser_default.getStreamXls(file.fullUri, {
+              sheetName: prod.settings.sheetName,
+              startRow: prod.settings.startRow,
+              startColumn: prod.settings.startColumn
+            });
+            await (0, import_promises11.pipeline)(
+              xlsCsvStream,
+              import_fs13.default.createWriteStream(decodedPath)
+            );
+            const fileStats = await import_promises10.default.stat(decodedPath);
+            decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
+            decodedCount++;
+            continue;
+          }
+          if (inferredType === "XML") {
+            const fileContent = await import_promises10.default.readFile(file.fullUri, "utf-8");
+            const jsonData = XMLParser_default.xmlToJson(fileContent);
+            const records = normalizeXmlRows(jsonData);
+            if (records.length === 0) {
+              await import_promises10.default.writeFile(decodedPath, "", "utf-8");
+            } else {
+              const columns = [];
+              for (const record of records) {
+                for (const key of Object.keys(record)) {
+                  if (!columns.includes(key))
+                    columns.push(key);
+                }
+              }
+              if (columns.length === 0)
+                columns.push("value");
+              const lines = [CSVParser_default.stringifyRow(columns)];
+              for (const record of records) {
+                const row = columns.map((column) => csvSafeValue(record[column]));
+                lines.push(CSVParser_default.stringifyRow(row));
+              }
+              await import_promises10.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
+            }
+            const fileStats = await import_promises10.default.stat(decodedPath);
+            decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
+            decodedCount++;
+            continue;
+          }
+          decodedFiles.push(file);
+        }
+        if (decodedCount > 0)
+          Logger_default.log(`Producer "${prod.name}": decoded ${decodedCount} encoded file(s) to CSV`);
+        return {
+          ...readyResult,
+          response: {
+            ...response,
+            files: decodedFiles
+          }
+        };
+      }));
+      return decodedResults;
+    };
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")