@forzalabs/remora 1.2.7 → 1.2.9

package/index.js

@@ -18770,7 +18770,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.2.7",
+  cliVersion: "1.2.9",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -19851,6 +19851,7 @@ var XLSParserClass = class {
   }
 };
 var XLSParser = new XLSParserClass();
+var XLSParser_default = XLSParser;
 
 // ../../packages/parsing/src/XMLParser.ts
 var import_fast_xml_parser = require("fast-xml-parser");
@@ -19951,7 +19952,7 @@ var DeltaShareSourceDriver = class {
     this.readAll = async (request) => {
       Affirm_default(request, "Invalid download request");
       const table = this._resolveTable(request.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
       const hyparquet = await import("hyparquet");
       const lines = [];
       for (const deltaFile of deltaFiles) {
@@ -19965,7 +19966,7 @@ var DeltaShareSourceDriver = class {
       Affirm_default(request.options, "Invalid read options");
       Affirm_default(request.options.lineFrom !== void 0 && request.options.lineTo !== void 0, "Missing read range");
       const table = this._resolveTable(request.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
       const hyparquet = await import("hyparquet");
       const { options: { lineFrom, lineTo } } = request;
       const lines = [];
@@ -20066,9 +20067,9 @@ var DeltaShareSourceDriver = class {
 `);
       return true;
     };
-    this._getAllFilesInTable = async (table) => {
+    this._getAllFilesInTable = async (table, disableHistory = false) => {
       const url = this._getTableUrl(this._query, table);
-      const body = {
+      const body = disableHistory ? {} : {
         version: await this._getVersion(table)
       };
       const res = await fetch(url, {
@@ -20090,7 +20091,7 @@ var DeltaShareSourceDriver = class {
       Affirm_default(producer, "Invalid producer");
       Affirm_default(scope, "Invalid executor scope");
       const table = this._resolveTable(producer.settings.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, producer.settings.disableHistory);
       const hyparquet = await import("hyparquet");
       const delimiter = producer.settings.delimiter ?? ",";
       const files = [];
@@ -22699,6 +22700,7 @@ var ProducerEngineClass = class {
             fileKey,
             fileType: effectiveFileType,
             options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName, hasHeaderRow, startRow, startColumn },
+            disableHistory: producer.settings?.disableHistory,
             httpApi: producer.settings?.httpApi
           });
           break;
@@ -22707,6 +22709,7 @@ var ProducerEngineClass = class {
             fileKey,
             fileType: effectiveFileType,
             options: { sheetName, hasHeaderRow, startRow, startColumn },
+            disableHistory: producer.settings?.disableHistory,
             httpApi: producer.settings?.httpApi
           });
           break;
@@ -22864,8 +22867,10 @@ var ProducerEngineClass = class {
       for (const dimension of producerDimensions) {
         const key = dimension.alias ?? dimension.name;
         const index = myHeader.findIndex((x) => x === key);
-        if (index < 0)
+        if (index < 0) {
+          Logger_default.error(`Unable to find dimension "${dimension.name}" in: ${myHeader.join(" | ")}`);
           throw new Error(`The dimension "${dimension.name}" (with key "${key}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
+        }
         dimensions.push({
           index,
           name: dimension.name,
@@ -25430,7 +25435,9 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
         _progress.update({ phase: "Preparing source data", progress: 0 });
-        const sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
+        let sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
+        _progress.update({ phase: "Preparing source data", progress: 0.8 });
+        sourceFilesByProducer = await this.decodeSourceFiles(sourceFilesByProducer);
         _progress.update({ phase: "Preparing source data", progress: 1 });
         tracker.measure("ready-producers", performance.now() - counter);
         if (sourceFilesByProducer.length === 1 && sourceFilesByProducer[0].response?.files.length === 0)
@@ -25642,6 +25649,9 @@ var ExecutorOrchestratorClass = class {
       }
       return fileSize;
     };
+    /**
+     * Grabs the remote datasets and creates an internal .dataset to work on
+     */
     this.readySourceFiles = async (consumer, scope) => {
       const results = [];
       for (let i = 0; i < consumer.producers.length; i++) {
@@ -25654,6 +25664,95 @@ var ExecutorOrchestratorClass = class {
       }
       return results;
     };
+    this.decodeSourceFiles = async (readyResults) => {
+      const csvSafeValue = (value) => {
+        if (!Algo_default.hasVal(value))
+          return "";
+        if (typeof value === "object")
+          return JSON.stringify(value);
+        return String(value);
+      };
+      const normalizeXmlRows = (raw) => {
+        const rows = Array.isArray(raw) ? raw : [raw];
+        const normalized = [];
+        for (const row of rows) {
+          if (Array.isArray(row)) {
+            normalized.push({ value: JSON.stringify(row) });
+            continue;
+          }
+          if (typeof row === "object" && row !== null) {
+            normalized.push(row);
+            continue;
+          }
+          normalized.push({ value: row });
+        }
+        return normalized;
+      };
+      const decodedResults = await Promise.all(readyResults.map(async (readyResult) => {
+        const { prod, response } = readyResult;
+        const fileType = prod.settings.fileType?.toUpperCase();
+        const decodedFiles = [];
+        let decodedCount = 0;
+        for (const file of response.files) {
+          const inferredType = fileType ?? import_path21.default.extname(file.fullUri).replace(".", "").toUpperCase();
+          const decodedPath = `${file.fullUri}.decoded.csv`;
+          if (inferredType === "XLS" || inferredType === "XLSX") {
+            const xlsCsvStream = await XLSParser_default.getStreamXls(file.fullUri, {
+              sheetName: prod.settings.sheetName,
+              startRow: prod.settings.startRow,
+              startColumn: prod.settings.startColumn
+            });
+            await (0, import_promises11.pipeline)(
+              xlsCsvStream,
+              import_fs14.default.createWriteStream(decodedPath)
+            );
+            const fileStats = await import_promises10.default.stat(decodedPath);
+            decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
+            decodedCount++;
+            continue;
+          }
+          if (inferredType === "XML") {
+            const fileContent = await import_promises10.default.readFile(file.fullUri, "utf-8");
+            const jsonData = XMLParser_default.xmlToJson(fileContent);
+            const records = normalizeXmlRows(jsonData);
+            if (records.length === 0) {
+              await import_promises10.default.writeFile(decodedPath, "", "utf-8");
+            } else {
+              const columns = [];
+              for (const record of records) {
+                for (const key of Object.keys(record)) {
+                  if (!columns.includes(key))
+                    columns.push(key);
+                }
+              }
+              if (columns.length === 0)
+                columns.push("value");
+              const lines = [CSVParser_default.stringifyRow(columns)];
+              for (const record of records) {
+                const row = columns.map((column) => csvSafeValue(record[column]));
+                lines.push(CSVParser_default.stringifyRow(row));
+              }
+              await import_promises10.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
+            }
+            const fileStats = await import_promises10.default.stat(decodedPath);
+            decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
+            decodedCount++;
+            continue;
+          }
+          decodedFiles.push(file);
+        }
+        if (decodedCount > 0)
+          Logger_default.log(`Producer "${prod.name}": decoded ${decodedCount} encoded file(s) to CSV`);
+        return {
+          ...readyResult,
+          response: {
+            ...response,
+            files: decodedFiles
+          }
+        };
+      }));
+      return decodedResults;
+    };
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")

package/json_schemas/producer-schema.json

@@ -163,6 +163,10 @@
                     "type": "string",
                     "description": "The column delimiter for CSV or TXT files if different from the default (,)."
                 },
+                "disableHistory": {
+                    "type": "boolean",
+                    "description": "For Delta Share producers, skips sending the current table version in the query request body."
+                },
                 "hasHeaderRow": {
                     "type": "boolean",
                     "description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.2.7",
+    "version": "1.2.9",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -18764,7 +18764,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.2.7",
+  cliVersion: "1.2.9",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -20195,6 +20195,7 @@ var XLSParserClass = class {
   }
 };
 var XLSParser = new XLSParserClass();
+var XLSParser_default = XLSParser;
 
 // ../../packages/parsing/src/XMLParser.ts
 var import_fast_xml_parser = require("fast-xml-parser");
@@ -20295,7 +20296,7 @@ var DeltaShareSourceDriver = class {
     this.readAll = async (request) => {
       Affirm_default(request, "Invalid download request");
       const table = this._resolveTable(request.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
       const hyparquet = await import("hyparquet");
       const lines = [];
       for (const deltaFile of deltaFiles) {
@@ -20309,7 +20310,7 @@ var DeltaShareSourceDriver = class {
       Affirm_default(request.options, "Invalid read options");
       Affirm_default(request.options.lineFrom !== void 0 && request.options.lineTo !== void 0, "Missing read range");
       const table = this._resolveTable(request.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
       const hyparquet = await import("hyparquet");
       const { options: { lineFrom, lineTo } } = request;
       const lines = [];
@@ -20410,9 +20411,9 @@ var DeltaShareSourceDriver = class {
 `);
       return true;
     };
-    this._getAllFilesInTable = async (table) => {
+    this._getAllFilesInTable = async (table, disableHistory = false) => {
       const url = this._getTableUrl(this._query, table);
-      const body = {
+      const body = disableHistory ? {} : {
         version: await this._getVersion(table)
       };
       const res = await fetch(url, {
@@ -20434,7 +20435,7 @@ var DeltaShareSourceDriver = class {
       Affirm_default(producer, "Invalid producer");
       Affirm_default(scope, "Invalid executor scope");
       const table = this._resolveTable(producer.settings.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, producer.settings.disableHistory);
       const hyparquet = await import("hyparquet");
       const delimiter = producer.settings.delimiter ?? ",";
       const files = [];
@@ -22024,6 +22025,7 @@ var ProducerEngineClass = class {
             fileKey,
             fileType: effectiveFileType,
             options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName, hasHeaderRow, startRow, startColumn },
+            disableHistory: producer.settings?.disableHistory,
             httpApi: producer.settings?.httpApi
           });
           break;
@@ -22032,6 +22034,7 @@ var ProducerEngineClass = class {
             fileKey,
             fileType: effectiveFileType,
             options: { sheetName, hasHeaderRow, startRow, startColumn },
+            disableHistory: producer.settings?.disableHistory,
             httpApi: producer.settings?.httpApi
           });
           break;
@@ -22189,8 +22192,10 @@ var ProducerEngineClass = class {
       for (const dimension of producerDimensions) {
         const key = dimension.alias ?? dimension.name;
         const index = myHeader.findIndex((x) => x === key);
-        if (index < 0)
+        if (index < 0) {
+          Logger_default.error(`Unable to find dimension "${dimension.name}" in: ${myHeader.join(" | ")}`);
           throw new Error(`The dimension "${dimension.name}" (with key "${key}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
+        }
         dimensions.push({
           index,
           name: dimension.name,
@@ -25198,7 +25203,9 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
         _progress.update({ phase: "Preparing source data", progress: 0 });
-        const sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
+        let sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
+        _progress.update({ phase: "Preparing source data", progress: 0.8 });
+        sourceFilesByProducer = await this.decodeSourceFiles(sourceFilesByProducer);
         _progress.update({ phase: "Preparing source data", progress: 1 });
         tracker.measure("ready-producers", performance.now() - counter);
         if (sourceFilesByProducer.length === 1 && sourceFilesByProducer[0].response?.files.length === 0)
@@ -25410,6 +25417,9 @@ var ExecutorOrchestratorClass = class {
       }
       return fileSize;
     };
+    /**
+     * Grabs the remote datasets and creates an internal .dataset to work on
+     */
     this.readySourceFiles = async (consumer, scope) => {
       const results = [];
       for (let i = 0; i < consumer.producers.length; i++) {
@@ -25422,6 +25432,95 @@ var ExecutorOrchestratorClass = class {
       }
       return results;
     };
+    this.decodeSourceFiles = async (readyResults) => {
+      const csvSafeValue = (value) => {
+        if (!Algo_default.hasVal(value))
+          return "";
+        if (typeof value === "object")
+          return JSON.stringify(value);
+        return String(value);
+      };
+      const normalizeXmlRows = (raw) => {
+        const rows = Array.isArray(raw) ? raw : [raw];
+        const normalized = [];
+        for (const row of rows) {
+          if (Array.isArray(row)) {
+            normalized.push({ value: JSON.stringify(row) });
+            continue;
+          }
+          if (typeof row === "object" && row !== null) {
+            normalized.push(row);
+            continue;
+          }
+          normalized.push({ value: row });
+        }
+        return normalized;
+      };
+      const decodedResults = await Promise.all(readyResults.map(async (readyResult) => {
+        const { prod, response } = readyResult;
+        const fileType = prod.settings.fileType?.toUpperCase();
+        const decodedFiles = [];
+        let decodedCount = 0;
+        for (const file of response.files) {
+          const inferredType = fileType ?? import_path18.default.extname(file.fullUri).replace(".", "").toUpperCase();
+          const decodedPath = `${file.fullUri}.decoded.csv`;
+          if (inferredType === "XLS" || inferredType === "XLSX") {
+            const xlsCsvStream = await XLSParser_default.getStreamXls(file.fullUri, {
+              sheetName: prod.settings.sheetName,
+              startRow: prod.settings.startRow,
+              startColumn: prod.settings.startColumn
+            });
+            await (0, import_promises11.pipeline)(
+              xlsCsvStream,
+              import_fs13.default.createWriteStream(decodedPath)
+            );
+            const fileStats = await import_promises10.default.stat(decodedPath);
+            decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
+            decodedCount++;
+            continue;
+          }
+          if (inferredType === "XML") {
+            const fileContent = await import_promises10.default.readFile(file.fullUri, "utf-8");
+            const jsonData = XMLParser_default.xmlToJson(fileContent);
+            const records = normalizeXmlRows(jsonData);
+            if (records.length === 0) {
+              await import_promises10.default.writeFile(decodedPath, "", "utf-8");
+            } else {
+              const columns = [];
+              for (const record of records) {
+                for (const key of Object.keys(record)) {
+                  if (!columns.includes(key))
+                    columns.push(key);
+                }
+              }
+              if (columns.length === 0)
+                columns.push("value");
+              const lines = [CSVParser_default.stringifyRow(columns)];
+              for (const record of records) {
+                const row = columns.map((column) => csvSafeValue(record[column]));
+                lines.push(CSVParser_default.stringifyRow(row));
+              }
+              await import_promises10.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
+            }
+            const fileStats = await import_promises10.default.stat(decodedPath);
+            decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
+            decodedCount++;
+            continue;
+          }
+          decodedFiles.push(file);
+        }
+        if (decodedCount > 0)
+          Logger_default.log(`Producer "${prod.name}": decoded ${decodedCount} encoded file(s) to CSV`);
+        return {
+          ...readyResult,
+          response: {
+            ...response,
+            files: decodedFiles
+          }
+        };
+      }));
+      return decodedResults;
+    };
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")