@forzalabs/remora 0.1.3-nasco.3 → 0.1.5-nasco.3

package/engines/dataset/Dataset.js

@@ -35,7 +35,7 @@ const Helper_1 = __importDefault(require("../../helper/Helper"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const Environment_1 = __importDefault(require("../Environment"));
 class Dataset {
-    constructor(name, file, batchSize) {
+    constructor(name, file, batchSize, executionId) {
         var _a;
         this.getPath = () => this._path;
         this.setPath = (path) => {
@@ -43,6 +43,7 @@ class Dataset {
             return this;
         };
         this.getFile = () => this._file;
+        this.getExecutionId = () => this._executionId;
         this.getBatchSize = () => this._batchSize;
         this.setFirstLine = (firstLine) => {
             this._firstLine = firstLine;
@@ -170,7 +171,7 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${lineCount}: ${error}`);
+                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
                     }
                 }
             }
@@ -536,7 +537,7 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${lineCount}: ${error}`);
+                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
                     }
                 }
             }
@@ -627,7 +628,7 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${lineCount}: ${error}`);
+                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
                     }
                 }
             }
@@ -703,10 +704,18 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
-        this.setDimensinons = (dimensions) => {
+        this.setDimensions = (dimensions) => {
             this._dimensions = dimensions;
             return this;
         };
+        this.setSingleDimension = (newDimension, oldDimension) => {
+            (0, Affirm_1.default)(newDimension, `Invalid new dimension`);
+            (0, Affirm_1.default)(oldDimension, 'Invalid old dimension');
+            const current = this._dimensions.findIndex(x => x.index === oldDimension.index);
+            (0, Affirm_1.default)(current, `Trying to updata a dataset dimension that doesn't exist: ${oldDimension.name} index ${oldDimension.index}`);
+            this._dimensions.splice(current, 1, newDimension);
+            return this;
+        };
         /**
          * Update the record pool when dimensions change
          */
@@ -829,6 +838,7 @@ class Dataset {
         this._computeSize = () => fs_1.default.statSync(this._path).size / (1024 * 1024);
         this.name = name;
         this._file = file;
+        this._executionId = executionId;
         this._batchSize = (_a = batchSize !== null && batchSize !== void 0 ? batchSize : parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
         this._dimensions = [];
         this._firstLine = '';
@@ -843,8 +853,9 @@ class Dataset {
             .replace(/_{2,}/g, '_')
             .replace(/^_+|_+$/g, '')
             .toLowerCase();
-        this._path = path_1.default.join('./remora', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, datasetName, '.dataset');
-        this._tempPath = path_1.default.join('./remora/', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, datasetName, '.dataset_tmp');
+        const execFolder = executionId ? path_1.default.join(datasetName, executionId) : datasetName;
+        this._path = path_1.default.join('./remora', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, execFolder, '.dataset');
+        this._tempPath = path_1.default.join('./remora/', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, execFolder, '.dataset_tmp');
         this.ensureFile(this._path);
     }
 }

package/engines/dataset/DatasetManager.js

@@ -21,9 +21,15 @@ const ParseManager_1 = __importDefault(require("../parsing/ParseManager"));
 const Dataset_1 = __importDefault(require("./Dataset"));
 const promises_1 = require("stream/promises");
 const fs_1 = require("fs");
+const DeveloperEngine_1 = __importDefault(require("../ai/DeveloperEngine"));
 class DatasetManagerClass {
     constructor() {
-        this.create = (producer) => {
+        /**
+         * Create a new Dataset for a producer. If an executionId is provided, the dataset files will
+         * be isolated inside a sub-folder specific to that execution to avoid concurrency conflicts
+         * when the same producer / consumer is executed multiple times in parallel.
+         */
+        this.create = (producer, executionId) => {
             (0, Affirm_1.default)(producer, 'Invalid producer');
             const { name, settings: { delimiter, fileKey, fileType, hasHeaderRow, sheetName } } = producer;
             const dataset = new Dataset_1.default(name, {
@@ -32,7 +38,7 @@ class DatasetManagerClass {
                 hasHeaderRow,
                 sheetName,
                 delimiter
-            });
+            }, undefined, executionId);
             return dataset;
         };
         this.buildDimensions = (dataset_1, producer_1, ...args_1) => __awaiter(this, [dataset_1, producer_1, ...args_1], void 0, function* (dataset, producer, discover = false) {
@@ -43,7 +49,7 @@ class DatasetManagerClass {
             return this.buildDimensionsFromFirstLine(firstLine, dataset.getFile(), producer, discover);
         });
         this.buildDimensionsFromFirstLine = (firstLine_1, dsFile_1, producer_1, ...args_1) => __awaiter(this, [firstLine_1, dsFile_1, producer_1, ...args_1], void 0, function* (firstLine, dsFile, producer, discover = false) {
-            var _a, _b, _c, _d, _e, _f;
+            var _a, _b, _c, _d, _e, _f, _g, _h;
             (0, Affirm_1.default)(firstLine, `Invalid first line`);
             (0, Affirm_1.default)(dsFile, `Invalid dataset file`);
             (0, Affirm_1.default)(producer, `Invalid producer`);
@@ -54,10 +60,17 @@ class DatasetManagerClass {
                     const headerLine = firstLine;
                     const rawDimensions = ParseManager_1.default._extractHeader(headerLine, delimiterChar, producer, discover);
                     return {
-                        dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index, hidden: null })),
+                        dimensions: rawDimensions.map(x => ({
+                            key: x.name,
+                            name: x.saveAs,
+                            index: x.index,
+                            hidden: null,
+                            type: x.type
+                        })),
                         delimiter: delimiterChar
                     };
                 }
+                case 'PARQUET':
                 case 'JSONL':
                 case 'JSON': {
                     const source = Environment_1.default.getSource(producer.source);
@@ -67,7 +80,13 @@ class DatasetManagerClass {
                     if (discover) {
                         return {
                             delimiter: (_b = file.delimiter) !== null && _b !== void 0 ? _b : ',',
-                            dimensions: keys.map((x, i) => ({ hidden: false, index: i, key: x, name: x }))
+                            dimensions: keys.map((x, i) => ({
+                                hidden: false,
+                                index: i,
+                                key: x,
+                                name: x,
+                                type: DeveloperEngine_1.default.inferDimensionType(firstObject === null || firstObject === void 0 ? void 0 : firstObject[x])
+                            }))
                         };
                     }
                     const dimensions = [];
@@ -75,34 +94,61 @@ class DatasetManagerClass {
                         const columnKey = (_c = pColumn.aliasInProducer) !== null && _c !== void 0 ? _c : pColumn.nameInProducer;
                         const csvColumnIndex = keys.findIndex(x => x === columnKey);
                         (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
-                        dimensions.push({ index: csvColumnIndex, key: columnKey, name: pColumn.nameInProducer, hidden: null });
+                        dimensions.push({
+                            index: csvColumnIndex,
+                            key: columnKey,
+                            name: pColumn.nameInProducer,
+                            hidden: null,
+                            type: (_e = (_d = pColumn.dimension) === null || _d === void 0 ? void 0 : _d.type) !== null && _e !== void 0 ? _e : 'string'
+                        });
                     }
-                    const delimiterChar = (_d = file.delimiter) !== null && _d !== void 0 ? _d : ',';
+                    const delimiterChar = (_f = file.delimiter) !== null && _f !== void 0 ? _f : ',';
                     return { dimensions, delimiter: delimiterChar };
                 }
                 case 'TXT': {
                     if (!file.hasHeaderRow) {
                         // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
-                        const delimiterChar = (_e = file.delimiter) !== null && _e !== void 0 ? _e : ',';
+                        const delimiterChar = (_g = file.delimiter) !== null && _g !== void 0 ? _g : ',';
                         const source = Environment_1.default.getSource(producer.source);
                         const columns = FileCompiler_1.default.compileProducer(producer, source);
                         if (discover) {
                             // Since I don't have an header, and I'm discovering, I just create placeholder dimensions based on the same number of columns of the txt
                             return {
                                 delimiter: delimiterChar,
-                                dimensions: firstLine.split(delimiterChar).map((x, i) => ({ hidden: false, index: i, key: `Col ${i + 1}`, name: `Col ${i + 1}` }))
+                                dimensions: firstLine.split(delimiterChar).map((x, i) => ({
+                                    hidden: false,
+                                    index: i,
+                                    key: `Col ${i + 1}`,
+                                    name: `Col ${i + 1}`,
+                                    type: 'string'
+                                }))
                             };
                         }
                         return {
-                            dimensions: columns.map((x, i) => { var _a; return ({ key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer, name: x.nameInProducer, index: i, hidden: null }); }),
+                            dimensions: columns.map((x, i) => {
+                                var _a, _b, _c;
+                                return ({
+                                    key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer,
+                                    name: x.nameInProducer,
+                                    index: i,
+                                    hidden: null,
+                                    type: (_c = (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string'
+                                });
+                            }),
                             delimiter: delimiterChar
                         };
                     }
                     else {
-                        const delimiterChar = (_f = producer.settings.delimiter) !== null && _f !== void 0 ? _f : ',';
+                        const delimiterChar = (_h = producer.settings.delimiter) !== null && _h !== void 0 ? _h : ',';
                         const rawDimensions = ParseManager_1.default._extractHeader(firstLine, delimiterChar, producer, discover);
                         return {
-                            dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index, hidden: null })),
+                            dimensions: rawDimensions.map(x => ({
+                                key: x.name,
+                                name: x.saveAs,
+                                index: x.index,
+                                hidden: null,
+                                type: x.type
+                            })),
                             delimiter: delimiterChar
                         };
                     }

package/engines/dataset/DatasetRecord.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Algo_1 = __importDefault(require("../../core/Algo"));
+const TypeCaster_1 = __importDefault(require("../transform/TypeCaster"));
 class DatasetRecord {
     constructor(row, dimensions, delimiter) {
         this.parse = (row, delimiter, dimensions) => {
@@ -11,7 +12,7 @@ class DatasetRecord {
                 const parts = row.split(delimiter);
                 for (let i = 0; i < dimensions.length; i++) {
                     const dim = dimensions[i];
-                    this._value[dim.name] = parts[i];
+                    this._value[dim.name] = TypeCaster_1.default.cast(parts[i], dim.type);
                 }
             }
         };
@@ -35,7 +36,7 @@ class DatasetRecord {
             this.parse(row, delimiter, this._dimensions);
         };
         this.wholeUpdateDimension = (update) => {
-            var _a;
+            var _a, _b, _c, _d, _e;
             if (update.toDelete) {
                 // To remove
                 delete this._value[update.currentDimension.name];
@@ -46,7 +47,13 @@ class DatasetRecord {
             }
             else if (!update.currentDimension) {
                 // To create (at the right position)
-                const newDimension = { index: update.newPosition, key: update.newName, name: update.newName, hidden: update.newHidden };
+                const newDimension = {
+                    index: update.newPosition,
+                    key: update.newName,
+                    name: update.newName,
+                    hidden: update.newHidden,
+                    type: (_b = (_a = update.currentDimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string'
+                };
                 this._value[newDimension.name] = null;
                 this._dimensions = [...this._dimensions, newDimension];
             }
@@ -56,7 +63,13 @@ class DatasetRecord {
                 if (index < 0)
                     index = this._dimensions.findIndex(x => x.key === update.currentDimension.key);
                 const currentDim = this._dimensions[index];
-                const updatedDim = { name: update.newName, key: (_a = currentDim.key) !== null && _a !== void 0 ? _a : update.newName, hidden: update.newHidden, index: update.newPosition };
+                const updatedDim = {
+                    name: update.newName,
+                    key: (_c = currentDim.key) !== null && _c !== void 0 ? _c : update.newName,
+                    hidden: update.newHidden,
+                    index: update.newPosition,
+                    type: (_e = (_d = update.currentDimension) === null || _d === void 0 ? void 0 : _d.type) !== null && _e !== void 0 ? _e : 'string'
+                };
                 this._value[updatedDim.name] = this._value[currentDim.name];
                 if (updatedDim.name !== currentDim.name)
                     delete this._value[currentDim.name];

package/engines/dataset/ParallelDataset.js

@@ -20,9 +20,29 @@ const DatasetManager_1 = __importDefault(require("./DatasetManager"));
 const path_1 = __importDefault(require("path"));
 class ParallelDatasetClass {
     constructor() {
+        this.init = () => {
+            /**
+             * I need the init to be called after all the setup has been completed because I need the .env to be loaded
+             */
+            if (!this._filterPool || !this._projectionPool || !this._transformPool) {
+                const options = {
+                    workerThreadOpts: {
+                        resourceLimits: {
+                            maxOldGenerationSizeMb: Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB
+                        }
+                    }
+                };
+                const workerPath = this._getWorkerPath();
+                this._filterPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'FilterWorker.js'), options);
+                this._projectionPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'ProjectionWorker.js'), options);
+                this._transformPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'TransformWorker.js'), options);
+            }
+        };
         this._getWorkerPath = () => {
             // Get the current file's directory
             const currentDir = __dirname;
+            if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'development')
+                return path_1.default.resolve('./.build/workers');
             // Check if we're in a published npm package (no .build in path)
             if (!currentDir.includes('.build')) {
                 // We're in the published package, workers are relative to package root
@@ -47,6 +67,7 @@ class ParallelDatasetClass {
         this.filter = (dataset, filters) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, `Invalid dataset`);
             (0, Affirm_1.default)(filters, `Invalid filters`);
+            this.init();
             // Distribute the work of the filter among the various workers, trying to have them match the batch size
             const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
             dataset._startOperation('filter-parallel', { workerCount });
@@ -60,6 +81,7 @@ class ParallelDatasetClass {
                 const workerData = {
                     datasetDimensions: dataset.getDimensions(),
                     datasetFile: dataset.getFile(),
+                    executionId: dataset.getExecutionId(),
                     datasetName: dataset.name,
                     datasetDelimiter: dataset.getDelimiter(),
                     fromLine: fromLine,
@@ -77,13 +99,14 @@ class ParallelDatasetClass {
             yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
             dataset
                 .setDelimiter(results[0].datasetDelimiter)
-                .setDimensinons(results[0].datasetDimensions);
+                .setDimensions(results[0].datasetDimensions);
             dataset._finishOperation('filter-parallel');
             return dataset;
         });
         this.projection = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, `Invalid dataset`);
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            this.init();
             const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
             dataset._startOperation('projection-parallel', { workerCount });
             const threads = [];
@@ -96,6 +119,7 @@ class ParallelDatasetClass {
                 const workerData = {
                     datasetDimensions: dataset.getDimensions(),
                     datasetFile: dataset.getFile(),
+                    executionId: dataset.getExecutionId(),
                     datasetName: dataset.name,
                     datasetDelimiter: dataset.getDelimiter(),
                     fromLine: fromLine,
@@ -111,13 +135,14 @@ class ParallelDatasetClass {
             yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
             dataset
                 .setDelimiter(results[0].datasetDelimiter)
-                .setDimensinons(results[0].datasetDimensions);
+                .setDimensions(results[0].datasetDimensions);
             dataset._finishOperation('projection-parallel');
             return dataset;
         });
         this.transform = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, `Invalid dataset`);
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            this.init();
             const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
             dataset._startOperation('transform-parallel', { workerCount });
             const threads = [];
@@ -130,6 +155,7 @@ class ParallelDatasetClass {
                 const workerData = {
                     datasetDimensions: dataset.getDimensions(),
                     datasetFile: dataset.getFile(),
+                    executionId: dataset.getExecutionId(),
                     datasetName: dataset.name,
                     datasetDelimiter: dataset.getDelimiter(),
                     fromLine: fromLine,
@@ -145,14 +171,10 @@ class ParallelDatasetClass {
             yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
             dataset
                 .setDelimiter(results[0].datasetDelimiter)
-                .setDimensinons(results[0].datasetDimensions);
+                .setDimensions(results[0].datasetDimensions);
             dataset._finishOperation('transform-parallel');
             return dataset;
         });
-        const workerPath = this._getWorkerPath();
-        this._filterPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'FilterWorker.js'));
-        this._projectionPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'ProjectionWorker.js'));
-        this._transformPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'TransformWorker.js'));
     }
 }
 const ParallelDataset = new ParallelDatasetClass();

package/engines/execution/ExecutionEnvironment.js

@@ -28,7 +28,7 @@ const Algo_1 = __importDefault(require("../../core/Algo"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
 const ParallelDataset_1 = __importDefault(require("../dataset/ParallelDataset"));
 class ExecutionEnvironment {
-    constructor(consumer) {
+    constructor(consumer, executionId) {
         this.run = (options) => __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c, _d;
             (0, Affirm_1.default)(this._consumer, 'Invalid consumer');
@@ -64,7 +64,7 @@ class ExecutionEnvironment {
                             (0, Affirm_1.default)(planStep.producer, `Invalid producer in execute-SQL step`);
                             const driver = yield DriverFactory_1.default.instantiateSource(planStep.source);
                             const queryData = (yield driver.query(this._envData.finalSQL)).rows;
-                            let dataset = DatasetManager_1.default.create(planStep.producer);
+                            let dataset = DatasetManager_1.default.create(planStep.producer, this._executionId);
                             dataset = yield dataset.loadFromMemory(queryData, planStep.producer);
                             this._storeIntermidiate(planStep, dataset);
                             break;
@@ -74,7 +74,7 @@ class ExecutionEnvironment {
                             const { producer } = planStep;
                             const source = Environment_1.default.getSource(producer.source);
                             (0, Affirm_1.default)(source, `Source "${producer.source}" of producer "${producer.name}" not found.`);
-                            let dataset = DatasetManager_1.default.create(producer);
+                            let dataset = DatasetManager_1.default.create(producer, this._executionId);
                             dataset = yield dataset.load(source);
                             this._storeIntermidiate(planStep, dataset);
                             break;
@@ -102,7 +102,7 @@ class ExecutionEnvironment {
                         case 'export-file': {
                             (0, Affirm_1.default)(planStep.output, `Invalid output in export-file step`);
                             (0, Affirm_1.default)(this._resultingDataset, 'Invalid resulting dataset in export-file step');
-                            const res = yield FileExporter_1.default.export(this._consumer, planStep.output, this._resultingDataset);
+                            const res = yield FileExporter_1.default.export(this._consumer, planStep.output, this._resultingDataset, this._executionId);
                             result.fileUri = res;
                             break;
                         }
@@ -156,6 +156,13 @@ class ExecutionEnvironment {
                 if (ds)
                     Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getCount()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
                 Logger_1.default.log(`\tFailed step: ${currentStep.type}->\n\t${error}`);
+                // IMPORTANT: cleanup all the datasets to not leave any data around and to avoid memory leaks
+                const datasets = [
+                    ...this._producedData.map(x => x.dataset),
+                    this._resultingDataset
+                ].filter(Algo_1.default.hasVal);
+                const promises = datasets.map(x => x.destroy());
+                yield Promise.all(promises);
                 throw error;
             }
             Logger_1.default.log(`Completed execution of consumer:\n\tSize: ${result._stats.size}\n\tCycles: ${result._stats.cycles}\n\tTime: ${result._stats.elapsedMS}\n\tOperations: ${Logger_1.default.formatList(result._stats.operations)}`);
@@ -184,6 +191,8 @@ class ExecutionEnvironment {
         this._envData = { consumerSQL: null, executionRequestSQL: null, finalSQL: null };
         this._producedData = [];
         this._resultingDataset = null;
+        // A short unique id to isolate temp dataset files & output names
+        this._executionId = executionId;
     }
 }
 exports.default = ExecutionEnvironment;

package/engines/execution/ExecutionPlanner.js

@@ -108,7 +108,8 @@ class ExecutionPlannerClas {
                     break;
                 }
                 case 'local':
-                case 'aws-s3': {
+                case 'aws-s3':
+                case 'delta-share': {
                     plan.push({ type: 'load-dataset', producer });
                     plan.push({ type: 'prepare-dataset', producer });
                     if (producer.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))

package/engines/file/FileCompiler.js

@@ -7,9 +7,10 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 class FileCompilerClass {
     constructor() {
         this.compileProducer = (producer, source) => {
+            var _a;
             (0, Affirm_1.default)(producer, `Invalid producer`);
             (0, Affirm_1.default)(source, `Invalid source`);
-            (0, Affirm_1.default)(producer.settings.fileKey, `Missing required file key in producer settings`);
+            (0, Affirm_1.default)((_a = producer.settings.fileKey) !== null && _a !== void 0 ? _a : producer.settings.sqlTable, `Missing required file key in producer settings`);
             (0, Affirm_1.default)(producer.settings.fileType, `Missing required file type in producer settings`);
             (0, Affirm_1.default)(!producer.measures || producer.measures.length === 0, `Cannot use "measure" with a producer linked to a file (only dimensions are allowed).`);
             const columns = producer.dimensions.map(x => ({

package/engines/file/FileExporter.js

@@ -20,7 +20,7 @@ const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const Environment_1 = __importDefault(require("../Environment"));
 class FileExporterClass {
     constructor() {
-        this.export = (consumer, output, dataset) => __awaiter(this, void 0, void 0, function* () {
+        this.export = (consumer, output, dataset, executionId) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
             (0, Affirm_1.default)(output, `Invalid output`);
             (0, Affirm_1.default)(dataset, `Invalid export dataset`);
@@ -32,7 +32,7 @@ class FileExporterClass {
                 : output.format === 'JSON'
                     ? 'jsonl'
                     : 'txt';
-            const name = this._composeFileName(consumer, extension);
+            const name = this._composeFileName(consumer, output, extension, executionId);
             const uploadRes = yield driver.uploadStream({
                 dataset,
                 name,
@@ -67,7 +67,16 @@ class FileExporterClass {
             }
             return chunks;
         };
-        this._composeFileName = (consumer, extension) => `${consumer.name}_${Algo_1.default.replaceAll(DSTE_1.default.now().toISOString().split('.')[0], ':', '-')}.${extension}`;
+        this._composeFileName = (consumer, output, extension, executionId) => {
+            if (output.exportName && output.exportName.trim().length > 0) {
+                // Ensure no extension duplication
+                const sanitized = output.exportName.replace(/\.[^.]+$/, '');
+                return `${sanitized}.${extension}`;
+            }
+            const baseTs = Algo_1.default.replaceAll(DSTE_1.default.now().toISOString().split('.')[0], ':', '-');
+            const suffix = executionId ? `_${executionId}` : '';
+            return `${consumer.name}_${baseTs}${suffix}.${extension}`;
+        };
     }
 }
 const FileExporter = new FileExporterClass();

package/engines/parsing/ParseManager.js

@@ -9,7 +9,7 @@ const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
 class ParseManagerClass {
     constructor() {
         this._extractHeader = (headerLine, delimiter, producer, discover) => {
-            var _a;
+            var _a, _b, _c;
             (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);
             (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
             (0, Affirm_1.default)(producer, 'Invalid producer');
@@ -24,7 +24,12 @@ class ParseManagerClass {
                 const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;
                 const csvColumnIndex = headerColumns.findIndex(x => x === columnKey);
                 (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
-                csvColumns.push({ index: csvColumnIndex, name: columnKey, saveAs: pColumn.nameInProducer });
+                csvColumns.push({
+                    index: csvColumnIndex,
+                    name: columnKey,
+                    saveAs: pColumn.nameInProducer,
+                    type: (_c = (_b = pColumn.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string'
+                });
             }
             return csvColumns;
         };

package/engines/producer/ProducerEngine.js

@@ -34,7 +34,8 @@ class ProducerEngineClass {
                     (0, Affirm_1.default)(sql, `Invalid SQL from compilation for producer "${producer.name}"`);
                     return sql;
                 }
-                case 'aws-s3': {
+                case 'aws-s3':
+                case 'delta-share': {
                     const columns = FileCompiler_1.default.compileProducer(producer, source);
                     (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
                     break;
@@ -141,7 +142,8 @@ class ProducerEngineClass {
                     break;
                 }
                 case 'local':
-                case 'aws-s3': {
+                case 'aws-s3':
+                case 'delta-share': {
                     const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
                     dataset = yield dataset.loadFromMemory(fileData.data, producer, discover);
                     break;

package/engines/transform/JoinEngine.js

@@ -102,12 +102,16 @@ class JoinEngineClass {
                 fileType: 'CSV'
             });
             // Get dimensions for the result dataset based on consumer columns
-            const resultDimensions = consumerColumns.map((col, index) => ({
-                name: col.consumerAlias || col.consumerKey,
-                key: col.consumerAlias || col.consumerKey,
-                index,
-                hidden: null
-            }));
+            const resultDimensions = consumerColumns.map((col, index) => {
+                var _a, _b;
+                return ({
+                    name: col.consumerAlias || col.consumerKey,
+                    key: col.consumerAlias || col.consumerKey,
+                    index,
+                    type: (_b = (_a = col.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string',
+                    hidden: null
+                });
+            });
             // Initialize the result dataset with proper dimensions
             resultDataset.getDimensions().length = 0;
             resultDataset.getDimensions().push(...resultDimensions);

package/engines/transform/TransformationEngine.js

@@ -22,7 +22,7 @@ class TransformationEngineClass {
             (0, Affirm_1.default)(dataset, 'Invalid data');
             const fieldsToTransform = consumer.fields.filter(field => Algo_1.default.hasVal(field.transform));
             Affirm_1.default.hasItems(fieldsToTransform, 'No fields with transformations');
-            return yield dataset.map(record => {
+            yield dataset.map(record => {
                 var _a;
                 for (const field of fieldsToTransform) {
                     if (!field.transform)
@@ -54,6 +54,17 @@ class TransformationEngineClass {
                 }
                 return record;
             }, options);
+            /**
+             * Some transformations (for now only "cast") change the underlying type of the dataset dimension
+             * Here I update the dimension type of the dataset.
+             * TODO: I think that we may have a bug if you cast AND then do an operation on the number, since it reverts back to being a string in the same trnasformation chain, since the dimension type update is applied only at the end of all the transformations
+             */
+            for (const field of fieldsToTransform) {
+                if (!field.transform)
+                    continue;
+                this.applyDimensionsChanges(field.transform, field, dataset);
+            }
+            return dataset;
         });
         this.isFieldCombinationTransformation = (transformation) => {
             if (Array.isArray(transformation)) {
@@ -73,9 +84,12 @@ class TransformationEngineClass {
             }
             // Single transformation
             if ('cast' in transformations) {
-                const casted = TypeCaster_1.default.cast(value, transformations.cast);
-                if (isNaN(casted) && transformations.cast === 'number')
+                const { cast, format } = transformations;
+                const casted = TypeCaster_1.default.cast(value, cast, format);
+                if (cast === 'number' && isNaN(casted))
                     throw new Error(`Cannot cast non-numeric value in field '${field.key}'`);
+                if (cast === 'datetime' && casted instanceof Date && isNaN(casted.getTime()))
+                    throw new Error(`Cannot cast value to date in field '${field.key}'`);
                 return casted;
             }
             if ('multiply' in transformations) {
@@ -260,6 +274,24 @@ class TransformationEngineClass {
             }
             return false;
         };
+        this.applyDimensionsChanges = (transformations, field, dataset) => {
+            if (Array.isArray(transformations)) {
+                for (const transform of transformations) {
+                    this.applyDimensionsChanges(transform, field, dataset);
+                }
+                return dataset;
+            }
+            // Single transformation
+            if ('cast' in transformations) {
+                const { cast } = transformations;
+                let oldDimension = dataset.getDimensions().find(x => x.name === field.key);
+                if (!oldDimension)
+                    oldDimension = dataset.getDimensions().find(x => x.key === field.key);
+                const newDimension = Object.assign(Object.assign({}, structuredClone(oldDimension)), { type: cast });
+                dataset.setSingleDimension(newDimension, oldDimension);
+            }
+            return dataset;
+        };
     }
 }
 const TransformationEngine = new TransformationEngineClass();