@forzalabs/remora 0.1.4-nasco.3 → 0.1.5-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.1.4-nasco',
+    cliVersion: '0.1.5-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/consumer-schema.json

@@ -223,6 +223,10 @@
                         "type": "string",
                         "description": "The name of the source where the consumer will export its data when deployed/run"
                     },
+                    "exportName": {
+                        "type": "string",
+                        "description": "If the format is a file, forces the same name in the export file (extension is auto-added)"
+                    },
                     "trigger": {
                         "type": "object",
                         "description": "Triggers to perform the export (not just the usual 'Deploy')",
@@ -463,11 +467,11 @@
                         "cast": {
                             "type": "string",
                             "description": "Cast the value to a specific type",
-                            "enum": ["string", "number", "date", "boolean"]
+                            "enum": ["string", "number", "datetime", "boolean"]
                         },
                         "format": {
                             "type": "string",
-                            "description": "Optional format for date parsing or string formatting (supports tokens: yyyy, mm, dd)"
+                            "description": "Optional format for date parsing or string formatting (e.g. YYYY-MM-DD, DD/MM/YY)"
                         }
                     },
                     "required": ["cast"],

package/definitions/json_schemas/producer-schema.json

@@ -39,7 +39,8 @@
                         "enum": [
                             "string",
                             "number",
-                            "datetime"
+                            "datetime",
+                            "boolean"
                         ],
                         "description": "The data type of the dimension"
                     },

package/definitions/json_schemas/source-schema.json

@@ -23,7 +23,8 @@
                 "aws-dynamodb",
                 "aws-s3",
                 "postgres",
-                "local"
+                "local",
+                "delta-share"
             ],
             "description": "The type of data engine"
         },
@@ -66,6 +67,10 @@
                     "type": "string",
                     "description": "Database schema name"
                 },
+                "table": {
+                    "type": "string",
+                    "description": "Table name (used by some engines like delta-share)"
+                },
                 "port": {
                     "type": "string",
                     "description": "Port number for the connection"
@@ -101,6 +106,14 @@
                 "path": {
                     "type": "string",
                     "description": "The folder path"
+                },
+                "share": {
+                    "type": "string",
+                    "description": "Delta Sharing share name"
+                },
+                "bearerToken": {
+                    "type": "string",
+                    "description": "Delta Sharing bearer token used for authentication"
                 }
             },
             "required": ["method"]

package/documentation/README.md

@@ -106,6 +106,7 @@ Consumers transform and combine data from producers for specific use cases.
 | `outputs[].accellerated` | Whether to materialize for performance | `true`, `false` |
 | `outputs[].direct` | Whether to query directly without creating views | `true`, `false` |
 | `outputs[].exportDestination` | Where to export data | Must match a source `name` |
+| `outputs[].exportName` | Fixed file name (without extension) for file exports | String |
 | `outputs[].trigger.type` | How to trigger exports | `CRON`, `API` |
 | `outputs[].trigger.value` | Trigger expression | CRON expression (e.g., `0 0 * * *`) or endpoint path |
 | `metadata` | Custom tags | Object with string keys and values |

package/documentation/default_resources/consumer.json

@@ -11,8 +11,8 @@
       "joins": [
         {
           "otherName": "<primary producer name>",
-          "relationship": "<one-to-one | one-to-many | many-to-one>",
-          "sql": "<join condition>"
+          "relationship": "one-to-many",
+          "sql": "${P.id} = ${<primary producer name>.fk_id}"
         }
       ]
     }
@@ -29,17 +29,17 @@
     }
   ],
   "outputs": [
-    { "format": "<api | sql>" },
+    { "format": "API" },
     {
-      "format": "<json | csv | parquet>",
+      "format": "JSON",
       "exportDestination": "<export destination>"
     },
     {
-      "format": "<csv | json>",
+      "format": "CSV",
       "exportDestination": "<export destination>",
       "trigger": {
-        "type": "<cron | api>",
-        "value": "<cron expression or endpoint path>"
+        "type": "CRON",
+        "value": "0 0 * * *"
       }
     }
   ],

package/drivers/DeltaShareDriver.js

@@ -0,0 +1,178 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+/**
+ * Delta Share (Databricks Delta Sharing) Source Driver
+ */
+class DeltaShareSourceDriver {
+    constructor() {
+        this._query = '{prefix}/shares/{share}/schemas/{schema}/tables/{table}/query';
+        this._version = '{prefix}/shares/{share}/schemas/{schema}/tables/{table}/version';
+        this._tablesInShare = '{prefix}/shares/{share}/all-tables';
+        this._tablesInSchema = '{prefix}/shares/{share}/schemas/{schema}/tables';
+        this._schemasInShare = '{prefix}/shares/{share}/schemas';
+        this._shares = '{prefix}/shares';
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(source, 'Invalid source');
+            // Expected authentication shape for delta-share
+            const { authentication } = source;
+            (0, Affirm_1.default)(authentication, 'Invalid authentication for delta-share source');
+            this._shareUrl = authentication.host;
+            this._bearerToken = authentication.bearerToken || authentication.sessionToken || authentication.password;
+            this._share = authentication.share;
+            this._schema = authentication.schema;
+            this._table = authentication.table;
+            (0, Affirm_1.default)(this._shareUrl, 'Missing delta-share host (share server URL) in source.authentication.host');
+            (0, Affirm_1.default)(this._bearerToken, 'Missing delta-share bearer token in source.authentication.sessionToken (or password)');
+            (0, Affirm_1.default)(this._share, 'Missing delta-share "share" (use authentication.share or bucket)');
+            (0, Affirm_1.default)(this._schema, 'Missing delta-share schema in source.authentication.schema');
+            (0, Affirm_1.default)(this._table, 'Missing delta-share table in source.authentication.table (or database)');
+            this._source = source;
+            return this;
+        });
+        // Delta Sharing is not a SQL engine; expose explicit error
+        this.execute = (__sql) => __awaiter(this, void 0, void 0, function* () {
+            void __sql;
+            throw new Error('DeltaShareSourceDriver.execute is not supported: Delta Sharing is not a SQL engine');
+        });
+        this.query = (__sql, __values) => __awaiter(this, void 0, void 0, function* () {
+            void __sql;
+            void __values;
+            throw new Error('DeltaShareSourceDriver.query is not supported: Delta Sharing is not a SQL engine');
+        });
+        this.readAll = (request) => __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
+            (0, Affirm_1.default)(request, `Invalid download request`);
+            (0, Affirm_1.default)(!request.fileKey.includes('%'), `On a delta-share the file key can not include "%"`);
+            const deltaFiles = yield this._getAllFilesInTables(this._table);
+            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            const lines = [];
+            for (const deltaFile of deltaFiles) {
+                const byteLength = (_b = (_a = deltaFile.file.deltaSingleAction.add) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : (_c = deltaFile.file.deltaSingleAction.remove) === null || _c === void 0 ? void 0 : _c.size;
+                const file = yield asyncBufferFromUrl({ url: deltaFile.file.url, byteLength });
+                const parquetRecords = yield parquetReadObjects({ file: file });
+                lines.push(...parquetRecords.map(x => JSON.stringify(x)));
+            }
+            return lines;
+        });
+        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
+            (0, Affirm_1.default)(request, 'Invalid read request');
+            (0, Affirm_1.default)(request.options, 'Invalid read options');
+            (0, Affirm_1.default)(request.options.lineFrom !== undefined && request.options.lineTo !== undefined, 'Missing read range');
+            const deltaFiles = yield this._getAllFilesInTables(this._table);
+            const { options: { lineFrom, lineTo } } = request;
+            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            const lines = [];
+            let index = 0;
+            for (const deltaFile of deltaFiles) {
+                const byteLength = (_b = (_a = deltaFile.file.deltaSingleAction.add) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : (_c = deltaFile.file.deltaSingleAction.remove) === null || _c === void 0 ? void 0 : _c.size;
+                const file = yield asyncBufferFromUrl({ url: deltaFile.file.url, byteLength });
+                const parquetRecords = yield parquetReadObjects({ file: file });
+                for (const record of parquetRecords) {
+                    if (index >= lineFrom && index < lineTo)
+                        lines.push(JSON.stringify(record));
+                    index++;
+                    if (index >= lineTo)
+                        break;
+                }
+            }
+            return lines;
+        });
+        this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
+            (0, Affirm_1.default)(dataset, 'Invalid dataset');
+            const deltaFiles = yield this._getAllFilesInTables(this._table);
+            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            // For each file, download it with the hyparquet package, read lines, then save locally to create the dataset
+            let index = 0;
+            let totalLineCount = 0;
+            for (const deltaFile of deltaFiles) {
+                const byteLength = (_b = (_a = deltaFile.file.deltaSingleAction.add) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : (_c = deltaFile.file.deltaSingleAction.remove) === null || _c === void 0 ? void 0 : _c.size;
+                const file = yield asyncBufferFromUrl({ url: deltaFile.file.url, byteLength });
+                const parquetRecords = yield parquetReadObjects({ file: file });
+                if (index === 0 && parquetRecords.length > 0) {
+                    // I intentionally keep the first record as a JSON, so it can be used to extract the dimensions
+                    dataset.setFirstLine(JSON.stringify(parquetRecords[0]));
+                }
+                totalLineCount += yield DriverHelper_1.default.appendObjectsToUnifiedFile({
+                    append: index > 0,
+                    delimiter: dataset.getDelimiter(),
+                    destinationPath: dataset.getPath(),
+                    objects: parquetRecords
+                });
+                index++;
+            }
+            dataset.setCount(totalLineCount);
+            return dataset;
+        });
+        this.exist = (__producer) => __awaiter(this, void 0, void 0, function* () {
+            void __producer;
+            try {
+                yield this._getAllFilesInTables(this._table);
+                // If it doesn't exist, then it fails in the above function
+                return true;
+            }
+            catch (_a) {
+                return false;
+            }
+        });
+        this._getVersion = (table) => __awaiter(this, void 0, void 0, function* () {
+            const url = this._version
+                .replace('{prefix}', this._shareUrl)
+                .replace('{share}', this._share)
+                .replace('{schema}', this._schema)
+                .replace('{table}', table);
+            const res = yield fetch(url, {
+                method: 'GET',
+                headers: {
+                    Authorization: `Bearer ${this._bearerToken}`
+                }
+            });
+            (0, Affirm_1.default)(res.ok, `Error fetching version from the delta share: ${res.status} ${res.statusText}`);
+            const version = res.headers['delta-table-version'];
+            return version;
+        });
+        this._getAllFilesInTables = (table) => __awaiter(this, void 0, void 0, function* () {
+            const url = this._query
+                .replace('{prefix}', this._shareUrl)
+                .replace('{share}', this._share)
+                .replace('{schema}', this._schema)
+                .replace('{table}', table);
+            const body = {
+                version: yield this._getVersion(table)
+            };
+            const res = yield fetch(url, {
+                method: 'POST',
+                headers: {
+                    'Authorization': `Bearer ${this._bearerToken}`,
+                    'delta-sharing-capabilities': 'responseformat=delta;readerfeatures=deletionvectors'
+                },
+                body: JSON.stringify(body)
+            });
+            const rawText = yield res.text();
+            (0, Affirm_1.default)(res.ok, `Error fetching data from the delta share: ${res.status} ${res.statusText}; Message: ${rawText}`);
+            // By the protocol: the first is the profile, the second is the metadata, I'm interested from the third onwards
+            const deltaLines = rawText
+                .split('\n')
+                .filter(x => x.length > 0)
+                .slice(2)
+                .map(x => JSON.parse(x));
+            return deltaLines;
+        });
+    }
+}
+exports.default = DeltaShareSourceDriver;

package/drivers/DriverFactory.js

@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const LocalDriver_1 = require("./LocalDriver");
 const RedshiftDriver_1 = __importDefault(require("./RedshiftDriver"));
 const S3Driver_1 = require("./S3Driver");
+const DeltaShareDriver_1 = __importDefault(require("./DeltaShareDriver"));
 class DriverFactoryClass {
     constructor() {
         this.instantiateSource = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -30,6 +31,11 @@ class DriverFactoryClass {
                     yield driver.init(source);
                     return driver;
                 }
+                case 'delta-share': {
+                    const driver = new DeltaShareDriver_1.default();
+                    yield driver.init(source);
+                    return driver;
+                }
                 case 'local': {
                     const driver = new LocalDriver_1.LocalSourceDriver();
                     yield driver.init(source);

package/drivers/DriverHelper.js

@@ -122,6 +122,21 @@ const DriverHelper = {
         yield (0, promises_1.pipeline)(stream, headerValidationTransform, writeStream);
         return lineCount;
     }),
+    appendObjectsToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
+        (0, Affirm_1.default)(options, 'Invalid options');
+        const { append, destinationPath, objects, delimiter } = options;
+        const writeOptions = append ? { flags: 'a' } : {};
+        const writeStream = (0, fs_1.createWriteStream)(destinationPath, writeOptions);
+        let lineCount = 0;
+        const keys = Object.keys(objects[0]);
+        for (const obj of objects) {
+            const serialized = keys.map(k => obj[k]).join(delimiter) + '\n';
+            writeStream.write(serialized);
+            lineCount++;
+        }
+        writeStream.close();
+        return lineCount;
+    }),
     quickReadFile: (filePath, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
         var _a, e_1, _b, _c;
         const fileStream = (0, fs_1.createReadStream)(filePath);

package/engines/ai/DeveloperEngine.js

@@ -16,6 +16,9 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const ProducerEngine_1 = __importDefault(require("../producer/ProducerEngine"));
 const path_1 = __importDefault(require("path"));
 const promises_1 = __importDefault(require("fs/promises"));
+const dayjs_1 = __importDefault(require("dayjs"));
+const customParseFormat_1 = __importDefault(require("dayjs/plugin/customParseFormat"));
+dayjs_1.default.extend(customParseFormat_1.default);
 class DeveloperEngineClass {
     constructor() {
         this.discover = (producer) => __awaiter(this, void 0, void 0, function* () {
@@ -60,13 +63,99 @@ class DeveloperEngineClass {
                     return 'string';
             }
         };
+        // Infer the most likely type from a single JS value
+        // Returns one of: 'number' | 'boolean' | 'date' | 'datetime' | 'string' | 'array' | 'object' | 'null'
+        this.inferType = (value) => {
+            if (value === null || value === undefined)
+                return 'string';
+            // Arrays
+            if (Array.isArray(value))
+                return 'array';
+            // Booleans (including common string representations)
+            if (typeof value === 'boolean')
+                return 'boolean';
+            if (typeof value === 'string') {
+                const trimmed = value.trim();
+                const lower = trimmed.toLowerCase();
+                if (lower === 'true' || lower === 'false')
+                    return 'boolean';
+                // Numbers (numeric strings)
+                const numericRegex = /^-?\d+(?:\.\d+)?$/;
+                if (numericRegex.test(trimmed))
+                    return 'number';
+                // Timestamps (10 or 13 digits)
+                const tsRegex = /^-?\d{10}(?:\d{3})?$/;
+                if (tsRegex.test(trimmed)) {
+                    const n = Number(trimmed.length === 10 ? `${trimmed}000` : trimmed);
+                    const d = new Date(n);
+                    if (!isNaN(d.getTime()))
+                        return 'datetime';
+                }
+                // Dates with common formats
+                const dateFormats = [
+                    'YYYY-MM-DD',
+                    'YYYY/MM/DD',
+                    'DD/MM/YYYY',
+                    'MM/DD/YYYY',
+                    'YYYYMMDD',
+                    'DD-MMM-YYYY',
+                    'YYYY-MM-DD HH:mm',
+                    'YYYY-MM-DD HH:mm:ss',
+                    'YYYY-MM-DDTHH:mm',
+                    'YYYY-MM-DDTHH:mmZ',
+                    'YYYY-MM-DDTHH:mm:ss',
+                    'YYYY-MM-DDTHH:mm:ssZ',
+                    'YYYY-MM-DDTHH:mm:ss.SSSZ'
+                ];
+                for (const fmt of dateFormats) {
+                    const d = (0, dayjs_1.default)(trimmed, fmt, true);
+                    if (d.isValid()) {
+                        // If time components likely present, classify as datetime
+                        if (/T|\d+:\d+/.test(trimmed))
+                            return 'datetime';
+                        return 'date';
+                    }
+                }
+                // ISO 8601 without specifying format
+                const iso = (0, dayjs_1.default)(trimmed);
+                if (iso.isValid() && /\d{4}-\d{2}-\d{2}/.test(trimmed)) {
+                    if (/T|\d+:\d+/.test(trimmed))
+                        return 'datetime';
+                    return 'date';
+                }
+                return 'string';
+            }
+            if (typeof value === 'number')
+                return 'number';
+            if (typeof value === 'object') {
+                // Date instance
+                if (value instanceof Date && !isNaN(value.getTime()))
+                    return 'datetime';
+                return 'object';
+            }
+            // Fallback for bigint, symbol, function -> string
+            return 'string';
+        };
+        this.inferDimensionType = (value) => {
+            const type = this.inferType(value);
+            switch (type) {
+                case 'array':
+                case 'object': return 'string';
+                case 'boolean': return 'boolean';
+                case 'date':
+                case 'datetime': return 'datetime';
+                case 'number': return 'number';
+                case 'string': return 'string';
+                default: return 'string';
+            }
+        };
         this.extractFieldTypes = (records) => {
             if (!records || records.length === 0)
                 return [];
             const sample = records[0];
             return Object.entries(sample._value).map(([key, value]) => ({
                 name: key,
-                type: Array.isArray(value) ? 'array' : typeof value
+                type: this.inferType(value)
             }));
         };
         this.extractFieldClassification = (field) => {

package/engines/consumer/ConsumerEngine.js

@@ -132,7 +132,7 @@ class ConsumerEngineClass {
             (0, Affirm_1.default)(options, `Invalid execute consume options`);
             const { usageId } = UsageManager_1.default.startUsage(consumer, user);
             try {
-                const execution = new ExecutionEnvironment_1.default(consumer);
+                const execution = new ExecutionEnvironment_1.default(consumer, usageId);
                 const result = yield execution.run(options);
                 UsageManager_1.default.endUsage(usageId, result._stats.size);
                 return result;

package/engines/consumer/PostProcessor.js

@@ -62,7 +62,7 @@ class PostProcessorClass {
                 }
                 return record;
             }, options);
-            newDataset.setDimensinons(updatedDimensions);
+            newDataset.setDimensions(updatedDimensions);
             return newDataset;
         });
         /**
@@ -174,12 +174,16 @@ class PostProcessorClass {
                         normalizedRecord[fieldName] = (_a = splitRecord[fieldName]) !== null && _a !== void 0 ? _a : '';
                     }
                     // Create dimensions based on the expected field names
-                    const newDimensions = expectedFieldNames.map((key, index) => ({
-                        name: key,
-                        key: key,
-                        index: index,
-                        hidden: null
-                    }));
+                    const newDimensions = expectedFieldNames.map((key, index) => {
+                        var _a, _b, _c;
+                        return ({
+                            name: key,
+                            key: key,
+                            index: index,
+                            hidden: null,
+                            type: (_c = (_b = (_a = columns[index]) === null || _a === void 0 ? void 0 : _a.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string'
+                        });
+                    });
                     // Create the row string
                     const values = newDimensions.map(dim => {
                         const value = normalizedRecord[dim.name];
@@ -196,14 +200,17 @@ class PostProcessorClass {
             // Update the dataset dimensions to match the unpacked structure
             // TODO: 99% certain this will cause a bug
             if (columns.length > 0) {
-                const newDimensions = columns.map((col, index) => ({
-                    name: col.nameInProducer,
-                    key: col.nameInProducer,
-                    index: index,
-                    hidden: null
-                }));
-                // Update the dataset dimensions
-                resDataset['_dimensions'] = newDimensions;
+                const newDimensions = columns.map((col, index) => {
+                    var _a;
+                    return ({
+                        name: col.nameInProducer,
+                        key: col.nameInProducer,
+                        index: index,
+                        hidden: null,
+                        type: (_a = col.dimension) === null || _a === void 0 ? void 0 : _a.type
+                    });
+                });
+                resDataset.setDimensions(newDimensions);
             }
             return resDataset;
         });

package/engines/dataset/Dataset.js

@@ -35,7 +35,7 @@ const Helper_1 = __importDefault(require("../../helper/Helper"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const Environment_1 = __importDefault(require("../Environment"));
 class Dataset {
-    constructor(name, file, batchSize) {
+    constructor(name, file, batchSize, executionId) {
         var _a;
         this.getPath = () => this._path;
         this.setPath = (path) => {
@@ -43,6 +43,7 @@ class Dataset {
             return this;
         };
         this.getFile = () => this._file;
+        this.getExecutionId = () => this._executionId;
         this.getBatchSize = () => this._batchSize;
         this.setFirstLine = (firstLine) => {
             this._firstLine = firstLine;
@@ -170,7 +171,7 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${lineCount}: ${error}`);
+                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
                     }
                 }
             }
@@ -536,7 +537,7 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${lineCount}: ${error}`);
+                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
                     }
                 }
             }
@@ -627,7 +628,7 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${lineCount}: ${error}`);
+                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
                     }
                 }
             }
@@ -703,10 +704,18 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
-        this.setDimensinons = (dimensions) => {
+        this.setDimensions = (dimensions) => {
             this._dimensions = dimensions;
             return this;
         };
+        this.setSingleDimension = (newDimension, oldDimension) => {
+            (0, Affirm_1.default)(newDimension, `Invalid new dimension`);
+            (0, Affirm_1.default)(oldDimension, 'Invalid old dimension');
+            const current = this._dimensions.findIndex(x => x.index === oldDimension.index);
+            (0, Affirm_1.default)(current, `Trying to updata a dataset dimension that doesn't exist: ${oldDimension.name} index ${oldDimension.index}`);
+            this._dimensions.splice(current, 1, newDimension);
+            return this;
+        };
         /**
          * Update the record pool when dimensions change
          */
@@ -829,6 +838,7 @@ class Dataset {
         this._computeSize = () => fs_1.default.statSync(this._path).size / (1024 * 1024);
         this.name = name;
         this._file = file;
+        this._executionId = executionId;
         this._batchSize = (_a = batchSize !== null && batchSize !== void 0 ? batchSize : parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
         this._dimensions = [];
         this._firstLine = '';
@@ -843,8 +853,9 @@ class Dataset {
             .replace(/_{2,}/g, '_')
             .replace(/^_+|_+$/g, '')
             .toLowerCase();
-        this._path = path_1.default.join('./remora', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, datasetName, '.dataset');
-        this._tempPath = path_1.default.join('./remora/', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, datasetName, '.dataset_tmp');
+        const execFolder = executionId ? path_1.default.join(datasetName, executionId) : datasetName;
+        this._path = path_1.default.join('./remora', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, execFolder, '.dataset');
+        this._tempPath = path_1.default.join('./remora/', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, execFolder, '.dataset_tmp');
         this.ensureFile(this._path);
     }
 }