@forzalabs/remora 0.1.9 → 0.2.1

package/Constants.js

@@ -1,9 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.1.9',
-    lambdaVersion: 1,
-    port: 5069,
+    cliVersion: '0.2.0',
+    backendVersion: 1,
+    backendPort: 5088,
+    workerVersion: 2,
+    workerPort: 5069,
+    /**
+     * Column name for the dynamically injected source filename dimension.
+     * Prefixed with $ to indicate it's a system-generated dynamic value.
+     */
+    SOURCE_FILENAME_COLUMN: '$source_filename',
     defaults: {
         PRODUCER_TEMP_FOLDER: '.temp',
         SQL_MAX_QUERY_ROWS: 10000,

package/actions/deploy.js

@@ -51,7 +51,7 @@ const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
                 // Read the zip file as a buffer
                 const zipBuffer = fs_1.default.readFileSync(tempZipPath);
                 const host = process.env.REMORA_WORKER_HOST;
-                const version = Constants_1.default.lambdaVersion;
+                const version = Constants_1.default.workerVersion;
                 const workerAPI = `${host}/cli/v${version}/uploaddeployment`;
                 const formData = new FormData();
                 const blob = new Blob([zipBuffer], { type: 'application/zip' });

package/database/DatabaseEngine.js

@@ -21,7 +21,8 @@ class DatabaseEngineClass {
         this.db = () => this._db;
         this.connect = () => __awaiter(this, void 0, void 0, function* () {
             var _a;
-            // WARNING: this was changed during the deployment to ECS... I've reverted, but maybe it needs to be changed or looked into...
+            // WARNING: this was changed during the deployment to ECS...
+            // I've reverted it, but maybe it needs to be changed or looked into...
             this._uri = ((_a = process.env.MONGO_URI) !== null && _a !== void 0 ? _a : Helper_1.default.isDev())
                 ? 'mongodb://mongo:27017/remora'
                 : 'mongodb://localhost:27017/remora';
@@ -29,7 +30,7 @@ class DatabaseEngineClass {
             const errors = [];
             for (let i = 0; i < this.MAX_TRY_CONNECTION; i++) {
                 try {
-                    console.log(`Attempting to connect to mongo: "${this._uri}"`);
+                    console.log(`Attempting to connect to mongo: "${this._uri}" (${i})`);
                     yield this._client.connect();
                     this._db = this._client.db(Settings_1.default.db.name);
                     this._connected = true;
@@ -37,7 +38,7 @@ class DatabaseEngineClass {
                     break;
                 }
                 catch (error) {
-                    errors.push((i + 1) + ': connection to MongoDB throws this error:', error);
+                    errors.push((i + 1) + ': connection to MongoDB throws this error: ' + error);
                 }
             }
             if (!this._connected)
@@ -54,6 +55,7 @@ class DatabaseEngineClass {
         });
         this.query = (collectionName, filter, options) => __awaiter(this, void 0, void 0, function* () {
             try {
+                yield this._checkConnection();
                 const collection = this._db.collection(collectionName);
                 const result = yield collection.find(filter, options).toArray();
                 return result;
@@ -65,6 +67,7 @@ class DatabaseEngineClass {
         });
         this.aggregate = (collectionName, aggregation) => __awaiter(this, void 0, void 0, function* () {
             try {
+                yield this._checkConnection();
                 const collection = this._db.collection(collectionName);
                 return yield collection.aggregate(aggregation).toArray();
             }
@@ -75,6 +78,7 @@ class DatabaseEngineClass {
         });
         this.get = (collectionName, id) => __awaiter(this, void 0, void 0, function* () {
             try {
+                yield this._checkConnection();
                 const collection = this._db.collection(collectionName);
                 return yield collection.findOne({ _id: id });
             }
@@ -85,6 +89,7 @@ class DatabaseEngineClass {
         });
         this.findOne = (collectionName, query) => __awaiter(this, void 0, void 0, function* () {
             try {
+                yield this._checkConnection();
                 const collection = this._db.collection(collectionName);
                 return yield collection.findOne(query);
             }
@@ -95,6 +100,7 @@ class DatabaseEngineClass {
         });
         this.upsert = (collectionName, id, update) => __awaiter(this, void 0, void 0, function* () {
             try {
+                yield this._checkConnection();
                 const collection = this._db.collection(collectionName);
                 const result = yield collection.findOneAndUpdate({ _id: id }, { $set: update }, { upsert: true, returnDocument: 'after' });
                 return result;
@@ -106,6 +112,7 @@ class DatabaseEngineClass {
         });
         this.addToList = (collectionName, id, arrayField, arrayItem) => __awaiter(this, void 0, void 0, function* () {
             try {
+                yield this._checkConnection();
                 const collection = this._db.collection(collectionName);
                 const result = yield collection.findOneAndUpdate({ _id: id }, { $push: { [arrayField]: arrayItem } }, { returnDocument: 'after' });
                 return result;
@@ -117,6 +124,7 @@ class DatabaseEngineClass {
         });
         this.doUpdate = (collectionName, id, update) => __awaiter(this, void 0, void 0, function* () {
             try {
+                yield this._checkConnection();
                 const collection = this._db.collection(collectionName);
                 const result = yield collection.findOneAndUpdate({ _id: id }, update, { returnDocument: 'after' });
                 return result;
@@ -126,6 +134,13 @@ class DatabaseEngineClass {
                 throw error;
             }
         });
+        this._checkConnection = () => __awaiter(this, void 0, void 0, function* () {
+            if (this._connected)
+                return;
+            yield this.connect();
+            if (!this._connected)
+                throw new Error(`Can't to perform db operation: unable to connect to the database (${this._uri})`);
+        });
     }
 }
 const DatabaseEngine = new DatabaseEngineClass();

package/definitions/DatasetDefinitions.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/json_schemas/producer-schema.json

@@ -76,6 +76,10 @@
                             "none",
                             "{REMORA_MASK_IN_DEV}"
                         ]
+                    },
+                    "sourceFilename": {
+                        "type": "boolean",
+                        "description": "When true, this dimension will be populated with the source filename. Only valid for file-based producers (local, aws-s3) and only one dimension per producer can have this set to true. Useful when reading multiple files with wildcard patterns to track which file each row came from."
                     }
                 },
                 "required": [
@@ -130,7 +134,7 @@
                 },
                 "fileKey": {
                     "type": "string",
-                    "description": "If the source is a bucket, this is the file key that identifies the file to read"
+                    "description": "For S3/local sources: the file key/path that identifies the file to read. For HTTP API sources: the API endpoint path (e.g., '/api/v1/users')"
                 },
                 "fileType": {
                     "type": "string",
@@ -252,6 +256,40 @@
                 "fileType": "CSV"
             },
             "_version": 2
+        },
+        {
+            "name": "APIUsers",
+            "description": "Producer for user data from REST API",
+            "source": "REST API with Bearer Token",
+            "dimensions": [
+                {
+                    "name": "user_id",
+                    "type": "string",
+                    "pk": true
+                },
+                {
+                    "name": "username",
+                    "type": "string"
+                },
+                {
+                    "name": "email",
+                    "type": "string",
+                    "classification": [
+                        "PII",
+                        "GDPR"
+                    ],
+                    "mask": "mask"
+                },
+                {
+                    "name": "created_at",
+                    "type": "datetime"
+                }
+            ],
+            "settings": {
+                "fileKey": "/api/v1/users",
+                "fileType": "JSON"
+            },
+            "_version": 1
         }
     ]
 }

package/definitions/json_schemas/source-schema.json

@@ -24,7 +24,8 @@
                 "aws-s3",
                 "postgres",
                 "local",
-                "delta-share"
+                "delta-share",
+                "http-api"
             ],
             "description": "The type of data engine"
         },
@@ -39,7 +40,10 @@
                         "username-password",
                         "access-secret-key",
                         "arn",
-                        "implicit"
+                        "implicit",
+                        "bearer-token",
+                        "api-key",
+                        "none"
                     ],
                     "description": "The authentication method to use"
                 },
@@ -113,7 +117,47 @@
                 },
                 "bearerToken": {
                     "type": "string",
-                    "description": "Delta Sharing bearer token used for authentication"
+                    "description": "Bearer token used for authentication (Delta Sharing or HTTP API)"
+                },
+                "url": {
+                    "type": "string",
+                    "format": "uri",
+                    "description": "Base URL for HTTP API sources"
+                },
+                "headers": {
+                    "type": "object",
+                    "description": "Custom HTTP headers for API requests",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "queryParams": {
+                    "type": "object",
+                    "description": "Default query parameters for API requests",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "httpMethod": {
+                    "type": "string",
+                    "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"],
+                    "description": "HTTP method to use for API requests",
+                    "default": "GET"
+                },
+                "apiKey": {
+                    "type": "string",
+                    "description": "API key for api-key authentication method"
+                },
+                "apiKeyHeader": {
+                    "type": "string",
+                    "description": "Header name for API key (defaults to X-API-Key)",
+                    "default": "X-API-Key"
+                },
+                "timeout": {
+                    "type": "number",
+                    "description": "Request timeout in milliseconds",
+                    "default": 30000,
+                    "minimum": 1000
                 }
             },
             "required": ["method"]
@@ -172,6 +216,35 @@
                 "clusterId": "analytics-cluster"
             },
             "_version": 1
+        },
+        {
+            "name": "REST API with Bearer Token",
+            "description": "HTTP API source with bearer token authentication",
+            "engine": "http-api",
+            "authentication": {
+                "method": "bearer-token",
+                "url": "https://api.example.com",
+                "bearerToken": "{API_BEARER_TOKEN}",
+                "headers": {
+                    "Accept": "application/json"
+                },
+                "timeout": 30000
+            },
+            "_version": 1
+        },
+        {
+            "name": "Public REST API",
+            "description": "Public HTTP API with no authentication",
+            "engine": "http-api",
+            "authentication": {
+                "method": "none",
+                "url": "https://api.publicapis.org",
+                "headers": {
+                    "Accept": "application/json"
+                },
+                "httpMethod": "GET"
+            },
+            "_version": 1
         }
     ]
 }

package/drivers/DriverFactory.js

@@ -16,6 +16,7 @@ const LocalDriver_1 = require("./LocalDriver");
 const RedshiftDriver_1 = __importDefault(require("./RedshiftDriver"));
 const S3Driver_1 = require("./S3Driver");
 const DeltaShareDriver_1 = __importDefault(require("./DeltaShareDriver"));
+const HttpApiDriver_1 = require("./HttpApiDriver");
 class DriverFactoryClass {
     constructor() {
         this.instantiateSource = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -41,6 +42,11 @@ class DriverFactoryClass {
                     yield driver.init(source);
                     return driver;
                 }
+                case 'http-api': {
+                    const driver = new HttpApiDriver_1.HttpApiSourceDriver();
+                    yield driver.init(source);
+                    return driver;
+                }
                 default: throw new Error(`Invalid driver type "${source.engine}". This driver is not implemented yet`);
             }
         });

package/drivers/DriverHelper.js

@@ -28,12 +28,17 @@ const Affirm_1 = __importDefault(require("../core/Affirm"));
 const DriverHelper = {
     appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(options, 'Invalid options');
-        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow, delimiter } = options;
+        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow, delimiter, sourceFilename } = options;
         (0, Affirm_1.default)(headerLine, `Invalid header line`);
         const keys = (fileType === 'JSON' || fileType === 'JSONL')
             ? Object.keys(JSON.parse(headerLine))
             : [];
         const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
+        // When sourceFilename is set, the headerLine includes $source_filename at the end.
+        // For validation, we need to compare against the original header without this suffix.
+        const originalHeaderLine = sourceFilename
+            ? headerLine.slice(0, headerLine.lastIndexOf(delimiter))
+            : headerLine;
         let isFirstLine = true;
         let hasValidatedHeader = shouldValidateHeader ? false : true;
         let leftoverData = '';
@@ -50,8 +55,8 @@ const DriverHelper = {
                     const line = lines[i];
                     // Header validation for first line
                     if (!hasValidatedHeader && isFirstLine && i === 0) {
-                        if (shouldValidateHeader && headerLine && headerLine.trim() !== '' && line.trim() !== headerLine.trim()) {
-                            const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${line}\n\t-main: ${headerLine}`;
+                        if (shouldValidateHeader && originalHeaderLine && originalHeaderLine.trim() !== '' && line.trim() !== originalHeaderLine.trim()) {
+                            const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${line}\n\t-main: ${originalHeaderLine}`;
                             Logger_1.default.log(msg);
                             return callback(new Error(msg));
                         }
@@ -77,7 +82,7 @@ const DriverHelper = {
                 // Process any remaining data
                 if (leftoverData.trim()) {
                     if (shouldIncludeLine(leftoverData, -1)) {
-                        callback(null, Buffer.from(processLine(leftoverData)));
+                        callback(null, Buffer.from(processLine(leftoverData) + '\n'));
                     }
                     else {
                         callback(null, null);
@@ -101,21 +106,28 @@ const DriverHelper = {
         };
         const processLine = (line) => {
             lineCount++;
+            let processedLine;
             switch (fileType) {
                 case 'JSON':
                 case 'JSONL': {
                     try {
                         const parsed = JSON.parse(line);
-                        return keys.map(k => parsed[k]).join(delimiter);
+                        processedLine = keys.map(k => parsed[k]).join(delimiter);
                     }
                     catch (error) {
                         Logger_1.default.log(`Failed parsing in JSON line -> file: ${fileKey}; index: ${globalIndex}; line: ${line}; err: ${error === null || error === void 0 ? void 0 : error.name}`);
                         throw error;
                     }
+                    break;
                 }
                 default:
-                    return line;
+                    processedLine = line;
+            }
+            // If sourceFilename is provided, append it to each line
+            if (sourceFilename) {
+                processedLine = processedLine + delimiter + sourceFilename;
             }
+            return processedLine;
         };
         const writeOptions = append ? { flags: 'a' } : {};
         const writeStream = (0, fs_1.createWriteStream)(destinationPath, writeOptions);

package/drivers/HttpApiDriver.js

@@ -0,0 +1,204 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpApiSourceDriver = void 0;
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
+const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+class HttpApiSourceDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(source, 'Invalid source');
+            (0, Affirm_1.default)(source.authentication, 'Invalid authentication');
+            (0, Affirm_1.default)(source.authentication.url, 'HTTP API source requires a URL in authentication.url');
+            this._source = source;
+            this._baseUrl = SecretManager_1.default.replaceSecret(source.authentication.url);
+            this._httpMethod = source.authentication.httpMethod || 'GET';
+            this._timeout = source.authentication.timeout || 30000; // 30 seconds default
+            this._headers = source.authentication.headers ? Object.assign({}, source.authentication.headers) : {};
+            this._queryParams = source.authentication.queryParams ? Object.assign({}, source.authentication.queryParams) : {};
+            // Handle different authentication methods
+            switch (source.authentication.method) {
+                case 'bearer-token': {
+                    (0, Affirm_1.default)(source.authentication.bearerToken, 'Bearer token authentication requires bearerToken');
+                    this._headers['Authorization'] = `Bearer ${SecretManager_1.default.replaceSecret(source.authentication.bearerToken)}`;
+                    break;
+                }
+                case 'api-key': {
+                    (0, Affirm_1.default)(source.authentication.apiKey, 'API key authentication requires apiKey');
+                    const apiKeyHeader = source.authentication.apiKeyHeader || 'X-API-Key';
+                    this._headers[apiKeyHeader] = SecretManager_1.default.replaceSecret(source.authentication.apiKey);
+                    break;
+                }
+                case 'username-password': {
+                    (0, Affirm_1.default)(source.authentication.user && source.authentication.password, 'Username-password authentication requires user and password');
+                    const credentials = Buffer.from(`${SecretManager_1.default.replaceSecret(source.authentication.user)}:${SecretManager_1.default.replaceSecret(source.authentication.password)}`).toString('base64');
+                    this._headers['Authorization'] = `Basic ${credentials}`;
+                    break;
+                }
+                case 'none':
+                    // No authentication required
+                    break;
+                default:
+                    throw new Error(`Authentication method "${source.authentication.method}" is not supported for HTTP API sources`);
+            }
+            // Test connection
+            try {
+                yield this._makeRequest(this._baseUrl);
+                Logger_1.default.log(`HTTP API connection to ${this._baseUrl} successful`);
+            }
+            catch (error) {
+                throw new Error(`Failed to connect to HTTP API at ${this._baseUrl}: ${error.message}`);
+            }
+            return this;
+        });
+        this._makeRequest = (url, options) => __awaiter(this, void 0, void 0, function* () {
+            const method = (options === null || options === void 0 ? void 0 : options.method) || this._httpMethod;
+            const headers = Object.assign(Object.assign({}, this._headers), options === null || options === void 0 ? void 0 : options.additionalHeaders);
+            const queryParams = Object.assign(Object.assign({}, this._queryParams), options === null || options === void 0 ? void 0 : options.additionalQueryParams);
+            // Build URL with query parameters
+            const urlWithParams = new URL(url);
+            Object.entries(queryParams).forEach(([key, value]) => {
+                urlWithParams.searchParams.append(key, value);
+            });
+            const fetchOptions = {
+                method,
+                headers,
+                signal: AbortSignal.timeout(this._timeout)
+            };
+            if ((options === null || options === void 0 ? void 0 : options.body) && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
+                fetchOptions.body = typeof options.body === 'string'
+                    ? options.body
+                    : JSON.stringify(options.body);
+                if (!headers['Content-Type']) {
+                    headers['Content-Type'] = 'application/json';
+                }
+            }
+            const response = yield fetch(urlWithParams.toString(), fetchOptions);
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            const contentType = response.headers.get('content-type');
+            if (contentType === null || contentType === void 0 ? void 0 : contentType.includes('application/json')) {
+                return yield response.json();
+            }
+            else {
+                return yield response.text();
+            }
+        });
+        this.execute = (_sql) => __awaiter(this, void 0, void 0, function* () {
+            void _sql;
+            throw new Error('SQL execution is not supported for HTTP API sources. Use query() or readAll() instead.');
+        });
+        this.query = (_sql, _values) => __awaiter(this, void 0, void 0, function* () {
+            void _sql;
+            void _values;
+            throw new Error('SQL queries are not supported for HTTP API sources. Use readAll() to fetch data from the API.');
+        });
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const endpoint = producer.settings.fileKey || '';
+                const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
+                yield this._makeRequest(url, { method: 'HEAD' });
+                return true;
+            }
+            catch (error) {
+                if (error.message.includes('404')) {
+                    return false;
+                }
+                throw error;
+            }
+        });
+        this.readAll = (request, values) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(request, 'Invalid read request');
+            (0, Affirm_1.default)(request.fileKey, 'Invalid file key (endpoint path)');
+            const endpoint = request.fileKey;
+            const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
+            // Convert IQueryParameter[] to query params if provided
+            const additionalQueryParams = {};
+            if (values && values.length > 0) {
+                values.forEach(param => {
+                    additionalQueryParams[param.name] = param.value;
+                });
+            }
+            const data = yield this._makeRequest(url, { additionalQueryParams });
+            // Convert response to string array (lines)
+            return this._extractObjectsFromResponse(data, request.httpApi).map(x => JSON.stringify(x));
+        });
+        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(request, 'Invalid read request');
+            (0, Affirm_1.default)(request.options, 'Invalid read request options');
+            const allLines = yield this.readAll(request);
+            const { lineFrom, lineTo } = request.options;
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                return allLines.slice(lineFrom, lineTo);
+            }
+            return allLines;
+        });
+        this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(dataset, 'Invalid dataset');
+            const file = dataset.getFile();
+            (0, Affirm_1.default)(file, 'Invalid dataset file');
+            (0, Affirm_1.default)(file.fileKey, 'Invalid file key (endpoint path)');
+            const endpoint = file.fileKey;
+            const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
+            const data = yield this._makeRequest(url);
+            const apiObjects = this._extractObjectsFromResponse(data, file.httpApi);
+            dataset.setFirstLine(JSON.stringify(apiObjects[0]));
+            const totalLineCount = yield DriverHelper_1.default.appendObjectsToUnifiedFile({
+                append: true,
+                delimiter: dataset.getDelimiter(),
+                destinationPath: dataset.getPath(),
+                objects: apiObjects
+            });
+            dataset.setCount(totalLineCount);
+            return dataset;
+        });
+        this._extractObjectsFromResponse = (data, httpApi) => {
+            let itemsData = [];
+            if (httpApi && httpApi.dataProperty && httpApi.dataProperty.length > 0) {
+                itemsData = data[httpApi.dataProperty];
+            }
+            else {
+                if (typeof data === 'string') {
+                    itemsData = data.split('\n').filter(line => line.trim().length > 0);
+                }
+                else if (Array.isArray(data)) {
+                    itemsData = data;
+                }
+                else if (typeof data === 'object' && data !== null) {
+                    const dataObj = data;
+                    if (dataObj.data && Array.isArray(dataObj.data)) {
+                        itemsData = dataObj.data;
+                    }
+                    else if (dataObj.results && Array.isArray(dataObj.results)) {
+                        itemsData = dataObj.results;
+                    }
+                    else if (dataObj.items && Array.isArray(dataObj.items)) {
+                        itemsData = dataObj.items;
+                    }
+                    else {
+                        // Single object, return as single line
+                        itemsData = [data];
+                    }
+                }
+            }
+            return itemsData;
+        };
+    }
+}
+exports.HttpApiSourceDriver = HttpApiSourceDriver;
+exports.default = HttpApiSourceDriver;

package/drivers/LocalDriver.js

@@ -65,6 +65,7 @@ const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"))
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+const Constants_1 = __importDefault(require("../Constants"));
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -123,7 +124,8 @@ class LocalSourceDriver {
             (0, Affirm_1.default)(file, 'Invalid dataset file');
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false) {
+            const includeSourceFilename = file.includeSourceFilename === true;
+            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false, sourceFilename) {
                 const sourceFilePath = path_1.default.join(this._path, fileKey);
                 (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
                 // Copy and validate header in a single stream pass
@@ -136,7 +138,8 @@ class LocalSourceDriver {
                     headerLine,
                     fileType: file.fileType,
                     hasHeaderRow: file.hasHeaderRow,
-                    delimiter: dataset.getDelimiter()
+                    delimiter: dataset.getDelimiter(),
+                    sourceFilename
                 });
             });
             const { fileKey } = file;
@@ -145,21 +148,32 @@ class LocalSourceDriver {
                 Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
                 Affirm_1.default.hasItems(allFileKeys, `The file key "${fileKey}" doesn't have any matches in path "${this._path}".`);
                 // Get header line from the first file
-                const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
+                let headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
+                // If including source filename, append a placeholder column name to the header
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
                 dataset.setFirstLine(headerLine);
                 let totalLineCount = 0;
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    totalLineCount += yield copyLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
+                    const currentFileKey = allFileKeys[i];
+                    // Pass the filename (just the basename) if includeSourceFilename is enabled
+                    const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
+                    totalLineCount += yield copyLocally(currentFileKey, headerLine, i > 0, sourceFilename); // Append mode for subsequent files
                 }
                 dataset.setCount(totalLineCount);
                 return dataset;
             }
             else {
-                // For single file, no header validation needed
-                const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
+                // For single file, include the filename if configured
+                let headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
                 dataset.setFirstLine(headerLine);
-                const totalLineCount = yield copyLocally(fileKey, headerLine, false);
+                const sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
+                const totalLineCount = yield copyLocally(fileKey, headerLine, false, sourceFilename);
                 dataset.setCount(totalLineCount);
                 return dataset;
             }

package/drivers/S3Driver.js

@@ -24,6 +24,7 @@ const client_s3_1 = require("@aws-sdk/client-s3");
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 const readline_1 = __importDefault(require("readline"));
+const path_1 = __importDefault(require("path"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
 const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
@@ -32,6 +33,7 @@ const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"))
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
+const Constants_1 = __importDefault(require("../Constants"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -213,7 +215,8 @@ class S3SourceDriver {
             (0, Affirm_1.default)(file, 'Invalid dataset file');
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const downloadLocally = (fileUrl_1, headerLine_1, ...args_1) => __awaiter(this, [fileUrl_1, headerLine_1, ...args_1], void 0, function* (fileUrl, headerLine, appendMode = false) {
+            const includeSourceFilename = file.includeSourceFilename === true;
+            const downloadLocally = (fileUrl_1, headerLine_1, ...args_1) => __awaiter(this, [fileUrl_1, headerLine_1, ...args_1], void 0, function* (fileUrl, headerLine, appendMode = false, sourceFilename) {
                 // Download and validate header in a single stream pass
                 const command = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,
@@ -230,7 +233,8 @@ class S3SourceDriver {
                     headerLine,
                     fileType: file.fileType,
                     hasHeaderRow: file.hasHeaderRow,
-                    delimiter: dataset.getDelimiter()
+                    delimiter: dataset.getDelimiter(),
+                    sourceFilename
                 });
             });
             const { fileKey } = file;
@@ -246,18 +250,25 @@ class S3SourceDriver {
                 const firstFileResponse = yield this._client.send(firstFileCommand);
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
-                const headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                let headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                // If including source filename, append a placeholder column name to the header
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
                 dataset.setFirstLine(headerLine);
                 let totalLineCount = 0;
                 // Download files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    totalLineCount += yield downloadLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
+                    const currentFileKey = allFileKeys[i];
+                    // Pass the filename (just the basename) if includeSourceFilename is enabled
+                    const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
+                    totalLineCount += yield downloadLocally(currentFileKey, headerLine, i > 0, sourceFilename); // Append mode for subsequent files
                 }
                 dataset.setCount(totalLineCount);
                 return dataset;
             }
             else {
-                // Get header line from the first file
+                // Get header line from the single file
                 const firstFileCommand = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,
                     Key: fileKey
@@ -265,10 +276,15 @@ class S3SourceDriver {
                 const firstFileResponse = yield this._client.send(firstFileCommand);
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
-                const headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                let headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                // If including source filename, append a placeholder column name to the header
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
                 dataset.setFirstLine(headerLine);
-                // For single file, no header validation needed
-                const totalLineCount = yield downloadLocally(fileKey, headerLine);
+                // Pass the filename if includeSourceFilename is enabled
+                const sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
+                const totalLineCount = yield downloadLocally(fileKey, headerLine, false, sourceFilename);
                 dataset.setCount(totalLineCount);
                 return dataset;
             }

package/engines/dataset/DatasetManager.js

@@ -22,6 +22,7 @@ const Dataset_1 = __importDefault(require("./Dataset"));
 const promises_1 = require("stream/promises");
 const fs_1 = require("fs");
 const DeveloperEngine_1 = __importDefault(require("../ai/DeveloperEngine"));
+const Constants_1 = __importDefault(require("../../Constants"));
 class DatasetManagerClass {
     constructor() {
         /**
@@ -30,14 +31,19 @@ class DatasetManagerClass {
          * when the same producer / consumer is executed multiple times in parallel.
          */
         this.create = (producer, executionId) => {
+            var _a, _b;
             (0, Affirm_1.default)(producer, 'Invalid producer');
-            const { name, settings: { delimiter, fileKey, fileType, hasHeaderRow, sheetName } } = producer;
+            const { name, settings: { delimiter, fileKey, fileType, hasHeaderRow, sheetName, httpApi } } = producer;
+            // Check if any dimension has sourceFilename flag set to true
+            const hasSourceFilenameDimension = (_b = (_a = producer.dimensions) === null || _a === void 0 ? void 0 : _a.some(d => d.sourceFilename === true)) !== null && _b !== void 0 ? _b : false;
             const dataset = new Dataset_1.default(name, {
                 fileKey,
                 fileType,
                 hasHeaderRow,
                 sheetName,
-                delimiter
+                delimiter,
+                httpApi,
+                includeSourceFilename: hasSourceFilenameDimension
             }, undefined, executionId);
             return dataset;
         };
@@ -49,7 +55,7 @@ class DatasetManagerClass {
             return this.buildDimensionsFromFirstLine(firstLine, dataset.getFile(), producer, discover);
         });
         this.buildDimensionsFromFirstLine = (firstLine_1, dsFile_1, producer_1, ...args_1) => __awaiter(this, [firstLine_1, dsFile_1, producer_1, ...args_1], void 0, function* (firstLine, dsFile, producer, discover = false) {
-            var _a, _b, _c, _d, _e, _f, _g, _h;
+            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
             (0, Affirm_1.default)(firstLine, `Invalid first line`);
             (0, Affirm_1.default)(dsFile, `Invalid dataset file`);
             (0, Affirm_1.default)(producer, `Invalid producer`);
@@ -77,6 +83,12 @@ class DatasetManagerClass {
                     const columns = FileCompiler_1.default.compileProducer(producer, source);
                     const firstObject = JSON.parse(firstLine);
                     const keys = Object.keys(firstObject);
+                    // If includeSourceFilename is enabled, the driver has added $source_filename column
+                    // We need to add it to the keys list so dimensions can reference it
+                    const includeSourceFilename = file.includeSourceFilename === true;
+                    if (includeSourceFilename) {
+                        keys.push(Constants_1.default.SOURCE_FILENAME_COLUMN);
+                    }
                     if (discover) {
                         return {
                             delimiter: (_b = file.delimiter) !== null && _b !== void 0 ? _b : ',',
@@ -91,7 +103,21 @@ class DatasetManagerClass {
                     }
                     const dimensions = [];
                     for (const pColumn of columns) {
-                        const columnKey = (_c = pColumn.aliasInProducer) !== null && _c !== void 0 ? _c : pColumn.nameInProducer;
+                        // Handle sourceFilename dimension specially - it maps to the $source_filename column added by the driver
+                        if (((_c = pColumn.dimension) === null || _c === void 0 ? void 0 : _c.sourceFilename) === true) {
+                            if (includeSourceFilename) {
+                                const sourceFilenameIndex = keys.findIndex(x => x === Constants_1.default.SOURCE_FILENAME_COLUMN);
+                                dimensions.push({
+                                    index: sourceFilenameIndex,
+                                    key: Constants_1.default.SOURCE_FILENAME_COLUMN,
+                                    name: pColumn.nameInProducer,
+                                    hidden: null,
+                                    type: (_e = (_d = pColumn.dimension) === null || _d === void 0 ? void 0 : _d.type) !== null && _e !== void 0 ? _e : 'string'
+                                });
+                            }
+                            continue;
+                        }
+                        const columnKey = (_f = pColumn.aliasInProducer) !== null && _f !== void 0 ? _f : pColumn.nameInProducer;
                         const csvColumnIndex = keys.findIndex(x => x === columnKey);
                         (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
                         dimensions.push({
@@ -99,47 +125,64 @@ class DatasetManagerClass {
                             key: columnKey,
                             name: pColumn.nameInProducer,
                             hidden: null,
-                            type: (_e = (_d = pColumn.dimension) === null || _d === void 0 ? void 0 : _d.type) !== null && _e !== void 0 ? _e : 'string'
+                            type: (_h = (_g = pColumn.dimension) === null || _g === void 0 ? void 0 : _g.type) !== null && _h !== void 0 ? _h : 'string'
                         });
                     }
-                    const delimiterChar = (_f = file.delimiter) !== null && _f !== void 0 ? _f : ',';
+                    const delimiterChar = (_j = file.delimiter) !== null && _j !== void 0 ? _j : ',';
                     return { dimensions, delimiter: delimiterChar };
                 }
                 case 'TXT': {
                     if (!file.hasHeaderRow) {
                         // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
-                        const delimiterChar = (_g = file.delimiter) !== null && _g !== void 0 ? _g : ',';
+                        const delimiterChar = (_k = file.delimiter) !== null && _k !== void 0 ? _k : ',';
                         const source = Environment_1.default.getSource(producer.source);
                         const columns = FileCompiler_1.default.compileProducer(producer, source);
+                        const includeSourceFilename = file.includeSourceFilename === true;
                         if (discover) {
                             // Since I don't have an header, and I'm discovering, I just create placeholder dimensions based on the same number of columns of the txt
+                            const colValues = firstLine.split(delimiterChar);
+                            const dimensions = colValues.map((x, i) => ({
+                                hidden: false,
+                                index: i,
+                                key: `Col ${i + 1}`,
+                                name: `Col ${i + 1}`,
+                                type: 'string'
+                            }));
                             return {
                                 delimiter: delimiterChar,
-                                dimensions: firstLine.split(delimiterChar).map((x, i) => ({
-                                    hidden: false,
-                                    index: i,
-                                    key: `Col ${i + 1}`,
-                                    name: `Col ${i + 1}`,
-                                    type: 'string'
-                                }))
+                                dimensions
                             };
                         }
+                        // Filter out sourceFilename columns for index-based mapping, but track them for later
+                        const regularColumns = columns.filter(x => { var _a; return ((_a = x.dimension) === null || _a === void 0 ? void 0 : _a.sourceFilename) !== true; });
+                        const sourceFilenameColumn = columns.find(x => { var _a; return ((_a = x.dimension) === null || _a === void 0 ? void 0 : _a.sourceFilename) === true; });
+                        const dimensions = regularColumns.map((x, i) => {
+                            var _a, _b, _c;
+                            return ({
+                                key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer,
+                                name: x.nameInProducer,
+                                index: i,
+                                hidden: null,
+                                type: (_c = (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string'
+                            });
+                        });
+                        // Add sourceFilename dimension at the end if enabled
+                        if (sourceFilenameColumn && includeSourceFilename) {
+                            dimensions.push({
+                                key: Constants_1.default.SOURCE_FILENAME_COLUMN,
+                                name: sourceFilenameColumn.nameInProducer,
+                                index: regularColumns.length, // Index after all regular columns
+                                hidden: null,
+                                type: (_m = (_l = sourceFilenameColumn.dimension) === null || _l === void 0 ? void 0 : _l.type) !== null && _m !== void 0 ? _m : 'string'
+                            });
+                        }
                         return {
-                            dimensions: columns.map((x, i) => {
-                                var _a, _b, _c;
-                                return ({
-                                    key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer,
-                                    name: x.nameInProducer,
-                                    index: i,
-                                    hidden: null,
-                                    type: (_c = (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string'
-                                });
-                            }),
+                            dimensions,
                             delimiter: delimiterChar
                         };
                     }
                     else {
-                        const delimiterChar = (_h = producer.settings.delimiter) !== null && _h !== void 0 ? _h : ',';
+                        const delimiterChar = (_o = producer.settings.delimiter) !== null && _o !== void 0 ? _o : ',';
                         const rawDimensions = ParseManager_1.default._extractHeader(firstLine, delimiterChar, producer, discover);
                         return {
                             dimensions: rawDimensions.map(x => ({

package/engines/dataset/DatasetRecord.js

@@ -13,7 +13,8 @@ class DatasetRecord {
                 const parts = CSVParser_1.default.parseRow(row, delimiter);
                 for (let i = 0; i < dimensions.length; i++) {
                     const dim = dimensions[i];
-                    this._value[dim.name] = TypeCaster_1.default.cast(parts[i], dim.type, dim.format);
+                    // Use dim.index to get the correct column from the file, not the loop index
+                    this._value[dim.name] = TypeCaster_1.default.cast(parts[dim.index], dim.type, dim.format);
                 }
             }
         };

package/engines/execution/ExecutionPlanner.js

@@ -112,7 +112,8 @@ class ExecutionPlannerClas {
                 }
                 case 'local':
                 case 'aws-s3':
-                case 'delta-share': {
+                case 'delta-share':
+                case 'http-api': {
                     plan.push({ type: 'load-dataset', producer });
                     plan.push({ type: 'prepare-dataset', producer });
                     if (producer.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))

package/engines/parsing/ParseManager.js

@@ -7,10 +7,11 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Environment_1 = __importDefault(require("../Environment"));
 const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
 const CSVParser_1 = __importDefault(require("./CSVParser"));
+const Constants_1 = __importDefault(require("../../Constants"));
 class ParseManagerClass {
     constructor() {
         this._extractHeader = (headerLine, delimiter, producer, discover) => {
-            var _a, _b, _c;
+            var _a, _b, _c, _d, _e, _f;
             (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);
             (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
             (0, Affirm_1.default)(producer, 'Invalid producer');
@@ -22,14 +23,29 @@ class ParseManagerClass {
                 columns = headerColumns.map(x => ({ nameInProducer: x }));
             const csvColumns = [];
             for (const pColumn of columns) {
-                const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;
+                // Skip sourceFilename dimensions - they don't exist in the source file
+                // They are added dynamically by the driver when reading the file
+                if (((_a = pColumn.dimension) === null || _a === void 0 ? void 0 : _a.sourceFilename) === true) {
+                    // Find the index of $source_filename in the header (it was added by the driver)
+                    const sourceFilenameIndex = headerColumns.findIndex(x => x === Constants_1.default.SOURCE_FILENAME_COLUMN);
+                    if (sourceFilenameIndex > -1) {
+                        csvColumns.push({
+                            index: sourceFilenameIndex,
+                            name: Constants_1.default.SOURCE_FILENAME_COLUMN,
+                            saveAs: pColumn.nameInProducer,
+                            type: (_c = (_b = pColumn.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string'
+                        });
+                    }
+                    continue;
+                }
+                const columnKey = (_d = pColumn.aliasInProducer) !== null && _d !== void 0 ? _d : pColumn.nameInProducer;
                 const csvColumnIndex = headerColumns.findIndex(x => x === columnKey);
                 (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
                 csvColumns.push({
                     index: csvColumnIndex,
                     name: columnKey,
                     saveAs: pColumn.nameInProducer,
-                    type: (_c = (_b = pColumn.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string'
+                    type: (_f = (_e = pColumn.dimension) === null || _e === void 0 ? void 0 : _e.type) !== null && _f !== void 0 ? _f : 'string'
                 });
             }
             return csvColumns;

package/engines/producer/ProducerEngine.js

@@ -65,7 +65,7 @@ class ProducerEngineClass {
             }
         });
         this.readFile = (producer, options) => __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b, _c;
             (0, Affirm_1.default)(producer, 'Invalid producer');
             (0, Affirm_1.default)(options, 'Invalid options');
             if (options.readmode === 'lines')
@@ -79,16 +79,25 @@ class ProducerEngineClass {
             let lines = [];
             switch (options.readmode) {
                 case 'lines':
-                    lines = yield driver.readLinesInRange({ fileKey, fileType, options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName, hasHeaderRow } });
+                    lines = yield driver.readLinesInRange({
+                        fileKey,
+                        fileType,
+                        options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName, hasHeaderRow },
+                        httpApi: (_a = producer.settings) === null || _a === void 0 ? void 0 : _a.httpApi
+                    });
                     break;
                 case 'all':
-                    lines = yield driver.readAll({ fileKey, fileType, options: { sheetName, hasHeaderRow } });
+                    lines = yield driver.readAll({
+                        fileKey, fileType,
+                        options: { sheetName, hasHeaderRow },
+                        httpApi: (_b = producer.settings) === null || _b === void 0 ? void 0 : _b.httpApi
+                    });
                     break;
                 case 'download':
                     dataset = yield driver.download(dataset);
                     break;
             }
-            switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
+            switch ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) {
                 case 'CSV':
                 case 'TXT':
                     return { data: lines, dataset, dataType: 'lines-of-text' };

package/engines/validation/Validator.js

@@ -51,6 +51,23 @@ class ValidatorClass {
                     errors.push(`Missing parameter "source" in producer`);
                 if (producer.dimensions.some(x => x.name.includes('{') || x.name.includes('[')))
                     errors.push(`Invalid dimension name found in producer "${producer.name}": can't use characters "{" or "[" in dimension names`);
+                // Validate sourceFilename dimension usage
+                const sourceFilenameDimensions = producer.dimensions.filter(x => x.sourceFilename === true);
+                if (sourceFilenameDimensions.length > 1) {
+                    errors.push(`Producer "${producer.name}" has multiple dimensions with sourceFilename=true. Only one dimension can have this flag.`);
+                }
+                if (sourceFilenameDimensions.length > 0) {
+                    const source = Environment_1.default.getSource(producer.source);
+                    if (source) {
+                        const validEngines = ['local', 'aws-s3'];
+                        if (!validEngines.includes(source.engine)) {
+                            errors.push(`Producer "${producer.name}" has a dimension with sourceFilename=true but the source engine "${source.engine}" doesn't support this feature. Only "local" and "aws-s3" sources support sourceFilename.`);
+                        }
+                        if (!producer.settings.fileKey && !producer.settings.fileType) {
+                            errors.push(`Producer "${producer.name}" has a dimension with sourceFilename=true but is not a file-based producer. sourceFilename requires fileKey and fileType to be set.`);
+                        }
+                    }
+                }
             }
             catch (e) {
                 if (errors.length === 0)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.1.9",
+    "version": "0.2.1",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -61,6 +61,7 @@
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "seedrandom": "^3.0.5",
+        "uuid": "^13.0.0",
         "workerpool": "^9.3.3",
         "xlsx": "^0.18.5",
         "zod": "^3.24.2"