@forzalabs/remora 1.0.13 → 1.0.15

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '1.0.1',
+    cliVersion: '1.0.14',
     backendVersion: 1,
     backendPort: 5088,
     workerVersion: 2,
@@ -12,6 +12,7 @@ const CONSTANTS = {
      */
     SOURCE_FILENAME_COLUMN: '$source_filename',
     defaults: {
+        REMORA_PATH: './remora',
         PRODUCER_TEMP_FOLDER: '.temp',
         SQL_MAX_QUERY_ROWS: 10000,
         STRING_MAX_CHARACTERS_LENGTH: 10000000,

package/database/DatabaseEngine.js

@@ -20,17 +20,18 @@ class DatabaseEngineClass {
         this.MAX_TRY_CONNECTION = 3;
         this.db = () => this._db;
         this.connect = () => __awaiter(this, void 0, void 0, function* () {
-            var _a;
             // WARNING: this was changed during the deployment to ECS...
             // I've reverted it, but maybe it needs to be changed or looked into...
+            var _a;
             this._uri = ((_a = process.env.MONGO_URI) !== null && _a !== void 0 ? _a : Helper_1.default.isDev())
                 ? 'mongodb://mongo:27017/remora'
                 : 'mongodb://localhost:27017/remora';
             this._client = new mongodb_1.MongoClient(this._uri);
             const errors = [];
+            this._client = new mongodb_1.MongoClient(this._uri);
             for (let i = 0; i < this.MAX_TRY_CONNECTION; i++) {
                 try {
-                    console.log(`Attempting to connect to mongo: "${this._uri}" (${i})`);
+                    console.log(`Attempting to connect to mongo "${this._uri}`);
                     yield this._client.connect();
                     this._db = this._client.db(Settings_1.default.db.name);
                     this._connected = true;

package/definitions/json_schemas/consumer-schema.json

@@ -57,29 +57,6 @@
                             "additionalProperties": false
                         }
                     },
-                    "custom": {
-                        "type": "object",
-                        "description": "Custom code to extract fields from producers, transform them, and return results",
-                        "properties": {
-                            "language": {
-                                "type": "string",
-                                "enum": [
-                                    "js",
-                                    "python"
-                                ],
-                                "description": "The language of the custom code"
-                            },
-                            "code": {
-                                "type": "string",
-                                "description": "The custom code to execute"
-                            }
-                        },
-                        "required": [
-                            "language",
-                            "code"
-                        ],
-                        "additionalProperties": false
-                    },
                     "union": {
                         "type": "boolean",
                         "description": "Merges the data from the various producers in a single dataset. They must have the same output dimensions. If true, then you can't set any joins on any producer, since all producers are merged in a single dataset."
@@ -392,6 +369,45 @@
                 "distinct": {
                     "type": "boolean",
                     "description": "If true, then the result set will only contain DISTINCT values"
+                },
+                "distinctOn": {
+                    "type": "object",
+                    "description": "Performs a distinct operation on specific key(s) and applies collision resolution rules to determine which record to keep when duplicates are found",
+                    "properties": {
+                        "keys": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "minItems": 1,
+                            "description": "The field(s) to use for determining uniqueness. Use the 'alias' if specified. Can be a single field or multiple fields for composite keys."
+                        },
+                        "resolution": {
+                            "type": "object",
+                            "description": "Rules to determine which record to keep when duplicates are found",
+                            "properties": {
+                                "strategy": {
+                                    "type": "string",
+                                    "enum": ["first", "last", "min", "max"],
+                                    "description": "Strategy for resolving which record to keep. 'first': Keep the first record based on orderBy field. 'last': Keep the last record based on orderBy field. 'min': Keep the record with the minimum value in orderBy field. 'max': Keep the record with the maximum value in orderBy field."
+                                },
+                                "orderBy": {
+                                    "type": "string",
+                                    "description": "Required for 'first', 'last', 'min', 'max' strategies. The field to use for ordering/comparison when selecting the record to keep."
+                                },
+                                "direction": {
+                                    "type": "string",
+                                    "enum": ["asc", "desc"],
+                                    "default": "asc",
+                                    "description": "For 'first' and 'last' strategies, the sort direction."
+                                }
+                            },
+                            "required": ["strategy"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["keys", "resolution"],
+                    "additionalProperties": false
                 }
             },
             "additionalProperties": false

package/drivers/DeltaShareDriver.js

@@ -177,8 +177,8 @@ class DeltaShareSourceDriver {
                 .map(x => JSON.parse(x));
             return deltaLines;
         });
-        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
-            void producer;
+        this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
+            void request;
             throw new Error('DeltaShareSourceDriver.ready is not supported: Delta Sharing does not support readiness checks');
         });
     }

package/drivers/HttpApiDriver.js

@@ -198,10 +198,10 @@ class HttpApiSourceDriver {
             }
             return itemsData;
         };
-        this.ready = (producer) => {
-            void producer;
+        this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
+            void request;
             throw new Error('Not implemented yet');
-        };
+        });
     }
 }
 exports.HttpApiSourceDriver = HttpApiSourceDriver;

package/drivers/RedshiftDriver.js

@@ -175,8 +175,8 @@ class RedshiftDriver {
             }
             return records;
         };
-        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
-            void producer;
+        this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
+            void request;
             throw new Error('Not implemented yet');
         });
     }

package/drivers/files/LocalDestinationDriver.js

@@ -57,7 +57,6 @@ const promises_1 = __importDefault(require("fs/promises"));
 const readline_1 = __importDefault(require("readline"));
 const path_1 = __importDefault(require("path"));
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const FileExporter_1 = __importDefault(require("../../engines/file/FileExporter"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
 class LocalDestinationDriver {
     constructor() {
@@ -70,57 +69,6 @@ class LocalDestinationDriver {
             this._path = source.authentication['path'];
             return this;
         });
-        this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(options, 'Invalid upload options');
-            (0, Affirm_1.default)(options.name, 'File name is required');
-            (0, Affirm_1.default)(options.content != null, 'File content is required');
-            const folder = this._path;
-            try {
-                if (!fs.existsSync(folder))
-                    fs.mkdirSync(folder, { recursive: true });
-                const filePath = path_1.default.join(folder, options.name);
-                fs.writeFileSync(filePath, options.content);
-                return { bucket: folder, key: filePath, res: true };
-            }
-            catch (error) {
-                throw new Error(`Failed to upload local file "${options.name}": ${error.message}`);
-            }
-        });
-        this.uploadStream = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(options, `Invalid upload options`);
-            const { dataset, name, recordProjection } = options;
-            (0, Affirm_1.default)(dataset, 'No streaming dataset');
-            (0, Affirm_1.default)(name, 'No filename provided for upload stream');
-            (0, Affirm_1.default)(recordProjection, 'No recordProjection for upload stream');
-            const folder = this._path;
-            try {
-                if (!fs.existsSync(folder))
-                    fs.mkdirSync(folder, { recursive: true });
-                const filePath = path_1.default.join(folder, options.name);
-                fs.writeFileSync(filePath, '');
-                yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
-                    const chunks = FileExporter_1.default.prepareBatch(batch, options);
-                    for (const chunk of chunks)
-                        fs.appendFileSync(filePath, chunk);
-                }));
-                return { bucket: folder, key: filePath, res: true };
-            }
-            catch (error) {
-                // Clean up the partial file if it exists
-                const filePath = path_1.default.join(folder, options.name);
-                if (fs.existsSync(filePath)) {
-                    try {
-                        fs.unlinkSync(filePath);
-                    }
-                    catch (cleanupError) {
-                        console.error(`Failed to clean up partial file after error: ${cleanupError.message}`);
-                        throw cleanupError;
-                    }
-                }
-                throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
-            }
-        });
         this.saveFile = (fileKey, content) => {
             (0, Affirm_1.default)(this._path, 'Path not initialized');
             (0, Affirm_1.default)(fileKey, 'Invalid file key');
@@ -150,9 +98,6 @@ class LocalDestinationDriver {
             const fileContent = yield s3Driver.downloadFile(sourceFileKey);
             yield this.saveFile(destinationFileKey, fileContent);
         });
-        this.ready = (destinationPath) => __awaiter(this, void 0, void 0, function* () {
-            return fs.createWriteStream(destinationPath);
-        });
         this.move = (fromPath, toName) => __awaiter(this, void 0, void 0, function* () {
             try {
                 const toFilePath = path_1.default.join(this._path, toName);

package/drivers/files/LocalSourceDriver.js

@@ -374,8 +374,9 @@ class LocalSourceDriver {
             }
             fs.renameSync(sourceFilePath, destinationFilePath);
         };
-        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
+        this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(request, 'Invalid ready request');
+            const { producer } = request;
             // TODO: extra logic for encoded files (xml, xls, ...) to be decoded and prepared locally as a plain CSV
             // then return the uri to this new temporary file
             const { fileKey } = producer.settings;

package/drivers/s3/S3DestinationDriver.js

@@ -8,6 +8,26 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = Object.create((typeof AsyncIterator === "function" ? AsyncIterator : Object).prototype), verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -15,7 +35,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const client_s3_1 = require("@aws-sdk/client-s3");
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const SecretManager_1 = __importDefault(require("../../engines/SecretManager"));
-const FileExporter_1 = __importDefault(require("../../engines/file/FileExporter"));
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const readline_1 = __importDefault(require("readline"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -33,32 +55,74 @@ class S3DestinationDriver {
             // TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
             return this;
         });
-        this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(options, `Invalid upload options`);
-            const { content, name } = options;
-            const commandParams = {
+        this.copyFromS3 = (sourceBucket, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(sourceBucket, 'Invalid source bucket');
+            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            yield this._client.send(new client_s3_1.CopyObjectCommand({
+                CopySource: `${sourceBucket}/${sourceFileKey}`,
+                Bucket: this._bucketName,
+                Key: destinationFileKey
+            }));
+        });
+        this.saveFile = (fileKey, content) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(content, 'Invalid content');
+            yield this._client.send(new client_s3_1.PutObjectCommand({
                 Bucket: this._bucketName,
-                Key: name,
+                Key: fileKey,
                 Body: content
+            }));
+        });
+        this.move = (fromPath, toName) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(fromPath, 'Invalid source path');
+            (0, Affirm_1.default)(toName, 'Invalid destination name');
+            (0, Affirm_1.default)(fs_1.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
+            const readStream = fs_1.default.createReadStream(fromPath);
+            return this._multipartUpload(toName, readStream);
+        });
+        this.transformAndMove = (fromPath, transform, toName) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(fromPath, 'Invalid source path');
+            (0, Affirm_1.default)(transform, 'Invalid transform function');
+            (0, Affirm_1.default)(toName, 'Invalid destination name');
+            (0, Affirm_1.default)(fs_1.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
+            const reader = fs_1.default.createReadStream(fromPath);
+            const lineReader = readline_1.default.createInterface({ input: reader, crlfDelay: Infinity });
+            const transformedLines = function () {
+                return __asyncGenerator(this, arguments, function* () {
+                    var _a, e_1, _b, _c;
+                    try {
+                        for (var _d = true, lineReader_1 = __asyncValues(lineReader), lineReader_1_1; lineReader_1_1 = yield __await(lineReader_1.next()), _a = lineReader_1_1.done, !_a; _d = true) {
+                            _c = lineReader_1_1.value;
+                            _d = false;
+                            const line = _c;
+                            yield yield __await(transform(line) + '\n');
+                        }
+                    }
+                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                    finally {
+                        try {
+                            if (!_d && !_a && (_b = lineReader_1.return)) yield __await(_b.call(lineReader_1));
+                        }
+                        finally { if (e_1) throw e_1.error; }
+                    }
+                });
             };
-            const command = new client_s3_1.PutObjectCommand(commandParams);
-            const res = yield this._client.send(command);
-            (0, Affirm_1.default)(res.$metadata.httpStatusCode === 200, `Failed to upload the file "${name}" to the bucket "${this._bucketName}": status code ${res.$metadata.httpStatusCode}`);
-            return { res: true, key: name, bucket: this._bucketName };
+            return this._multipartUpload(toName, transformedLines());
         });
-        this.uploadStream = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(options, `Invalid upload options`);
-            const { dataset, name, recordProjection } = options;
-            (0, Affirm_1.default)(dataset, 'No streaming dataset');
-            (0, Affirm_1.default)(name, 'No filename provided for upload stream');
-            (0, Affirm_1.default)(recordProjection, 'No recordProjection for upload stream');
+        this._multipartUpload = (toName, dataSource) => __awaiter(this, void 0, void 0, function* () {
+            var _a, dataSource_1, dataSource_1_1;
+            var _b, e_2, _c, _d;
+            let uploadId;
             try {
                 // Create the multipart upload
                 const createMultipartUploadRes = yield this._client.send(new client_s3_1.CreateMultipartUploadCommand({
                     Bucket: this._bucketName,
-                    Key: name
+                    Key: toName
                 }));
-                const uploadId = createMultipartUploadRes.UploadId;
+                uploadId = createMultipartUploadRes.UploadId;
                 (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
                 const uploadedParts = [];
                 let partNumber = 1;
@@ -67,7 +131,7 @@ class S3DestinationDriver {
                 const uploadPart = (buffer) => __awaiter(this, void 0, void 0, function* () {
                     const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
                         Bucket: this._bucketName,
-                        Key: name,
+                        Key: toName,
                         UploadId: uploadId,
                         PartNumber: partNumber,
                         Body: buffer
@@ -78,18 +142,28 @@ class S3DestinationDriver {
                     });
                     partNumber++;
                 });
-                yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
-                    const chunks = FileExporter_1.default.prepareBatch(batch, options);
-                    for (const chunk of chunks) {
-                        const chunkBuffer = Buffer.from(chunk);
+                try {
+                    for (_a = true, dataSource_1 = __asyncValues(dataSource); dataSource_1_1 = yield dataSource_1.next(), _b = dataSource_1_1.done, !_b; _a = true) {
+                        _d = dataSource_1_1.value;
+                        _a = false;
+                        const chunk = _d;
+                        const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
                         accumulatedBuffer = Buffer.concat([accumulatedBuffer, chunkBuffer]);
                         // If accumulated buffer is at least 5MB, upload it as a part
-                        if (accumulatedBuffer.length >= MIN_PART_SIZE) {
-                            yield uploadPart(accumulatedBuffer);
-                            accumulatedBuffer = Buffer.alloc(0);
+                        while (accumulatedBuffer.length >= MIN_PART_SIZE) {
+                            const partBuffer = accumulatedBuffer.subarray(0, MIN_PART_SIZE);
+                            accumulatedBuffer = accumulatedBuffer.subarray(MIN_PART_SIZE);
+                            yield uploadPart(partBuffer);
                         }
                     }
-                }));
+                }
+                catch (e_2_1) { e_2 = { error: e_2_1 }; }
+                finally {
+                    try {
+                        if (!_a && !_b && (_c = dataSource_1.return)) yield _c.call(dataSource_1);
+                    }
+                    finally { if (e_2) throw e_2.error; }
+                }
                 // Upload any remaining data as the final part (even if smaller than 5MB)
                 if (accumulatedBuffer.length > 0) {
                     yield uploadPart(accumulatedBuffer);
@@ -97,63 +171,27 @@ class S3DestinationDriver {
                 // Complete the multipart upload
                 const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
                     Bucket: this._bucketName,
-                    Key: options.name,
+                    Key: toName,
                     UploadId: uploadId,
                     MultipartUpload: {
                         Parts: uploadedParts
                     }
                 }));
-                (0, Affirm_1.default)(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${options.name}": status code ${completeRes.$metadata.httpStatusCode}`);
-                return { res: true, key: options.name, bucket: this._bucketName };
+                (0, Affirm_1.default)(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${toName}": status code ${completeRes.$metadata.httpStatusCode}`);
+                return { res: true, key: path_1.default.join(this._bucketName, toName), bucket: this._bucketName };
             }
             catch (error) {
                 // If anything fails, make sure to abort the multipart upload
-                if (error.UploadId) {
+                if (uploadId) {
                     yield this._client.send(new client_s3_1.AbortMultipartUploadCommand({
                         Bucket: this._bucketName,
-                        Key: options.name,
-                        UploadId: error.UploadId
+                        Key: toName,
+                        UploadId: uploadId
                     }));
                 }
                 throw error;
             }
         });
-        this.copyFromS3 = (sourceBucket, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
-            (0, Affirm_1.default)(sourceBucket, 'Invalid source bucket');
-            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
-            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
-            yield this._client.send(new client_s3_1.CopyObjectCommand({
-                CopySource: `${sourceBucket}/${sourceFileKey}`,
-                Bucket: this._bucketName,
-                Key: destinationFileKey
-            }));
-        });
-        this.saveFile = (fileKey, content) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
-            (0, Affirm_1.default)(fileKey, 'Invalid file key');
-            (0, Affirm_1.default)(content, 'Invalid content');
-            yield this._client.send(new client_s3_1.PutObjectCommand({
-                Bucket: this._bucketName,
-                Key: fileKey,
-                Body: content
-            }));
-        });
-        this.ready = (destinationPath) => {
-            void destinationPath;
-            throw new Error('Not implemented yet');
-        };
-        this.move = (fromPath, toName) => {
-            void fromPath;
-            void toName;
-            throw new Error('Not implemented yet');
-        };
-        this.transformAndMove = (fromPath, transform, toName) => {
-            void fromPath;
-            void toName;
-            void transform;
-            throw new Error('Not implemented yet');
-        };
     }
 }
 exports.default = S3DestinationDriver;

package/drivers/s3/S3SourceDriver.js

@@ -22,8 +22,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const client_s3_1 = require("@aws-sdk/client-s3");
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const SecretManager_1 = __importDefault(require("../../engines/SecretManager"));
+const promises_1 = require("stream/promises");
 const readline_1 = __importDefault(require("readline"));
 const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
 const XMLParser_1 = __importDefault(require("../../engines/parsing/XMLParser"));
@@ -33,6 +35,7 @@ const DriverHelper_1 = __importDefault(require("../DriverHelper"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
 const Constants_1 = __importDefault(require("../../Constants"));
 const XLSParser_1 = __importDefault(require("../../engines/parsing/XLSParser"));
+const ExecutorScope_1 = __importDefault(require("../../executors/ExecutorScope"));
 class S3SourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -446,10 +449,47 @@ class S3SourceDriver {
                 Key: destinationFileKey
             }));
         });
-        this.ready = (producer) => {
-            void producer;
-            throw new Error('Not implemented yet');
-        };
+        this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(request, 'Invalid producer');
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            const { producer, scope } = request;
+            const { fileKey } = producer.settings;
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const streamToFile = (s3Key, localPath) => __awaiter(this, void 0, void 0, function* () {
+                const command = new client_s3_1.GetObjectCommand({
+                    Bucket: this._bucketName,
+                    Key: s3Key
+                });
+                const response = yield this._client.send(command);
+                (0, Affirm_1.default)(response.Body, `Failed to fetch object "${s3Key}" from S3`);
+                // Ensure the directory for the file exists
+                const fileDir = path_1.default.dirname(localPath);
+                if (!fs_1.default.existsSync(fileDir)) {
+                    fs_1.default.mkdirSync(fileDir, { recursive: true });
+                }
+                const writeStream = fs_1.default.createWriteStream(localPath);
+                yield (0, promises_1.pipeline)(response.Body, writeStream);
+            });
+            if (fileKey.includes('%')) {
+                const allFileKeys = yield this.listFiles(fileKey);
+                Affirm_1.default.hasItems(allFileKeys, `The file key pattern "${fileKey}" doesn't have any matches in bucket "${this._bucketName}".`);
+                // Stream each file to local temp storage sequentially to avoid overwhelming the connection
+                const allFilePaths = [];
+                for (const s3Key of allFileKeys) {
+                    const localPath = ExecutorScope_1.default.getProducerPath(scope, producer, s3Key);
+                    ExecutorScope_1.default.ensurePath(localPath);
+                    yield streamToFile(s3Key, localPath);
+                    allFilePaths.push(localPath);
+                }
+                return { files: allFilePaths.map(x => ({ fullUri: x })) };
+            }
+            else {
+                const localPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
+                ExecutorScope_1.default.ensurePath(localPath);
+                yield streamToFile(fileKey, localPath);
+                return { files: [{ fullUri: localPath }] };
+            }
+        });
     }
 }
 exports.default = S3SourceDriver;

package/engines/Environment.js

@@ -45,7 +45,7 @@ class EnvironmentClass {
                         }
                     }
                     catch (error) {
-                        throw new Error(`Error loading configuration from ${configPath}: ${error.message}`);
+                        throw new Error(`Error loading from ${path_1.default.resolve(remoraPath)} configuration from ${configPath}: ${error.message}`);
                     }
                 }
                 return configs;

package/engines/parsing/LineParser.js

@@ -46,6 +46,25 @@ class LineParserClass {
                     throw new Error(`File type ${fileType} not implemented yet.`);
             }
         };
+        /**
+         * Used ONLY to parse internal records (inside the .dataset) since I know they are already prepared in the correct way
+         */
+        this._internalParseCSV = (line, fields, delimiter) => {
+            var _a, _b;
+            const parts = CSVParser_1.default.parseRow(line, delimiter !== null && delimiter !== void 0 ? delimiter : ',');
+            const record = {};
+            for (const [index, field] of fields.entries()) {
+                const fieldKey = field.finalKey;
+                record[fieldKey] = TypeCaster_1.default.cast(parts[index], (_b = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string');
+            }
+            return record;
+        };
+        /**
+         * Used ONLY to parse internal records (inside the .dataset) since I know they are already prepared in the correct way
+         */
+        this._internalParseJSON = (line) => {
+            return JSON.parse(line);
+        };
     }
 }
 const LineParser = new LineParserClass();

package/engines/validation/Validator.js

@@ -119,11 +119,11 @@ class ValidatorClass {
                 // For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
                 const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
                 if (uniqEngines.length !== 1)
-                    errors.push(`Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
+                    errors.push(`Sources with different engines are used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
                 // For now we also only support consumers that have producers ALL having the same exact source
                 const uniqNames = Algo_1.default.uniqBy(sources, 'name');
                 if (uniqNames.length !== 1)
-                    errors.push(`Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
+                    errors.push(`Producers with different sources are used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
                 if (consumer.filters && consumer.filters.length > 0) {
                     if (consumer.filters.some(x => x.sql && x.rule))
                         errors.push(`A single consumer can't have both filters based on SQL and filters based on rules.`);
@@ -188,6 +188,11 @@ class ValidatorClass {
                             errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
                     }
                 }
+                // Validate distinct
+                if (consumer.options) {
+                    if (Algo_1.default.hasVal(consumer.options.distinct) && Algo_1.default.hasVal(consumer.options.distinctOn))
+                        errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+                }
             }
             catch (e) {
                 if (errors.length === 0)