@forzalabs/remora 0.0.12

package/documentation/default_resources/project.json

@@ -0,0 +1,14 @@
+{
+    "name": "PROJECT_NAME_EXAMPLE",
+    "version": "1.0.0",
+    "description": "DESCRIPTION_OF_THIS_PROJECT",
+    "consumers": ["/consumers"],
+    "producers": ["/producers"],
+    "sources": ["/sources"],
+    "schemas": ["/schemas"],
+    "settings": {
+        "SQL_MAX_QUERY_ROWS": 10000,
+        "DEFAULT_TIMEOUT_SECONDS": 300,
+        "LOG_LEVEL": "INFO"
+    }
+}

package/documentation/default_resources/schema.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "<http://json-schema.org/draft-07/schema#>",
+  "title": "<schema title>",
+  "type": "<object>",
+  "version": 1,
+  "description": "<schema description>",
+  "properties": {
+    "<required id field>": {
+      "type": "<number | string | etc.>",
+      "description": "<field description>"
+    },
+    "<required name field>": {
+      "type": "<string>",
+      "description": "<field description>"
+    },
+    "<status field with enum>": {
+      "type": "<string>",
+      "description": "<field description>",
+      "enum": [
+        "<example status 1>",
+        "<example status 2>",
+        "<example status 3>"
+      ]
+    },
+    "<datetime field example>": {
+      "type": "<string>",
+      "format": "<date-time>",
+      "description": "<field description>"
+    }
+  },
+  "required": [
+    "<required id field>",
+    "<required name field>"
+  ],
+  "additionalProperties": false
+}

package/documentation/default_resources/source.json

@@ -0,0 +1,15 @@
+{
+  "name": "<source name>",
+  "description": "<source description>",
+  "_version": 1,
+  "authentication": {
+    "method": "<iam | username-password | access-secret-key | arn | implicit>",
+    "schema": "<database_schema_name - db schema>",
+    "accessKey": "<access key var from env>",
+    "secretKey": "<secret key var from env>",
+    "region": "<aws region>",
+    "database": "<db name>",
+    "workgroup": "<db workgroup>"
+  },
+  "engine": "<aws-redshift | aws-dynamodb | aws-s3 | postgres | local>"
+}

package/drivers/DriverFactory.js

@@ -0,0 +1,56 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const LocalDriver_1 = __importDefault(require("./LocalDriver"));
+const RedshiftDriver_1 = __importDefault(require("./RedshiftDriver"));
+const S3Driver_1 = __importDefault(require("./S3Driver"));
+const S3SourceDriver_1 = __importDefault(require("./S3SourceDriver"));
+class DriverFactoryClass {
+    constructor() {
+        this.instantiateSource = (source) => __awaiter(this, void 0, void 0, function* () {
+            switch (source.engine) {
+                // TODO: implement all the other engines
+                case 'aws-redshift': {
+                    const driver = new RedshiftDriver_1.default();
+                    yield driver.init(source);
+                    return driver;
+                }
+                case 'aws-s3': {
+                    const driver = new S3SourceDriver_1.default();
+                    yield driver.init(source);
+                    return driver;
+                }
+                case 'local': {
+                    const driver = new LocalDriver_1.default();
+                    yield driver.init(source);
+                    return driver;
+                }
+                default: throw new Error(`Invalid driver type "${source.engine}". This driver is not implemented yet`);
+            }
+        });
+        this.instantiateDestination = (source) => __awaiter(this, void 0, void 0, function* () {
+            switch (source.engine) {
+                // TODO: implement all the other engines
+                case 'aws-s3': {
+                    const driver = new S3Driver_1.default();
+                    yield driver.init(source);
+                    return driver;
+                }
+                default: throw new Error(`Invalid driver type "${source.engine}". This driver is not implemented yet`);
+            }
+        });
+    }
+}
+const DriverFactory = new DriverFactoryClass();
+exports.default = DriverFactory;

package/drivers/LocalDriver.js

@@ -0,0 +1,122 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const path_1 = __importDefault(require("path"));
+const readline_1 = __importDefault(require("readline"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+class LocalDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            const fileURL = source.authentication['path'];
+            (0, Affirm_1.default)(fileURL, 'file path is not entered in configuration');
+            const exist = fs.existsSync(fileURL);
+            (0, Affirm_1.default)(exist, 'The path entered is incorrect: ' + fileURL);
+            this._path = source.authentication['path'];
+            return this;
+        });
+        this.download = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(request, `Invalid download request`);
+            (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
+            const fileUrl = path_1.default.join(this._path, request.fileKey);
+            const data = yield fs.promises.readFile(fileUrl, 'utf-8');
+            return data;
+        });
+        this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(readOptions, 'Invalid read options');
+            (0, Affirm_1.default)(readOptions.fileKey, 'Invalid file key');
+            const fileUrl = path_1.default.join(this._path, readOptions.fileKey);
+            const { lineFrom, lineTo } = readOptions;
+            const stream = fs.createReadStream(fileUrl);
+            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            const lines = [];
+            let lineCounter = 0;
+            try {
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    if (lineCounter >= lineFrom && lineCounter < lineTo) {
+                        lines.push(line);
+                    }
+                    lineCounter++;
+                    if (lineCounter >= lineTo)
+                        break;
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            reader.close();
+            stream.close();
+            return lines;
+        });
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            const fileKey = producer.settings.fileKey;
+            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
+            const fileUrl = path_1.default.join(this._path, fileKey);
+            return fs.existsSync(fileUrl);
+        });
+    }
+}
+exports.default = LocalDriver;

package/drivers/RedshiftDriver.js

@@ -0,0 +1,179 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const client_redshift_data_1 = require("@aws-sdk/client-redshift-data");
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+class RedshiftDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(source, `Invalid source`);
+            this._SQL_MAX_QUERY_ROWS = parseInt(Environment_1.default.get('SQL_MAX_QUERY_ROWS'));
+            this._dbName = source.authentication['database'];
+            this._workgroup = source.authentication['workgroup'];
+            const config = {
+                region: source.authentication['region'],
+                credentials: {
+                    accessKeyId: source.authentication['accessKey'],
+                    secretAccessKey: source.authentication['secretKey']
+                    // sessionToken: 'IQoJb3JpZ2luX2VjEDQaCXVzLWVhc3QtMSJIMEYCIQDXHjVYaGyCWCMgYWmnYt652z02H+w0jia8l/ckgu+gSAIhAOaKqkWbitaywpIy4l3lXQfQqW/fYhd187jRsL0kkm+wKokDCCwQABoMNzMwMzM1MTgwOTE5Igz36Yz4UuTQ6KuP5lcq5gKi1zczRVbAI6fnKQGNqg2k18/wJFpQgINNW+Ae9BoFofarAlOkXO5qeloP/5dGMufPVu/hy4tqaDi0Jpz8Zpa1JJjj6Q4l7XZi0+SpCkWwoVRsM/8ojMzowTWnUzan17I0jnG4TQKVEABQjQEACbJMbPwHNm1cN+dEY14cTyA36yJ0ma+8GDLECYnBM4klHKxzYJbD2tdwSgW4KAZ2ttk9+AnrKJUVH4hDI4cPBRv8uIUKGGfzD8cRpMGCByYeLVPc0o5IFCZGqkHncRlAU7dH0xZyGyCeDo3Z3819Ll8vJ41RN4RQOwcWrmL1Ne73KRgH32Ya9xH5j052WxERJPouCk+Lj307XmL0YwpkiQtV7cK5bQQRHtihviGkiK4maZLXXy7aUPQQG57xc0Q3HJXdI6LiXhvKvPKQm2iwAQS+FUZeekf1msT6MkVWd1ibYyhMzFeHBBPvVDlc4NFtLH6rGKAGgkuaMLGunrwGOqUB+nVvTAgcT7amelZRma0TRJsfFezBCAXXWNurD/6qAAKpKgmP+ouE+gH+SmG67IbQqIzK/5dc53sMV7dXDdG4/XGvmWfTph1iHZN/c1bd/rYPfVzy3K3ybTe2Djj0IaTbNndxVPfWIU6cC6cJ6ahEMvzokNnKNUqdDnu/Z1S2xiOfV4P/9vTg7O1XuLUylsCzveeYWRoZJPY4XCUnOgNlqQ/ZwKTC'
+                }
+            };
+            try {
+                this._client = new client_redshift_data_1.RedshiftData(config);
+                // This is just to check that the connection was successful and warm up the connection
+                yield this.query('SELECT :diss AS number', [{ name: ':diss', value: '1' }]);
+            }
+            catch (error) {
+                const myError = error;
+                if (myError.__type === 'UnrecognizedClientException')
+                    throw new Error(`Invalid AWS credentials for source "${source.name}": ${myError.message}`);
+                else
+                    throw error;
+            }
+            return this;
+        });
+        this.execute = (sql) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            (0, Affirm_1.default)(sql, `Invalid SQL`);
+            const input = {
+                Sql: sql,
+                Database: this._dbName,
+                WorkgroupName: this._workgroup
+            };
+            const executeStatementRes = yield this._client.executeStatement(input);
+            const queryId = (_a = executeStatementRes.Id) !== null && _a !== void 0 ? _a : '';
+            const res = (yield this.executeStatement(queryId, { retry: 5 }))[0];
+            console.log(res);
+            return {
+                rows: res.result
+            };
+        });
+        this.query = (sql, values) => __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            (0, Affirm_1.default)(sql, `Invalid SQL`);
+            const wrapperSql = `SELECT * FROM (${sql}) LIMIT ${this._SQL_MAX_QUERY_ROWS}`;
+            const input = {
+                Sql: wrapperSql,
+                Database: this._dbName,
+                WorkgroupName: this._workgroup,
+                Parameters: (_a = values === null || values === void 0 ? void 0 : values.map(x => ({ name: Algo_1.default.replaceAll(x.name, ':', ''), value: x.value }))) !== null && _a !== void 0 ? _a : undefined
+            };
+            const executeStatementRes = yield this._client.executeStatement(input);
+            const queryId = (_b = executeStatementRes.Id) !== null && _b !== void 0 ? _b : '';
+            // TODO: handle errors (permission, wrong query, ...)
+            const res = (yield this.executeStatement(queryId, { retry: 5 }))[0];
+            console.log(res);
+            return {
+                rows: res.result
+            };
+        });
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            const sql = `SELECT * FROM "${producer.settings.sqlTable}" LIMIT 1`;
+            const input = {
+                Sql: sql,
+                Database: this._dbName,
+                WorkgroupName: this._workgroup
+            };
+            const executeStatementRes = yield this._client.executeStatement(input);
+            const queryId = (_a = executeStatementRes.Id) !== null && _a !== void 0 ? _a : '';
+            try {
+                yield this.executeStatement(queryId, { retry: 5 });
+                return true;
+            }
+            catch (e) {
+                return false;
+            }
+        });
+        this.executeStatement = (queryId, options) => __awaiter(this, void 0, void 0, function* () {
+            const MAX_WAIT_CYCLES = 20;
+            for (let i = 0; i < MAX_WAIT_CYCLES; i++) {
+                const statement = yield this._client.describeStatement({ Id: queryId });
+                (0, Affirm_1.default)(statement, 'SQL query failed: describe statement call failed.');
+                (0, Affirm_1.default)(statement.Status !== 'FAILED', `SQL query failed: ${queryId}:\n\tSQL: ${statement.QueryString}\n\tError: ${statement.Error}.`);
+                const { Status: status, SubStatements: subStatements } = statement;
+                if (status === 'FINISHED') {
+                    if (subStatements && subStatements.length > 0) {
+                        const result = [];
+                        for (let i = 0; i < subStatements.length; i++) {
+                            const subStatement = subStatements[i];
+                            result.push({
+                                statement: { hasResultSet: statement.HasResultSet, queryString: statement.QueryString, resultRows: statement.ResultRows, resultSize: statement.ResultSize },
+                                subStatement: { hasResultSet: subStatement.HasResultSet, queryString: subStatement.QueryString, resultRows: subStatement.ResultRows, resultSize: subStatement.ResultSize },
+                                result: yield this.getStatementResult(subStatement)
+                            });
+                        }
+                        return result;
+                    }
+                    else {
+                        return [{
+                                statement: { hasResultSet: statement.HasResultSet, queryString: statement.QueryString, resultRows: statement.ResultRows, resultSize: statement.ResultSize },
+                                result: yield this.getStatementResult(statement)
+                            }];
+                    }
+                }
+                yield Algo_1.default.sleep(1);
+            }
+            if (options && options.retry > 0)
+                return yield this.executeStatement(queryId, Object.assign(Object.assign({}, options), { retry: options.retry - 1 }));
+            else
+                throw new Error('SQL timeout: query failed because timeout exceeded max wait cycles');
+        });
+        this.getStatementResult = (statement) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(statement, `SQL failed: can't extract result from invalid statement`);
+            if (statement.HasResultSet) {
+                const result = yield this._client.getStatementResult({ Id: statement.Id });
+                (0, Affirm_1.default)(result, 'SQL failed: invalid result statement');
+                (0, Affirm_1.default)(result.Records, 'SQL failed: no records in result statement');
+                return this.parseQueryResult(result);
+            }
+            else {
+                return [];
+            }
+        });
+        this.parseQueryResult = (queryResult) => {
+            (0, Affirm_1.default)(queryResult, `Parse SQL failed: invalid query result`);
+            (0, Affirm_1.default)(queryResult.ColumnMetadata, `Parse SQL failed: invalid column metadata in query result`);
+            (0, Affirm_1.default)(queryResult.Records, `Parse SQL failed: invalid records in query result`);
+            const columnNames = queryResult.ColumnMetadata.map(column => column.name);
+            const records = [];
+            for (const record of queryResult.Records) {
+                const recordData = {};
+                for (let i = 0; i < columnNames.length; i++) {
+                    const columnName = columnNames[i];
+                    (0, Affirm_1.default)(columnName, `Parse SQL failed: invalid column name`);
+                    const value = record[i];
+                    if (value.longValue !== undefined)
+                        recordData[columnName] = value.longValue;
+                    else if (value.stringValue !== undefined)
+                        recordData[columnName] = value.stringValue;
+                    else if (value.booleanValue !== undefined)
+                        recordData[columnName] = value.booleanValue;
+                    else if (value.doubleValue !== undefined)
+                        recordData[columnName] = value.doubleValue;
+                    else if (value.isNull === true)
+                        recordData[columnName] = null;
+                    else
+                        recordData[columnName] = null;
+                }
+                records.push(recordData);
+            }
+            return records;
+        };
+    }
+}
+exports.default = RedshiftDriver;

package/drivers/S3Driver.js

@@ -0,0 +1,47 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const client_s3_1 = require("@aws-sdk/client-s3");
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+class S3Driver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            this._bucketName = source.authentication['bucketName'];
+            const config = {
+                region: source.authentication['region'],
+                credentials: {
+                    accessKeyId: source.authentication['accessKey'],
+                    secretAccessKey: source.authentication['secretKey']
+                }
+            };
+            this._client = new client_s3_1.S3Client(config);
+            // TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
+            return this;
+        });
+        this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(options, `Invalid upload options`);
+            const { content, name } = options;
+            const commandParams = {
+                Bucket: this._bucketName,
+                Key: name,
+                Body: content
+            };
+            const command = new client_s3_1.PutObjectCommand(commandParams);
+            const res = yield this._client.send(command);
+            (0, Affirm_1.default)(res.$metadata.httpStatusCode === 200, `Failed to upload the file "${name}" to the bucket "${this._bucketName}": status code ${res.$metadata.httpStatusCode}`);
+            return { res: true, key: name, bucket: this._bucketName };
+        });
+    }
+}
+exports.default = S3Driver;

package/drivers/S3SourceDriver.js

@@ -0,0 +1,127 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const client_s3_1 = require("@aws-sdk/client-s3");
+const readline_1 = __importDefault(require("readline"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+class S3SourceDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            this._bucketName = source.authentication['bucketName'];
+            const config = {
+                region: source.authentication['region'],
+                credentials: {
+                    accessKeyId: source.authentication['accessKey'],
+                    secretAccessKey: source.authentication['secretKey']
+                }
+            };
+            this._client = new client_s3_1.S3Client(config);
+            // TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
+            return this;
+        });
+        this.download = (request) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(request, `Invalid download request`);
+            (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
+            const { fileKey } = request;
+            const bucket = this._bucketName;
+            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
+                Bucket: bucket,
+                Key: fileKey
+            }));
+            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
+            const stream = response.Body;
+            const chunks = [];
+            try {
+                for (var _d = true, stream_1 = __asyncValues(stream), stream_1_1; stream_1_1 = yield stream_1.next(), _a = stream_1_1.done, !_a; _d = true) {
+                    _c = stream_1_1.value;
+                    _d = false;
+                    const chunk = _c;
+                    chunks.push(chunk);
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = stream_1.return)) yield _b.call(stream_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            return Buffer.concat(chunks).toString('utf-8');
+        });
+        this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_2, _b, _c;
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(readOptions, 'Invalid read options');
+            const { fileKey, lineFrom, lineTo } = readOptions;
+            const bucket = this._bucketName;
+            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
+                Bucket: bucket,
+                Key: fileKey
+            }));
+            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
+            const stream = response.Body;
+            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            const lines = [];
+            let lineCounter = 0;
+            try {
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    if (lineCounter >= lineFrom && lineCounter < lineTo)
+                        lines.push(line);
+                    if (lineCounter > lineTo)
+                        break;
+                    lineCounter++;
+                }
+            }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
+                }
+                finally { if (e_2) throw e_2.error; }
+            }
+            reader.close();
+            return lines;
+        });
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(producer, 'Invalid read producer');
+            const bucket = this._bucketName;
+            const fileKey = producer.settings.fileKey;
+            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
+            try {
+                yield this._client.send(new client_s3_1.HeadObjectCommand({ Bucket: bucket, Key: fileKey }));
+                return true;
+            }
+            catch (error) {
+                if (((_a = error.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode) === 404 || error.name === 'NotFound')
+                    return false;
+                throw error;
+            }
+        });
+    }
+}
+exports.default = S3SourceDriver;

package/engines/CryptoEngine.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const crypto_1 = __importDefault(require("crypto"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+class CryptoEngineClass {
+    constructor() {
+        this.hashQuery = (maskType, fieldReference, fieldName) => {
+            if (!Algo_1.default.hasVal(maskType))
+                return;
+            switch (maskType) {
+                case 'hash':
+                    return `SHA2(${fieldReference}, 256) AS "${fieldName}"`;
+                case 'crypt':
+                    throw new Error('Not implemented yet');
+                case 'mask':
+                    throw new Error('Not implemented yet');
+                default:
+                    throw new Error('This type is not ');
+            }
+        };
+        this.valueToHash = (value) => {
+            return crypto_1.default.createHash('sha256').update(JSON.stringify(value)).digest('hex');
+        };
+        this.hashValue = (maskType, value) => {
+            if (!Algo_1.default.hasVal(value))
+                return value;
+            if (!Algo_1.default.hasVal(maskType))
+                return value;
+            switch (maskType) {
+                case 'hash':
+                    return this.valueToHash(value);
+                case 'crypt':
+                    throw new Error('Not implemented yet');
+                case 'mask':
+                    throw new Error('Not implemented yet');
+                default:
+                    throw new Error(`This type doesn't exist`);
+            }
+        };
+    }
+}
+const CryptoEngine = new CryptoEngineClass();
+exports.default = CryptoEngine;