@forzalabs/remora 0.0.26 → 0.0.28

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.26',
+    cliVersion: '0.0.28',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/producer-schema.json

@@ -132,13 +132,24 @@
                     "enum": [
                         "JSON",
                         "JSONL",
-                        "CSV"
+                        "CSV", 
+                        "TXT",
+                        "XLS",
+                        "XLSX"
                     ],
                     "description": "The type of file to read"
                 },
                 "delimiter": {
                     "type": "string",
-                    "description": "The column delimiter for the CSV file if different from the default (,)."
+                    "description": "The column delimiter for CSV or TXT files if different from the default (,)."
+                },
+                "hasHeaderRow": {
+                    "type": "boolean",
+                    "description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."
+                },
+                "sheetName": {
+                    "type": "string", 
+                    "description": "For Excel files (.xls/.xlsx), specifies the name of the sheet to read data from. If not specified, the first sheet will be used."
                 }
             },
             "additionalProperties": false

package/definitions/json_schemas/source-schema.json

@@ -97,6 +97,10 @@
                 "clusterId": {
                     "type": "string",
                     "description": "Redshift cluster identifier"
+                },
+                "path": {
+                    "type": "string",
+                    "description": "The folder path"
                 }
             },
             "required": ["method"]

package/drivers/DriverFactory.js

@@ -1,37 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -45,10 +12,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const LocalDriver_1 = __importStar(require("./LocalDriver"));
+const LocalDriver_1 = require("./LocalDriver");
 const RedshiftDriver_1 = __importDefault(require("./RedshiftDriver"));
-const S3Driver_1 = __importDefault(require("./S3Driver"));
-const S3SourceDriver_1 = __importDefault(require("./S3SourceDriver"));
+const S3Driver_1 = require("./S3Driver");
 class DriverFactoryClass {
     constructor() {
         this.instantiateSource = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -60,12 +26,12 @@ class DriverFactoryClass {
                     return driver;
                 }
                 case 'aws-s3': {
-                    const driver = new S3SourceDriver_1.default();
+                    const driver = new S3Driver_1.S3SourceDriver();
                     yield driver.init(source);
                     return driver;
                 }
                 case 'local': {
-                    const driver = new LocalDriver_1.default();
+                    const driver = new LocalDriver_1.LocalSourceDriver();
                     yield driver.init(source);
                     return driver;
                 }
@@ -76,12 +42,12 @@ class DriverFactoryClass {
             switch (source.engine) {
                 // TODO: implement all the other engines
                 case 'aws-s3': {
-                    const driver = new S3Driver_1.default();
+                    const driver = new S3Driver_1.S3DestinationDriver();
                     yield driver.init(source);
                     return driver;
                 }
                 case 'local': {
-                    const driver = new LocalDriver_1.LocalDriverDestination();
+                    const driver = new LocalDriver_1.LocalDestinationDriver();
                     yield driver.init(source);
                     return driver;
                 }

package/drivers/LocalDriver.js

@@ -52,12 +52,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalDriverDestination = void 0;
+exports.LocalDestinationDriver = exports.LocalSourceDriver = void 0;
 const fs = __importStar(require("fs"));
 const path_1 = __importDefault(require("path"));
 const readline_1 = __importDefault(require("readline"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
-class LocalDriver {
+const Algo_1 = __importDefault(require("../core/Algo"));
+const xlsx_1 = __importDefault(require("xlsx"));
+class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             const fileURL = source.authentication['path'];
@@ -68,79 +70,109 @@ class LocalDriver {
             return this;
         });
         this.download = (request) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
             (0, Affirm_1.default)(this._path, `Invalid path`);
             (0, Affirm_1.default)(request, `Invalid download request`);
             (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
-            const fileUrl = path_1.default.join(this._path, request.fileKey);
-            const stream = fs.createReadStream(fileUrl);
-            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
-            const lines = [];
-            try {
-                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
-                    _c = reader_1_1.value;
-                    _d = false;
-                    const line = _c;
-                    lines.push(line);
-                }
+            (0, Affirm_1.default)(request.fileType, `Invalid file type for download request`);
+            const { fileKey, options } = request;
+            const fileUrl = path_1.default.join(this._path, fileKey);
+            switch (request.fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    return yield this._readLines(fileUrl);
+                case 'XLS':
+                case 'XLSX':
+                    return yield this._readExcelLines(fileUrl, options === null || options === void 0 ? void 0 : options.sheetName);
             }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
-                }
-                finally { if (e_1) throw e_1.error; }
+        });
+        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(request, 'Invalid read options');
+            (0, Affirm_1.default)(request.fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(request.fileType, `Invalid file type`);
+            (0, Affirm_1.default)(request.options, `Invalid request options`);
+            Affirm_1.default.hasValue(request.options.lineFrom, `Invalid request options line from`);
+            Affirm_1.default.hasValue(request.options.lineTo, `Invalid request options line to`);
+            const { fileKey, fileType, options: { lineFrom, lineTo, sheetName } } = request;
+            const fileUrl = path_1.default.join(this._path, fileKey);
+            switch (fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    return yield this._readLines(fileUrl, lineFrom, lineTo);
+                case 'XLS':
+                case 'XLSX':
+                    return yield this._readExcelLines(fileUrl, sheetName, lineFrom, lineTo);
             }
-            reader.close();
-            stream.close();
-            return lines;
         });
-        this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_2, _b, _c;
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(readOptions, 'Invalid read options');
-            (0, Affirm_1.default)(readOptions.fileKey, 'Invalid file key');
-            const fileUrl = path_1.default.join(this._path, readOptions.fileKey);
-            const { lineFrom, lineTo } = readOptions;
-            const stream = fs.createReadStream(fileUrl);
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            const fileKey = producer.settings.fileKey;
+            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
+            const fileUrl = path_1.default.join(this._path, fileKey);
+            return fs.existsSync(fileUrl);
+        });
+        this._readLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const stream = fs.createReadStream(fileUri);
             const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
             const lines = [];
             let lineCounter = 0;
             try {
-                for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
-                    _c = reader_2_1.value;
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
                     _d = false;
                     const line = _c;
-                    if (lineCounter >= lineFrom && lineCounter < lineTo) {
+                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                        if (lineCounter >= lineFrom && lineCounter < lineTo) {
+                            lines.push(line);
+                        }
+                        lineCounter++;
+                        if (lineCounter >= lineTo)
+                            break;
+                    }
+                    else {
                         lines.push(line);
                     }
-                    lineCounter++;
-                    if (lineCounter >= lineTo)
-                        break;
                 }
             }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
             finally {
                 try {
-                    if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
                 }
-                finally { if (e_2) throw e_2.error; }
+                finally { if (e_1) throw e_1.error; }
             }
             reader.close();
             stream.close();
             return lines;
         });
-        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(producer, `Invalid producer`);
-            const fileKey = producer.settings.fileKey;
-            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
-            const fileUrl = path_1.default.join(this._path, fileKey);
-            return fs.existsSync(fileUrl);
+        this._readExcelLines = (fileUri, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            const excel = xlsx_1.default.readFile(fileUri);
+            let targetSheetName = sheetName;
+            if (!targetSheetName) {
+                (0, Affirm_1.default)(excel.SheetNames.length > 0, 'The Excel file has no sheets.');
+                targetSheetName = excel.SheetNames[0];
+            }
+            else {
+                (0, Affirm_1.default)(excel.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
+            }
+            const sheet = excel.Sheets[targetSheetName];
+            const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
+            const lines = csv.split('\n');
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
+                return lines.slice(lineFrom, lineTo + 1);
+            else
+                return lines;
         });
     }
 }
-class LocalDriverDestination {
+exports.LocalSourceDriver = LocalSourceDriver;
+class LocalDestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(source, `Invalid source`);
@@ -206,5 +238,4 @@ class LocalDriverDestination {
         });
     }
 }
-exports.LocalDriverDestination = LocalDriverDestination;
-exports.default = LocalDriver;
+exports.LocalDestinationDriver = LocalDestinationDriver;

package/drivers/S3Driver.js

@@ -8,14 +8,25 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3SourceDriver = exports.S3DestinationDriver = void 0;
 const client_s3_1 = require("@aws-sdk/client-s3");
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
-class S3Driver {
+const readline_1 = __importDefault(require("readline"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+const xlsx_1 = __importDefault(require("xlsx"));
+class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             this._bucketName = source.authentication['bucket'];
@@ -99,4 +110,151 @@ class S3Driver {
         });
     }
 }
-exports.default = S3Driver;
+exports.S3DestinationDriver = S3DestinationDriver;
+class S3SourceDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            this._bucketName = source.authentication['bucket'];
+            const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
+            const config = {
+                region: source.authentication['region'],
+                credentials: {
+                    accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
+                    secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
+                    sessionToken: sessionToken ? sessionToken : undefined
+                }
+            };
+            this._client = new client_s3_1.S3Client(config);
+            // TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
+            return this;
+        });
+        this.download = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(request, `Invalid download request`);
+            (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
+            const { fileKey, fileType, options } = request;
+            const bucket = this._bucketName;
+            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
+                Bucket: bucket,
+                Key: fileKey
+            }));
+            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
+            const stream = response.Body;
+            switch (fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    return yield this._readLines(stream);
+                case 'XLS':
+                case 'XLSX':
+                    return yield this._readExcelLines(stream, options === null || options === void 0 ? void 0 : options.sheetName);
+            }
+        });
+        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(request, 'Invalid read request');
+            (0, Affirm_1.default)(request.options, 'Invalid read request options');
+            const { fileKey, fileType, options: { sheetName, lineFrom, lineTo } } = request;
+            const bucket = this._bucketName;
+            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
+                Bucket: bucket,
+                Key: fileKey
+            }));
+            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
+            const stream = response.Body;
+            switch (fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    return yield this._readLines(stream, lineFrom, lineTo);
+                case 'XLS':
+                case 'XLSX':
+                    return yield this._readExcelLines(stream, sheetName, lineFrom, lineTo);
+            }
+        });
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
+            (0, Affirm_1.default)(producer, 'Invalid read producer');
+            const bucket = this._bucketName;
+            const fileKey = producer.settings.fileKey;
+            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
+            try {
+                yield this._client.send(new client_s3_1.HeadObjectCommand({ Bucket: bucket, Key: fileKey }));
+                return true;
+            }
+            catch (error) {
+                if (((_a = error.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode) === 404 || error.name === 'NotFound')
+                    return false;
+                throw error;
+            }
+        });
+        this._readLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            const lines = [];
+            let lineCounter = 0;
+            try {
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                        if (lineCounter >= lineFrom && lineCounter < lineTo) {
+                            lines.push(line);
+                        }
+                        lineCounter++;
+                        if (lineCounter >= lineTo)
+                            break;
+                    }
+                    else {
+                        lines.push(line);
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            reader.close();
+            return lines;
+        });
+        this._readExcelLines = (stream, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, stream_1, stream_1_1;
+            var _b, e_2, _c, _d;
+            (0, Affirm_1.default)(sheetName, `Invalid sheetname`);
+            const chunks = [];
+            try {
+                for (_a = true, stream_1 = __asyncValues(stream); stream_1_1 = yield stream_1.next(), _b = stream_1_1.done, !_b; _a = true) {
+                    _d = stream_1_1.value;
+                    _a = false;
+                    const chunk = _d;
+                    chunks.push(chunk);
+                }
+            }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            finally {
+                try {
+                    if (!_a && !_b && (_c = stream_1.return)) yield _c.call(stream_1);
+                }
+                finally { if (e_2) throw e_2.error; }
+            }
+            const buffer = Buffer.concat(chunks);
+            const excel = xlsx_1.default.read(buffer, { type: 'buffer' });
+            (0, Affirm_1.default)(excel.SheetNames.includes(sheetName), `The sheet "${sheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
+            const sheet = excel.Sheets[sheetName];
+            const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
+            const lines = csv.split('\n');
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
+                return lines.slice(lineFrom, lineTo + 1);
+            else
+                return lines;
+        });
+    }
+}
+exports.S3SourceDriver = S3SourceDriver;

package/engines/ParseManager.js

@@ -18,9 +18,9 @@ class ParseManagerClass {
             (0, Affirm_1.default)(lines, 'Invalid csv lines');
             Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
             const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
-            const headerRow = lines[0];
-            const rows = lines.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
-            const headerColumns = this._extractHeader(headerRow, delimiterChar, producer, discover);
+            const { header, records } = this._getClassifiedRows(lines, delimiterChar, producer);
+            const headerColumns = this._extractHeader(header, delimiterChar, producer, discover);
+            const rows = records.map(x => x.split(delimiterChar).map(k => k.trim()));
             const result = [];
             for (const row of rows) {
                 const rowObject = {};
@@ -32,6 +32,23 @@ class ParseManagerClass {
             }
             return result;
         };
+        this._getClassifiedRows = (lines, delimiterChar, producer) => {
+            if (producer.settings.fileType === 'TXT' && !producer.settings.hasHeaderRow) {
+                // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
+                const source = Environment_1.default.getSource(producer.source);
+                const columns = FileCompiler_1.default.compileProducer(producer, source);
+                return {
+                    header: columns.map(x => x.nameInProducer).join(delimiterChar),
+                    records: lines
+                };
+            }
+            else {
+                return {
+                    header: lines[0],
+                    records: lines.slice(1)
+                };
+            }
+        };
         this._extractHeader = (headerLine, delimiter, producer, discover) => {
             var _a;
             (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);

package/engines/ProducerEngine.js

@@ -91,19 +91,24 @@ class ProducerEngineClass {
             (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
             const driver = yield DriverFactory_1.default.instantiateSource(source);
             (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+            const { settings: { fileKey, fileType, sheetName } } = producer;
             let lines = [];
             if (options.readmode === 'lines')
-                lines = yield driver.readLinesInRange({ fileKey: producer.settings.fileKey, lineFrom: options.lines.from, lineTo: options.lines.to });
+                lines = yield driver.readLinesInRange({ fileKey, fileType, options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName } });
             else
-                lines = yield driver.download({ fileKey: producer.settings.fileKey });
+                lines = yield driver.download({ fileKey, fileType, options: { sheetName } });
             switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
-                case 'CSV': {
+                case 'CSV':
+                case 'TXT':
+                    return { data: lines, dataType: 'lines-of-text' };
+                case 'XLS':
+                case 'XLSX':
                     return { data: lines, dataType: 'lines-of-text' };
-                }
                 case 'JSONL':
                 case 'JSON': {
-                    if (lines.length === 1)
+                    if (lines.length === 1) {
                         lines = lines[0].split('\n');
+                    }
                     const json = lines.map(x => JSON.parse(x));
                     return { data: json, dataType: 'array-of-json' };
                 }
@@ -112,7 +117,7 @@ class ProducerEngineClass {
             }
         });
         this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10, discover = false) {
-            var _a, _b, _c;
+            var _a;
             (0, Affirm_1.default)(producer, 'Invalid producer');
             (0, Affirm_1.default)(sampleSize > 0, 'Sample size must be greater than 0');
             const source = Environment_1.default.getSource(producer.source);
@@ -131,18 +136,29 @@ class ProducerEngineClass {
                 case 'local':
                 case 'aws-s3': {
                     const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
-                    if (((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) === 'CSV') {
-                        sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer, discover);
-                    }
-                    else if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'JSON' || ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) === 'JSONL') {
-                        // With JSON or JSONL the readFile function already parses the strings
-                        if (typeof fileData.data[0] === 'object')
+                    const fileType = (_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase();
+                    switch (fileType) {
+                        case 'CSV':
+                        case 'TXT':
+                        case 'XLS':
+                        case 'XLSX': {
+                            sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer, discover);
+                            break;
+                        }
+                        case 'JSON':
+                        case 'JSONL': {
+                            // With JSON or JSONL the readFile function already parses the strings
+                            if (typeof fileData.data[0] === 'object')
+                                sampleData = fileData.data;
+                            else
+                                sampleData = fileData.data.map(line => JSON.parse(line));
+                            sampleData = sampleData.slice(0, sampleSize);
+                            break;
+                        }
+                        default: {
                             sampleData = fileData.data;
-                        else
-                            sampleData = fileData.data.map(line => JSON.parse(line));
-                    }
-                    else {
-                        sampleData = fileData.data;
+                            break;
+                        }
                     }
                     break;
                 }

package/engines/ai/LLM.js

@@ -202,14 +202,14 @@ class LLM {
                         $schema: zod_2.z.string().describe('The schema of the producer. This should always be the same.'),
                         name: zod_2.z.string(),
                         description: zod_2.z.string(),
+                        source: zod_2.z.string().describe('The name of the source linked to this producer.'),
+                        settings: zod_2.z.object({
+                            fileKey: zod_2.z.string().describe('The name of the file'),
+                            fileType: zod_2.z.string().describe('The file extension (CSV | JSONL | JSON)')
+                        }),
                         dimensions: zod_2.z.array(zod_2.z.object({
                             name: zod_2.z.string(),
                             // alias: z.string().optional(),
-                            source: zod_2.z.string().describe('The name of the source linked to this producer.'),
-                            settings: zod_2.z.object({
-                                fileKey: zod_2.z.string().describe('The name of the file'),
-                                fileType: zod_2.z.string().describe('The file extension (CSV | JSONL | JSON)')
-                            }),
                             description: zod_2.z.string().optional(),
                             type: zod_2.z.enum(['string', 'number', 'datetime']),
                             pk: zod_2.z.boolean().optional(),

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.26",
+    "version": "0.0.28",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -9,6 +9,7 @@
     },
     "scripts": {
         "sync": "cd ../dev_ops && npm run sync",
+        "tsc-check": "npx tsc --noemit",
         "init": "npx tsx ./src/index.ts init",
         "version": "npx tsx ./src/index.ts -v",
         "run": "npx tsx ./src/index.ts run",
@@ -53,6 +54,7 @@
         "ora": "^5.4.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
+        "xlsx": "^0.18.5",
         "zod": "^3.24.2"
     },
     "devDependencies": {