@forzalabs/remora 0.0.59-nasco.3 → 0.0.61-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.59-nasco',
+    cliVersion: '0.0.61-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/drivers/DriverHelper.js

@@ -29,7 +29,9 @@ const DriverHelper = {
     appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(options, 'Invalid options');
         const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow, delimiter } = options;
-        const keys = (fileType === 'JSON' || fileType === 'JSONL') ? Object.keys(headerLine) : [];
+        const keys = (fileType === 'JSON' || fileType === 'JSONL')
+            ? Object.keys(JSON.parse(headerLine))
+            : [];
         const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
         let isFirstLine = true;
         let hasValidatedHeader = shouldValidateHeader ? false : true;
@@ -99,8 +101,14 @@ const DriverHelper = {
             switch (fileType) {
                 case 'JSON':
                 case 'JSONL': {
-                    const parsed = JSON.parse(line);
-                    return keys.map(k => parsed[k]).join(delimiter);
+                    try {
+                        const parsed = JSON.parse(line);
+                        return keys.map(k => parsed[k]).join(delimiter);
+                    }
+                    catch (error) {
+                        Logger_1.default.log(`Failed parsing line in JSON - index: ${globalIndex}; line: ${line}; err: ${error === null || error === void 0 ? void 0 : error.name}`);
+                        throw error;
+                    }
                 }
                 default:
                     return line;

package/drivers/LocalDriver.js

@@ -156,7 +156,7 @@ class LocalSourceDriver {
                 // For single file, no header validation needed
                 const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
                 dataset.setFirstLine(headerLine);
-                yield copyLocally(fileKey, '', false);
+                yield copyLocally(fileKey, headerLine, false);
                 return dataset;
             }
         });
@@ -347,42 +347,6 @@ class LocalDestinationDriver {
                 throw new Error(`Failed to upload local file "${options.name}": ${error.message}`);
             }
         });
-        this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(options, 'Invalid upload options');
-            (0, Affirm_1.default)(options.name, 'File name is required');
-            (0, Affirm_1.default)(options.contents && Array.isArray(options.contents), 'Contents must be an array');
-            (0, Affirm_1.default)(options.contents.length > 0, 'Contents array cannot be empty');
-            const folder = this._path;
-            try {
-                if (!fs.existsSync(folder)) {
-                    fs.mkdirSync(folder, { recursive: true });
-                }
-                const filePath = path_1.default.join(folder, options.name);
-                // Create or truncate the file first
-                fs.writeFileSync(filePath, '');
-                // Append each chunk
-                for (const chunk of options.contents) {
-                    (0, Affirm_1.default)(typeof chunk === 'string', 'Each chunk must be a string');
-                    fs.appendFileSync(filePath, chunk);
-                }
-                return { bucket: folder, key: filePath, res: true };
-            }
-            catch (error) {
-                // Clean up the partial file if it exists
-                const filePath = path_1.default.join(folder, options.name);
-                if (fs.existsSync(filePath)) {
-                    try {
-                        fs.unlinkSync(filePath);
-                    }
-                    catch (cleanupError) {
-                        console.error(`Failed to clean up partial file after error: ${cleanupError.message}`);
-                        throw cleanupError;
-                    }
-                }
-                throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
-            }
-        });
         this.uploadStream = (options) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(options, `Invalid upload options`);
             const { dataset, name, recordProjection } = options;

package/drivers/S3Driver.js

@@ -62,58 +62,6 @@ class S3DestinationDriver {
             (0, Affirm_1.default)(res.$metadata.httpStatusCode === 200, `Failed to upload the file "${name}" to the bucket "${this._bucketName}": status code ${res.$metadata.httpStatusCode}`);
             return { res: true, key: name, bucket: this._bucketName };
         });
-        this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(options, `Invalid upload options`);
-            (0, Affirm_1.default)(options.contents && options.contents.length > 0, 'No contents provided for multipart upload');
-            (0, Affirm_1.default)(options.name, 'No filename provided for multipart upload');
-            try {
-                // Create the multipart upload
-                const createMultipartUploadRes = yield this._client.send(new client_s3_1.CreateMultipartUploadCommand({
-                    Bucket: this._bucketName,
-                    Key: options.name
-                }));
-                const uploadId = createMultipartUploadRes.UploadId;
-                (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
-                // Upload each part
-                const uploadPromises = options.contents.map((content, index) => __awaiter(this, void 0, void 0, function* () {
-                    const partNumber = index + 1;
-                    const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
-                        Bucket: this._bucketName,
-                        Key: options.name,
-                        UploadId: uploadId,
-                        PartNumber: partNumber,
-                        Body: Buffer.from(content)
-                    }));
-                    return {
-                        PartNumber: partNumber,
-                        ETag: uploadPartRes.ETag
-                    };
-                }));
-                const uploadedParts = yield Promise.all(uploadPromises);
-                // Complete the multipart upload
-                const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
-                    Bucket: this._bucketName,
-                    Key: options.name,
-                    UploadId: uploadId,
-                    MultipartUpload: {
-                        Parts: uploadedParts
-                    }
-                }));
-                (0, Affirm_1.default)(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${options.name}": status code ${completeRes.$metadata.httpStatusCode}`);
-                return { res: true, key: options.name, bucket: this._bucketName };
-            }
-            catch (error) {
-                // If anything fails, make sure to abort the multipart upload
-                if (error.UploadId) {
-                    yield this._client.send(new client_s3_1.AbortMultipartUploadCommand({
-                        Bucket: this._bucketName,
-                        Key: options.name,
-                        UploadId: error.UploadId
-                    }));
-                }
-                throw error;
-            }
-        });
         this.uploadStream = (options) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(options, `Invalid upload options`);
             const { dataset, name, recordProjection } = options;
@@ -282,7 +230,7 @@ class S3SourceDriver {
                 const headerLine = yield this.getFirstLineFromStream(firstFileStream);
                 dataset.setFirstLine(headerLine);
                 // For single file, no header validation needed
-                yield downloadLocally(fileKey, '');
+                yield downloadLocally(fileKey, headerLine);
                 return dataset;
             }
         });

package/engines/dataset/DatasetManager.js

@@ -41,7 +41,7 @@ class DatasetManagerClass {
             return this.buildDimensionsFromFirstLine(firstLine, dataset.getFile(), producer, discover);
         });
         this.buildDimensionsFromFirstLine = (firstLine_1, dsFile_1, producer_1, ...args_1) => __awaiter(this, [firstLine_1, dsFile_1, producer_1, ...args_1], void 0, function* (firstLine, dsFile, producer, discover = false) {
-            var _a, _b, _c, _d, _e;
+            var _a, _b, _c, _d, _e, _f;
             (0, Affirm_1.default)(firstLine, `Invalid first line`);
             (0, Affirm_1.default)(dsFile, `Invalid dataset file`);
             (0, Affirm_1.default)(producer, `Invalid producer`);
@@ -62,29 +62,42 @@ class DatasetManagerClass {
                     const columns = FileCompiler_1.default.compileProducer(producer, source);
                     const firstObject = JSON.parse(firstLine);
                     const keys = Object.keys(firstObject);
+                    if (discover) {
+                        return {
+                            delimiter: (_b = file.delimiter) !== null && _b !== void 0 ? _b : ',',
+                            dimensions: keys.map((x, i) => ({ hidden: false, index: i, key: x, name: x }))
+                        };
+                    }
                     const dimensions = [];
                     for (const pColumn of columns) {
-                        const columnKey = (_b = pColumn.aliasInProducer) !== null && _b !== void 0 ? _b : pColumn.nameInProducer;
+                        const columnKey = (_c = pColumn.aliasInProducer) !== null && _c !== void 0 ? _c : pColumn.nameInProducer;
                         const csvColumnIndex = keys.findIndex(x => x === columnKey);
                         (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
                         dimensions.push({ index: csvColumnIndex, key: columnKey, name: pColumn.nameInProducer, hidden: null });
                     }
-                    const delimiterChar = (_c = file.delimiter) !== null && _c !== void 0 ? _c : ',';
+                    const delimiterChar = (_d = file.delimiter) !== null && _d !== void 0 ? _d : ',';
                     return { dimensions, delimiter: delimiterChar };
                 }
                 case 'TXT': {
                     if (!file.hasHeaderRow) {
                         // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
-                        const delimiterChar = (_d = file.delimiter) !== null && _d !== void 0 ? _d : ',';
+                        const delimiterChar = (_e = file.delimiter) !== null && _e !== void 0 ? _e : ',';
                         const source = Environment_1.default.getSource(producer.source);
                         const columns = FileCompiler_1.default.compileProducer(producer, source);
+                        if (discover) {
+                            // Since I don't have an header, and I'm discovering, I just create placeholder dimensions based on the same number of columns of the txt
+                            return {
+                                delimiter: delimiterChar,
+                                dimensions: firstLine.split(delimiterChar).map((x, i) => ({ hidden: false, index: i, key: `Col ${i + 1}`, name: `Col ${i + 1}` }))
+                            };
+                        }
                         return {
                             dimensions: columns.map((x, i) => { var _a; return ({ key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer, name: x.nameInProducer, index: i, hidden: null }); }),
                             delimiter: delimiterChar
                         };
                     }
                     else {
-                        const delimiterChar = (_e = producer.settings.delimiter) !== null && _e !== void 0 ? _e : ',';
+                        const delimiterChar = (_f = producer.settings.delimiter) !== null && _f !== void 0 ? _f : ',';
                         const rawDimensions = ParseManager_1.default._extractHeader(firstLine, delimiterChar, producer, discover);
                         return {
                             dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index, hidden: null })),

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.59-nasco.3",
+    "version": "0.0.61-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,