@forzalabs/remora 0.0.60-nasco.3 → 0.0.61-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.59-nasco',
+    cliVersion: '0.0.61-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/drivers/DriverHelper.js

@@ -29,7 +29,9 @@ const DriverHelper = {
     appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(options, 'Invalid options');
         const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow, delimiter } = options;
-        const keys = (fileType === 'JSON' || fileType === 'JSONL') ? Object.keys(headerLine) : [];
+        const keys = (fileType === 'JSON' || fileType === 'JSONL')
+            ? Object.keys(JSON.parse(headerLine))
+            : [];
         const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
         let isFirstLine = true;
         let hasValidatedHeader = shouldValidateHeader ? false : true;
@@ -99,8 +101,14 @@ const DriverHelper = {
             switch (fileType) {
                 case 'JSON':
                 case 'JSONL': {
-                    const parsed = JSON.parse(line);
-                    return keys.map(k => parsed[k]).join(delimiter);
+                    try {
+                        const parsed = JSON.parse(line);
+                        return keys.map(k => parsed[k]).join(delimiter);
+                    }
+                    catch (error) {
+                        Logger_1.default.log(`Failed parsing line in JSON - index: ${globalIndex}; line: ${line}; err: ${error === null || error === void 0 ? void 0 : error.name}`);
+                        throw error;
+                    }
                 }
                 default:
                     return line;

package/drivers/LocalDriver.js

@@ -156,7 +156,7 @@ class LocalSourceDriver {
                 // For single file, no header validation needed
                 const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
                 dataset.setFirstLine(headerLine);
-                yield copyLocally(fileKey, '', false);
+                yield copyLocally(fileKey, headerLine, false);
                 return dataset;
             }
         });
@@ -347,42 +347,6 @@ class LocalDestinationDriver {
                 throw new Error(`Failed to upload local file "${options.name}": ${error.message}`);
             }
         });
-        this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(options, 'Invalid upload options');
-            (0, Affirm_1.default)(options.name, 'File name is required');
-            (0, Affirm_1.default)(options.contents && Array.isArray(options.contents), 'Contents must be an array');
-            (0, Affirm_1.default)(options.contents.length > 0, 'Contents array cannot be empty');
-            const folder = this._path;
-            try {
-                if (!fs.existsSync(folder)) {
-                    fs.mkdirSync(folder, { recursive: true });
-                }
-                const filePath = path_1.default.join(folder, options.name);
-                // Create or truncate the file first
-                fs.writeFileSync(filePath, '');
-                // Append each chunk
-                for (const chunk of options.contents) {
-                    (0, Affirm_1.default)(typeof chunk === 'string', 'Each chunk must be a string');
-                    fs.appendFileSync(filePath, chunk);
-                }
-                return { bucket: folder, key: filePath, res: true };
-            }
-            catch (error) {
-                // Clean up the partial file if it exists
-                const filePath = path_1.default.join(folder, options.name);
-                if (fs.existsSync(filePath)) {
-                    try {
-                        fs.unlinkSync(filePath);
-                    }
-                    catch (cleanupError) {
-                        console.error(`Failed to clean up partial file after error: ${cleanupError.message}`);
-                        throw cleanupError;
-                    }
-                }
-                throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
-            }
-        });
         this.uploadStream = (options) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(options, `Invalid upload options`);
             const { dataset, name, recordProjection } = options;

package/drivers/S3Driver.js

@@ -62,58 +62,6 @@ class S3DestinationDriver {
             (0, Affirm_1.default)(res.$metadata.httpStatusCode === 200, `Failed to upload the file "${name}" to the bucket "${this._bucketName}": status code ${res.$metadata.httpStatusCode}`);
             return { res: true, key: name, bucket: this._bucketName };
         });
-        this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(options, `Invalid upload options`);
-            (0, Affirm_1.default)(options.contents && options.contents.length > 0, 'No contents provided for multipart upload');
-            (0, Affirm_1.default)(options.name, 'No filename provided for multipart upload');
-            try {
-                // Create the multipart upload
-                const createMultipartUploadRes = yield this._client.send(new client_s3_1.CreateMultipartUploadCommand({
-                    Bucket: this._bucketName,
-                    Key: options.name
-                }));
-                const uploadId = createMultipartUploadRes.UploadId;
-                (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
-                // Upload each part
-                const uploadPromises = options.contents.map((content, index) => __awaiter(this, void 0, void 0, function* () {
-                    const partNumber = index + 1;
-                    const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
-                        Bucket: this._bucketName,
-                        Key: options.name,
-                        UploadId: uploadId,
-                        PartNumber: partNumber,
-                        Body: Buffer.from(content)
-                    }));
-                    return {
-                        PartNumber: partNumber,
-                        ETag: uploadPartRes.ETag
-                    };
-                }));
-                const uploadedParts = yield Promise.all(uploadPromises);
-                // Complete the multipart upload
-                const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
-                    Bucket: this._bucketName,
-                    Key: options.name,
-                    UploadId: uploadId,
-                    MultipartUpload: {
-                        Parts: uploadedParts
-                    }
-                }));
-                (0, Affirm_1.default)(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${options.name}": status code ${completeRes.$metadata.httpStatusCode}`);
-                return { res: true, key: options.name, bucket: this._bucketName };
-            }
-            catch (error) {
-                // If anything fails, make sure to abort the multipart upload
-                if (error.UploadId) {
-                    yield this._client.send(new client_s3_1.AbortMultipartUploadCommand({
-                        Bucket: this._bucketName,
-                        Key: options.name,
-                        UploadId: error.UploadId
-                    }));
-                }
-                throw error;
-            }
-        });
         this.uploadStream = (options) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(options, `Invalid upload options`);
             const { dataset, name, recordProjection } = options;
@@ -282,7 +230,7 @@ class S3SourceDriver {
                 const headerLine = yield this.getFirstLineFromStream(firstFileStream);
                 dataset.setFirstLine(headerLine);
                 // For single file, no header validation needed
-                yield downloadLocally(fileKey, '');
+                yield downloadLocally(fileKey, headerLine);
                 return dataset;
             }
         });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.60-nasco.3",
+    "version": "0.0.61-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,