@forzalabs/remora 0.0.61-nasco.3 → 0.0.63-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.61-nasco',
+    cliVersion: '0.0.63-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/drivers/DriverHelper.js

@@ -29,6 +29,7 @@ const DriverHelper = {
     appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(options, 'Invalid options');
         const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow, delimiter } = options;
+        (0, Affirm_1.default)(headerLine, `Invalid header line`);
         const keys = (fileType === 'JSON' || fileType === 'JSONL')
             ? Object.keys(JSON.parse(headerLine))
             : [];

package/drivers/S3Driver.js

@@ -78,24 +78,38 @@ class S3DestinationDriver {
                 (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
                 const uploadedParts = [];
                 let partNumber = 1;
+                const MIN_PART_SIZE = 5 * 1024 * 1024; // 5MB
+                let accumulatedBuffer = Buffer.alloc(0);
+                const uploadPart = (buffer) => __awaiter(this, void 0, void 0, function* () {
+                    const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
+                        Bucket: this._bucketName,
+                        Key: name,
+                        UploadId: uploadId,
+                        PartNumber: partNumber,
+                        Body: buffer
+                    }));
+                    uploadedParts.push({
+                        PartNumber: partNumber,
+                        ETag: uploadPartRes.ETag
+                    });
+                    partNumber++;
+                });
                 yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
                     const chunks = FileExporter_1.default.prepareBatch(batch, options);
                     for (const chunk of chunks) {
-                        const body = Buffer.from(chunk);
-                        const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
-                            Bucket: this._bucketName,
-                            Key: name,
-                            UploadId: uploadId,
-                            PartNumber: partNumber,
-                            Body: body
-                        }));
-                        uploadedParts.push({
-                            PartNumber: partNumber,
-                            ETag: uploadPartRes.ETag
-                        });
-                        partNumber++;
+                        const chunkBuffer = Buffer.from(chunk);
+                        accumulatedBuffer = Buffer.concat([accumulatedBuffer, chunkBuffer]);
+                        // If accumulated buffer is at least 5MB, upload it as a part
+                        if (accumulatedBuffer.length >= MIN_PART_SIZE) {
+                            yield uploadPart(accumulatedBuffer);
+                            accumulatedBuffer = Buffer.alloc(0);
+                        }
                     }
                 }));
+                // Upload any remaining data as the final part (even if smaller than 5MB)
+                if (accumulatedBuffer.length > 0) {
+                    yield uploadPart(accumulatedBuffer);
+                }
                 // Complete the multipart upload
                 const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
                     Bucket: this._bucketName,

package/engines/CryptoEngine.js

@@ -8,6 +8,8 @@ const Algo_1 = __importDefault(require("../core/Algo"));
 const RandomEngine_1 = __importDefault(require("./RandomEngine"));
 class CryptoEngineClass {
     constructor() {
+        var _a;
+        this._salt = '';
         this.hashQuery = (maskType, fieldReference, fieldName) => {
             if (!Algo_1.default.hasVal(maskType))
                 return;
@@ -25,30 +27,32 @@ class CryptoEngineClass {
             }
         };
         this.valueToHash = (value) => {
-            return crypto_1.default.createHash('sha256').update(JSON.stringify(value)).digest('hex');
+            const textValue = JSON.stringify(value);
+            return crypto_1.default.createHash('sha256').update(textValue).digest('hex');
         };
         this.hashValue = (maskType, value, valueType) => {
             if (!Algo_1.default.hasVal(value))
                 return value;
             if (!Algo_1.default.hasVal(maskType))
                 return value;
+            const saltedValue = this._salt + value;
             switch (maskType) {
                 case 'hash':
-                    return this.valueToHash(value);
+                    return this.valueToHash(saltedValue);
                 case 'random': {
                     switch (valueType) {
                         case 'datetime': return RandomEngine_1.default.rngDate();
                         case 'number': return RandomEngine_1.default.rng();
-                        case 'string': return this.valueToHash(value);
+                        case 'string': return this.valueToHash(saltedValue);
                         default:
                             throw new Error('Not implemented yet');
                     }
                 }
                 case 'seeded-random': {
                     switch (valueType) {
-                        case 'datetime': return RandomEngine_1.default.sRngDate(value);
-                        case 'number': return RandomEngine_1.default.sRng(value);
-                        case 'string': return this.valueToHash(value);
+                        case 'datetime': return RandomEngine_1.default.sRngDate(saltedValue);
+                        case 'number': return RandomEngine_1.default.sRng(saltedValue);
+                        case 'string': return this.valueToHash(saltedValue);
                         default:
                             throw new Error('Not implemented yet');
                     }
@@ -63,6 +67,7 @@ class CryptoEngineClass {
                     throw new Error(`This type doesn't exist`);
             }
         };
+        this._salt = (_a = process.env.REMORA_SALT) !== null && _a !== void 0 ? _a : '';
     }
 }
 const CryptoEngine = new CryptoEngineClass();

package/engines/RandomEngine.js

@@ -22,11 +22,23 @@ class RandomEngineClass {
                 return rng();
         };
         this.rngDate = (min, max) => {
-            const randomNumber = this.rng(min, max);
+            let myMin = min;
+            let myMax = max;
+            if (!Algo_1.default.hasVal(min) && !Algo_1.default.hasVal(max)) {
+                myMin = new Date('1900-01-01').getTime();
+                myMax = new Date().getTime();
+            }
+            const randomNumber = this.rng(myMin, myMax);
             return new Date(randomNumber).toJSON();
         };
         this.sRngDate = (seed, min, max) => {
-            const randomNumber = this.sRng(seed, min, max);
+            let myMin = min;
+            let myMax = max;
+            if (!Algo_1.default.hasVal(min) && !Algo_1.default.hasVal(max)) {
+                myMin = new Date('1900-01-01').getTime();
+                myMax = new Date().getTime();
+            }
+            const randomNumber = this.sRng(seed, myMin, myMax);
             return new Date(randomNumber).toJSON();
         };
     }

package/engines/dataset/DatasetRecord.js

@@ -19,7 +19,10 @@ class DatasetRecord {
         this.isEmpty = () => { var _a; return ((_a = this._row) === null || _a === void 0 ? void 0 : _a.trim().length) === 0; };
         this.getRaw = () => this._row;
         this.getValue = (dimension) => this._value[dimension];
-        this.setValue = (dimension, value) => this._value[dimension] = value;
+        this.setValue = (dimension, value) => {
+            this._value[dimension] = value;
+            return this;
+        };
         /**
          * Reinitialize the record with new data instead of creating a new instance
          * This is used for object pooling optimization

package/engines/file/FileExporter.js

@@ -63,7 +63,7 @@ class FileExporterClass {
             const chunks = [];
             for (let i = 0; i < records.length; i += chunkSize) {
                 const chunk = records.slice(i, i + chunkSize);
-                chunks.push(chunk.join(separator));
+                chunks.push(chunk.join(separator) + separator);
             }
             return chunks;
         };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.61-nasco.3",
+    "version": "0.0.63-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -9,7 +9,7 @@
     },
     "scripts": {
         "sync": "cd ../dev_ops && npm run sync",
-        "dev": "clear && npx tsx scripts/dev.ts",
+        "dev": "clear && npx tsx scripts/threaded.ts",
         "tsc-check": "npx tsc --noemit",
         "init": "npx tsx ./src/index.ts init",
         "version": "npx tsx ./src/index.ts -v",