@trayio/cdk-runtime 2.8.0 → 2.10.0

package/dist/connector/operation/HttpOperationExecution.d.ts

@@ -21,6 +21,7 @@ export declare class HttpOperationExecution<AUTH extends OperationHandlerAuth<un
     private parseBodyAsJson;
     private parseEmptyBody;
     private parseBodyAsFile;
-    private parseeBodyAsText;
+    private parseBodyAsText;
+    private generateUploadOptions;
 }
 //# sourceMappingURL=HttpOperationExecution.d.ts.map

package/dist/connector/operation/HttpOperationExecution.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"HttpOperationExecution.d.ts","sourceRoot":"","sources":["../../../src/connector/operation/HttpOperationExecution.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EAGtB,MAAM,sDAAsD,CAAC;AAC9D,OAAO,EAEN,oBAAoB,EAKpB,MAAM,0DAA0D,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAe7D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAI1D,qBAAa,sBAAsB,CAClC,IAAI,SAAS,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,EACnD,EAAE,EACF,GAAG,CACF,YAAW,kBAAkB,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC;IAE7C,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,WAAW,CAAc;gBAGhC,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,oBAAoB,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC;IAQvC,OAAO,CACZ,GAAG,EAAE,uBAAuB,CAAC,IAAI,CAAC,EAClC,KAAK,EAAE,EAAE,GACP,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAiDvC,OAAO,CAAC,6BAA6B;IA2BrC,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAM5B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,oBAAoB;IAc5B,OAAO,CAAC,aAAa;IAiCrB,OAAO,CAAC,eAAe;IAmBvB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,eAAe;IAwCvB,OAAO,CAAC,gBAAgB;CAexB"}
+{"version":3,"file":"HttpOperationExecution.d.ts","sourceRoot":"","sources":["../../../src/connector/operation/HttpOperationExecution.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EAGtB,MAAM,sDAAsD,CAAC;AAC9D,OAAO,EAEN,oBAAoB,EAKpB,MAAM,0DAA0D,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAe7D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAExD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAO1D,qBAAa,sBAAsB,CAClC,IAAI,SAAS,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,EACnD,EAAE,EACF,GAAG,CACF,YAAW,kBAAkB,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC;IAE7C,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,WAAW,CAAc;gBAGhC,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,oBAAoB,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC;IAQvC,OAAO,CACZ,GAAG,EAAE,uBAAuB,CAAC,IAAI,CAAC,EAClC,KAAK,EAAE,EAAE,GACP,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAiDvC,OAAO,CAAC,6BAA6B;IA2BrC,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAM5B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,oBAAoB;IAc5B,OAAO,CAAC,aAAa;IAiCrB,OAAO,CAAC,eAAe;IAmBvB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,eAAe;IA2CvB,OAAO,CAAC,eAAe;IAiBvB,OAAO,CAAC,qBAAqB;CAwC7B"}

package/dist/connector/operation/HttpOperationExecution.js

@@ -44,6 +44,7 @@ const O = __importStar(require("fp-ts/Option"));
 const function_1 = require("fp-ts/function");
 const BufferExtensions_1 = require("@trayio/commons/buffer/BufferExtensions");
 const mime = require('mime');
+const { AWS_LAMBDA_FUNCTION_MEMORY_SIZE, CONNECTOR_MAX_ALLOCATED_RAM_MB } = process.env;
 class HttpOperationExecution {
     constructor(httpClient, fileStorage, handler) {
         this.jsonSerialization = new JsonSerialization_1.JsonSerialization();
@@ -131,7 +132,7 @@ class HttpOperationExecution {
                 case Http_1.HttpContentType.MultipartRequestBody:
                     return TE.right(OperationHandler_1.OperationHandlerResult.failure(OperationHandler_1.OperationHandlerError.connectorError(`Invalid Response Type ${contentType}`)));
                 case Http_1.HttpContentType.Text:
-                    return this.parseeBodyAsText(operationResponse);
+                    return this.parseBodyAsText(operationResponse);
                 default:
                     return this.parseBodyAsJson(operationResponse);
             }
@@ -148,18 +149,19 @@ class HttpOperationExecution {
         return TE.right(operationResponse.responseParser(undefined));
     }
     parseBodyAsFile(operationResponse) {
-        var _a;
         const contentType = (operationResponse.response.headers[Http_1.HttpHeader.ContentType.toLowerCase()]);
         const fileExtension = mime.getExtension(contentType);
+        const contentLength = (operationResponse.response.headers[Http_1.HttpHeader.ContentLength.toLowerCase()]);
+        const uploadOptions = this.generateUploadOptions(contentLength);
         return (0, function_1.pipe)(this.fileStorage.write({
             content: operationResponse.response.body,
             key: fileExtension ? `${(0, uuid_1.v4)()}.${fileExtension}` : (0, uuid_1.v4)(),
             metadata: {
                 name: (0, uuid_1.v4)(),
-                size: parseFloat((_a = (operationResponse.response.headers[Http_1.HttpHeader.ContentLength.toLowerCase()])) !== null && _a !== void 0 ? _a : undefined),
+                size: parseFloat(contentLength !== null && contentLength !== void 0 ? contentLength : undefined),
                 contentType,
             },
-        }), TE.chain((file) => (0, function_1.pipe)(this.fileStorage.getSharedUrl(file.key), TE.map((sharedUrl) => {
+        }, O.some(uploadOptions)), TE.chain((file) => (0, function_1.pipe)(this.fileStorage.getSharedUrl(file.key), TE.map((sharedUrl) => {
             var _a, _b;
             return operationResponse.responseParser({
                 name: file.metadata.name,
@@ -169,12 +171,43 @@ class HttpOperationExecution {
             });
         }))));
     }
-    parseeBodyAsText(operationResponse) {
+    parseBodyAsText(operationResponse) {
         const arrayBufferTE = BufferExtensions_1.BufferExtensions.readableToArrayBuffer(operationResponse.response.body);
         return (0, function_1.pipe)(arrayBufferTE, TE.chain((arrayBuffer) => {
             const deserializeBody = new TextDecoder().decode(arrayBuffer);
             return TE.right(operationResponse.responseParser(deserializeBody));
         }));
     }
+    // This method is used to respect lambda memory limits as per the old falfel implementation
+    generateUploadOptions(contentLength) {
+        const allocatedRam = AWS_LAMBDA_FUNCTION_MEMORY_SIZE || CONNECTOR_MAX_ALLOCATED_RAM_MB;
+        const availableRAM = allocatedRam ? parseInt(allocatedRam, 10) : 128; // 128MB default
+        const uploadMinSize = 1024 * 1024 * 8, // 8MB
+        maxRam = 1024 * 1024 * availableRAM;
+        /*
+            128 / 16 = 8 (which is UPLOAD_MIN_SIZE)
+            TARGET_SIZE should either be 8MB (since default assumed RAM is 128MB) or
+            1/16 of of the availableRAM, which ever is greater. The TARGET_SIZE, set
+            as `partSize`, will always be used in conjuction with `queueSize`.
+        */
+        const sixteenthOfRAM = Math.floor(maxRam / 16);
+        const targetSize = sixteenthOfRAM < uploadMinSize ? uploadMinSize : sixteenthOfRAM;
+        const uploadOptions = {
+            // 8MB (minimum) * 4 parallel = upto 32MB/s RAM usage
+            partSize: targetSize,
+            queueSize: 4,
+            acl: 'bucket-owner-full-control',
+        };
+        if (!contentLength) {
+            if (CONNECTOR_MAX_ALLOCATED_RAM_MB) {
+                uploadOptions.queueSize = 12;
+            }
+        }
+        else if (parseInt(contentLength, 10) > targetSize &&
+            parseInt(contentLength, 10) > targetSize * 8) {
+            uploadOptions.queueSize = 12;
+        }
+        return uploadOptions;
+    }
 }
 exports.HttpOperationExecution = HttpOperationExecution;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trayio/cdk-runtime",
-  "version": "2.8.0",
+  "version": "2.10.0",
   "description": "A Runtime that executes connector operations defined using the CDK DSL",
   "exports": {
     "./*": "./dist/*.js"
@@ -14,10 +14,10 @@
     "node": ">=18.x"
   },
   "dependencies": {
-    "@trayio/axios": "2.8.0",
-    "@trayio/cdk-dsl": "2.8.0",
-    "@trayio/express": "2.8.0",
-    "@trayio/winston": "2.8.0",
+    "@trayio/axios": "2.10.0",
+    "@trayio/cdk-dsl": "2.10.0",
+    "@trayio/express": "2.10.0",
+    "@trayio/winston": "2.10.0",
     "mime": "3.0.0",
     "uuid": "9.0.0"
   },