@trayio/cdk-runtime 2.9.0 → 2.11.0

package/dist/connector/operation/HttpOperationExecution.d.ts

@@ -21,6 +21,7 @@ export declare class HttpOperationExecution<AUTH extends OperationHandlerAuth<un
     private parseBodyAsJson;
     private parseEmptyBody;
     private parseBodyAsFile;
-    private parseeBodyAsText;
+    private parseBodyAsText;
+    private generateUploadOptions;
 }
 //# sourceMappingURL=HttpOperationExecution.d.ts.map

package/dist/connector/operation/HttpOperationExecution.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"HttpOperationExecution.d.ts","sourceRoot":"","sources":["../../../src/connector/operation/HttpOperationExecution.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EAGtB,MAAM,sDAAsD,CAAC;AAC9D,OAAO,EAEN,oBAAoB,EAKpB,MAAM,0DAA0D,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAe7D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAI1D,qBAAa,sBAAsB,CAClC,IAAI,SAAS,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,EACnD,EAAE,EACF,GAAG,CACF,YAAW,kBAAkB,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC;IAE7C,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,WAAW,CAAc;gBAGhC,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,oBAAoB,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC;IAQvC,OAAO,CACZ,GAAG,EAAE,uBAAuB,CAAC,IAAI,CAAC,EAClC,KAAK,EAAE,EAAE,GACP,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAiDvC,OAAO,CAAC,6BAA6B;IA2BrC,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAM5B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,oBAAoB;IAc5B,OAAO,CAAC,aAAa;IAiCrB,OAAO,CAAC,eAAe;IAmBvB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,eAAe;IA2CvB,OAAO,CAAC,gBAAgB;CAexB"}
+{"version":3,"file":"HttpOperationExecution.d.ts","sourceRoot":"","sources":["../../../src/connector/operation/HttpOperationExecution.ts"],"names":[],"mappings":"AASA,OAAO,EACN,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EAGtB,MAAM,sDAAsD,CAAC;AAC9D,OAAO,EAGN,oBAAoB,EAKpB,MAAM,0DAA0D,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAW7D,OAAO,EAAQ,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAO1D,qBAAa,sBAAsB,CAClC,IAAI,SAAS,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,EACnD,EAAE,EACF,GAAG,CACF,YAAW,kBAAkB,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC;IAE7C,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,WAAW,CAAc;gBAGhC,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,oBAAoB,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC;IAQvC,OAAO,CACZ,GAAG,EAAE,uBAAuB,CAAC,IAAI,CAAC,EAClC,KAAK,EAAE,EAAE,GACP,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAiDvC,OAAO,CAAC,6BAA6B;IA+BrC,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAyD5B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,oBAAoB;IAc5B,OAAO,CAAC,aAAa;IAiCrB,OAAO,CAAC,eAAe;IAmBvB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,eAAe;IA2CvB,OAAO,CAAC,eAAe;IAiBvB,OAAO,CAAC,qBAAqB;CAwC7B"}

package/dist/connector/operation/HttpOperationExecution.js

@@ -33,17 +33,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HttpOperationExecution = void 0;
-const OperationHandler_1 = require("@trayio/cdk-dsl/connector/operation/OperationHandler");
-const HttpOperationHandler_1 = require("@trayio/cdk-dsl/connector/operation/HttpOperationHandler");
-const Http_1 = require("@trayio/commons/http/Http");
-const uuid_1 = require("uuid");
-const JsonSerialization_1 = require("@trayio/commons/serialization/JsonSerialization");
 const TE = __importStar(require("fp-ts/TaskEither"));
 const E = __importStar(require("fp-ts/Either"));
 const O = __importStar(require("fp-ts/Option"));
 const function_1 = require("fp-ts/function");
+const Array_1 = require("fp-ts/Array");
+const uuid_1 = require("uuid");
+const OperationHandler_1 = require("@trayio/cdk-dsl/connector/operation/OperationHandler");
+const HttpOperationHandler_1 = require("@trayio/cdk-dsl/connector/operation/HttpOperationHandler");
+const Http_1 = require("@trayio/commons/http/Http");
+const JsonSerialization_1 = require("@trayio/commons/serialization/JsonSerialization");
 const BufferExtensions_1 = require("@trayio/commons/buffer/BufferExtensions");
 const mime = require('mime');
+const { AWS_LAMBDA_FUNCTION_MEMORY_SIZE, CONNECTOR_MAX_ALLOCATED_RAM_MB } = process.env;
 class HttpOperationExecution {
     constructor(httpClient, fileStorage, handler) {
         this.jsonSerialization = new JsonSerialization_1.JsonSerialization();
@@ -94,7 +96,40 @@ class HttpOperationExecution {
         return TE.right(BufferExtensions_1.BufferExtensions.arrayBufferToReadable(serializedBody));
     }
     serializeAsMultipart(body) {
-        throw new Error('unimplemented');
+        const fileStreamsFromReferencesTasks = Object.keys(body.files).map((fileKey) => {
+            const fileRef = body.files[fileKey];
+            return (0, function_1.pipe)(this.httpClient.execute(Http_1.HttpMethod.Get, fileRef.url, {
+                headers: {
+                    [Http_1.HttpHeader.ContentType]: Http_1.HttpContentType.OctetStream,
+                },
+                pathParams: {},
+                queryString: {},
+                body: BufferExtensions_1.BufferExtensions.arrayBufferToReadable(new ArrayBuffer(0)),
+            }), TE.chain((response) => {
+                if (response.statusCode >= 300) {
+                    return TE.left(Error('error downloading file from source'));
+                }
+                const parsedSized = Number.parseInt(response.headers.ContentLength ||
+                    response.headers['content-length'], 10);
+                return TE.right({
+                    key: fileKey,
+                    metadata: {
+                        name: fileRef.name,
+                        size: Number.isNaN(parsedSized) ? undefined : parsedSized,
+                        contentType: fileRef.mime_type,
+                    },
+                    content: response.body,
+                });
+            }));
+        });
+        const filesTask = (0, Array_1.sequence)(TE.MonadTask)(fileStreamsFromReferencesTasks);
+        return (0, function_1.pipe)(filesTask, TE.map((files) => files.reduce((acc, file) => {
+            acc[file.key] = file;
+            return acc;
+        }, {})), TE.map((multipartFiles) => ({
+            fields: body.fields,
+            files: multipartFiles,
+        })));
     }
     serializeAsFile(operationRequest) {
         const downloadFilefromReference = this.httpClient.execute(Http_1.HttpMethod.Get, operationRequest.body.url, {
@@ -131,7 +166,7 @@ class HttpOperationExecution {
                 case Http_1.HttpContentType.MultipartRequestBody:
                     return TE.right(OperationHandler_1.OperationHandlerResult.failure(OperationHandler_1.OperationHandlerError.connectorError(`Invalid Response Type ${contentType}`)));
                 case Http_1.HttpContentType.Text:
-                    return this.parseeBodyAsText(operationResponse);
+                    return this.parseBodyAsText(operationResponse);
                 default:
                     return this.parseBodyAsJson(operationResponse);
             }
@@ -148,18 +183,19 @@ class HttpOperationExecution {
         return TE.right(operationResponse.responseParser(undefined));
     }
     parseBodyAsFile(operationResponse) {
-        var _a;
         const contentType = (operationResponse.response.headers[Http_1.HttpHeader.ContentType.toLowerCase()]);
         const fileExtension = mime.getExtension(contentType);
+        const contentLength = (operationResponse.response.headers[Http_1.HttpHeader.ContentLength.toLowerCase()]);
+        const uploadOptions = this.generateUploadOptions(contentLength);
         return (0, function_1.pipe)(this.fileStorage.write({
             content: operationResponse.response.body,
             key: fileExtension ? `${(0, uuid_1.v4)()}.${fileExtension}` : (0, uuid_1.v4)(),
             metadata: {
                 name: (0, uuid_1.v4)(),
-                size: parseFloat((_a = (operationResponse.response.headers[Http_1.HttpHeader.ContentLength.toLowerCase()])) !== null && _a !== void 0 ? _a : undefined),
+                size: parseFloat(contentLength !== null && contentLength !== void 0 ? contentLength : undefined),
                 contentType,
             },
-        }, O.none), TE.chain((file) => (0, function_1.pipe)(this.fileStorage.getSharedUrl(file.key), TE.map((sharedUrl) => {
+        }, O.some(uploadOptions)), TE.chain((file) => (0, function_1.pipe)(this.fileStorage.getSharedUrl(file.key), TE.map((sharedUrl) => {
             var _a, _b;
             return operationResponse.responseParser({
                 name: file.metadata.name,
@@ -169,12 +205,43 @@ class HttpOperationExecution {
             });
         }))));
     }
-    parseeBodyAsText(operationResponse) {
+    parseBodyAsText(operationResponse) {
         const arrayBufferTE = BufferExtensions_1.BufferExtensions.readableToArrayBuffer(operationResponse.response.body);
         return (0, function_1.pipe)(arrayBufferTE, TE.chain((arrayBuffer) => {
             const deserializeBody = new TextDecoder().decode(arrayBuffer);
             return TE.right(operationResponse.responseParser(deserializeBody));
         }));
     }
+    // This method is used to respect lambda memory limits as per the old falfel implementation
+    generateUploadOptions(contentLength) {
+        const allocatedRam = AWS_LAMBDA_FUNCTION_MEMORY_SIZE || CONNECTOR_MAX_ALLOCATED_RAM_MB;
+        const availableRAM = allocatedRam ? parseInt(allocatedRam, 10) : 128; // 128MB default
+        const uploadMinSize = 1024 * 1024 * 8, // 8MB
+        maxRam = 1024 * 1024 * availableRAM;
+        /*
+            128 / 16 = 8 (which is UPLOAD_MIN_SIZE)
+            TARGET_SIZE should either be 8MB (since default assumed RAM is 128MB) or
+            1/16 of of the availableRAM, which ever is greater. The TARGET_SIZE, set
+            as `partSize`, will always be used in conjuction with `queueSize`.
+        */
+        const sixteenthOfRAM = Math.floor(maxRam / 16);
+        const targetSize = sixteenthOfRAM < uploadMinSize ? uploadMinSize : sixteenthOfRAM;
+        const uploadOptions = {
+            // 8MB (minimum) * 4 parallel = upto 32MB/s RAM usage
+            partSize: targetSize,
+            queueSize: 4,
+            acl: 'bucket-owner-full-control',
+        };
+        if (!contentLength) {
+            if (CONNECTOR_MAX_ALLOCATED_RAM_MB) {
+                uploadOptions.queueSize = 12;
+            }
+        }
+        else if (parseInt(contentLength, 10) > targetSize &&
+            parseInt(contentLength, 10) > targetSize * 8) {
+            uploadOptions.queueSize = 12;
+        }
+        return uploadOptions;
+    }
 }
 exports.HttpOperationExecution = HttpOperationExecution;

package/dist/connector/operation/OperationExecutionGateway.unit.test.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"OperationExecutionGateway.unit.test.d.ts","sourceRoot":"","sources":["../../../src/connector/operation/OperationExecutionGateway.unit.test.ts"],"names":[],"mappings":"AAOA,OAAO,8BAA8B,CAAC"}
+{"version":3,"file":"OperationExecutionGateway.unit.test.d.ts","sourceRoot":"","sources":["../../../src/connector/operation/OperationExecutionGateway.unit.test.ts"],"names":[],"mappings":"AAmBA,OAAO,8BAA8B,CAAC"}

package/dist/connector/operation/OperationExecutionGateway.unit.test.js

@@ -23,18 +23,18 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const OperationHandlerSetup_1 = require("@trayio/cdk-dsl/connector/operation/OperationHandlerSetup");
-const OperationHandlerTest_1 = require("@trayio/cdk-dsl/connector/operation/OperationHandlerTest");
-const OperationHandler_1 = require("@trayio/cdk-dsl/connector/operation/OperationHandler");
-require("./OperationHandlerTestRunner");
+const stream_1 = require("stream");
 const T = __importStar(require("fp-ts/Task"));
 const TE = __importStar(require("fp-ts/TaskEither"));
 const function_1 = require("fp-ts/function");
+const OperationHandlerSetup_1 = require("@trayio/cdk-dsl/connector/operation/OperationHandlerSetup");
+const OperationHandlerTest_1 = require("@trayio/cdk-dsl/connector/operation/OperationHandlerTest");
+const OperationHandler_1 = require("@trayio/cdk-dsl/connector/operation/OperationHandler");
 const Http_1 = require("@trayio/commons/http/Http");
-const stream_1 = require("stream");
 const ExpressHttpServer_1 = require("@trayio/express/http/ExpressHttpServer");
 const ExpressHttpController_1 = require("@trayio/express/http/ExpressHttpController");
 const NodeFsFileStorage_1 = require("@trayio/commons/file/NodeFsFileStorage");
+require("./OperationHandlerTestRunner");
 const fileStorage = new NodeFsFileStorage_1.NodeFsFileStorage('/tmp');
 class TestControllerHttp {
     constructor() {
@@ -80,6 +80,26 @@ class TestControllerHttp {
                     })));
                 },
             },
+            {
+                path: `/multipart-upload`,
+                method: Http_1.HttpMethod.Post,
+                execute: (request) => {
+                    const { body } = request;
+                    expect(body).toEqual(expect.objectContaining({
+                        fields: expect.objectContaining({ test: 'Hello world!' }),
+                        files: expect.objectContaining({
+                            'test-image': expect.any(Object),
+                        }),
+                    }));
+                    return T.of({
+                        body: stream_1.Readable.from(JSON.stringify({
+                            success: true,
+                        })),
+                        headers: {},
+                        statusCode: 200,
+                    });
+                },
+            },
         ];
     }
 }
@@ -152,6 +172,37 @@ describe('OperationExecutionGateway', () => {
             .finallyDoNothing())
             .nothingAfterAll());
     });
+    describe('post multipart (with file)', () => {
+        const multipartOperation = OperationHandlerSetup_1.OperationHandlerSetup.configureHandler((handler) => handler.usingHttp((http) => http
+            .post('http://localhost:3000/multipart-upload')
+            .handleRequest((_ctx, _input, request) => request.withBodyAsMultipart({
+            fields: {
+                test: 'Hello world!',
+            },
+            files: {
+                'test-image': {
+                    name: 'testImage.png',
+                    url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/6/61/Wikipedia-logo-transparent.png/526px-Wikipedia-logo-transparent.png',
+                    mime_type: 'image/png',
+                    expires: 0,
+                },
+            },
+        }))
+            .handleResponse((_ctx, _input, response) => response.parseWithBodyAsJson())));
+        OperationHandlerTest_1.OperationHandlerTestSetup.configureHandlerTest(multipartOperation, (handlerTest) => handlerTest
+            .usingHandlerContext('test')
+            .nothingBeforeAll()
+            .testCase('should send a multipart body (both data and file)', (testCase) => testCase
+            .usingHandlerContext('test')
+            .givenNothing()
+            .when(() => ({}))
+            .then(({ output }) => {
+            const outputValue = OperationHandler_1.OperationHandlerResult.getSuccessfulValueOrFail(output);
+            expect(outputValue.success).toEqual(true);
+        })
+            .finallyDoNothing())
+            .nothingAfterAll());
+    });
     describe('upload an image', () => {
         const uploadImageOperation = OperationHandlerSetup_1.OperationHandlerSetup.configureHandler((handler) => handler.usingHttp((http) => http
             .post('http://localhost:3000/image-upload')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trayio/cdk-runtime",
-  "version": "2.9.0",
+  "version": "2.11.0",
   "description": "A Runtime that executes connector operations defined using the CDK DSL",
   "exports": {
     "./*": "./dist/*.js"
@@ -14,10 +14,10 @@
     "node": ">=18.x"
   },
   "dependencies": {
-    "@trayio/axios": "2.9.0",
-    "@trayio/cdk-dsl": "2.9.0",
-    "@trayio/express": "2.9.0",
-    "@trayio/winston": "2.9.0",
+    "@trayio/axios": "2.11.0",
+    "@trayio/cdk-dsl": "2.11.0",
+    "@trayio/express": "2.11.0",
+    "@trayio/winston": "2.11.0",
     "mime": "3.0.0",
     "uuid": "9.0.0"
   },