@devrev/airsync-sdk 2.0.0 → 2.1.0-beta.0

package/README.md

@@ -1,6 +1,6 @@
 # AirSync SDK
 
-[![Coverage Status](https://coveralls.io/repos/github/devrev/adaas-sdk/badge.svg?branch=main&t=s4Otlm)](https://coveralls.io/github/devrev/adaas-sdk?branch=main)
+[![Coverage Status](https://coveralls.io/repos/github/devrev/airsync-sdk/badge.svg?branch=main&t=s4Otlm)](https://coveralls.io/github/devrev/airsync-sdk?branch=main)
 
 TypeScript SDK for building AirSync snap-ins (connectors) that sync data between external systems and DevRev. It handles the AirSync control protocol for you: worker threads, timeouts, state, artifact uploads, attachment streaming and loading.
 

package/dist/common/constants.d.ts

@@ -4,6 +4,7 @@ export declare const STATELESS_LOADING_EVENT_TYPES: EventType[];
 export declare const STATELESS_EVENT_TYPES: EventType[];
 export declare const ARTIFACT_BATCH_SIZE = 2000;
 export declare const MAX_DEVREV_ARTIFACT_SIZE: number;
+export declare const S3_MULTIPART_CHUNK_SIZE: number;
 export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
 export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
 export declare const MAX_EVENT_SIZE_BYTES = 200000;

package/dist/common/constants.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAG/D,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}

package/dist/common/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UNBOUNDED_DATE_TIME_VALUE = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = void 0;
+exports.UNBOUNDED_DATE_TIME_VALUE = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.S3_MULTIPART_CHUNK_SIZE = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = void 0;
 const extraction_1 = require("../types/extraction");
 const helpers_1 = require("./helpers");
 exports.STATELESS_EXTRACTION_EVENT_TYPES = [
@@ -18,6 +18,9 @@ exports.STATELESS_EVENT_TYPES = [
 ];
 exports.ARTIFACT_BATCH_SIZE = 2000;
 exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
+// S3 multipart uploads require every non-final part to be at least 5 MiB. A
+// 10 MiB part gives us a useful buffer while keeping the memory overhead low.
+exports.S3_MULTIPART_CHUNK_SIZE = 10 * 1024 * 1024;
 exports.MAX_DEVREV_FILENAME_LENGTH = 256;
 exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
 // Max SQS message size is 250KB, we want to leave some room for the other data in the message

package/dist/multithreading/adapters/extraction-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAoDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAwGjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}
+{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAoDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA4IjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}

package/dist/multithreading/adapters/extraction-adapter.js

@@ -230,44 +230,70 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
             ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
             'application/octet-stream';
         const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
-        const fileSize = contentLength ? parseInt(contentLength) : undefined;
-        const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
-        if (artifactUrlError) {
-            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
-            return {
-                status: 'error',
-                error: {
-                    message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
-                    fileSize,
-                },
-            };
-        }
-        const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
-        if (uploadedArtifactError) {
-            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
-            return {
-                status: 'error',
-                error: {
-                    message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
-                        (0, logger_1.serializeError)(uploadedArtifactError),
-                    fileSize,
-                },
-            };
+        const parsedFileSize = contentLength
+            ? Number.parseInt(contentLength, 10)
+            : undefined;
+        const fileSize = parsedFileSize !== undefined && Number.isFinite(parsedFileSize)
+            ? parsedFileSize
+            : undefined;
+        let artifactId;
+        if (fileSize === undefined) {
+            // Unknown-length streams cannot be uploaded using a presigned POST with
+            // a guessed Content-Length. Stream them through S3 multipart upload.
+            const { response: multipartUpload, error: multipartUploadError } = await this.uploader.streamArtifactMultipart(attachment.file_name, fileType, httpStream);
+            if (multipartUploadError || !multipartUpload) {
+                return {
+                    status: 'error',
+                    error: {
+                        message: `Error while uploading attachment for attachment ID ${attachment.id}. Skipping attachment. ` +
+                            (0, logger_1.serializeError)(multipartUploadError !== null && multipartUploadError !== void 0 ? multipartUploadError : new Error('Multipart upload returned no response.')),
+                        fileSize,
+                    },
+                };
+            }
+            artifactId = multipartUpload.artifact_id;
         }
-        const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
-        if (confirmArtifactUploadError) {
-            return {
-                status: 'error',
-                error: {
-                    message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
-                        (0, logger_1.serializeError)(confirmArtifactUploadError),
-                    fileSize,
-                },
-            };
+        else {
+            // Known-size attachments retain the existing single-request upload flow.
+            const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
+            if (artifactUrlError || !artifactUrlResponse) {
+                (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+                return {
+                    status: 'error',
+                    error: {
+                        message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError !== null && artifactUrlError !== void 0 ? artifactUrlError : new Error('No artifact upload response.'))}`,
+                        fileSize,
+                    },
+                };
+            }
+            const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
+            if (uploadedArtifactError) {
+                (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+                return {
+                    status: 'error',
+                    error: {
+                        message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
+                            (0, logger_1.serializeError)(uploadedArtifactError),
+                        fileSize,
+                    },
+                };
+            }
+            const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
+            if (confirmArtifactUploadError) {
+                return {
+                    status: 'error',
+                    error: {
+                        message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
+                            (0, logger_1.serializeError)(confirmArtifactUploadError),
+                        fileSize,
+                    },
+                };
+            }
+            artifactId = artifactUrlResponse.artifact_id;
         }
         const ssorAttachment = {
             id: {
-                devrev: artifactUrlResponse.artifact_id,
+                devrev: artifactId,
                 external: attachment.id,
             },
             parent_id: {

package/dist/uploader/uploader.d.ts

@@ -1,7 +1,7 @@
 import { AxiosResponse } from 'axios';
 import { NormalizedAttachment } from '../repo/repo.interfaces';
 import { HttpStreamResponse } from '../types/extraction';
-import { ArtifactToUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
+import { ArtifactToUpload, MultipartUpload, UploaderFactoryInterface, UploaderResult, UploadResponse } from './uploader.interfaces';
 export declare class Uploader {
     private isLocalDevelopment?;
     private devrevApiEndpoint;
@@ -17,6 +17,26 @@ export declare class Uploader {
     uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<UploaderResult<AxiosResponse>>;
     /** Streams a file from an HTTP response directly to the artifact's upload URL. */
     streamArtifact(artifact: ArtifactToUpload, fileStream: HttpStreamResponse): Promise<UploaderResult<AxiosResponse>>;
+    /**
+     * Creates an S3 multipart upload for an artifact whose size is not known.
+     * The returned URL is for part 1; subsequent URLs are obtained with
+     * getNextMultipartUploadPart.
+     */
+    createMultipartUpload(filename: string, fileType: string, partSize?: number): Promise<UploaderResult<MultipartUpload>>;
+    /** Gets a presigned URL for the next multipart upload part. */
+    getNextMultipartUploadPart(artifactId: string, cursor: string, partSize?: number): Promise<UploaderResult<MultipartUpload>>;
+    /** Finalizes and publishes an S3 multipart artifact upload. */
+    completeMultipartUpload(artifactId: string, cursor: string): Promise<UploaderResult<AxiosResponse>>;
+    /**
+     * Streams an attachment of unknown size through an S3 multipart upload.
+     * At most one 10 MiB part is buffered at a time. The final part may be
+     * smaller than S3's 5 MiB minimum.
+     */
+    streamArtifactMultipart(filename: string, fileType: string, fileStream: HttpStreamResponse): Promise<UploaderResult<{
+        artifact_id: string;
+    }>>;
+    /** Uploads one raw part to its presigned S3 URL. */
+    private uploadMultipartPart;
     confirmArtifactUpload(artifactId: string): Promise<{
         response?: AxiosResponse;
         error?: unknown;

package/dist/uploader/uploader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAOtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EAEL,gBAAgB,EAChB,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwEpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
+{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAUtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAWzD,OAAO,EAEL,gBAAgB,EAChB,eAAe,EACf,wBAAwB,EACxB,cAAc,EACd,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,wBAAwB;IAW1E,mFAAmF;IAC7E,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwEpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C,mFAAmF;IAC7E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC,kFAAkF;IAC5E,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAqCzC;;;;OAIG;IACG,qBAAqB,CACzB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,0BAA0B,CAC9B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,SAA0B,GACjC,OAAO,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAkB3C,+DAA+D;IACzD,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAoBzC;;;;OAIG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,kBAAkB,GAC7B,OAAO,CAAC,cAAc,CAAC;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA6HnD,oDAAoD;YACtC,mBAAmB;IAsD3B,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAgCF,oEAAoE;IAC9D,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAuDY,sBAAsB;YAsBtB,gBAAgB;IAc9B,yEAAyE;IACnE,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}

package/dist/uploader/uploader.interfaces.d.ts

@@ -40,6 +40,12 @@ export interface ArtifactToUpload {
         value: string;
     }[];
 }
+/** Multipart upload state returned by the artifacts upload endpoints. */
+export interface MultipartUpload {
+    id: string;
+    url: string;
+    cursor: string;
+}
 export interface UploadResponse {
     artifact?: Artifact;
     error?: ErrorRecord;

package/dist/uploader/uploader.interfaces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}
+{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}

package/dist/uploader/uploader.js

@@ -4,6 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
+const http_1 = __importDefault(require("http"));
+const https_1 = __importDefault(require("https"));
 const form_data_1 = __importDefault(require("form-data"));
 const js_jsonl_1 = require("js-jsonl");
 const constants_1 = require("../common/constants");
@@ -136,6 +138,201 @@ class Uploader {
             return { error };
         }
     }
+    /**
+     * Creates an S3 multipart upload for an artifact whose size is not known.
+     * The returned URL is for part 1; subsequent URLs are obtained with
+     * getNextMultipartUploadPart.
+     */
+    async createMultipartUpload(filename, fileType, partSize = constants_1.S3_MULTIPART_CHUNK_SIZE) {
+        const url = `${this.devrevApiEndpoint}/internal/artifacts.upload.create`;
+        try {
+            const response = await client_1.axiosClient.post(url, {
+                file_name: (0, uploader_helpers_1.truncateFilename)(filename),
+                file_type: fileType,
+                part_size: partSize,
+            }, { headers: Object.assign({}, this.defaultHeaders) });
+            return { response: response.data };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    /** Gets a presigned URL for the next multipart upload part. */
+    async getNextMultipartUploadPart(artifactId, cursor, partSize = constants_1.S3_MULTIPART_CHUNK_SIZE) {
+        const url = `${this.devrevApiEndpoint}/internal/artifacts.upload.next`;
+        try {
+            const response = await client_1.axiosClient.post(url, {
+                id: artifactId,
+                cursor,
+                part_size: partSize,
+            }, { headers: Object.assign({}, this.defaultHeaders) });
+            return { response: response.data };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    /** Finalizes and publishes an S3 multipart artifact upload. */
+    async completeMultipartUpload(artifactId, cursor) {
+        const url = `${this.devrevApiEndpoint}/internal/artifacts.upload.complete`;
+        try {
+            const response = await client_1.axiosClient.post(url, {
+                id: artifactId,
+                cursor,
+            }, { headers: Object.assign({}, this.defaultHeaders) });
+            if (response.status >= 200 && response.status < 300) {
+                return { response };
+            }
+            return { error: response };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    /**
+     * Streams an attachment of unknown size through an S3 multipart upload.
+     * At most one 10 MiB part is buffered at a time. The final part may be
+     * smaller than S3's 5 MiB minimum.
+     */
+    async streamArtifactMultipart(filename, fileType, fileStream) {
+        try {
+            const iterator = fileStream.data[Symbol.asyncIterator]();
+            let pendingChunk;
+            // Read a complete part before requesting its presigned URL. The gateway
+            // signs Content-Length using part_size, so the requested part size must
+            // be the exact byte length that will be sent (including the final part).
+            const readPart = async () => {
+                const chunks = [];
+                let length = 0;
+                while (length < constants_1.S3_MULTIPART_CHUNK_SIZE) {
+                    let chunk = pendingChunk;
+                    pendingChunk = undefined;
+                    if (!chunk) {
+                        const next = await iterator.next();
+                        if (next.done) {
+                            break;
+                        }
+                        chunk = Buffer.isBuffer(next.value)
+                            ? next.value
+                            : Buffer.from(next.value);
+                    }
+                    if (chunk.length === 0) {
+                        continue;
+                    }
+                    const capacity = constants_1.S3_MULTIPART_CHUNK_SIZE - length;
+                    if (chunk.length <= capacity) {
+                        chunks.push(chunk);
+                        length += chunk.length;
+                    }
+                    else {
+                        chunks.push(chunk.subarray(0, capacity));
+                        pendingChunk = chunk.subarray(capacity);
+                        length += capacity;
+                    }
+                }
+                return length > 0 ? Buffer.concat(chunks, length) : undefined;
+            };
+            let part = await readPart();
+            if (!part) {
+                throw new Error('Cannot complete a multipart upload with an empty file.');
+            }
+            const { response: createdUpload, error: createError } = await this.createMultipartUpload(filename, fileType, part.length);
+            if (createError || !createdUpload) {
+                return {
+                    error: createError !== null && createError !== void 0 ? createError : new Error('Multipart upload creation returned no response.'),
+                };
+            }
+            const artifactId = createdUpload.id;
+            let cursor = createdUpload.cursor;
+            let uploadUrl = createdUpload.url;
+            if (!artifactId || !cursor || !uploadUrl) {
+                return {
+                    error: new Error('Multipart upload creation returned incomplete upload information.'),
+                };
+            }
+            while (part) {
+                const { response, error } = await this.uploadMultipartPart(uploadUrl, part, part.length);
+                if (error || !response) {
+                    return {
+                        error: error !== null && error !== void 0 ? error : new Error('Multipart part upload returned no response.'),
+                    };
+                }
+                const nextPart = part.length === constants_1.S3_MULTIPART_CHUNK_SIZE
+                    ? await readPart()
+                    : undefined;
+                if (!nextPart) {
+                    break;
+                }
+                const { response: nextUpload, error: nextError } = await this.getNextMultipartUploadPart(artifactId, cursor, nextPart.length);
+                if (nextError || !(nextUpload === null || nextUpload === void 0 ? void 0 : nextUpload.url) || !nextUpload.cursor) {
+                    return {
+                        error: nextError !== null && nextError !== void 0 ? nextError : new Error('Could not obtain the next multipart upload URL.'),
+                    };
+                }
+                part = nextPart;
+                uploadUrl = nextUpload.url;
+                cursor = nextUpload.cursor;
+            }
+            const { error: completeError } = await this.completeMultipartUpload(artifactId, cursor);
+            if (completeError) {
+                return { error: completeError };
+            }
+            return { response: { artifact_id: artifactId } };
+        }
+        catch (error) {
+            return { error };
+        }
+        finally {
+            (0, uploader_helpers_1.destroyHttpStream)(fileStream);
+        }
+    }
+    /** Uploads one raw part to its presigned S3 URL. */
+    async uploadMultipartPart(uploadUrl, part, contentLength) {
+        try {
+            // Use Node's native request instead of Axios for the presigned request.
+            // Axios may normalize, remove, or recalculate Content-Length; that is
+            // not allowed when the presigned URL includes this header in its
+            // signature.
+            const parsedUrl = new URL(uploadUrl);
+            const transport = parsedUrl.protocol === 'https:' ? https_1.default : http_1.default;
+            const response = await new Promise((resolve, reject) => {
+                const request = transport.request(parsedUrl, {
+                    method: 'PUT',
+                    headers: {
+                        'Content-Length': String(contentLength),
+                    },
+                }, (incomingResponse) => {
+                    const chunks = [];
+                    incomingResponse.on('data', (chunk) => chunks.push(chunk));
+                    incomingResponse.on('end', () => {
+                        var _a, _b;
+                        const data = Buffer.concat(chunks);
+                        const status = (_a = incomingResponse.statusCode) !== null && _a !== void 0 ? _a : 0;
+                        const axiosResponse = {
+                            data,
+                            status,
+                            statusText: (_b = incomingResponse.statusMessage) !== null && _b !== void 0 ? _b : '',
+                            headers: incomingResponse.headers,
+                            config: {},
+                            request,
+                        };
+                        if (status >= 200 && status < 400) {
+                            resolve(axiosResponse);
+                        }
+                        else {
+                            reject(new Error(`Multipart part upload returned HTTP ${status}.`));
+                        }
+                    });
+                });
+                request.once('error', reject);
+                request.end(part);
+            });
+            return { response };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
     async confirmArtifactUpload(artifactId) {
         const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.confirm-upload`;
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/airsync-sdk",
-  "version": "2.0.0",
+  "version": "2.1.0-beta.0",
   "description": "Typescript SDK for building AirSync snap-ins on the DevRev platform.",
   "type": "commonjs",
   "main": "./dist/index.js",