@devrev/ts-adaas 1.1.5 → 1.1.6

package/README.md

@@ -2,6 +2,11 @@
 
 ## Release Notes
 
+### v1.1.6
+
+- Add exponential retry and handle rate-limiting towards DevRev.
+- Gracefully handle failure to upload extracted attachments.
+
 ### v1.1.5
 
 - Increase `delayFactor` and number of retries for the exponential backoff retry mechanism for HTTP requests.

package/dist/common/constants.d.ts

@@ -4,7 +4,7 @@ export declare const ALLOWED_EXTRACTION_EVENT_TYPES: EventType[];
 export declare const ALLOWED_LOADING_EVENT_TYPES: EventType[];
 export declare const ALLOWED_EVENT_TYPES: EventType[];
 export declare const ARTIFACT_BATCH_SIZE = 2000;
-export declare const MAX_DEVREV_ARTIFACT_SIZE = 536870912;
+export declare const MAX_DEVREV_ARTIFACT_SIZE = 262144000;
 export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
     EXTERNAL_DOMAIN_METADATA: string;
     ATTACHMENTS: string;

package/dist/common/constants.js

@@ -31,7 +31,7 @@ exports.ALLOWED_EVENT_TYPES = [
     ...exports.ALLOWED_LOADING_EVENT_TYPES,
 ];
 exports.ARTIFACT_BATCH_SIZE = 2000;
-exports.MAX_DEVREV_ARTIFACT_SIZE = 536870912; // 512MB
+exports.MAX_DEVREV_ARTIFACT_SIZE = 262144000; // 250MB
 exports.AIRDROP_DEFAULT_ITEM_TYPES = {
     EXTERNAL_DOMAIN_METADATA: 'external_domain_metadata',
     ATTACHMENTS: 'attachments',

package/dist/deprecated/uploader/index.js

@@ -22,12 +22,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
-const axios_1 = __importDefault(require("axios"));
+const axios_devrev_client_1 = require("../../http/axios-devrev-client");
 const typescript_sdk_1 = require("@devrev/typescript-sdk");
 const fs_1 = __importStar(require("fs"));
 const helpers_1 = require("../common/helpers");
@@ -111,7 +108,7 @@ class Uploader {
     ) {
         const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
         try {
-            const response = await axios_1.default.post(preparedArtifact.url, formData, {
+            const response = await axios_devrev_client_1.axiosDevRevClient.post(preparedArtifact.url, formData, {
                 headers: {
                     'Content-Type': 'multipart/form',
                 },

package/dist/http/axios-devrev-client.d.ts

@@ -0,0 +1,3 @@
+import axios from 'axios';
+declare const axiosDevRevClient: import("axios").AxiosInstance;
+export { axios, axiosDevRevClient };

package/dist/http/axios-devrev-client.js

@@ -0,0 +1,37 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.axiosDevRevClient = exports.axios = void 0;
+const axios_1 = __importDefault(require("axios"));
+exports.axios = axios_1.default;
+const axios_retry_1 = __importDefault(require("axios-retry"));
+const axiosDevRevClient = axios_1.default.create();
+exports.axiosDevRevClient = axiosDevRevClient;
+(0, axios_retry_1.default)(axiosDevRevClient, {
+    retries: 5,
+    retryDelay: (retryCount, error) => {
+        var _a, _b;
+        console.warn('Retry attempt: ' + retryCount + 'to url: ' + ((_a = error.config) === null || _a === void 0 ? void 0 : _a.url) + '.');
+        if (error.response) {
+            const retry_after = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers['retry-after'];
+            if (retry_after) {
+                return retry_after;
+            }
+        }
+        // Exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
+        return axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
+    },
+    retryCondition: (error) => {
+        var _a;
+        return (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) ||
+            ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429);
+    },
+    onMaxRetryTimesExceeded(error, retryCount) {
+        var _a;
+        console.log(`Max retries attempted: ${retryCount}`);
+        (_a = error.config) === null || _a === void 0 ? true : delete _a.headers.Authorization;
+        delete error.request._header;
+    },
+});

package/dist/repo/repo.js

@@ -48,7 +48,7 @@ class Repo {
             }
             // Add the new records to the items array
             this.items.push(...recordsToPush);
-            console.log(`Extracted ${recordsToPush.length} new items of type ${this.itemType}. Total number of items in repo: ${this.items.length}.`);
+            console.info(`Extracted ${recordsToPush.length} new items of type ${this.itemType}. Total number of items in repo: ${this.items.length}.`);
             // Upload in batches while the number of items exceeds the batch size
             while (this.items.length >= constants_1.ARTIFACT_BATCH_SIZE) {
                 // Slice out a batch of ARTIFACT_BATCH_SIZE items to upload

package/dist/uploader/uploader.d.ts

@@ -1,7 +1,7 @@
-import { AxiosResponse } from 'axios';
 import { betaSDK } from '@devrev/typescript-sdk';
 import { NormalizedAttachment } from '../repo/repo.interfaces';
 import { UploadResponse, UploaderFactoryInterface } from './uploader.interfaces';
+import { AxiosResponse } from 'axios';
 export declare class Uploader {
     private event;
     private betaDevrevSdk;

package/dist/uploader/uploader.js

@@ -28,7 +28,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
 const fs_1 = __importStar(require("fs"));
-const axios_client_1 = require("../http/axios-client");
+const axios_devrev_client_1 = require("../http/axios-devrev-client");
 const zlib_1 = __importDefault(require("zlib"));
 const js_jsonl_1 = require("js-jsonl");
 const form_data_1 = __importDefault(require("form-data"));
@@ -98,7 +98,7 @@ class Uploader {
             return response.data;
         }
         catch (error) {
-            if (axios_client_1.axios.isAxiosError(error)) {
+            if (axios_devrev_client_1.axios.isAxiosError(error)) {
                 console.error('Error while preparing artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
@@ -115,13 +115,13 @@ class Uploader {
         }
         formData.append('file', file);
         try {
-            const response = await axios_client_1.axiosClient.post(preparedArtifact.url, formData, {
+            const response = await axios_devrev_client_1.axiosDevRevClient.post(preparedArtifact.url, formData, {
                 headers: Object.assign({}, formData.getHeaders()),
             });
             return response;
         }
         catch (error) {
-            if (axios_client_1.axios.isAxiosError(error)) {
+            if (axios_devrev_client_1.axios.isAxiosError(error)) {
                 console.error('Error while uploading artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
@@ -137,8 +137,11 @@ class Uploader {
             formData.append(field.key, field.value);
         }
         formData.append('file', fileStreamResponse.data);
+        if (fileStreamResponse.headers['content-length'] > constants_1.MAX_DEVREV_ARTIFACT_SIZE) {
+            return;
+        }
         try {
-            const response = await axios_client_1.axiosClient.post(preparedArtifact.url, formData, {
+            const response = await axios_devrev_client_1.axiosDevRevClient.post(preparedArtifact.url, formData, {
                 headers: Object.assign(Object.assign({}, formData.getHeaders()), (!fileStreamResponse.headers['content-length'] && {
                     'Content-Length': constants_1.MAX_DEVREV_ARTIFACT_SIZE,
                 })),
@@ -146,7 +149,7 @@ class Uploader {
             return response;
         }
         catch (error) {
-            if (axios_client_1.axios.isAxiosError(error)) {
+            if (axios_devrev_client_1.axios.isAxiosError(error)) {
                 console.error('Error while streaming artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
@@ -199,13 +202,13 @@ class Uploader {
     }
     async downloadArtifact(artifactUrl) {
         try {
-            const response = await axios_client_1.axiosClient.get(artifactUrl, {
+            const response = await axios_devrev_client_1.axiosDevRevClient.get(artifactUrl, {
                 responseType: 'arraybuffer',
             });
             return response.data;
         }
         catch (error) {
-            if (axios_client_1.axios.isAxiosError(error)) {
+            if (axios_devrev_client_1.axios.isAxiosError(error)) {
                 console.error('Error while downloading artifact from URL.', (0, logger_1.serializeAxiosError)(error));
             }
             else {

package/dist/workers/worker-adapter.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkerAdapter = void 0;
 exports.createWorkerAdapter = createWorkerAdapter;
-const axios_client_1 = require("../http/axios-client");
+const axios_devrev_client_1 = require("../http/axios-devrev-client");
 const extraction_1 = require("../types/extraction");
 const loading_1 = require("../types/loading");
 const constants_1 = require("../common/constants");
@@ -340,7 +340,7 @@ class WorkerAdapter {
                         console.log('Updated sync mapper record', JSON.stringify(updateSyncMapperRecordResponse.data));
                     }
                     catch (error) {
-                        if (axios_client_1.axios.isAxiosError(error)) {
+                        if (axios_devrev_client_1.axios.isAxiosError(error)) {
                             console.error('Failed to update sync mapper record', (0, logger_1.serializeAxiosError)(error));
                             return {
                                 error: {
@@ -385,7 +385,7 @@ class WorkerAdapter {
             // Update mapper (optional)
         }
         catch (error) {
-            if (axios_client_1.axios.isAxiosError(error)) {
+            if (axios_devrev_client_1.axios.isAxiosError(error)) {
                 if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
                     // Create item
                     const { id, delay, error } = await itemTypeToLoad.create({
@@ -411,7 +411,7 @@ class WorkerAdapter {
                             };
                         }
                         catch (error) {
-                            if (axios_client_1.axios.isAxiosError(error)) {
+                            if (axios_devrev_client_1.axios.isAxiosError(error)) {
                                 console.error('Failed to create sync mapper record', (0, logger_1.serializeAxiosError)(error));
                                 return {
                                     error: {
@@ -469,7 +469,7 @@ class WorkerAdapter {
      * or error information if there was an error
      */
     async streamAttachments({ stream, }) {
-        var _a, _b, _c, _d, _e, _f, _g;
+        var _a, _b, _c, _d, _e, _f;
         const repos = [
             {
                 itemType: 'ssor_attachment',
@@ -487,9 +487,8 @@ class WorkerAdapter {
             if (error) {
                 return { error };
             }
-            console.log('this.state.toDevRev?.attachmentsMetadata :>> ', (_c = this.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata);
             if (attachments) {
-                const attachmentsToProcess = attachments.slice((_e = (_d = this.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata) === null || _e === void 0 ? void 0 : _e.lastProcessed, attachments.length);
+                const attachmentsToProcess = attachments.slice((_d = (_c = this.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata) === null || _d === void 0 ? void 0 : _d.lastProcessed, attachments.length);
                 for (const attachment of attachmentsToProcess) {
                     const { httpStream, delay, error } = await stream({
                         item: attachment,
@@ -503,19 +502,26 @@ class WorkerAdapter {
                         return { delay };
                     }
                     if (httpStream) {
-                        const fileType = ((_f = httpStream.headers) === null || _f === void 0 ? void 0 : _f['content-type']) ||
+                        const fileType = ((_e = httpStream.headers) === null || _e === void 0 ? void 0 : _e['content-type']) ||
                             'application/octet-stream';
                         const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
                         if (!preparedArtifact) {
-                            return {
-                                error: { message: 'Error while preparing artifact.' },
-                            };
+                            console.warn('Error while preparing artifact for attachment ID ' +
+                                attachment.id +
+                                '. Skipping attachment');
+                            if (this.state.toDevRev) {
+                                this.state.toDevRev.attachmentsMetadata.lastProcessed++;
+                            }
+                            continue;
                         }
                         const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
                         if (!uploadedArtifact) {
-                            return {
-                                error: { message: 'Error while streaming artifact.' },
-                            };
+                            console.warn('Error while preparing artifact for attachment ID ' +
+                                attachment.id);
+                            if (this.state.toDevRev) {
+                                this.state.toDevRev.attachmentsMetadata.lastProcessed++;
+                            }
+                            continue;
                         }
                         const ssorAttachment = {
                             id: {
@@ -529,7 +535,7 @@ class WorkerAdapter {
                                 external: attachment.author_id,
                             },
                         };
-                        await ((_g = this.getRepo('ssor_attachment')) === null || _g === void 0 ? void 0 : _g.push([ssorAttachment]));
+                        await ((_f = this.getRepo('ssor_attachment')) === null || _f === void 0 ? void 0 : _f.push([ssorAttachment]));
                         if (this.state.toDevRev) {
                             this.state.toDevRev.attachmentsMetadata.lastProcessed++;
                         }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.1.5",
-  "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
+  "version": "1.1.6",
+  "description": "DevRev ADaaS (AirDrop-as-a-Service) Typescript SDK.",
   "type": "commonjs",
   "main": "./dist/index.js",
   "typings": "./dist/index.d.ts",