@devrev/ts-adaas 1.1.4 → 1.1.5

package/README.md

@@ -2,71 +2,77 @@
 
 ## Release Notes
 
+### v1.1.5
+
+- Increase `delayFactor` and number of retries for the exponential backoff retry mechanism for HTTP requests.
+- Provide an inject function for streaming attachments.
+- Fix the attachments streaming bug.
+
 ### v1.1.4
 
-- Provide log line and stack trace on runtime worker errors.
+- Provide log lines and stack traces for runtime worker errors.
 
 ### v1.1.3
 
-- Exported `axios` and `axiosClient` with exponential backoff retry mechanism for HTTP requests and omitting Authorization headers from Axios errors.
-- Resolved issues with circular structure logging.
-- Fixed attachments metadata normalization bug.
-- Improved repository logging.
+- Export `axios` and `axiosClient` with the exponential backoff retry mechanism for HTTP requests and omit Authorization headers from Axios errors.
+- Resolve circular structure logging issues.
+- Fix the attachments metadata normalization bug.
+- Improve repository logging.
 
-#### v1.1.2
+### v1.1.2
 
-- Unified incoming and outgoing event context.
-- Added `dev_oid` to logger tags.
+- Unify incoming and outgoing event context.
+- Add `dev_oid` to logger tags.
 
-#### v1.1.1
+### v1.1.1
 
-- Added default workers for loading deletion events.
+- Add default workers for loading deletion events.
 
-#### v1.1.0
+### v1.1.0
 
-- Support for sync from DevRev to external system. Known limitations: no support for loading attachments.
+- Support sync from DevRev to the external system. (Known limitations: no support for loading attachments.)
 
-#### v1.0.4
+### v1.0.4
 
 - Fix logging from worker threads.
 
-#### v1.0.3
+### v1.0.3
 
-- Added Release notes.
+- Add release notes.
 
-#### v1.0.2
+### v1.0.2
 
-- Bug fixes and improvements in local development.
+- Fix bugs and improve local development.
 - Expose `formatAxiosError` function for error handling.
 
-#### v1.0.1
+### v1.0.1
 
-- Bug fixes and improvements in logging.
+- Fix bugs and improve logging.
 
-#### v1.0.0
+### v1.0.0
 
-- Allow extractions to use full lambda runtime and gracefully handle execution context timeout.
-- Simplified metadata and data normalization and uploading with repo implementation.
-- Default handling of attachment extraction phase in ADaaS SDK library.
-- Reduced file size, streamlined process by gzip compression.
-- Bug fixes and improvements in error handling.
+- Enable extractions to use the full lambda runtime and gracefully handle execution context timeout.
+- Simplify metadata and data normalization and uploading with the repo implementation.
+- Provide default handling of the attachment extraction phase in the ADaaS SDK library.
+- Reduce file size and streamline processes with gzip compression.
+- Fix bugs and improve error handling.
 
-#### v0.0.3
+### v0.0.3
 
-- Support for new recipe management
+- Support new recipe management.
 
-#### v0.0.2
+### v0.0.2
 
-- Support for the State API
-- HTTP client for API requests
-- Local development environment creates local artifact files
-- Improvements in logging
+- Support the State API.
+- Provide an HTTP client for API requests.
+- Create local artifact files in the local development environment.
+- Improve logging.
 
-#### v0.0.1
+### v0.0.1
 
-- Demo implementation of ADaaS snap-in
-- Adapter for ADaaS control protocol with helper functions
-- Uploader for uploading artifacts
+- Implement a demo of the ADaaS snap-in.
+- Add an adapter for the ADaaS control protocol with helper functions.
+- Provide an uploader for uploading artifacts.
 
 # Overview
 

package/dist/http/axios-client.js

@@ -9,13 +9,13 @@ exports.axios = axios_1.default;
 const axios_retry_1 = __importDefault(require("axios-retry"));
 const axiosClient = axios_1.default.create();
 exports.axiosClient = axiosClient;
-// Exponential backoff algorithm: Retry 3 times and there will be a delay of more than 1 * no. of retries second + random number of milliseconds between each retry.
 (0, axios_retry_1.default)(axiosClient, {
-    retries: 3,
+    retries: 5,
     retryDelay: (retryCount, error) => {
         var _a;
-        console.log(`Retry attempt: ${retryCount} of ${(_a = error.config) === null || _a === void 0 ? void 0 : _a.url}.`);
-        return axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
+        console.warn('Retry attempt: ' + retryCount + 'to url: ' + ((_a = error.config) === null || _a === void 0 ? void 0 : _a.url) + '.');
+        // Exponential backoff algorithm: 1 * 2 ^ retryCount * 5000ms
+        return axios_retry_1.default.exponentialDelay(retryCount, error, 5000);
     },
     retryCondition: (error) => {
         var _a, _b, _c, _d;

package/dist/repo/repo.interfaces.d.ts

@@ -36,6 +36,7 @@ export interface NormalizedAttachment {
     file_name: string;
     author_id: string;
     parent_id: string;
+    inline?: boolean;
 }
 /**
  * Item is an interface that defines the structure of an item.

package/dist/state/state.interfaces.js

@@ -1,3 +1,2 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-;

package/dist/types/extraction.d.ts

@@ -1,7 +1,9 @@
 import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { ErrorRecord } from './common';
-import { DonV2, LoaderReport } from './loading';
+import { DonV2, LoaderReport, RateLimited } from './loading';
+import { NormalizedAttachment } from 'repo/repo.interfaces';
+import { AxiosResponse } from 'axios';
 /**
  * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
  * The external extractor can use these events to know what to do next in the extraction process.
@@ -218,3 +220,18 @@ export interface LoaderEvent {
     event_context: EventContext;
     event_data?: EventData;
 }
+export type ExternalSystemAttachmentStreamingFunction = ({ item, event, }: ExternalSystemAttachmentStreamingParams) => Promise<ExternalSystemAttachmentStreamingResponse>;
+export interface ExternalSystemAttachmentStreamingParams {
+    item: NormalizedAttachment;
+    event: AirdropEvent;
+}
+export interface ExternalSystemAttachmentStreamingResponse {
+    httpStream?: AxiosResponse;
+    error?: ErrorRecord;
+    delay?: number;
+}
+export interface StreamAttachmentsResponse {
+    error?: ErrorRecord;
+    report?: LoaderReport;
+    rateLimit?: RateLimited;
+}

package/dist/types/index.d.ts

@@ -1,5 +1,5 @@
 export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
-export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, } from './extraction';
+export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, } from './extraction';
 export { LoaderEventType, ExternalSystemItem, ExternalSystemItemLoadingResponse, ExternalSystemItemLoadingParams, } from './loading';
 export { NormalizedItem, NormalizedAttachment, RepoInterface, } from '../repo/repo.interfaces';
 export { AdapterState } from '../state/state.interfaces';

package/dist/uploader/uploader.d.ts

@@ -1,4 +1,7 @@
-import { UploadResponse, StreamAttachmentsResponse, UploaderFactoryInterface } from './uploader.interfaces';
+import { AxiosResponse } from 'axios';
+import { betaSDK } from '@devrev/typescript-sdk';
+import { NormalizedAttachment } from '../repo/repo.interfaces';
+import { UploadResponse, UploaderFactoryInterface } from './uploader.interfaces';
 export declare class Uploader {
     private event;
     private betaDevrevSdk;
@@ -14,26 +17,22 @@ export declare class Uploader {
      * or error information if there was an error
      */
     upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
-    private prepareArtifact;
+    prepareArtifact(filename: string, fileType: string): Promise<betaSDK.ArtifactsPrepareResponse | void>;
     private uploadToArtifact;
-    private streamToArtifact;
-    /**
-     * Streams the attachments to the DevRev platform.
-     * The attachments are streamed to the platform and the artifact information is returned.
-     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
-     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
-     * or error information if there was an error
-     */
-    streamAttachments({ attachmentsMetadataArtifactId, }: {
-        attachmentsMetadataArtifactId: string;
-    }): Promise<StreamAttachmentsResponse>;
+    streamToArtifact(preparedArtifact: betaSDK.ArtifactsPrepareResponse, fileStreamResponse: any): Promise<AxiosResponse | void>;
+    getAttachmentsFromArtifactId({ artifact, }: {
+        artifact: string;
+    }): Promise<{
+        attachments?: NormalizedAttachment[];
+        error?: {
+            message: string;
+        };
+    }>;
     private getArtifactDownloadUrl;
     private downloadArtifact;
     private compressGzip;
     private decompressGzip;
     private parseJsonl;
-    private stream;
-    private getFileStreamResponse;
     getJsonObjectByArtifactId({ artifactId, isGzipped, }: {
         artifactId: string;
         isGzipped?: boolean;

package/dist/uploader/uploader.js

@@ -27,8 +27,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
-const axios_client_1 = require("../http/axios-client");
 const fs_1 = __importStar(require("fs"));
+const axios_client_1 = require("../http/axios-client");
 const zlib_1 = __importDefault(require("zlib"));
 const js_jsonl_1 = require("js-jsonl");
 const form_data_1 = __importDefault(require("form-data"));
@@ -155,17 +155,9 @@ class Uploader {
             return;
         }
     }
-    /**
-     * Streams the attachments to the DevRev platform.
-     * The attachments are streamed to the platform and the artifact information is returned.
-     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
-     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
-     * or error information if there was an error
-     */
-    async streamAttachments({ attachmentsMetadataArtifactId, }) {
-        console.log('Started streaming attachments to the platform.');
+    async getAttachmentsFromArtifactId({ artifact, }) {
         // 1. Get the URL of the attachments metadata artifact
-        const artifactUrl = await this.getArtifactDownloadUrl(attachmentsMetadataArtifactId);
+        const artifactUrl = await this.getArtifactDownloadUrl(artifact);
         if (!artifactUrl) {
             return {
                 error: { message: 'Error while getting artifact download URL.' },
@@ -192,23 +184,7 @@ class Uploader {
                 error: { message: 'Error while parsing jsonl object.' },
             };
         }
-        // 5. Stream each attachment to the platform, and push the ssorAttachment to the ssorAttachments array
-        const ssorAttachments = [];
-        for (const attachmentMetadata of jsonObject) {
-            const { ssorAttachment, error } = await this.stream(attachmentMetadata);
-            if (error || !ssorAttachment) {
-                console.warn('Error while streaming attachment', error);
-                continue;
-            }
-            ssorAttachments.push(ssorAttachment);
-        }
-        if (!ssorAttachments.length) {
-            console.warn('No attachments were streamed to the platform.');
-            return {
-                error: { message: 'No attachments were streamed to the platform.' },
-            };
-        }
-        return { ssorAttachments };
+        return { attachments: jsonObject };
     }
     async getArtifactDownloadUrl(artifactId) {
         try {
@@ -262,63 +238,6 @@ class Uploader {
             console.error('Error while parsing jsonl object.', error);
         }
     }
-    async stream(attachmentMetadata) {
-        var _a;
-        const { id: externalId, file_name: filename, url, parent_id: parentId, author_id: actorId, } = attachmentMetadata;
-        const fileStreamResponse = await this.getFileStreamResponse(url);
-        if (!fileStreamResponse) {
-            return {
-                error: { message: 'Error while fetching attachment from URL' },
-            };
-        }
-        const fileType = ((_a = fileStreamResponse.headers) === null || _a === void 0 ? void 0 : _a['content-type']) ||
-            'application/octet-stream';
-        const preparedArtifact = await this.prepareArtifact(filename, fileType);
-        if (!preparedArtifact) {
-            return {
-                error: { message: 'Error while preparing artifact.' },
-            };
-        }
-        const uploadedArtifact = await this.streamToArtifact(preparedArtifact, fileStreamResponse);
-        if (!uploadedArtifact) {
-            return {
-                error: { message: 'Error while streaming artifact.' },
-            };
-        }
-        const ssorAttachment = {
-            id: {
-                devrev: preparedArtifact.id,
-                external: externalId,
-            },
-            parent_id: {
-                external: parentId,
-            },
-            actor_id: {
-                external: actorId,
-            },
-        };
-        console.log('Successful stream of attachment: ', ssorAttachment);
-        return { ssorAttachment };
-    }
-    async getFileStreamResponse(url) {
-        try {
-            const fileStreamResponse = await axios_client_1.axiosClient.get(url, {
-                responseType: 'stream',
-                headers: {
-                    Authorization: this.event.payload.connection_data.key,
-                },
-            });
-            return fileStreamResponse;
-        }
-        catch (error) {
-            if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Error while fetching attachment from URL.', (0, logger_1.serializeAxiosError)(error));
-            }
-            else {
-                console.error('Error while fetching attachment from URL.', error);
-            }
-        }
-    }
     async getJsonObjectByArtifactId({ artifactId, isGzipped = false, }) {
         const artifactUrl = await this.getArtifactDownloadUrl(artifactId);
         if (!artifactUrl) {

package/dist/workers/default-workers/attachments-extraction.js

@@ -1,42 +1,43 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const index_1 = require("../../index");
-const uploader_1 = require("../../uploader/uploader");
-const repos = [
-    {
-        itemType: 'ssor_attachment',
-    },
-];
+const axios_client_1 = require("../../http/axios-client");
+const getAttachmentStream = async ({ item, }) => {
+    const { id, url } = item;
+    try {
+        const fileStreamResponse = await axios_client_1.axiosClient.get(url, {
+            responseType: 'stream',
+        });
+        return { httpStream: fileStreamResponse };
+    }
+    catch (error) {
+        if (axios_client_1.axios.isAxiosError(error)) {
+            console.error('Error while fetching attachment from URL.', (0, index_1.serializeAxiosError)(error));
+        }
+        else {
+            console.error('Error while fetching attachment from URL.', error);
+        }
+        return {
+            error: {
+                message: 'Error while fetching attachment ' + id + ' from URL.',
+            },
+        };
+    }
+};
 (0, index_1.processTask)({
     task: async ({ adapter }) => {
-        var _a, _b, _c, _d, _e;
-        if (!((_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds) ||
-            adapter.state.toDevRev.attachmentsMetadata.artifactIds.length === 0) {
-            console.log('No attachments to extract, skipping.');
-            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
-            return;
-        }
-        adapter.initializeRepos(repos);
-        const uploader = new uploader_1.Uploader({
-            event: adapter.event,
-            options: adapter.options,
+        const { error, delay } = await adapter.streamAttachments({
+            stream: getAttachmentStream,
         });
-        for (const attachmentsMetadataArtifactId of (_b = adapter.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds) {
-            const { ssorAttachments, error } = await uploader.streamAttachments({
-                attachmentsMetadataArtifactId,
+        if (delay) {
+            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDelay, {
+                delay,
+            });
+        }
+        else if (error) {
+            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
+                error,
             });
-            if (error || !ssorAttachments) {
-                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
-                    error,
-                });
-                return;
-            }
-            await ((_c = adapter.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push(ssorAttachments));
-            (_d = adapter.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata.artifactIds.shift();
-            adapter.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
-            if (((_e = adapter.state.toDevRev) === null || _e === void 0 ? void 0 : _e.attachmentsMetadata.artifactIds.length) === 0) {
-                break;
-            }
         }
         await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
     },

package/dist/workers/worker-adapter.d.ts

@@ -1,4 +1,4 @@
-import { AirdropEvent, ExtractorEventType, EventData } from '../types/extraction';
+import { AirdropEvent, ExtractorEventType, EventData, ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
 import { LoaderEventType } from '../types/loading';
 import { AdapterState } from '../state/state.interfaces';
 import { Artifact } from '../uploader/uploader.interfaces';
@@ -58,4 +58,28 @@ export declare class WorkerAdapter<ConnectorState> {
         item: ExternalSystemItem;
         itemTypeToLoad: ItemTypeToLoad;
     }): Promise<LoadItemResponse>;
+    /**
+     * Streams the attachments to the DevRev platform.
+     * The attachments are streamed to the platform and the artifact information is returned.
+     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
+     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
+     * or error information if there was an error
+     */
+    streamAttachments({ stream, }: {
+        stream: ExternalSystemAttachmentStreamingFunction;
+    }): Promise<{
+        report: {};
+        error?: undefined;
+        delay?: undefined;
+    } | {
+        error: {
+            message: string;
+        };
+        report?: undefined;
+        delay?: undefined;
+    } | {
+        delay: number;
+        report?: undefined;
+        error?: undefined;
+    }>;
 }

package/dist/workers/worker-adapter.js

@@ -461,5 +461,87 @@ class WorkerAdapter {
             };
         }
     }
+    /**
+     * Streams the attachments to the DevRev platform.
+     * The attachments are streamed to the platform and the artifact information is returned.
+     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
+     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
+     * or error information if there was an error
+     */
+    async streamAttachments({ stream, }) {
+        var _a, _b, _c, _d, _e, _f, _g;
+        const repos = [
+            {
+                itemType: 'ssor_attachment',
+            },
+        ];
+        this.initializeRepos(repos);
+        for (const attachmentsMetadataArtifactId of ((_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds) || []) {
+            if (((_b = this.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.length) === 0) {
+                return { report: {} };
+            }
+            console.log('Started streaming attachments to the platform.');
+            const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
+                artifact: attachmentsMetadataArtifactId,
+            });
+            if (error) {
+                return { error };
+            }
+            console.log('this.state.toDevRev?.attachmentsMetadata :>> ', (_c = this.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata);
+            if (attachments) {
+                const attachmentsToProcess = attachments.slice((_e = (_d = this.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata) === null || _e === void 0 ? void 0 : _e.lastProcessed, attachments.length);
+                for (const attachment of attachmentsToProcess) {
+                    const { httpStream, delay, error } = await stream({
+                        item: attachment,
+                        event: this.event,
+                    });
+                    if (error) {
+                        console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
+                        continue;
+                    }
+                    else if (delay) {
+                        return { delay };
+                    }
+                    if (httpStream) {
+                        const fileType = ((_f = httpStream.headers) === null || _f === void 0 ? void 0 : _f['content-type']) ||
+                            'application/octet-stream';
+                        const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
+                        if (!preparedArtifact) {
+                            return {
+                                error: { message: 'Error while preparing artifact.' },
+                            };
+                        }
+                        const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
+                        if (!uploadedArtifact) {
+                            return {
+                                error: { message: 'Error while streaming artifact.' },
+                            };
+                        }
+                        const ssorAttachment = {
+                            id: {
+                                devrev: preparedArtifact.id,
+                                external: attachment.id,
+                            },
+                            parent_id: {
+                                external: attachment.parent_id,
+                            },
+                            actor_id: {
+                                external: attachment.author_id,
+                            },
+                        };
+                        await ((_g = this.getRepo('ssor_attachment')) === null || _g === void 0 ? void 0 : _g.push([ssorAttachment]));
+                        if (this.state.toDevRev) {
+                            this.state.toDevRev.attachmentsMetadata.lastProcessed++;
+                        }
+                    }
+                }
+            }
+            if (this.state.toDevRev) {
+                this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
+                this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
+            }
+        }
+        return { report: {} };
+    }
 }
 exports.WorkerAdapter = WorkerAdapter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",