@devrev/ts-adaas 1.1.3 → 1.1.5

package/README.md

@@ -2,67 +2,77 @@
 
 ## Release Notes
 
+### v1.1.5
+
+- Increase `delayFactor` and number of retries for the exponential backoff retry mechanism for HTTP requests.
+- Provide an inject function for streaming attachments.
+- Fix the attachments streaming bug.
+
+### v1.1.4
+
+- Provide log lines and stack traces for runtime worker errors.
+
 ### v1.1.3
 
-- Exported `axios` and `axiosClient` with exponential backoff retry mechanism for HTTP requests and omitting Authorization headers from Axios errors.
-- Resolved issues with circular structure logging.
-- Fixed attachments metadata normalization bug.
-- Improved repository logging.
+- Export `axios` and `axiosClient` with the exponential backoff retry mechanism for HTTP requests and omit Authorization headers from Axios errors.
+- Resolve circular structure logging issues.
+- Fix the attachments metadata normalization bug.
+- Improve repository logging.
 
-#### v1.1.2
+### v1.1.2
 
-- Unified incoming and outgoing event context.
-- Added `dev_oid` to logger tags.
+- Unify incoming and outgoing event context.
+- Add `dev_oid` to logger tags.
 
-#### v1.1.1
+### v1.1.1
 
-- Added default workers for loading deletion events.
+- Add default workers for loading deletion events.
 
-#### v1.1.0
+### v1.1.0
 
-- Support for sync from DevRev to external system. Known limitations: no support for loading attachments.
+- Support sync from DevRev to the external system. (Known limitations: no support for loading attachments.)
 
-#### v1.0.4
+### v1.0.4
 
 - Fix logging from worker threads.
 
-#### v1.0.3
+### v1.0.3
 
-- Added Release notes.
+- Add release notes.
 
-#### v1.0.2
+### v1.0.2
 
-- Bug fixes and improvements in local development.
+- Fix bugs and improve local development.
 - Expose `formatAxiosError` function for error handling.
 
-#### v1.0.1
+### v1.0.1
 
-- Bug fixes and improvements in logging.
+- Fix bugs and improve logging.
 
-#### v1.0.0
+### v1.0.0
 
-- Allow extractions to use full lambda runtime and gracefully handle execution context timeout.
-- Simplified metadata and data normalization and uploading with repo implementation.
-- Default handling of attachment extraction phase in ADaaS SDK library.
-- Reduced file size, streamlined process by gzip compression.
-- Bug fixes and improvements in error handling.
+- Enable extractions to use the full lambda runtime and gracefully handle execution context timeout.
+- Simplify metadata and data normalization and uploading with the repo implementation.
+- Provide default handling of the attachment extraction phase in the ADaaS SDK library.
+- Reduce file size and streamline processes with gzip compression.
+- Fix bugs and improve error handling.
 
-#### v0.0.3
+### v0.0.3
 
-- Support for new recipe management
+- Support new recipe management.
 
-#### v0.0.2
+### v0.0.2
 
-- Support for the State API
-- HTTP client for API requests
-- Local development environment creates local artifact files
-- Improvements in logging
+- Support the State API.
+- Provide an HTTP client for API requests.
+- Create local artifact files in the local development environment.
+- Improve logging.
 
-#### v0.0.1
+### v0.0.1
 
-- Demo implementation of ADaaS snap-in
-- Adapter for ADaaS control protocol with helper functions
-- Uploader for uploading artifacts
+- Implement a demo of the ADaaS snap-in.
+- Add an adapter for the ADaaS control protocol with helper functions.
+- Provide an uploader for uploading artifacts.
 
 # Overview
 

package/dist/common/control-protocol.js

@@ -23,7 +23,7 @@ const emit = async ({ event, eventType, data, }) => {
         }
         catch (error) {
             if (axios_client_1.axios.isAxiosError(error)) {
-                console.error(`Failed to emit event with event type ${eventType}.`, (0, logger_1.formatAxiosError)(error));
+                console.error(`Failed to emit event with event type ${eventType}.`, (0, logger_1.serializeAxiosError)(error));
             }
             else {
                 // TODO: Stop it through UI or think about retrying this request. Implement exponential retry mechanism.

package/dist/common/install-initial-domain-mapping.js

@@ -42,7 +42,7 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
             }
             catch (error) {
                 if (axios_client_1.axios.isAxiosError(error)) {
-                    console.error('Error while creating recipe blueprint', (0, logger_1.formatAxiosError)(error));
+                    console.error('Error while creating recipe blueprint', (0, logger_1.serializeAxiosError)(error));
                 }
                 else {
                     console.error('Error while creating recipe blueprint', error);
@@ -64,7 +64,7 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
         }
         catch (error) {
             if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Error while installing initial domain mapping', (0, logger_1.formatAxiosError)(error));
+                console.error('Error while installing initial domain mapping', (0, logger_1.serializeAxiosError)(error));
             }
             else {
                 console.error('Error while installing initial domain mapping', error);
@@ -74,7 +74,7 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
     }
     catch (error) {
         if (axios_client_1.axios.isAxiosError(error)) {
-            console.error('Error while fetching snap in', (0, logger_1.formatAxiosError)(error));
+            console.error('Error while fetching snap in', (0, logger_1.serializeAxiosError)(error));
         }
         else {
             console.error('Error while fetching snap in', error);

package/dist/http/axios-client.js

@@ -9,13 +9,13 @@ exports.axios = axios_1.default;
 const axios_retry_1 = __importDefault(require("axios-retry"));
 const axiosClient = axios_1.default.create();
 exports.axiosClient = axiosClient;
-// Exponential backoff algorithm: Retry 3 times and there will be a delay of more than 1 * no. of retries second + random number of milliseconds between each retry.
 (0, axios_retry_1.default)(axiosClient, {
-    retries: 3,
+    retries: 5,
     retryDelay: (retryCount, error) => {
         var _a;
-        console.log(`Retry attempt: ${retryCount} of ${(_a = error.config) === null || _a === void 0 ? void 0 : _a.url}.`);
-        return axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
+        console.warn('Retry attempt: ' + retryCount + 'to url: ' + ((_a = error.config) === null || _a === void 0 ? void 0 : _a.url) + '.');
+        // Exponential backoff algorithm: 1 * 2 ^ retryCount * 5000ms
+        return axios_retry_1.default.exponentialDelay(retryCount, error, 5000);
     },
     retryCondition: (error) => {
         var _a, _b, _c, _d;

package/dist/index.d.ts

@@ -9,3 +9,4 @@ export { processTask } from './workers/process-task';
 export { spawn } from './workers/spawn';
 export * from './types/workers';
 export { formatAxiosError } from './logger/logger';
+export { serializeAxiosError } from './logger/logger';

package/dist/index.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatAxiosError = exports.spawn = exports.processTask = exports.WorkerAdapter = void 0;
+exports.serializeAxiosError = exports.formatAxiosError = exports.spawn = exports.processTask = exports.WorkerAdapter = void 0;
 __exportStar(require("./deprecated/adapter"), exports);
 __exportStar(require("./deprecated/demo-extractor"), exports);
 __exportStar(require("./deprecated/uploader"), exports);
@@ -30,3 +30,5 @@ Object.defineProperty(exports, "spawn", { enumerable: true, get: function () { r
 __exportStar(require("./types/workers"), exports);
 var logger_1 = require("./logger/logger");
 Object.defineProperty(exports, "formatAxiosError", { enumerable: true, get: function () { return logger_1.formatAxiosError; } });
+var logger_2 = require("./logger/logger");
+Object.defineProperty(exports, "serializeAxiosError", { enumerable: true, get: function () { return logger_2.serializeAxiosError; } });

package/dist/logger/logger.d.ts

@@ -1,6 +1,6 @@
 import { Console } from 'node:console';
 import { LoggerFactoryInterface, LogLevel, PrintableState } from './logger.interfaces';
-import { AxiosError } from 'axios';
+import { AxiosError, RawAxiosResponseHeaders } from 'axios';
 export declare class Logger extends Console {
     private options?;
     constructor({ event, options }: LoggerFactoryInterface);
@@ -11,4 +11,23 @@ export declare class Logger extends Console {
     error(...args: unknown[]): void;
 }
 export declare function getPrintableState(state: Record<string, any>): PrintableState;
+/**
+ * @deprecated
+ */
 export declare function formatAxiosError(error: AxiosError): object;
+export declare const serializeError: (error: unknown) => Error;
+export declare function serializeAxiosError(error: AxiosError): {
+    config: {
+        method: string | undefined;
+        params: any;
+        url: string | undefined;
+    };
+    isAxiosError: boolean;
+    isCorsOrNoNetworkError: boolean;
+    response: {
+        data: unknown;
+        headers: RawAxiosResponseHeaders;
+        status: number;
+        statusText: string;
+    } | null;
+};

package/dist/logger/logger.js

@@ -3,9 +3,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Logger = void 0;
+exports.serializeError = exports.Logger = void 0;
 exports.getPrintableState = getPrintableState;
 exports.formatAxiosError = formatAxiosError;
+exports.serializeAxiosError = serializeAxiosError;
 const lambda_log_1 = __importDefault(require("lambda-log"));
 const node_console_1 = require("node:console");
 const logger_interfaces_1 = require("./logger.interfaces");
@@ -85,17 +86,42 @@ function getPrintableState(state) {
     // Process the state object directly since it's guaranteed to be an object
     return processValue(state);
 }
+/**
+ * @deprecated
+ */
 function formatAxiosError(error) {
-    var _a, _b, _c, _d;
-    if (error.response) {
-        return {
-            status: error.response.status,
-            data: error.response.data,
-            method: (_a = error.config) === null || _a === void 0 ? void 0 : _a.method,
-            baseURL: (_b = error.config) === null || _b === void 0 ? void 0 : _b.baseURL,
-            url: (_c = error.config) === null || _c === void 0 ? void 0 : _c.url,
-            payload: (_d = error.config) === null || _d === void 0 ? void 0 : _d.data,
-        };
+    return serializeAxiosError(error);
+}
+const serializeError = (error) => {
+    let serializedError = error;
+    try {
+        serializedError = JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error)));
+    }
+    catch (err) {
+        console.error('Failed to serialize error object for logger', err);
     }
-    return error;
+    return serializedError;
+};
+exports.serializeError = serializeError;
+function serializeAxiosError(error) {
+    var _a, _b, _c;
+    const response = error.response
+        ? {
+            data: error.response.data,
+            headers: error.response.headers,
+            status: error.response.status,
+            statusText: error.response.statusText,
+        }
+        : null;
+    const config = {
+        method: (_a = error.config) === null || _a === void 0 ? void 0 : _a.method,
+        params: (_b = error.config) === null || _b === void 0 ? void 0 : _b.params,
+        url: (_c = error.config) === null || _c === void 0 ? void 0 : _c.url,
+    };
+    return {
+        config,
+        isAxiosError: true,
+        isCorsOrNoNetworkError: !error.response,
+        response,
+    };
 }

package/dist/logger/logger.test.js

@@ -30,7 +30,7 @@ it('getPrintableState should return printable state', () => {
         },
     });
 });
-it('formatAxiosError should return formatted error', () => {
+it('serializeAxiosError should return formatted error', () => {
     const error = {
         response: {
             status: 500,
@@ -40,13 +40,20 @@ it('formatAxiosError should return formatted error', () => {
             method: 'GET',
         },
     };
-    const formattedError = (0, logger_1.formatAxiosError)(error);
+    const formattedError = (0, logger_1.serializeAxiosError)(error);
     expect(formattedError).toEqual({
-        status: 500,
-        data: 'Internal server error',
-        method: 'GET',
-        baseURL: undefined,
-        url: undefined,
-        payload: undefined,
+        config: {
+            method: 'GET',
+            params: undefined,
+            url: undefined,
+        },
+        isAxiosError: true,
+        isCorsOrNoNetworkError: false,
+        response: {
+            data: 'Internal server error',
+            headers: undefined,
+            status: 500,
+            statusText: undefined,
+        },
     });
 });

package/dist/repo/repo.interfaces.d.ts

@@ -36,6 +36,7 @@ export interface NormalizedAttachment {
     file_name: string;
     author_id: string;
     parent_id: string;
+    inline?: boolean;
 }
 /**
  * Item is an interface that defines the structure of an item.

package/dist/state/state.interfaces.js

@@ -1,3 +1,2 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-;

package/dist/state/state.js

@@ -72,7 +72,7 @@ class State {
         }
         catch (error) {
             if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Failed to update state.', (0, logger_1.formatAxiosError)(error));
+                console.error('Failed to update state.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
                 console.error('Failed to update state.', error);

package/dist/types/extraction.d.ts

@@ -1,7 +1,9 @@
 import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { ErrorRecord } from './common';
-import { DonV2, LoaderReport } from './loading';
+import { DonV2, LoaderReport, RateLimited } from './loading';
+import { NormalizedAttachment } from 'repo/repo.interfaces';
+import { AxiosResponse } from 'axios';
 /**
  * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
  * The external extractor can use these events to know what to do next in the extraction process.
@@ -218,3 +220,18 @@ export interface LoaderEvent {
     event_context: EventContext;
     event_data?: EventData;
 }
+export type ExternalSystemAttachmentStreamingFunction = ({ item, event, }: ExternalSystemAttachmentStreamingParams) => Promise<ExternalSystemAttachmentStreamingResponse>;
+export interface ExternalSystemAttachmentStreamingParams {
+    item: NormalizedAttachment;
+    event: AirdropEvent;
+}
+export interface ExternalSystemAttachmentStreamingResponse {
+    httpStream?: AxiosResponse;
+    error?: ErrorRecord;
+    delay?: number;
+}
+export interface StreamAttachmentsResponse {
+    error?: ErrorRecord;
+    report?: LoaderReport;
+    rateLimit?: RateLimited;
+}

package/dist/types/index.d.ts

@@ -1,5 +1,5 @@
 export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
-export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, } from './extraction';
+export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, } from './extraction';
 export { LoaderEventType, ExternalSystemItem, ExternalSystemItemLoadingResponse, ExternalSystemItemLoadingParams, } from './loading';
 export { NormalizedItem, NormalizedAttachment, RepoInterface, } from '../repo/repo.interfaces';
 export { AdapterState } from '../state/state.interfaces';

package/dist/uploader/uploader.d.ts

@@ -1,4 +1,7 @@
-import { UploadResponse, StreamAttachmentsResponse, UploaderFactoryInterface } from './uploader.interfaces';
+import { AxiosResponse } from 'axios';
+import { betaSDK } from '@devrev/typescript-sdk';
+import { NormalizedAttachment } from '../repo/repo.interfaces';
+import { UploadResponse, UploaderFactoryInterface } from './uploader.interfaces';
 export declare class Uploader {
     private event;
     private betaDevrevSdk;
@@ -14,26 +17,22 @@ export declare class Uploader {
      * or error information if there was an error
      */
     upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
-    private prepareArtifact;
+    prepareArtifact(filename: string, fileType: string): Promise<betaSDK.ArtifactsPrepareResponse | void>;
     private uploadToArtifact;
-    private streamToArtifact;
-    /**
-     * Streams the attachments to the DevRev platform.
-     * The attachments are streamed to the platform and the artifact information is returned.
-     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
-     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
-     * or error information if there was an error
-     */
-    streamAttachments({ attachmentsMetadataArtifactId, }: {
-        attachmentsMetadataArtifactId: string;
-    }): Promise<StreamAttachmentsResponse>;
+    streamToArtifact(preparedArtifact: betaSDK.ArtifactsPrepareResponse, fileStreamResponse: any): Promise<AxiosResponse | void>;
+    getAttachmentsFromArtifactId({ artifact, }: {
+        artifact: string;
+    }): Promise<{
+        attachments?: NormalizedAttachment[];
+        error?: {
+            message: string;
+        };
+    }>;
     private getArtifactDownloadUrl;
     private downloadArtifact;
     private compressGzip;
     private decompressGzip;
     private parseJsonl;
-    private stream;
-    private getFileStreamResponse;
     getJsonObjectByArtifactId({ artifactId, isGzipped, }: {
         artifactId: string;
         isGzipped?: boolean;

package/dist/uploader/uploader.js

@@ -27,8 +27,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
-const axios_client_1 = require("../http/axios-client");
 const fs_1 = __importStar(require("fs"));
+const axios_client_1 = require("../http/axios-client");
 const zlib_1 = __importDefault(require("zlib"));
 const js_jsonl_1 = require("js-jsonl");
 const form_data_1 = __importDefault(require("form-data"));
@@ -99,7 +99,7 @@ class Uploader {
         }
         catch (error) {
             if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Error while preparing artifact.', (0, logger_1.formatAxiosError)(error));
+                console.error('Error while preparing artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
                 console.error('Error while preparing artifact.', error);
@@ -122,7 +122,7 @@ class Uploader {
         }
         catch (error) {
             if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Error while uploading artifact.', (0, logger_1.formatAxiosError)(error));
+                console.error('Error while uploading artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
                 console.error('Error while uploading artifact.', error);
@@ -147,7 +147,7 @@ class Uploader {
         }
         catch (error) {
             if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Error while streaming artifact.', (0, logger_1.formatAxiosError)(error));
+                console.error('Error while streaming artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
                 console.error('Error while streaming artifact.', error);
@@ -155,17 +155,9 @@ class Uploader {
             return;
         }
     }
-    /**
-     * Streams the attachments to the DevRev platform.
-     * The attachments are streamed to the platform and the artifact information is returned.
-     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
-     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
-     * or error information if there was an error
-     */
-    async streamAttachments({ attachmentsMetadataArtifactId, }) {
-        console.log('Started streaming attachments to the platform.');
+    async getAttachmentsFromArtifactId({ artifact, }) {
         // 1. Get the URL of the attachments metadata artifact
-        const artifactUrl = await this.getArtifactDownloadUrl(attachmentsMetadataArtifactId);
+        const artifactUrl = await this.getArtifactDownloadUrl(artifact);
         if (!artifactUrl) {
             return {
                 error: { message: 'Error while getting artifact download URL.' },
@@ -192,23 +184,7 @@ class Uploader {
                 error: { message: 'Error while parsing jsonl object.' },
             };
         }
-        // 5. Stream each attachment to the platform, and push the ssorAttachment to the ssorAttachments array
-        const ssorAttachments = [];
-        for (const attachmentMetadata of jsonObject) {
-            const { ssorAttachment, error } = await this.stream(attachmentMetadata);
-            if (error || !ssorAttachment) {
-                console.warn('Error while streaming attachment', error);
-                continue;
-            }
-            ssorAttachments.push(ssorAttachment);
-        }
-        if (!ssorAttachments.length) {
-            console.warn('No attachments were streamed to the platform.');
-            return {
-                error: { message: 'No attachments were streamed to the platform.' },
-            };
-        }
-        return { ssorAttachments };
+        return { attachments: jsonObject };
     }
     async getArtifactDownloadUrl(artifactId) {
         try {
@@ -230,7 +206,7 @@ class Uploader {
         }
         catch (error) {
             if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Error while downloading artifact from URL.', (0, logger_1.formatAxiosError)(error));
+                console.error('Error while downloading artifact from URL.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
                 console.error('Error while downloading artifact from URL.', error);
@@ -262,63 +238,6 @@ class Uploader {
             console.error('Error while parsing jsonl object.', error);
         }
     }
-    async stream(attachmentMetadata) {
-        var _a;
-        const { id: externalId, file_name: filename, url, parent_id: parentId, author_id: actorId, } = attachmentMetadata;
-        const fileStreamResponse = await this.getFileStreamResponse(url);
-        if (!fileStreamResponse) {
-            return {
-                error: { message: 'Error while fetching attachment from URL' },
-            };
-        }
-        const fileType = ((_a = fileStreamResponse.headers) === null || _a === void 0 ? void 0 : _a['content-type']) ||
-            'application/octet-stream';
-        const preparedArtifact = await this.prepareArtifact(filename, fileType);
-        if (!preparedArtifact) {
-            return {
-                error: { message: 'Error while preparing artifact.' },
-            };
-        }
-        const uploadedArtifact = await this.streamToArtifact(preparedArtifact, fileStreamResponse);
-        if (!uploadedArtifact) {
-            return {
-                error: { message: 'Error while streaming artifact.' },
-            };
-        }
-        const ssorAttachment = {
-            id: {
-                devrev: preparedArtifact.id,
-                external: externalId,
-            },
-            parent_id: {
-                external: parentId,
-            },
-            actor_id: {
-                external: actorId,
-            },
-        };
-        console.log('Successful stream of attachment: ', ssorAttachment);
-        return { ssorAttachment };
-    }
-    async getFileStreamResponse(url) {
-        try {
-            const fileStreamResponse = await axios_client_1.axiosClient.get(url, {
-                responseType: 'stream',
-                headers: {
-                    Authorization: this.event.payload.connection_data.key,
-                },
-            });
-            return fileStreamResponse;
-        }
-        catch (error) {
-            if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Error while fetching attachment from URL.', (0, logger_1.formatAxiosError)(error));
-            }
-            else {
-                console.error('Error while fetching attachment from URL.', error);
-            }
-        }
-    }
     async getJsonObjectByArtifactId({ artifactId, isGzipped = false, }) {
         const artifactUrl = await this.getArtifactDownloadUrl(artifactId);
         if (!artifactUrl) {

package/dist/workers/create-worker.js

@@ -16,7 +16,7 @@ async function createWorker(workerData) {
                 workerData,
             });
             worker.on(workers_1.WorkerEvent.WorkerError, (error) => {
-                logger.error('Worker error', error);
+                logger.error('Worker error', (0, logger_1.serializeError)(error));
                 reject();
             });
             worker.on(workers_1.WorkerEvent.WorkerOnline, () => {

package/dist/workers/default-workers/attachments-extraction.js

@@ -1,42 +1,43 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const index_1 = require("../../index");
-const uploader_1 = require("../../uploader/uploader");
-const repos = [
-    {
-        itemType: 'ssor_attachment',
-    },
-];
+const axios_client_1 = require("../../http/axios-client");
+const getAttachmentStream = async ({ item, }) => {
+    const { id, url } = item;
+    try {
+        const fileStreamResponse = await axios_client_1.axiosClient.get(url, {
+            responseType: 'stream',
+        });
+        return { httpStream: fileStreamResponse };
+    }
+    catch (error) {
+        if (axios_client_1.axios.isAxiosError(error)) {
+            console.error('Error while fetching attachment from URL.', (0, index_1.serializeAxiosError)(error));
+        }
+        else {
+            console.error('Error while fetching attachment from URL.', error);
+        }
+        return {
+            error: {
+                message: 'Error while fetching attachment ' + id + ' from URL.',
+            },
+        };
+    }
+};
 (0, index_1.processTask)({
     task: async ({ adapter }) => {
-        var _a, _b, _c, _d, _e;
-        if (!((_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds) ||
-            adapter.state.toDevRev.attachmentsMetadata.artifactIds.length === 0) {
-            console.log('No attachments to extract, skipping.');
-            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
-            return;
-        }
-        adapter.initializeRepos(repos);
-        const uploader = new uploader_1.Uploader({
-            event: adapter.event,
-            options: adapter.options,
+        const { error, delay } = await adapter.streamAttachments({
+            stream: getAttachmentStream,
         });
-        for (const attachmentsMetadataArtifactId of (_b = adapter.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds) {
-            const { ssorAttachments, error } = await uploader.streamAttachments({
-                attachmentsMetadataArtifactId,
+        if (delay) {
+            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDelay, {
+                delay,
+            });
+        }
+        else if (error) {
+            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
+                error,
             });
-            if (error || !ssorAttachments) {
-                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
-                    error,
-                });
-                return;
-            }
-            await ((_c = adapter.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push(ssorAttachments));
-            (_d = adapter.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata.artifactIds.shift();
-            adapter.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
-            if (((_e = adapter.state.toDevRev) === null || _e === void 0 ? void 0 : _e.attachmentsMetadata.artifactIds.length) === 0) {
-                break;
-            }
         }
         await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
     },

package/dist/workers/worker-adapter.d.ts

@@ -1,4 +1,4 @@
-import { AirdropEvent, ExtractorEventType, EventData } from '../types/extraction';
+import { AirdropEvent, ExtractorEventType, EventData, ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
 import { LoaderEventType } from '../types/loading';
 import { AdapterState } from '../state/state.interfaces';
 import { Artifact } from '../uploader/uploader.interfaces';
@@ -58,4 +58,28 @@ export declare class WorkerAdapter<ConnectorState> {
         item: ExternalSystemItem;
         itemTypeToLoad: ItemTypeToLoad;
     }): Promise<LoadItemResponse>;
+    /**
+     * Streams the attachments to the DevRev platform.
+     * The attachments are streamed to the platform and the artifact information is returned.
+     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
+     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
+     * or error information if there was an error
+     */
+    streamAttachments({ stream, }: {
+        stream: ExternalSystemAttachmentStreamingFunction;
+    }): Promise<{
+        report: {};
+        error?: undefined;
+        delay?: undefined;
+    } | {
+        error: {
+            message: string;
+        };
+        report?: undefined;
+        delay?: undefined;
+    } | {
+        delay: number;
+        report?: undefined;
+        error?: undefined;
+    }>;
 }

package/dist/workers/worker-adapter.js

@@ -341,7 +341,7 @@ class WorkerAdapter {
                     }
                     catch (error) {
                         if (axios_client_1.axios.isAxiosError(error)) {
-                            console.error('Failed to update sync mapper record', (0, logger_1.formatAxiosError)(error));
+                            console.error('Failed to update sync mapper record', (0, logger_1.serializeAxiosError)(error));
                             return {
                                 error: {
                                     message: error.message,
@@ -412,7 +412,7 @@ class WorkerAdapter {
                         }
                         catch (error) {
                             if (axios_client_1.axios.isAxiosError(error)) {
-                                console.error('Failed to create sync mapper record', (0, logger_1.formatAxiosError)(error));
+                                console.error('Failed to create sync mapper record', (0, logger_1.serializeAxiosError)(error));
                                 return {
                                     error: {
                                         message: error.message,
@@ -445,7 +445,7 @@ class WorkerAdapter {
                     }
                 }
                 else {
-                    console.error('Failed to get sync mapper record', (0, logger_1.formatAxiosError)(error));
+                    console.error('Failed to get sync mapper record', (0, logger_1.serializeAxiosError)(error));
                     return {
                         error: {
                             message: error.message,
@@ -461,5 +461,87 @@ class WorkerAdapter {
             };
         }
     }
+    /**
+     * Streams the attachments to the DevRev platform.
+     * The attachments are streamed to the platform and the artifact information is returned.
+     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
+     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
+     * or error information if there was an error
+     */
+    async streamAttachments({ stream, }) {
+        var _a, _b, _c, _d, _e, _f, _g;
+        const repos = [
+            {
+                itemType: 'ssor_attachment',
+            },
+        ];
+        this.initializeRepos(repos);
+        for (const attachmentsMetadataArtifactId of ((_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds) || []) {
+            if (((_b = this.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.length) === 0) {
+                return { report: {} };
+            }
+            console.log('Started streaming attachments to the platform.');
+            const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
+                artifact: attachmentsMetadataArtifactId,
+            });
+            if (error) {
+                return { error };
+            }
+            console.log('this.state.toDevRev?.attachmentsMetadata :>> ', (_c = this.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata);
+            if (attachments) {
+                const attachmentsToProcess = attachments.slice((_e = (_d = this.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata) === null || _e === void 0 ? void 0 : _e.lastProcessed, attachments.length);
+                for (const attachment of attachmentsToProcess) {
+                    const { httpStream, delay, error } = await stream({
+                        item: attachment,
+                        event: this.event,
+                    });
+                    if (error) {
+                        console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
+                        continue;
+                    }
+                    else if (delay) {
+                        return { delay };
+                    }
+                    if (httpStream) {
+                        const fileType = ((_f = httpStream.headers) === null || _f === void 0 ? void 0 : _f['content-type']) ||
+                            'application/octet-stream';
+                        const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
+                        if (!preparedArtifact) {
+                            return {
+                                error: { message: 'Error while preparing artifact.' },
+                            };
+                        }
+                        const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
+                        if (!uploadedArtifact) {
+                            return {
+                                error: { message: 'Error while streaming artifact.' },
+                            };
+                        }
+                        const ssorAttachment = {
+                            id: {
+                                devrev: preparedArtifact.id,
+                                external: attachment.id,
+                            },
+                            parent_id: {
+                                external: attachment.parent_id,
+                            },
+                            actor_id: {
+                                external: attachment.author_id,
+                            },
+                        };
+                        await ((_g = this.getRepo('ssor_attachment')) === null || _g === void 0 ? void 0 : _g.push([ssorAttachment]));
+                        if (this.state.toDevRev) {
+                            this.state.toDevRev.attachmentsMetadata.lastProcessed++;
+                        }
+                    }
+                }
+            }
+            if (this.state.toDevRev) {
+                this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
+                this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
+            }
+        }
+        return { report: {} };
+    }
 }
 exports.WorkerAdapter = WorkerAdapter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.1.3",
+  "version": "1.1.5",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",