@devrev/ts-adaas 1.2.0-beta.1 → 1.2.2

package/dist/workers/default-workers/attachments-extraction.js

@@ -12,10 +12,12 @@ const getAttachmentStream = async ({ item, }) => {
     }
     catch (error) {
         if (axios_client_1.axios.isAxiosError(error)) {
-            console.error('Error while fetching attachment from URL.', (0, index_1.serializeAxiosError)(error));
+            console.warn(`Error while fetching attachment ${id} from URL.`, (0, index_1.serializeAxiosError)(error));
+            console.warn('Failed attachment metadata', item);
         }
         else {
-            console.error('Error while fetching attachment from URL.', error);
+            console.warn(`Error while fetching attachment ${id} from URL.`, error);
+            console.warn('Failed attachment metadata', item);
         }
         return {
             error: {
@@ -26,20 +28,27 @@ const getAttachmentStream = async ({ item, }) => {
 };
 (0, index_1.processTask)({
     task: async ({ adapter }) => {
-        const { error, delay } = await adapter.streamAttachments({
-            stream: getAttachmentStream,
-        });
-        if (delay) {
-            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDelay, {
-                delay,
+        try {
+            const response = await adapter.streamAttachments({
+                stream: getAttachmentStream,
             });
+            if (response === null || response === void 0 ? void 0 : response.delay) {
+                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDelay, {
+                    delay: response.delay,
+                });
+            }
+            else if (response === null || response === void 0 ? void 0 : response.error) {
+                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
+                    error: response.error,
+                });
+            }
+            else {
+                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
+            }
         }
-        else if (error) {
-            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
-                error,
-            });
+        catch (error) {
+            console.error('An error occured while processing a task.', error);
         }
-        await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
     },
     onTimeout: async ({ adapter }) => {
         await adapter.postState();

package/dist/workers/default-workers/load-attachments.js

@@ -6,8 +6,8 @@ const types_1 = require("../../types");
     task: async ({ adapter }) => {
         await adapter.emit(types_1.LoaderEventType.UnknownEventType, {
             error: {
-                message: 'Event type ' + adapter.event.payload.event_type + ' not supported.'
-            }
+                message: 'Event type ' + adapter.event.payload.event_type + ' not supported.',
+            },
         });
     },
     onTimeout: async ({ adapter }) => {

package/dist/workers/process-task.js

@@ -22,7 +22,6 @@ function processTask({ task, onTimeout, }) {
                 const adapter = new worker_adapter_1.WorkerAdapter({
                     event,
                     adapterState,
-                    parentPort: node_worker_threads_1.parentPort,
                     options,
                 });
                 node_worker_threads_1.parentPort.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {

package/dist/workers/spawn.d.ts

@@ -10,16 +10,15 @@ import { SpawnFactoryInterface, SpawnInterface } from '../types/workers';
  * @param {string} workerPath - The path to the worker file
  * @returns {Promise<Spawn>} - A new instance of Spawn class
  */
-export declare function spawn<ConnectorState>({ event, initialState, workerPath, options, }: SpawnFactoryInterface<ConnectorState>): Promise<boolean | PromiseLike<boolean>>;
+export declare function spawn<ConnectorState>({ event, initialState, workerPath, options, }: SpawnFactoryInterface<ConnectorState>): Promise<void>;
 export declare class Spawn {
     private event;
-    private hasWorkerEmitted;
+    private alreadyEmitted;
     private defaultLambdaTimeout;
     private lambdaTimeout;
-    private worker;
-    private resolve;
     private timer;
     private logger;
+    private resolve;
     constructor({ event, worker, options, resolve }: SpawnInterface);
     private exitFromMainThread;
 }

package/dist/workers/spawn.js

@@ -1,7 +1,11 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Spawn = void 0;
 exports.spawn = spawn;
+const axios_1 = __importDefault(require("axios"));
 const extraction_1 = require("../types/extraction");
 const control_protocol_1 = require("../common/control-protocol");
 const helpers_1 = require("../common/helpers");
@@ -70,7 +74,6 @@ async function spawn({ event, initialState, workerPath, options, }) {
     const script = getWorkerPath({
         event,
         connectorWorkerPath: workerPath,
-        options,
     });
     if (script) {
         try {
@@ -90,47 +93,58 @@ async function spawn({ event, initialState, workerPath, options, }) {
             });
         }
         catch (error) {
-            logger.error('Worker error while processing task.', error);
-            return false;
+            logger.error('Worker error while processing task', error);
         }
     }
     else {
-        await (0, control_protocol_1.emit)({
-            event,
-            eventType: extraction_1.ExtractorEventType.UnknownEventType,
-            data: {
-                error: {
-                    message: 'Unrecognized event type in spawn ' +
-                        event.payload.event_type +
-                        '.',
+        console.error('Script was not found for event type: ' + event.payload.event_type + '.');
+        try {
+            await (0, control_protocol_1.emit)({
+                event,
+                eventType: extraction_1.ExtractorEventType.UnknownEventType,
+                data: {
+                    error: {
+                        message: 'Unrecognized event type in spawn ' +
+                            event.payload.event_type +
+                            '.',
+                    },
                 },
-            },
-        });
-        return false;
+            });
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error)) {
+                console.error('Error while emitting event', (0, logger_1.serializeAxiosError)(error));
+            }
+            else {
+                console.error('Error while emitting event', error);
+            }
+        }
     }
 }
 class Spawn {
     constructor({ event, worker, options, resolve }) {
         this.defaultLambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
-        this.hasWorkerEmitted = false;
+        this.alreadyEmitted = false;
         this.event = event;
+        this.logger = new logger_1.Logger({ event, options });
         this.lambdaTimeout = (options === null || options === void 0 ? void 0 : options.timeout)
             ? Math.min(options.timeout, this.defaultLambdaTimeout)
             : this.defaultLambdaTimeout;
         this.resolve = resolve;
+        // if lambda timeout is reached, then send a message to the worker to gracefully exit
         this.timer = setTimeout(async () => {
-            this.logger.log('Lambda timeout reached. Exiting.');
-            if (this.worker) {
-                this.worker.postMessage({
+            this.logger.log('Lambda timeout reached. Sending a message to the worker to gracefully exit.');
+            if (worker) {
+                worker.postMessage({
                     subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
                 });
             }
             else {
+                console.log("Worker doesn't exist. Exiting from main thread.");
                 await this.exitFromMainThread();
             }
         }, this.lambdaTimeout);
-        this.logger = new logger_1.Logger({ event, options });
-        this.worker = worker;
+        // if worker exits with process.exit(code) then we need to clear the timer and exit from main thread
         worker.on(workers_1.WorkerEvent.WorkerExit, async (code) => {
             this.logger.info('Worker exited with exit code: ' + code + '.');
             if (this.timer) {
@@ -139,45 +153,57 @@ class Spawn {
             await this.exitFromMainThread();
         });
         worker.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {
-            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
-                this.logger.info('Worker has emitted message to ADaaS.');
-                this.hasWorkerEmitted = true;
-            }
-            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageDone) {
-                this.logger.info('Worker has completed work.');
-                clearTimeout(this.timer);
-                await this.exitFromMainThread();
-            }
-        });
-        worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
             var _a, _b;
+            // if worker send a log message, then log it from the main thread with logger
             if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageLog) {
                 const args = (_a = message.payload) === null || _a === void 0 ? void 0 : _a.args;
                 const level = (_b = message.payload) === null || _b === void 0 ? void 0 : _b.level;
                 this.logger.logFn(args, level);
             }
+            // if worker sends a message that it has completed work, then clear the timer and exit from main thread
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageDone) {
+                this.logger.info('Worker has completed with executing the task.');
+                if (this.timer) {
+                    clearTimeout(this.timer);
+                }
+                await this.exitFromMainThread();
+            }
+            // if worker sends a message that it has emitted an event, then set alreadyEmitted to true
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
+                this.logger.info('Worker has emitted message to ADaaS.');
+                this.alreadyEmitted = true;
+            }
         });
     }
     async exitFromMainThread() {
-        if (this.hasWorkerEmitted) {
-            this.resolve(true);
+        if (this.alreadyEmitted) {
+            this.resolve();
             return;
         }
+        this.alreadyEmitted = true;
         const timeoutEventType = (0, helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
-        if (timeoutEventType !== null) {
+        if (timeoutEventType) {
             const { eventType } = timeoutEventType;
-            await (0, control_protocol_1.emit)({
-                eventType,
-                event: this.event,
-                data: {
-                    error: {
-                        message: 'Worker has not emitted anything. Exited.',
+            try {
+                await (0, control_protocol_1.emit)({
+                    eventType,
+                    event: this.event,
+                    data: {
+                        error: {
+                            message: 'Worker has not emitted anything. Exited.',
+                        },
                     },
-                },
-            }).then(() => {
-                this.logger.error('Worker has not emitted anything. Exited.');
-                this.resolve(true);
-            });
+                });
+                this.resolve();
+            }
+            catch (error) {
+                if (axios_1.default.isAxiosError(error)) {
+                    console.error('Error while emitting event', (0, logger_1.serializeAxiosError)(error));
+                }
+                else {
+                    console.error('Error while emitting event', error);
+                }
+            }
         }
     }
 }

package/dist/workers/worker-adapter.d.ts

@@ -1,12 +1,12 @@
-import { AirdropEvent, ExtractorEventType, EventData, ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
+import { AirdropEvent, ExtractorEventType, EventData, ExternalSystemAttachmentStreamingFunction, ExternalSystemAttachmentProcessors, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../types/extraction';
 import { ExternalSystemAttachment, ExternalSystemLoadingFunction, FileToLoad, LoaderEventType } from '../types/loading';
 import { AdapterState } from '../state/state.interfaces';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { WorkerAdapterInterface, WorkerAdapterOptions } from '../types/workers';
 import { Repo } from '../repo/repo';
-import { RepoInterface } from '../repo/repo.interfaces';
+import { NormalizedAttachment, RepoInterface } from '../repo/repo.interfaces';
 import { ExternalSystemItem, ItemTypesToLoadParams, ItemTypeToLoad, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../types/loading';
-export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
+export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
 /**
  * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
  * as parameter in processTask and onTimeout functions. The class provides
@@ -27,14 +27,13 @@ export declare class WorkerAdapter<ConnectorState> {
     private adapterState;
     private _artifacts;
     private hasWorkerEmitted;
-    private parentPort;
     private isTimeout;
     private repos;
     private loaderReports;
     private _processedFiles;
     private mappers;
     private uploader;
-    constructor({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>);
+    constructor({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>);
     get state(): AdapterState<ConnectorState>;
     set state(value: AdapterState<ConnectorState>);
     get reports(): LoaderReport[];
@@ -64,6 +63,7 @@ export declare class WorkerAdapter<ConnectorState> {
         item: ExternalSystemItem;
         itemTypeToLoad: ItemTypeToLoad;
     }): Promise<LoadItemResponse>;
+    processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
     loadAttachment({ item, create, }: {
         item: ExternalSystemAttachment;
         create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
@@ -71,25 +71,12 @@ export declare class WorkerAdapter<ConnectorState> {
     /**
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.
-     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
-     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
+     * @param {{ stream, processors }: { stream: ExternalSystemAttachmentStreamingFunction, processors?: ExternalSystemAttachmentProcessors  }} Params - The parameters to stream the attachments
+     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssoAttachment artifact information
      * or error information if there was an error
      */
-    streamAttachments({ stream, }: {
+    streamAttachments<NewBatch>({ stream, processors, }: {
         stream: ExternalSystemAttachmentStreamingFunction;
-    }): Promise<{
-        report: {};
-        error?: undefined;
-        delay?: undefined;
-    } | {
-        error: {
-            message: string;
-        };
-        report?: undefined;
-        delay?: undefined;
-    } | {
-        delay: number;
-        report?: undefined;
-        error?: undefined;
-    }>;
+        processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
+    }): Promise<StreamAttachmentsReturnType>;
 }