@devrev/ts-adaas 1.2.0-beta → 1.2.1

package/README.md

@@ -2,6 +2,13 @@
 
 ## Release Notes
 
+### v1.2.1
+
+- Reduced the `delayFactor` to minimize unnecessary delays.
+- Correct the setting of the `lastSyncStarted` timestamp.
+- Improve logging for attachment extraction and loading.
+- Fix several bugs related to the control protocol.
+
 ### v1.2.0
 
 - Add support for loading attachments from DevRev to external system.
@@ -85,6 +92,8 @@
 
 # Overview
 
+[![Coverage Status](https://coveralls.io/repos/github/devrev/adaas-sdk/badge.svg?branch=main&t=s4Otlm)](https://coveralls.io/github/devrev/adaas-sdk?branch=main)
+
 The ADaaS (Airdrop-as-a-Service) Library for TypeScript helps developers build Snap-ins that integrate with DevRev’s ADaaS platform. This library simplifies the workflow for handling data extraction and loading, event-driven actions, state management, and artifact handling.
 
 It provides features such as:
@@ -359,7 +368,7 @@ This phase is defined in `load-attachments.ts` and is responsible for loading th
 Loading is done by providing the create function to create attachments in the external system.
 
 ```typescript
-  processTask({
+processTask({
   task: async ({ adapter }) => {
     const { reports, processed_files } = await adapter.loadAttachments({
       create,
@@ -378,7 +387,6 @@ Loading is done by providing the create function to create attachments in the ex
     });
   },
 });
-
 ```
 
 The loading function `create` provides loading to the external system, to make API calls to the external system to create the attachments and handle errors and external system's rate limiting.

package/dist/common/control-protocol.d.ts

@@ -1,3 +1,4 @@
+import { AxiosResponse } from 'axios';
 import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
 import { LoaderEventType } from '../types/loading';
 export interface EmitInterface {
@@ -5,4 +6,4 @@ export interface EmitInterface {
     eventType: ExtractorEventType | LoaderEventType;
     data?: EventData;
 }
-export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<void | Error>;
+export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<AxiosResponse>;

package/dist/common/control-protocol.js

@@ -2,35 +2,19 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.emit = void 0;
 const axios_client_1 = require("../http/axios-client");
-const logger_1 = require("../logger/logger");
 const emit = async ({ event, eventType, data, }) => {
     const newEvent = {
         event_type: eventType,
         event_context: event.payload.event_context,
         event_data: Object.assign({}, data),
     };
-    return new Promise(async (resolve, reject) => {
-        console.info('Emitting event', JSON.stringify(newEvent));
-        try {
-            await axios_client_1.axiosClient.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
-                headers: {
-                    Accept: 'application/json, text/plain, */*',
-                    Authorization: event.context.secrets.service_account_token,
-                    'Content-Type': 'application/json',
-                },
-            });
-            resolve();
-        }
-        catch (error) {
-            if (axios_client_1.axios.isAxiosError(error)) {
-                console.error(`Failed to emit event with event type ${eventType}.`, (0, logger_1.serializeAxiosError)(error));
-            }
-            else {
-                // TODO: Stop it through UI or think about retrying this request. Implement exponential retry mechanism.
-                console.error(`Failed to emit event with event type ${eventType}.`, error);
-            }
-            reject();
-        }
+    console.info('Emitting event', JSON.stringify(newEvent));
+    return axios_client_1.axiosClient.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
+        headers: {
+            Accept: 'application/json, text/plain, */*',
+            Authorization: event.context.secrets.service_account_token,
+            'Content-Type': 'application/json',
+        },
     });
 };
 exports.emit = emit;

package/dist/http/axios-client.js

@@ -14,8 +14,8 @@ exports.axiosClient = axiosClient;
     retryDelay: (retryCount, error) => {
         var _a;
         console.warn('Retry attempt: ' + retryCount + 'to url: ' + ((_a = error.config) === null || _a === void 0 ? void 0 : _a.url) + '.');
-        // Exponential backoff algorithm: 1 * 2 ^ retryCount * 5000ms
-        return axios_retry_1.default.exponentialDelay(retryCount, error, 5000);
+        // Exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
+        return axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
     },
     retryCondition: (error) => {
         var _a, _b, _c, _d;

package/dist/repo/repo.interfaces.d.ts

@@ -36,7 +36,7 @@ export interface NormalizedAttachment {
     file_name: string;
     author_id: string;
     parent_id: string;
-    inline?: boolean;
+    grand_parent_id?: number;
 }
 /**
  * Item is an interface that defines the structure of an item.

package/dist/repo/repo.js

@@ -36,6 +36,10 @@ class Repo {
     }
     async push(items) {
         let recordsToPush;
+        if (!items || items.length === 0) {
+            console.log(`No items to push for type ${this.itemType}. Skipping push.`);
+            return true;
+        }
         // Normalize items if needed
         if (this.normalize &&
             this.itemType != constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA &&
@@ -47,7 +51,6 @@ class Repo {
         }
         // Add the new records to the items array
         this.items.push(...recordsToPush);
-        console.info(`Extracted ${recordsToPush.length} new items of type ${this.itemType}. Total number of items in repo: ${this.items.length}.`);
         // Upload in batches while the number of items exceeds the batch size
         while (this.items.length >= constants_1.ARTIFACT_BATCH_SIZE) {
             // Slice out a batch of ARTIFACT_BATCH_SIZE items to upload

package/dist/state/state.js

@@ -16,6 +16,11 @@ async function createAdapterState({ event, initialState, options, }) {
     });
     if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
         await as.fetchState(newInitialState);
+        if (event.payload.event_type === extraction_1.EventType.ExtractionDataStart &&
+            !as.state.lastSyncStarted) {
+            as.state.lastSyncStarted = new Date().toISOString();
+            console.log(`Setting lastSyncStarted to ${as.state.lastSyncStarted}.`);
+        }
     }
     return as;
 }
@@ -29,7 +34,7 @@ class State {
                     },
                 }
                 : {
-                    lastSyncStarted: new Date().toISOString(),
+                    lastSyncStarted: '',
                     lastSuccessfulSyncStarted: '',
                     toDevRev: {
                         attachmentsMetadata: {
@@ -107,7 +112,7 @@ class State {
             if (axios_client_1.axios.isAxiosError(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
                 const state = Object.assign(Object.assign({}, initialState), this.initialSdkState);
                 this.state = state;
-                console.log('State not found, returning initial state. Current state:', (0, logger_1.getPrintableState)(this.state));
+                console.log('State not found, returning initial state. Current state', (0, logger_1.getPrintableState)(this.state));
                 await this.postState(this.state);
                 return this.state;
             }

package/dist/types/extraction.d.ts

@@ -2,8 +2,9 @@ import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { ErrorRecord } from './common';
 import { DonV2, LoaderReport, RateLimited } from './loading';
-import { NormalizedAttachment } from 'repo/repo.interfaces';
+import { NormalizedAttachment } from '../repo/repo.interfaces';
 import { AxiosResponse } from 'axios';
+import { WorkerAdapter } from '../workers/worker-adapter';
 /**
  * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
  * The external extractor can use these events to know what to do next in the extraction process.
@@ -164,6 +165,12 @@ export interface EventData {
     processed_files?: string[];
     stats_file?: string;
 }
+/**
+ * WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS.
+ */
+export interface WorkerMetadata {
+    adaas_library_version: string;
+}
 /**
  * DomainObject is an interface that defines the structure of a domain object that can be extracted.
  * It must contain a name, a next chunk ID, the pages, the last modified date, whether it is done, and the count.
@@ -213,6 +220,7 @@ export interface ExtractorEvent {
     event_type: string;
     event_context: EventContext;
     event_data?: EventData;
+    worker_metadata?: WorkerMetadata;
 }
 /**
  * LoaderEvent
@@ -221,6 +229,7 @@ export interface LoaderEvent {
     event_type: string;
     event_context: EventContext;
     event_data?: EventData;
+    worker_metadata?: WorkerMetadata;
 }
 export type ExternalSystemAttachmentStreamingFunction = ({ item, event, }: ExternalSystemAttachmentStreamingParams) => Promise<ExternalSystemAttachmentStreamingResponse>;
 export interface ExternalSystemAttachmentStreamingParams {
@@ -237,3 +246,30 @@ export interface StreamAttachmentsResponse {
     report?: LoaderReport;
     rateLimit?: RateLimited;
 }
+export type ProcessAttachmentReturnType = {
+    delay?: number;
+    error?: {
+        message: string;
+    };
+} | undefined;
+export type StreamAttachmentsReturnType = {
+    delay?: number;
+    error?: ErrorRecord;
+} | undefined;
+export type ExternalSystemAttachmentReducerFunction<Batch, NewBatch, ConnectorState> = ({ attachments, adapter, }: {
+    attachments: Batch;
+    adapter: WorkerAdapter<ConnectorState>;
+}) => NewBatch;
+export type ExternalProcessAttachmentFunction = ({ attachment, stream, }: {
+    attachment: NormalizedAttachment;
+    stream: ExternalSystemAttachmentStreamingFunction;
+}) => Promise<ProcessAttachmentReturnType>;
+export type ExternalSystemAttachmentIteratorFunction<NewBatch, ConnectorState> = ({ reducedAttachments, adapter, stream, }: {
+    reducedAttachments: NewBatch;
+    adapter: WorkerAdapter<ConnectorState>;
+    stream: ExternalSystemAttachmentStreamingFunction;
+}) => Promise<ProcessAttachmentReturnType>;
+export interface ExternalSystemAttachmentProcessors<ConnectorState, Batch, NewBatch> {
+    reducer: ExternalSystemAttachmentReducerFunction<Batch, NewBatch, ConnectorState>;
+    iterator: ExternalSystemAttachmentIteratorFunction<NewBatch, ConnectorState>;
+}

package/dist/types/index.d.ts

@@ -1,5 +1,5 @@
 export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
-export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, } from './extraction';
+export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, ExternalProcessAttachmentFunction, ExternalSystemAttachmentReducerFunction, ExternalSystemAttachmentIteratorFunction, } from './extraction';
 export { LoaderEventType, ExternalSystemItem, ExternalSystemItemLoadingResponse, ExternalSystemItemLoadingParams, ExternalSystemAttachment, } from './loading';
 export { NormalizedItem, NormalizedAttachment, RepoInterface, } from '../repo/repo.interfaces';
 export { AdapterState } from '../state/state.interfaces';

package/dist/types/loading.d.ts

@@ -28,6 +28,8 @@ export interface ExternalSystemAttachment {
     created_date: string;
     modified_by_id: string;
     modified_date: string;
+    parent_id?: string;
+    grand_parent_id?: string;
 }
 export interface ExternalSystemItem {
     id: {

package/dist/types/workers.d.ts

@@ -30,7 +30,6 @@ export interface WorkerAdapterOptions {
     isLocalDevelopment?: boolean;
     timeout?: number;
 }
-export type SpawnResolve = (value: boolean | PromiseLike<boolean>) => void;
 /**
  * SpawnInterface is an interface for Spawn class.
  * @interface SpawnInterface
@@ -42,7 +41,7 @@ export interface SpawnInterface {
     event: AirdropEvent;
     worker: Worker;
     options?: WorkerAdapterOptions;
-    resolve: SpawnResolve;
+    resolve: (value: void | PromiseLike<void>) => void;
 }
 /**
  * SpawnFactoryInterface is an interface for Spawn class factory.

package/dist/uploader/uploader.js

@@ -96,7 +96,7 @@ class Uploader {
             item_type: itemType,
             item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1,
         };
-        console.log('Successful upload of artifact: ', artifact);
+        console.log('Successful upload of artifact', artifact);
         return { artifact };
     }
     async prepareArtifact(filename, fileType) {

package/dist/workers/default-workers/attachments-extraction.js

@@ -12,10 +12,12 @@ const getAttachmentStream = async ({ item, }) => {
     }
     catch (error) {
         if (axios_client_1.axios.isAxiosError(error)) {
-            console.error('Error while fetching attachment from URL.', (0, index_1.serializeAxiosError)(error));
+            console.warn(`Error while fetching attachment ${id} from URL.`, (0, index_1.serializeAxiosError)(error));
+            console.warn('Failed attachment metadata', item);
         }
         else {
-            console.error('Error while fetching attachment from URL.', error);
+            console.warn(`Error while fetching attachment ${id} from URL.`, error);
+            console.warn('Failed attachment metadata', item);
         }
         return {
             error: {
@@ -26,20 +28,27 @@ const getAttachmentStream = async ({ item, }) => {
 };
 (0, index_1.processTask)({
     task: async ({ adapter }) => {
-        const { error, delay } = await adapter.streamAttachments({
-            stream: getAttachmentStream,
-        });
-        if (delay) {
-            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDelay, {
-                delay,
+        try {
+            const response = await adapter.streamAttachments({
+                stream: getAttachmentStream,
             });
+            if (response === null || response === void 0 ? void 0 : response.delay) {
+                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDelay, {
+                    delay: response.delay,
+                });
+            }
+            else if (response === null || response === void 0 ? void 0 : response.error) {
+                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
+                    error: response.error,
+                });
+            }
+            else {
+                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
+            }
         }
-        else if (error) {
-            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
-                error,
-            });
+        catch (error) {
+            console.error('An error occured while processing a task.', error);
         }
-        await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
     },
     onTimeout: async ({ adapter }) => {
         await adapter.postState();

package/dist/workers/default-workers/load-attachments.js

@@ -1,12 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const process_task_1 = require("workers/process-task");
-const types_1 = require("types");
+const process_task_1 = require("../process-task");
+const types_1 = require("../../types");
 (0, process_task_1.processTask)({
     task: async ({ adapter }) => {
-        await adapter.emit(types_1.LoaderEventType.AttachmentLoadingDone, {
-            reports: adapter.reports,
-            processed_files: adapter.processedFiles,
+        await adapter.emit(types_1.LoaderEventType.UnknownEventType, {
+            error: {
+                message: 'Event type ' + adapter.event.payload.event_type + ' not supported.'
+            }
         });
     },
     onTimeout: async ({ adapter }) => {

package/dist/workers/default-workers/load-data.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const process_task_1 = require("workers/process-task");
+const process_task_1 = require("../process-task");
 const loading_1 = require("../../types/loading");
 (0, process_task_1.processTask)({
     task: async ({ adapter }) => {

package/dist/workers/spawn.d.ts

@@ -10,16 +10,15 @@ import { SpawnFactoryInterface, SpawnInterface } from '../types/workers';
  * @param {string} workerPath - The path to the worker file
  * @returns {Promise<Spawn>} - A new instance of Spawn class
  */
-export declare function spawn<ConnectorState>({ event, initialState, workerPath, options, }: SpawnFactoryInterface<ConnectorState>): Promise<boolean | PromiseLike<boolean>>;
+export declare function spawn<ConnectorState>({ event, initialState, workerPath, options, }: SpawnFactoryInterface<ConnectorState>): Promise<void>;
 export declare class Spawn {
     private event;
-    private hasWorkerEmitted;
+    private alreadyEmitted;
     private defaultLambdaTimeout;
     private lambdaTimeout;
-    private worker;
-    private resolve;
     private timer;
     private logger;
+    private resolve;
     constructor({ event, worker, options, resolve }: SpawnInterface);
     private exitFromMainThread;
 }

package/dist/workers/spawn.js

@@ -1,7 +1,11 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Spawn = void 0;
 exports.spawn = spawn;
+const axios_1 = __importDefault(require("axios"));
 const extraction_1 = require("../types/extraction");
 const control_protocol_1 = require("../common/control-protocol");
 const helpers_1 = require("../common/helpers");
@@ -90,47 +94,58 @@ async function spawn({ event, initialState, workerPath, options, }) {
             });
         }
         catch (error) {
-            logger.error('Worker error while processing task.', error);
-            return false;
+            logger.error('Worker error while processing task', error);
         }
     }
     else {
-        await (0, control_protocol_1.emit)({
-            event,
-            eventType: extraction_1.ExtractorEventType.UnknownEventType,
-            data: {
-                error: {
-                    message: 'Unrecognized event type in spawn ' +
-                        event.payload.event_type +
-                        '.',
+        console.error('Script was not found for event type: ' + event.payload.event_type + '.');
+        try {
+            await (0, control_protocol_1.emit)({
+                event,
+                eventType: extraction_1.ExtractorEventType.UnknownEventType,
+                data: {
+                    error: {
+                        message: 'Unrecognized event type in spawn ' +
+                            event.payload.event_type +
+                            '.',
+                    },
                 },
-            },
-        });
-        return false;
+            });
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error)) {
+                console.error('Error while emitting event', (0, logger_1.serializeAxiosError)(error));
+            }
+            else {
+                console.error('Error while emitting event', error);
+            }
+        }
     }
 }
 class Spawn {
     constructor({ event, worker, options, resolve }) {
         this.defaultLambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
-        this.hasWorkerEmitted = false;
+        this.alreadyEmitted = false;
         this.event = event;
+        this.logger = new logger_1.Logger({ event, options });
         this.lambdaTimeout = (options === null || options === void 0 ? void 0 : options.timeout)
             ? Math.min(options.timeout, this.defaultLambdaTimeout)
             : this.defaultLambdaTimeout;
         this.resolve = resolve;
+        // if lambda timeout is reached, then send a message to the worker to gracefully exit
         this.timer = setTimeout(async () => {
-            this.logger.log('Lambda timeout reached. Exiting.');
-            if (this.worker) {
-                this.worker.postMessage({
+            this.logger.log('Lambda timeout reached. Sending a message to the worker to gracefully exit.');
+            if (worker) {
+                worker.postMessage({
                     subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
                 });
             }
             else {
+                console.log("Worker doesn't exist. Exiting from main thread.");
                 await this.exitFromMainThread();
             }
         }, this.lambdaTimeout);
-        this.logger = new logger_1.Logger({ event, options });
-        this.worker = worker;
+        // if worker exits with process.exit(code) then we need to clear the timer and exit from main thread
         worker.on(workers_1.WorkerEvent.WorkerExit, async (code) => {
             this.logger.info('Worker exited with exit code: ' + code + '.');
             if (this.timer) {
@@ -139,45 +154,57 @@ class Spawn {
             await this.exitFromMainThread();
         });
         worker.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {
-            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
-                this.logger.info('Worker has emitted message to ADaaS.');
-                this.hasWorkerEmitted = true;
-            }
-            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageDone) {
-                this.logger.info('Worker has completed work.');
-                clearTimeout(this.timer);
-                await this.exitFromMainThread();
-            }
-        });
-        worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
             var _a, _b;
+            // if worker send a log message, then log it from the main thread with logger
             if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageLog) {
                 const args = (_a = message.payload) === null || _a === void 0 ? void 0 : _a.args;
                 const level = (_b = message.payload) === null || _b === void 0 ? void 0 : _b.level;
                 this.logger.logFn(args, level);
             }
+            // if worker sends a message that it has completed work, then clear the timer and exit from main thread
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageDone) {
+                this.logger.info('Worker has completed with executing the task.');
+                if (this.timer) {
+                    clearTimeout(this.timer);
+                }
+                await this.exitFromMainThread();
+            }
+            // if worker sends a message that it has emitted an event, then set alreadyEmitted to true
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
+                this.logger.info('Worker has emitted message to ADaaS.');
+                this.alreadyEmitted = true;
+            }
         });
     }
     async exitFromMainThread() {
-        if (this.hasWorkerEmitted) {
-            this.resolve(true);
+        if (this.alreadyEmitted) {
+            this.resolve();
             return;
         }
+        this.alreadyEmitted = true;
         const timeoutEventType = (0, helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
-        if (timeoutEventType !== null) {
+        if (timeoutEventType) {
             const { eventType } = timeoutEventType;
-            await (0, control_protocol_1.emit)({
-                eventType,
-                event: this.event,
-                data: {
-                    error: {
-                        message: 'Worker has not emitted anything. Exited.',
+            try {
+                await (0, control_protocol_1.emit)({
+                    eventType,
+                    event: this.event,
+                    data: {
+                        error: {
+                            message: 'Worker has not emitted anything. Exited.',
+                        },
                     },
-                },
-            }).then(() => {
-                this.logger.error('Worker has not emitted anything. Exited.');
-                this.resolve(true);
-            });
+                });
+                this.resolve();
+            }
+            catch (error) {
+                if (axios_1.default.isAxiosError(error)) {
+                    console.error('Error while emitting event', (0, logger_1.serializeAxiosError)(error));
+                }
+                else {
+                    console.error('Error while emitting event', error);
+                }
+            }
         }
     }
 }

package/dist/workers/worker-adapter.d.ts

@@ -1,10 +1,10 @@
-import { AirdropEvent, ExtractorEventType, EventData, ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
+import { AirdropEvent, ExtractorEventType, EventData, ExternalSystemAttachmentStreamingFunction, ExternalSystemAttachmentProcessors, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../types/extraction';
 import { ExternalSystemAttachment, ExternalSystemLoadingFunction, FileToLoad, LoaderEventType } from '../types/loading';
 import { AdapterState } from '../state/state.interfaces';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { WorkerAdapterInterface, WorkerAdapterOptions } from '../types/workers';
 import { Repo } from '../repo/repo';
-import { RepoInterface } from '../repo/repo.interfaces';
+import { NormalizedAttachment, RepoInterface } from '../repo/repo.interfaces';
 import { ExternalSystemItem, ItemTypesToLoadParams, ItemTypeToLoad, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../types/loading';
 export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
 /**
@@ -64,6 +64,7 @@ export declare class WorkerAdapter<ConnectorState> {
         item: ExternalSystemItem;
         itemTypeToLoad: ItemTypeToLoad;
     }): Promise<LoadItemResponse>;
+    processAttachment: (attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction) => Promise<ProcessAttachmentReturnType>;
     loadAttachment({ item, create, }: {
         item: ExternalSystemAttachment;
         create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
@@ -71,25 +72,12 @@ export declare class WorkerAdapter<ConnectorState> {
     /**
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.
-     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
-     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
+     * @param {{ stream, processors }: { stream: ExternalSystemAttachmentStreamingFunction, processors?: ExternalSystemAttachmentProcessors  }} Params - The parameters to stream the attachments
+     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssoAttachment artifact information
      * or error information if there was an error
      */
-    streamAttachments({ stream, }: {
+    streamAttachments<NewBatch>({ stream, processors, }: {
         stream: ExternalSystemAttachmentStreamingFunction;
-    }): Promise<{
-        report: {};
-        error?: undefined;
-        delay?: undefined;
-    } | {
-        error: {
-            message: string;
-        };
-        report?: undefined;
-        delay?: undefined;
-    } | {
-        delay: number;
-        report?: undefined;
-        error?: undefined;
-    }>;
+        processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
+    }): Promise<StreamAttachmentsReturnType>;
 }

package/dist/workers/worker-adapter.js

@@ -39,6 +39,49 @@ function createWorkerAdapter({ event, adapterState, parentPort, options, }) {
 class WorkerAdapter {
     constructor({ event, adapterState, parentPort, options, }) {
         this.repos = [];
+        this.processAttachment = async (attachment, stream) => {
+            var _a, _b;
+            const { httpStream, delay, error } = await stream({
+                item: attachment,
+                event: this.event,
+            });
+            if (error) {
+                console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
+                return { error };
+            }
+            else if (delay) {
+                return { delay };
+            }
+            if (httpStream) {
+                const fileType = ((_a = httpStream.headers) === null || _a === void 0 ? void 0 : _a['content-type']) || 'application/octet-stream';
+                const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
+                if (!preparedArtifact) {
+                    console.warn('Error while preparing artifact for attachment ID ' +
+                        attachment.id +
+                        '. Skipping attachment');
+                    return;
+                }
+                const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
+                if (!uploadedArtifact) {
+                    console.warn('Error while preparing artifact for attachment ID ' + attachment.id);
+                    return;
+                }
+                const ssorAttachment = {
+                    id: {
+                        devrev: preparedArtifact.id,
+                        external: attachment.id,
+                    },
+                    parent_id: {
+                        external: attachment.parent_id,
+                    },
+                    actor_id: {
+                        external: attachment.author_id,
+                    },
+                };
+                await ((_b = this.getRepo('ssor_attachment')) === null || _b === void 0 ? void 0 : _b.push([ssorAttachment]));
+            }
+            return;
+        };
         this.event = event;
         this.options = options;
         this.adapterState = adapterState;
@@ -125,6 +168,7 @@ class WorkerAdapter {
         // If the extraction is done, we want to save the timestamp of the last successful sync
         if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
             this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
+            this.state.lastSyncStarted = '';
         }
         // We want to save the state every time we emit an event, except for the start and delete events
         if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
@@ -147,9 +191,15 @@ class WorkerAdapter {
             };
             this.artifacts = [];
             this.parentPort.postMessage(message);
+            this.hasWorkerEmitted = true;
         }
         catch (error) {
-            console.error('Error while emitting event with event type: ' + newEventType + '.', error);
+            if (axios_devrev_client_1.axios.isAxiosError(error)) {
+                console.error('Error while emitting event with event type: ' + newEventType + '.', (0, logger_1.serializeAxiosError)(error));
+            }
+            else {
+                console.error('Error while emitting event with event type: ' + newEventType + '.', error);
+            }
             this.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
         }
     }
@@ -518,90 +568,66 @@ class WorkerAdapter {
     /**
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.
-     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
-     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
+     * @param {{ stream, processors }: { stream: ExternalSystemAttachmentStreamingFunction, processors?: ExternalSystemAttachmentProcessors  }} Params - The parameters to stream the attachments
+     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssoAttachment artifact information
      * or error information if there was an error
      */
-    async streamAttachments({ stream, }) {
-        var _a, _b, _c, _d, _e, _f;
+    async streamAttachments({ stream, processors, }) {
+        var _a, _b, _c;
         const repos = [
             {
                 itemType: 'ssor_attachment',
             },
         ];
         this.initializeRepos(repos);
-        for (const attachmentsMetadataArtifactId of ((_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds) || []) {
-            if (((_b = this.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.length) === 0) {
-                return { report: {} };
-            }
-            console.log('Started streaming attachments to the platform.');
+        const attachmentsState = (((_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds) || []).slice();
+        console.log('Attachments metadata artifact IDs', attachmentsState);
+        for (const attachmentsMetadataArtifactId of attachmentsState) {
+            console.log(`Started processing attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
             const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
                 artifact: attachmentsMetadataArtifactId,
             });
             if (error) {
+                console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
                 return { error };
             }
-            if (attachments) {
-                const attachmentsToProcess = attachments.slice((_d = (_c = this.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata) === null || _d === void 0 ? void 0 : _d.lastProcessed, attachments.length);
+            if (!attachments || attachments.length === 0) {
+                console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                continue;
+            }
+            if (processors) {
+                console.log(`Using custom processors for attachments.`);
+                const { reducer, iterator } = processors;
+                const reducedAttachments = reducer({ attachments, adapter: this });
+                const response = await iterator({
+                    reducedAttachments,
+                    adapter: this,
+                    stream,
+                });
+                if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
+                    return response;
+                }
+            }
+            else {
+                console.log(`Using default processors for attachments.`);
+                const attachmentsToProcess = attachments.slice((_c = (_b = this.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata) === null || _c === void 0 ? void 0 : _c.lastProcessed, attachments.length);
                 for (const attachment of attachmentsToProcess) {
-                    const { httpStream, delay, error } = await stream({
-                        item: attachment,
-                        event: this.event,
-                    });
-                    if (error) {
-                        console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
-                        continue;
-                    }
-                    else if (delay) {
-                        return { delay };
+                    const response = await this.processAttachment(attachment, stream);
+                    if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
+                        return response;
                     }
-                    if (httpStream) {
-                        const fileType = ((_e = httpStream.headers) === null || _e === void 0 ? void 0 : _e['content-type']) ||
-                            'application/octet-stream';
-                        const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
-                        if (!preparedArtifact) {
-                            console.warn('Error while preparing artifact for attachment ID ' +
-                                attachment.id +
-                                '. Skipping attachment');
-                            if (this.state.toDevRev) {
-                                this.state.toDevRev.attachmentsMetadata.lastProcessed++;
-                            }
-                            continue;
-                        }
-                        const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
-                        if (!uploadedArtifact) {
-                            console.warn('Error while preparing artifact for attachment ID ' +
-                                attachment.id);
-                            if (this.state.toDevRev) {
-                                this.state.toDevRev.attachmentsMetadata.lastProcessed++;
-                            }
-                            continue;
-                        }
-                        const ssorAttachment = {
-                            id: {
-                                devrev: preparedArtifact.id,
-                                external: attachment.id,
-                            },
-                            parent_id: {
-                                external: attachment.parent_id,
-                            },
-                            actor_id: {
-                                external: attachment.author_id,
-                            },
-                        };
-                        await ((_f = this.getRepo('ssor_attachment')) === null || _f === void 0 ? void 0 : _f.push([ssorAttachment]));
-                        if (this.state.toDevRev) {
-                            this.state.toDevRev.attachmentsMetadata.lastProcessed++;
-                        }
+                    if (this.state.toDevRev) {
+                        this.state.toDevRev.attachmentsMetadata.lastProcessed += 1;
                     }
                 }
             }
             if (this.state.toDevRev) {
+                console.log(`Finished processing attachments for artifact ID. Setting last processed to 0 and removing artifact ID from state.`);
                 this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
                 this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
             }
         }
-        return { report: {} };
+        return;
     }
 }
 exports.WorkerAdapter = WorkerAdapter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.2.0-beta",
+  "version": "1.2.1",
   "description": "DevRev ADaaS (AirDrop-as-a-Service) Typescript SDK.",
   "type": "commonjs",
   "main": "./dist/index.js",

package/dist/workers/default-workers/data-loading.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/workers/default-workers/data-loading.js

@@ -1,19 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const process_task_1 = require("workers/process-task");
-const loading_1 = require("../../types/loading");
-(0, process_task_1.processTask)({
-    task: async ({ adapter }) => {
-        await adapter.emit(loading_1.LoaderEventType.DataLoadingDone, {
-            reports: adapter.reports,
-            processed_files: adapter.processedFiles,
-        });
-    },
-    onTimeout: async ({ adapter }) => {
-        await adapter.postState();
-        await adapter.emit(loading_1.LoaderEventType.DataLoadingProgress, {
-            reports: adapter.reports,
-            processed_files: adapter.processedFiles,
-        });
-    },
-});

package/dist/workers/default-workers/loader-state-deletion.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/workers/default-workers/loader-state-deletion.js

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const index_1 = require("../../index");
-(0, index_1.processTask)({
-    task: async ({ adapter }) => {
-        await adapter.emit(index_1.LoaderEventType.LoaderStateDeletionDone);
-    },
-    onTimeout: async ({ adapter }) => {
-        await adapter.emit(index_1.LoaderEventType.LoaderStateDeletionError, {
-            error: {
-                message: 'Failed to delete data. Lambda timeout.',
-            },
-        });
-    },
-});