@devrev/ts-adaas 1.1.6 → 1.2.0-beta

package/README.md

@@ -2,6 +2,10 @@
 
 ## Release Notes
 
+### v1.2.0
+
+- Add support for loading attachments from DevRev to external system.
+
 ### v1.1.6
 
 - Add exponential retry and handle rate-limiting towards DevRev.
@@ -307,9 +311,9 @@ export interface NormalizedAttachment {
 
 ## Loading phases
 
-### 1. Data Loading
+### 1. Loading Data
 
-This phase is defined in `data-loading.ts` and is responsible for loading the data to the external system.
+This phase is defined in `load-data.ts` and is responsible for loading the data to the external system.
 
 Loading is done by providing an ordered list of itemTypes to load and their respective create and update functions.
 
@@ -347,3 +351,36 @@ Loading is done by providing an ordered list of itemTypes to load and their resp
 The loading functions `create` and `update` provide loading to the external system. They provide denormalization of the records to the schema of the external system and provide HTTP calls to the external system. Both loading functions must handle rate limiting for the external system and handle errors.
 
 Functions return an ID and modified date of the record in the external system, or specify rate-liming offset or errors, if the record could not be created or updated.
+
+### 2. Loading Attachments
+
+This phase is defined in `load-attachments.ts` and is responsible for loading the attachments to the external system.
+
+Loading is done by providing the create function to create attachments in the external system.
+
+```typescript
+  processTask({
+  task: async ({ adapter }) => {
+    const { reports, processed_files } = await adapter.loadAttachments({
+      create,
+    });
+
+    await adapter.emit(LoaderEventType.AttachmentLoadingDone, {
+      reports,
+      processed_files,
+    });
+  },
+  onTimeout: async ({ adapter }) => {
+    await adapter.postState();
+    await adapter.emit(LoaderEventType.AttachmentLoadingProgress, {
+      reports: adapter.reports,
+      processed_files: adapter.processedFiles,
+    });
+  },
+});
+
+```
+
+The loading function `create` provides loading to the external system, to make API calls to the external system to create the attachments and handle errors and external system's rate limiting.
+
+Functions return an ID and modified date of the record in the external system, specify rate-liming back-off, or log errors, if the attachment could not be created.

package/dist/common/constants.js

@@ -8,7 +8,7 @@ exports.STATELESS_EVENT_TYPES = [
     extraction_1.EventType.ExtractionDataDelete,
     extraction_1.EventType.ExtractionAttachmentsDelete,
     extraction_1.EventType.StartDeletingLoaderState,
-    extraction_1.EventType.StartDeletingLoaderAttachmentsState,
+    extraction_1.EventType.StartDeletingLoaderAttachmentState,
 ];
 exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
     extraction_1.EventType.ExtractionExternalSyncUnitsStart,
@@ -24,7 +24,7 @@ exports.ALLOWED_LOADING_EVENT_TYPES = [
     extraction_1.EventType.StartLoadingData,
     extraction_1.EventType.ContinueLoadingData,
     extraction_1.EventType.StartDeletingLoaderState,
-    extraction_1.EventType.StartDeletingLoaderAttachmentsState,
+    extraction_1.EventType.StartDeletingLoaderAttachmentState,
 ];
 exports.ALLOWED_EVENT_TYPES = [
     ...exports.ALLOWED_EXTRACTION_EVENT_TYPES,

package/dist/common/helpers.d.ts

@@ -1,13 +1,13 @@
 import { AirdropEvent, EventType, ExtractorEventType } from '../types/extraction';
-import { FileToLoad, ItemTypeToLoad, LoaderEventType, LoaderReport, StatsFileObject } from '../types/loading';
+import { FileToLoad, LoaderEventType, LoaderReport, StatsFileObject } from '../types/loading';
 export declare function getTimeoutErrorEventType(eventType: EventType): {
     eventType: ExtractorEventType | LoaderEventType;
 } | null;
 export declare function getSyncDirection({ event }: {
     event: AirdropEvent;
 }): string;
-export declare function getFilesToLoad({ itemTypesToLoad, statsFile, }: {
-    itemTypesToLoad: ItemTypeToLoad[];
+export declare function getFilesToLoad({ supportedItemTypes, statsFile, }: {
+    supportedItemTypes: string[];
     statsFile: StatsFileObject[];
 }): FileToLoad[];
 export declare function addReportToLoaderReport({ loaderReports, report, }: {

package/dist/common/helpers.js

@@ -41,25 +41,29 @@ function getTimeoutErrorEventType(eventType) {
                 eventType: loading_1.LoaderEventType.DataLoadingError,
             };
         case extraction_1.EventType.StartDeletingLoaderState:
-        case extraction_1.EventType.StartDeletingLoaderAttachmentsState:
             return {
                 eventType: loading_1.LoaderEventType.LoaderStateDeletionError,
             };
+        case extraction_1.EventType.StartDeletingLoaderAttachmentState:
+            return {
+                eventType: loading_1.LoaderEventType.LoaderAttachmentStateDeletionError,
+            };
         default:
             console.error('Event type not recognized in getTimeoutErrorEventType function: ' +
                 eventType);
-            return null;
+            return {
+                eventType: loading_1.LoaderEventType.UnknownEventType,
+            };
     }
 }
 function getSyncDirection({ event }) {
     return event.payload.event_context.mode;
 }
-function getFilesToLoad({ itemTypesToLoad, statsFile, }) {
+function getFilesToLoad({ supportedItemTypes, statsFile, }) {
     const filesToLoad = [];
-    if (itemTypesToLoad.length === 0 || statsFile.length === 0) {
+    if (supportedItemTypes.length === 0 || statsFile.length === 0) {
         return [];
     }
-    const supportedItemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
     const filteredStatsFile = statsFile.filter((file) => supportedItemTypes.includes(file.item_type));
     const orderedFiles = filteredStatsFile.sort((a, b) => {
         const aIndex = supportedItemTypes.indexOf(a.item_type);

package/dist/deprecated/uploader/index.js

@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
 const axios_devrev_client_1 = require("../../http/axios-devrev-client");
@@ -62,7 +72,7 @@ class Uploader {
      */
     async upload(filename, entity, fetchedObjects, filetype = 'application/jsonl+json') {
         if (this.local) {
-            this.downloadToLocal(filename, fetchedObjects);
+            await this.downloadToLocal(filename, fetchedObjects);
         }
         const preparedArtifact = await this.prepareArtifact(filename, filetype);
         if (!preparedArtifact) {

package/dist/http/client.js

@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HTTPClient = exports.defaultResponse = void 0;
 const axios_1 = __importStar(require("axios"));

package/dist/repo/repo.d.ts

@@ -10,5 +10,5 @@ export declare class Repo {
     constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
     getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
     upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
-    push(items: Item[]): Promise<void | ErrorRecord>;
+    push(items: Item[]): Promise<boolean | ErrorRecord>;
 }

package/dist/repo/repo.js

@@ -35,36 +35,33 @@ class Repo {
         }
     }
     async push(items) {
-        return new Promise(async (resolve, reject) => {
-            let recordsToPush;
-            // Normalize items if needed
-            if (this.normalize &&
-                this.itemType != constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA &&
-                this.itemType != constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT) {
-                recordsToPush = items.map((item) => this.normalize(item));
-            }
-            else {
-                recordsToPush = items;
+        let recordsToPush;
+        // Normalize items if needed
+        if (this.normalize &&
+            this.itemType != constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA &&
+            this.itemType != constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT) {
+            recordsToPush = items.map((item) => this.normalize(item));
+        }
+        else {
+            recordsToPush = items;
+        }
+        // Add the new records to the items array
+        this.items.push(...recordsToPush);
+        console.info(`Extracted ${recordsToPush.length} new items of type ${this.itemType}. Total number of items in repo: ${this.items.length}.`);
+        // Upload in batches while the number of items exceeds the batch size
+        while (this.items.length >= constants_1.ARTIFACT_BATCH_SIZE) {
+            // Slice out a batch of ARTIFACT_BATCH_SIZE items to upload
+            const batch = this.items.splice(0, constants_1.ARTIFACT_BATCH_SIZE);
+            try {
+                // Upload the batch
+                await this.upload(batch);
             }
-            // Add the new records to the items array
-            this.items.push(...recordsToPush);
-            console.info(`Extracted ${recordsToPush.length} new items of type ${this.itemType}. Total number of items in repo: ${this.items.length}.`);
-            // Upload in batches while the number of items exceeds the batch size
-            while (this.items.length >= constants_1.ARTIFACT_BATCH_SIZE) {
-                // Slice out a batch of ARTIFACT_BATCH_SIZE items to upload
-                const batch = this.items.splice(0, constants_1.ARTIFACT_BATCH_SIZE);
-                try {
-                    // Upload the batch
-                    await this.upload(batch);
-                }
-                catch (error) {
-                    console.error('Error while uploading batch', error);
-                    reject(error);
-                    return;
-                }
+            catch (error) {
+                console.error('Error while uploading batch', error);
+                return false;
             }
-            resolve();
-        });
+        }
+        return true;
     }
 }
 exports.Repo = Repo;

package/dist/tests/from_devrev/loading.test.js

@@ -52,19 +52,28 @@ describe('getFilesToLoad', () => {
     it('should return an empty array if statsFile is empty', () => {
         statsFile = [];
         const itemTypesToLoad = [];
-        const result = (0, helpers_1.getFilesToLoad)({ itemTypesToLoad, statsFile });
+        const result = (0, helpers_1.getFilesToLoad)({
+            supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
+            statsFile,
+        });
         expect(result).toEqual([]);
     });
     it('should return an empty array if itemTypesToLoad is empty', () => {
         const itemTypesToLoad = [];
-        const result = (0, helpers_1.getFilesToLoad)({ itemTypesToLoad, statsFile });
+        const result = (0, helpers_1.getFilesToLoad)({
+            supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
+            statsFile,
+        });
         expect(result).toEqual([]);
     });
     it('should return an empty array if statsFile has no matching items', () => {
         const itemTypesToLoad = [
             { itemType: 'users', create: jest.fn(), update: jest.fn() },
         ];
-        const result = (0, helpers_1.getFilesToLoad)({ itemTypesToLoad, statsFile });
+        const result = (0, helpers_1.getFilesToLoad)({
+            supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
+            statsFile,
+        });
         expect(result).toEqual([]);
     });
     it('should filter out files not in itemTypesToLoad and order them by itemTypesToLoad', () => {
@@ -72,7 +81,10 @@ describe('getFilesToLoad', () => {
             { itemType: 'attachments', create: jest.fn(), update: jest.fn() },
             { itemType: 'issues', create: jest.fn(), update: jest.fn() },
         ];
-        const result = (0, helpers_1.getFilesToLoad)({ itemTypesToLoad, statsFile });
+        const result = (0, helpers_1.getFilesToLoad)({
+            supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
+            statsFile,
+        });
         expect(result).toEqual([
             {
                 id: 'don:core:dvrv-us-1:devo/1:artifact/99',
@@ -113,7 +125,7 @@ describe('getFilesToLoad', () => {
             { itemType: 'issues', create: jest.fn(), update: jest.fn() },
         ];
         const result = (0, helpers_1.getFilesToLoad)({
-            itemTypesToLoad,
+            supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
             statsFile,
         });
         expect(result).toEqual([

package/dist/tests/test-worker.js

@@ -5,12 +5,10 @@ const extraction_1 = require("../types/extraction");
 (0, process_task_1.processTask)({
     task: async ({ adapter }) => {
         await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {});
-        return 0;
     },
     onTimeout: async ({ adapter }) => {
         await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError, {
             error: { message: 'External sync unit failed.' },
         });
-        return 0;
     },
 });

package/dist/types/extraction.d.ts

@@ -19,8 +19,10 @@ export declare enum EventType {
     ExtractionAttachmentsDelete = "EXTRACTION_ATTACHMENTS_DELETE",
     StartLoadingData = "START_LOADING_DATA",
     ContinueLoadingData = "CONTINUE_LOADING_DATA",
+    StartLoadingAttachments = "START_LOADING_ATTACHMENTS",
+    ContinueLoadingAttachments = "CONTINUE_LOADING_ATTACHMENTS",
     StartDeletingLoaderState = "START_DELETING_LOADER_STATE",
-    StartDeletingLoaderAttachmentsState = "START_DELETING_LOADER_ATTACHMENTS_STATE"
+    StartDeletingLoaderAttachmentState = "START_DELETING_LOADER_ATTACHMENT_STATE"
 }
 /**
  * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.

package/dist/types/extraction.js

@@ -19,8 +19,10 @@ var EventType;
     // Loading
     EventType["StartLoadingData"] = "START_LOADING_DATA";
     EventType["ContinueLoadingData"] = "CONTINUE_LOADING_DATA";
+    EventType["StartLoadingAttachments"] = "START_LOADING_ATTACHMENTS";
+    EventType["ContinueLoadingAttachments"] = "CONTINUE_LOADING_ATTACHMENTS";
     EventType["StartDeletingLoaderState"] = "START_DELETING_LOADER_STATE";
-    EventType["StartDeletingLoaderAttachmentsState"] = "START_DELETING_LOADER_ATTACHMENTS_STATE";
+    EventType["StartDeletingLoaderAttachmentState"] = "START_DELETING_LOADER_ATTACHMENT_STATE";
 })(EventType || (exports.EventType = EventType = {}));
 /**
  * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.

package/dist/types/index.d.ts

@@ -1,6 +1,6 @@
 export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
 export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, } from './extraction';
-export { LoaderEventType, ExternalSystemItem, ExternalSystemItemLoadingResponse, ExternalSystemItemLoadingParams, } from './loading';
+export { LoaderEventType, ExternalSystemItem, ExternalSystemItemLoadingResponse, ExternalSystemItemLoadingParams, ExternalSystemAttachment, } from './loading';
 export { NormalizedItem, NormalizedAttachment, RepoInterface, } from '../repo/repo.interfaces';
 export { AdapterState } from '../state/state.interfaces';
 export { Artifact, ArtifactsPrepareResponse, UploadResponse, StreamResponse, StreamAttachmentsResponse, SsorAttachment, } from '../uploader/uploader.interfaces';

package/dist/types/loading.d.ts

@@ -15,6 +15,29 @@ export interface FileToLoad {
     lineToProcess: number;
     completed: boolean;
 }
+export interface ExternalSystemAttachment {
+    reference_id: DonV2;
+    parent_type: string;
+    parent_reference_id: DonV2;
+    file_name: string;
+    file_type: string;
+    file_size: number;
+    url: string;
+    valid_until: string;
+    created_by_id: string;
+    created_date: string;
+    modified_by_id: string;
+    modified_date: string;
+}
+export interface ExternalSystemItem {
+    id: {
+        devrev: DonV2;
+        external?: string;
+    };
+    created_date: string;
+    modified_date: string;
+    data: any;
+}
 export interface ExternalSystemItem {
     id: {
         devrev: DonV2;
@@ -24,8 +47,8 @@ export interface ExternalSystemItem {
     modified_date: string;
     data: any;
 }
-export interface ExternalSystemItemLoadingParams {
-    item: ExternalSystemItem;
+export interface ExternalSystemItemLoadingParams<Type> {
+    item: Type;
     mappers: Mappers;
     event: AirdropEvent;
 }
@@ -40,11 +63,11 @@ export interface ExternalSystemItemLoadedItem {
     error?: string;
     modifiedDate?: string;
 }
-export type ExternalSystemLoadingFunction = ({ item, mappers, event, }: ExternalSystemItemLoadingParams) => Promise<ExternalSystemItemLoadingResponse>;
+export type ExternalSystemLoadingFunction<Item> = ({ item, mappers, event, }: ExternalSystemItemLoadingParams<Item>) => Promise<ExternalSystemItemLoadingResponse>;
 export interface ItemTypeToLoad {
     itemType: string;
-    create: ExternalSystemLoadingFunction;
-    update: ExternalSystemLoadingFunction;
+    create: ExternalSystemLoadingFunction<ExternalSystemItem>;
+    update: ExternalSystemLoadingFunction<ExternalSystemItem>;
 }
 export interface ItemTypesToLoadParams {
     itemTypesToLoad: ItemTypeToLoad[];
@@ -89,7 +112,13 @@ export declare enum LoaderEventType {
     DataLoadingDelay = "DATA_LOADING_DELAYED",
     DataLoadingDone = "DATA_LOADING_DONE",
     DataLoadingError = "DATA_LOADING_ERROR",
+    AttachmentLoadingProgress = "ATTACHMENT_LOADING_PROGRESS",
+    AttachmentLoadingDelayed = "ATTACHMENT_LOADING_DELAYED",
+    AttachmentLoadingDone = "ATTACHMENT_LOADING_DONE",
+    AttachmentLoadingError = "ATTACHMENT_LOADING_ERROR",
     LoaderStateDeletionDone = "LOADER_STATE_DELETION_DONE",
     LoaderStateDeletionError = "LOADER_STATE_DELETION_ERROR",
+    LoaderAttachmentStateDeletionDone = "LOADER_ATTACHMENT_STATE_DELETION_DONE",
+    LoaderAttachmentStateDeletionError = "LOADER_ATTACHMENT_STATE_DELETION_ERROR",
     UnknownEventType = "UNKNOWN_EVENT_TYPE"
 }

package/dist/types/loading.js

@@ -15,7 +15,13 @@ var LoaderEventType;
     LoaderEventType["DataLoadingDelay"] = "DATA_LOADING_DELAYED";
     LoaderEventType["DataLoadingDone"] = "DATA_LOADING_DONE";
     LoaderEventType["DataLoadingError"] = "DATA_LOADING_ERROR";
+    LoaderEventType["AttachmentLoadingProgress"] = "ATTACHMENT_LOADING_PROGRESS";
+    LoaderEventType["AttachmentLoadingDelayed"] = "ATTACHMENT_LOADING_DELAYED";
+    LoaderEventType["AttachmentLoadingDone"] = "ATTACHMENT_LOADING_DONE";
+    LoaderEventType["AttachmentLoadingError"] = "ATTACHMENT_LOADING_ERROR";
     LoaderEventType["LoaderStateDeletionDone"] = "LOADER_STATE_DELETION_DONE";
     LoaderEventType["LoaderStateDeletionError"] = "LOADER_STATE_DELETION_ERROR";
+    LoaderEventType["LoaderAttachmentStateDeletionDone"] = "LOADER_ATTACHMENT_STATE_DELETION_DONE";
+    LoaderEventType["LoaderAttachmentStateDeletionError"] = "LOADER_ATTACHMENT_STATE_DELETION_ERROR";
     LoaderEventType["UnknownEventType"] = "UNKNOWN_EVENT_TYPE";
 })(LoaderEventType || (exports.LoaderEventType = LoaderEventType = {}));

package/dist/types/workers.d.ts

@@ -79,8 +79,8 @@ export interface TaskAdapterInterface<ConnectorState> {
  * @param {function} onTimeout - The task to be executed on timeout, returns exit code
  */
 export interface ProcessTaskInterface<ConnectorState> {
-    task: (params: TaskAdapterInterface<ConnectorState>) => void;
-    onTimeout: (params: TaskAdapterInterface<ConnectorState>) => void;
+    task: (params: TaskAdapterInterface<ConnectorState>) => Promise<void>;
+    onTimeout: (params: TaskAdapterInterface<ConnectorState>) => Promise<void>;
 }
 /**
  * WorkerEvent represents the standard worker events.

package/dist/uploader/uploader.js

@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -55,10 +65,10 @@ class Uploader {
      */
     async upload(itemType, fetchedObjects) {
         if (this.isLocalDevelopment) {
-            this.downloadToLocal(itemType, fetchedObjects);
+            await this.downloadToLocal(itemType, fetchedObjects);
         }
         // 1. Compress the fetched objects to a gzipped jsonl object
-        const file = await this.compressGzip(js_jsonl_1.jsonl.stringify(fetchedObjects));
+        const file = this.compressGzip(js_jsonl_1.jsonl.stringify(fetchedObjects));
         if (!file) {
             return {
                 error: { message: 'Error while compressing jsonl object.' },
@@ -174,14 +184,14 @@ class Uploader {
             };
         }
         // 3. Decompress the gzipped jsonl object
-        const jsonlObject = await this.decompressGzip(gzippedJsonlObject);
+        const jsonlObject = this.decompressGzip(gzippedJsonlObject);
         if (!jsonlObject) {
             return {
                 error: { message: 'Error while decompressing gzipped jsonl object.' },
             };
         }
         // 4. Parse the jsonl object to get the attachment metadata
-        const jsonObject = (await this.parseJsonl(jsonlObject));
+        const jsonObject = this.parseJsonl(jsonlObject);
         if (!jsonObject) {
             return {
                 error: { message: 'Error while parsing jsonl object.' },
@@ -216,7 +226,7 @@ class Uploader {
             }
         }
     }
-    async compressGzip(jsonlObject) {
+    compressGzip(jsonlObject) {
         try {
             return zlib_1.default.gzipSync(jsonlObject);
         }
@@ -224,22 +234,23 @@ class Uploader {
             console.error('Error while compressing jsonl object.', error);
         }
     }
-    async decompressGzip(gzippedJsonlObject) {
+    decompressGzip(gzippedJsonlObject) {
         try {
-            const jsonlObject = await zlib_1.default.gunzipSync(gzippedJsonlObject);
+            const jsonlObject = zlib_1.default.gunzipSync(gzippedJsonlObject);
             return jsonlObject.toString();
         }
         catch (error) {
             console.error('Error while decompressing gzipped jsonl object.', error);
         }
     }
-    async parseJsonl(jsonlObject) {
+    parseJsonl(jsonlObject) {
         try {
             return js_jsonl_1.jsonl.parse(jsonlObject);
         }
         catch (error) {
             console.error('Error while parsing jsonl object.', error);
         }
+        return null;
     }
     async getJsonObjectByArtifactId({ artifactId, isGzipped = false, }) {
         const artifactUrl = await this.getArtifactDownloadUrl(artifactId);
@@ -251,7 +262,7 @@ class Uploader {
             return;
         }
         if (isGzipped) {
-            const decompressedArtifact = await this.decompressGzip(artifact);
+            const decompressedArtifact = this.decompressGzip(artifact);
             if (!decompressedArtifact) {
                 return;
             }

package/dist/workers/default-workers/delete-loader-attachment-state.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/delete-loader-attachment-state.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        await adapter.emit(index_1.LoaderEventType.LoaderAttachmentStateDeletionDone);
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(index_1.LoaderEventType.LoaderAttachmentStateDeletionError, {
+            error: {
+                message: 'Failed to delete attachment state. Timeout.',
+            },
+        });
+    },
+});

package/dist/workers/default-workers/delete-loader-state.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/delete-loader-state.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        await adapter.emit(index_1.LoaderEventType.LoaderStateDeletionDone);
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(index_1.LoaderEventType.LoaderStateDeletionError, {
+            error: {
+                message: 'Failed to delete data. Lambda timeout.',
+            },
+        });
+    },
+});

package/dist/workers/default-workers/load-attachments.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/load-attachments.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const process_task_1 = require("workers/process-task");
+const types_1 = require("types");
+(0, process_task_1.processTask)({
+    task: async ({ adapter }) => {
+        await adapter.emit(types_1.LoaderEventType.AttachmentLoadingDone, {
+            reports: adapter.reports,
+            processed_files: adapter.processedFiles,
+        });
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.postState();
+        await adapter.emit(types_1.LoaderEventType.AttachmentLoadingError, {
+            reports: adapter.reports,
+            processed_files: adapter.processedFiles,
+        });
+    },
+});

package/dist/workers/default-workers/load-data.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/load-data.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const process_task_1 = require("workers/process-task");
+const loading_1 = require("../../types/loading");
+(0, process_task_1.processTask)({
+    task: async ({ adapter }) => {
+        await adapter.emit(loading_1.LoaderEventType.DataLoadingDone, {
+            reports: adapter.reports,
+            processed_files: adapter.processedFiles,
+        });
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.postState();
+        await adapter.emit(loading_1.LoaderEventType.DataLoadingError, {
+            reports: adapter.reports,
+            processed_files: adapter.processedFiles,
+        });
+    },
+});

package/dist/workers/process-task.js

@@ -8,7 +8,7 @@ const workers_1 = require("../types/workers");
 const logger_1 = require("../logger/logger");
 function processTask({ task, onTimeout, }) {
     if (!node_worker_threads_1.isMainThread) {
-        (async () => {
+        void (async () => {
             const event = node_worker_threads_1.workerData.event;
             const initialState = node_worker_threads_1.workerData.initialState;
             const options = node_worker_threads_1.workerData.options;

package/dist/workers/spawn.js

@@ -6,13 +6,9 @@ const extraction_1 = require("../types/extraction");
 const control_protocol_1 = require("../common/control-protocol");
 const helpers_1 = require("../common/helpers");
 const logger_1 = require("../logger/logger");
-const constants_1 = require("../common/constants");
 const workers_1 = require("../types/workers");
 const create_worker_1 = require("./create-worker");
 function getWorkerPath({ event, connectorWorkerPath, }) {
-    if (!constants_1.ALLOWED_EVENT_TYPES.includes(event.payload.event_type)) {
-        return null;
-    }
     if (connectorWorkerPath)
         return connectorWorkerPath;
     let path = null;
@@ -41,23 +37,20 @@ function getWorkerPath({ event, connectorWorkerPath, }) {
         // Loading
         case extraction_1.EventType.StartLoadingData:
         case extraction_1.EventType.ContinueLoadingData:
-            path = __dirname + '/default-workers/data-loading';
+            path = __dirname + '/default-workers/load-data';
+            break;
+        case extraction_1.EventType.StartLoadingAttachments:
+        case extraction_1.EventType.ContinueLoadingAttachments:
+            path = __dirname + '/default-workers/load-attachments';
             break;
         case extraction_1.EventType.StartDeletingLoaderState:
-            path = __dirname + '/default-workers/loader-state-deletion';
+            path = __dirname + '/default-workers/delete-loader-state';
+            break;
+        case extraction_1.EventType.StartDeletingLoaderAttachmentState:
+            path = __dirname + '/default-workers/delete-loader-attachment-state';
             break;
         default:
-            (0, control_protocol_1.emit)({
-                event,
-                eventType: extraction_1.ExtractorEventType.UnknownEventType,
-                data: {
-                    error: {
-                        message: 'Unrecognized event type in spawn ' +
-                            event.payload.event_type +
-                            '.',
-                    },
-                },
-            });
+            path = null;
     }
     return path;
 }
@@ -102,7 +95,18 @@ async function spawn({ event, initialState, workerPath, options, }) {
         }
     }
     else {
-        throw new Error('Worker script not found.');
+        await (0, control_protocol_1.emit)({
+            event,
+            eventType: extraction_1.ExtractorEventType.UnknownEventType,
+            data: {
+                error: {
+                    message: 'Unrecognized event type in spawn ' +
+                        event.payload.event_type +
+                        '.',
+                },
+            },
+        });
+        return false;
     }
 }
 class Spawn {

package/dist/workers/worker-adapter.d.ts

@@ -1,5 +1,5 @@
 import { AirdropEvent, ExtractorEventType, EventData, ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
-import { LoaderEventType } from '../types/loading';
+import { ExternalSystemAttachment, ExternalSystemLoadingFunction, FileToLoad, LoaderEventType } from '../types/loading';
 import { AdapterState } from '../state/state.interfaces';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { WorkerAdapterInterface, WorkerAdapterOptions } from '../types/workers';
@@ -54,10 +54,20 @@ export declare class WorkerAdapter<ConnectorState> {
     uploadAllRepos(): Promise<void>;
     handleTimeout(): void;
     loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
+    getLoaderBatches({ supportedItemTypes, }: {
+        supportedItemTypes: string[];
+    }): Promise<FileToLoad[]>;
+    loadAttachments({ create, }: {
+        create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
+    }): Promise<LoadItemTypesResponse>;
     loadItem({ item, itemTypeToLoad, }: {
         item: ExternalSystemItem;
         itemTypeToLoad: ItemTypeToLoad;
     }): Promise<LoadItemResponse>;
+    loadAttachment({ item, create, }: {
+        item: ExternalSystemAttachment;
+        create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
+    }): Promise<LoadItemResponse>;
     /**
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.

package/dist/workers/worker-adapter.js

@@ -163,68 +163,22 @@ class WorkerAdapter {
     }
     async loadItemTypes({ itemTypesToLoad, }) {
         var _a, _b, _c;
-        const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
-        if (!statsFileArtifactId) {
-            console.error('Stats file artifact id not found in event data.');
-            await (0, control_protocol_1.emit)({
-                event: this.event,
-                eventType: loading_1.LoaderEventType.DataLoadingError,
-                data: {
-                    error: {
-                        message: 'Stats file artifact id not found in event data.',
-                    },
-                },
-            });
-            return {
-                reports: this.reports,
-                processed_files: this.processedFiles,
+        if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
+            this.adapterState.state.fromDevRev = {
+                filesToLoad: await this.getLoaderBatches({
+                    supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
+                }),
             };
         }
-        const statsFile = (await this.uploader.getJsonObjectByArtifactId({
-            artifactId: statsFileArtifactId,
-        }));
-        console.log('Stats file', statsFile);
-        if (!statsFile || statsFile.length === 0) {
-            console.warn('Stats file not found or empty.');
+        if (((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
+            console.warn('No files to load, returning.');
             return {
                 reports: this.reports,
                 processed_files: this.processedFiles,
             };
         }
-        if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
-            console.log('Recieved event type ' +
-                extraction_1.EventType.StartLoadingData +
-                '. Preparing files to load.');
-            const filesToLoad = (0, helpers_1.getFilesToLoad)({
-                itemTypesToLoad,
-                statsFile,
-            });
-            if (filesToLoad.length === 0) {
-                console.warn('No files to load, returning.');
-                return {
-                    reports: this.reports,
-                    processed_files: this.processedFiles,
-                };
-            }
-            this.adapterState.state.fromDevRev = { filesToLoad };
-        }
         console.log('Files to load in state', (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad);
-        if (!this.adapterState.state.fromDevRev) {
-            await (0, control_protocol_1.emit)({
-                event: this.event,
-                eventType: loading_1.LoaderEventType.DataLoadingError,
-                data: {
-                    error: {
-                        message: 'Unexpected state set in LOAD_DATA.',
-                    },
-                },
-            });
-            return {
-                reports: this.reports,
-                processed_files: this.processedFiles,
-            };
-        }
-        outerloop: for (const fileToLoad of (_c = this.adapterState.state.fromDevRev) === null || _c === void 0 ? void 0 : _c.filesToLoad) {
+        outerloop: for (const fileToLoad of ((_c = this.adapterState.state.fromDevRev) === null || _c === void 0 ? void 0 : _c.filesToLoad) || []) {
             const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
             if (!itemTypeToLoad) {
                 console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
@@ -249,20 +203,88 @@ class WorkerAdapter {
                     break outerloop;
                 }
                 for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
-                    const { report, rateLimit, error } = await this.loadItem({
+                    const { report, rateLimit } = await this.loadItem({
                         item: transformerFile[i],
                         itemTypeToLoad,
                     });
-                    if (error) {
+                    if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
                         await (0, control_protocol_1.emit)({
                             event: this.event,
-                            eventType: loading_1.LoaderEventType.DataLoadingError,
+                            eventType: loading_1.LoaderEventType.DataLoadingDelay,
                             data: {
-                                error,
+                                delay: rateLimit.delay,
+                                reports: this.reports,
+                                processed_files: this.processedFiles,
                             },
                         });
                         break outerloop;
                     }
+                    if (report) {
+                        (0, helpers_1.addReportToLoaderReport)({
+                            loaderReports: this.loaderReports,
+                            report,
+                        });
+                        fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
+                    }
+                }
+                fileToLoad.completed = true;
+                this._processedFiles.push(fileToLoad.id);
+            }
+        }
+        return {
+            reports: this.reports,
+            processed_files: this.processedFiles,
+        };
+    }
+    async getLoaderBatches({ supportedItemTypes, }) {
+        var _a;
+        const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
+        if (statsFileArtifactId) {
+            const statsFile = (await this.uploader.getJsonObjectByArtifactId({
+                artifactId: statsFileArtifactId,
+            }));
+            if (!statsFile || statsFile.length === 0) {
+                return [];
+            }
+            const filesToLoad = (0, helpers_1.getFilesToLoad)({
+                supportedItemTypes,
+                statsFile,
+            });
+            return filesToLoad;
+        }
+        return [];
+    }
+    async loadAttachments({ create, }) {
+        var _a, _b;
+        if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
+            this.adapterState.state.fromDevRev = {
+                filesToLoad: await this.getLoaderBatches({
+                    supportedItemTypes: ['attachment'],
+                }),
+            };
+        }
+        if (!this.adapterState.state.fromDevRev ||
+            ((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
+            return {
+                reports: this.reports,
+                processed_files: this.processedFiles,
+            };
+        }
+        outerloop: for (const fileToLoad of (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad) {
+            if (!fileToLoad.completed) {
+                const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
+                    artifactId: fileToLoad.id,
+                    isGzipped: true,
+                }));
+                if (!transformerFile) {
+                    console.error('Transformer file not found.');
+                    break outerloop;
+                }
+                for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
+                    const { report, rateLimit } = await this.loadAttachment({
+                        item: transformerFile[i],
+                        create,
+                    });
                     if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
                         await (0, control_protocol_1.emit)({
                             event: this.event,
@@ -461,6 +483,38 @@ class WorkerAdapter {
             };
         }
     }
+    async loadAttachment({ item, create, }) {
+        // Create item
+        const { id, delay, error } = await create({
+            item,
+            mappers: this.mappers,
+            event: this.event,
+        });
+        if (delay) {
+            return {
+                rateLimit: {
+                    delay,
+                },
+            };
+        }
+        else if (id) {
+            return {
+                report: {
+                    item_type: 'attachment',
+                    [loading_1.ActionType.CREATED]: 1,
+                },
+            };
+        }
+        else {
+            console.error('Failed to create item', error);
+            return {
+                report: {
+                    item_type: 'attachment',
+                    [loading_1.ActionType.FAILED]: 1,
+                },
+            };
+        }
+    }
     /**
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.1.6",
+  "version": "1.2.0-beta",
   "description": "DevRev ADaaS (AirDrop-as-a-Service) Typescript SDK.",
   "type": "commonjs",
   "main": "./dist/index.js",