@devrev/ts-adaas 1.2.1 → 1.2.3

package/README.md

@@ -2,6 +2,19 @@
 
 ## Release Notes
 
+### v1.2.3
+
+- Add `local` flag to use for local development of the ADaaS snap-ins.
+- Send library version, snap-in version and snap-in slug in headers while emitting.
+- Make `actor_id` field optional for `SsorAttachment` interface.
+- Fix bugs related to event handling, error logging.
+
+### v1.2.2
+
+- Add library version as a part of control protocol.
+- Improve axios client and adapter logging.
+- Fix bugs related to state handling.
+
 ### v1.2.1
 
 - Reduced the `delayFactor` to minimize unnecessary delays.

package/dist/common/constants.d.ts

@@ -10,3 +10,4 @@ export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
     ATTACHMENTS: string;
     SSOR_ATTACHMENT: string;
 };
+export declare const LIBRARY_VERSION: any;

package/dist/common/constants.js

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = void 0;
+exports.LIBRARY_VERSION = exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = void 0;
 const extraction_1 = require("../types/extraction");
+const helpers_1 = require("./helpers");
 exports.STATELESS_EVENT_TYPES = [
     extraction_1.EventType.ExtractionExternalSyncUnitsStart,
     extraction_1.EventType.ExtractionMetadataStart,
@@ -37,3 +38,4 @@ exports.AIRDROP_DEFAULT_ITEM_TYPES = {
     ATTACHMENTS: 'attachments',
     SSOR_ATTACHMENT: 'ssor_attachment',
 };
+exports.LIBRARY_VERSION = (0, helpers_1.getLibraryVersion)();

package/dist/common/control-protocol.js

@@ -2,11 +2,15 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.emit = void 0;
 const axios_client_1 = require("../http/axios-client");
+const constants_1 = require("./constants");
 const emit = async ({ event, eventType, data, }) => {
     const newEvent = {
         event_type: eventType,
         event_context: event.payload.event_context,
         event_data: Object.assign({}, data),
+        worker_metadata: {
+            adaas_library_version: constants_1.LIBRARY_VERSION,
+        },
     };
     console.info('Emitting event', JSON.stringify(newEvent));
     return axios_client_1.axiosClient.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
@@ -14,6 +18,9 @@ const emit = async ({ event, eventType, data, }) => {
             Accept: 'application/json, text/plain, */*',
             Authorization: event.context.secrets.service_account_token,
             'Content-Type': 'application/json',
+            'X-DevRev-Client-Platform': event.payload.event_context.snap_in_slug,
+            'X-DevRev-Client-Id': event.payload.event_context.snap_in_version_id,
+            'X-DevRev-Client-Version': constants_1.LIBRARY_VERSION,
         },
     });
 };

package/dist/common/helpers.d.ts

@@ -15,3 +15,4 @@ export declare function addReportToLoaderReport({ loaderReports, report, }: {
     report: LoaderReport;
 }): LoaderReport[];
 export declare function getCircularReplacer(): (key: any, value: any) => any;
+export declare function getLibraryVersion(): any;

package/dist/common/helpers.js

@@ -1,12 +1,48 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getTimeoutErrorEventType = getTimeoutErrorEventType;
 exports.getSyncDirection = getSyncDirection;
 exports.getFilesToLoad = getFilesToLoad;
 exports.addReportToLoaderReport = addReportToLoaderReport;
 exports.getCircularReplacer = getCircularReplacer;
+exports.getLibraryVersion = getLibraryVersion;
 const extraction_1 = require("../types/extraction");
 const loading_1 = require("../types/loading");
+const fs_1 = require("fs");
+const path = __importStar(require("path"));
 function getTimeoutErrorEventType(eventType) {
     switch (eventType) {
         case extraction_1.EventType.ExtractionMetadataStart:
@@ -44,6 +80,11 @@ function getTimeoutErrorEventType(eventType) {
             return {
                 eventType: loading_1.LoaderEventType.LoaderStateDeletionError,
             };
+        case extraction_1.EventType.StartLoadingAttachments:
+        case extraction_1.EventType.ContinueLoadingAttachments:
+            return {
+                eventType: loading_1.LoaderEventType.AttachmentLoadingError,
+            };
         case extraction_1.EventType.StartDeletingLoaderAttachmentState:
             return {
                 eventType: loading_1.LoaderEventType.LoaderAttachmentStateDeletionError,
@@ -120,3 +161,18 @@ function getCircularReplacer() {
         return value;
     };
 }
+// read adaas library version from package.json
+function getLibraryVersion() {
+    var _a;
+    try {
+        const version = (_a = JSON.parse((0, fs_1.readFileSync)(path.resolve(__dirname, '../../package.json'), 'utf8'))) === null || _a === void 0 ? void 0 : _a.version;
+        if (version) {
+            return version;
+        }
+        return '';
+    }
+    catch (error) {
+        console.error('Error reading adaas library version from package.json', error);
+        return '';
+    }
+}

package/dist/{http → deprecated/http}/client.d.ts

@@ -1,5 +1,5 @@
 import { RawAxiosRequestHeaders } from 'axios';
-import { HTTPResponse } from './types';
+import { HTTPResponse } from '../../http/types';
 export declare const defaultResponse: HTTPResponse;
 /**
  * HTTPClient class to make HTTP requests

package/dist/{http → deprecated/http}/client.js

@@ -35,7 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HTTPClient = exports.defaultResponse = void 0;
 const axios_1 = __importStar(require("axios"));
-const constants_1 = require("./constants");
+const constants_1 = require("../../http/constants");
 exports.defaultResponse = {
     data: {
         delay: 0,

package/dist/http/axios-client.js

@@ -12,30 +12,22 @@ exports.axiosClient = axiosClient;
 (0, axios_retry_1.default)(axiosClient, {
     retries: 5,
     retryDelay: (retryCount, error) => {
-        var _a;
-        console.warn('Retry attempt: ' + retryCount + 'to url: ' + ((_a = error.config) === null || _a === void 0 ? void 0 : _a.url) + '.');
-        // Exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
-        return axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
+        var _a, _b, _c;
+        // exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
+        const delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
+        console.warn(`Request to ${(_a = error.config) === null || _a === void 0 ? void 0 : _a.url} failed with response status code ${(_b = error.response) === null || _b === void 0 ? void 0 : _b.status}. Method ${(_c = error.config) === null || _c === void 0 ? void 0 : _c.method}. Retry count: ${retryCount}. Retrying in ${Math.round(delay / 1000)}s.`);
+        return delay;
     },
     retryCondition: (error) => {
-        var _a, _b, _c, _d;
-        if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) &&
-            ((_b = error.response) === null || _b === void 0 ? void 0 : _b.status) >= 500 &&
-            ((_c = error.response) === null || _c === void 0 ? void 0 : _c.status) <= 599) {
-            return true;
-        }
-        else if (((_d = error.response) === null || _d === void 0 ? void 0 : _d.status) === 429) {
-            console.log('Rate limit exceeded. Delay: ' + error.response.headers['retry-after']);
-            return false;
-        }
-        else {
-            return false;
-        }
-    },
-    onMaxRetryTimesExceeded(error, retryCount) {
         var _a;
-        console.log(`Max retries attempted: ${retryCount}`);
-        (_a = error.config) === null || _a === void 0 ? true : delete _a.headers.Authorization;
+        return (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
+            ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) !== 429);
+    },
+    onMaxRetryTimesExceeded(error) {
+        var _a, _b, _c, _d;
+        (_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
+        (_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
         delete error.request._header;
+        console.warn('Max retry times exceeded. Error', error);
     },
 });

package/dist/http/index.d.ts

@@ -1,3 +1,2 @@
-export * from './client';
 export * from './types';
 export * from './axios-client';

package/dist/http/index.js

@@ -14,6 +14,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./client"), exports);
 __exportStar(require("./types"), exports);
 __exportStar(require("./axios-client"), exports);

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 export * from './deprecated/adapter';
 export * from './deprecated/demo-extractor';
 export * from './deprecated/uploader';
+export * from './deprecated/http/client';
 export * from './types';
 export * from './http';
 export * from './common/install-initial-domain-mapping';

package/dist/index.js

@@ -18,6 +18,7 @@ exports.serializeAxiosError = exports.formatAxiosError = exports.spawn = exports
 __exportStar(require("./deprecated/adapter"), exports);
 __exportStar(require("./deprecated/demo-extractor"), exports);
 __exportStar(require("./deprecated/uploader"), exports);
+__exportStar(require("./deprecated/http/client"), exports);
 __exportStar(require("./types"), exports);
 __exportStar(require("./http"), exports);
 __exportStar(require("./common/install-initial-domain-mapping"), exports);

package/dist/repo/repo.interfaces.d.ts

@@ -34,8 +34,8 @@ export interface NormalizedAttachment {
     url: string;
     id: string;
     file_name: string;
-    author_id: string;
     parent_id: string;
+    author_id?: string;
     grand_parent_id?: number;
 }
 /**

package/dist/state/state.js

@@ -73,15 +73,16 @@ class State {
                 },
             });
             this.state = state || this.state;
-            console.log('State updated successfully to:', (0, logger_1.getPrintableState)(this.state));
+            console.log('State updated successfully to', (0, logger_1.getPrintableState)(this.state));
         }
         catch (error) {
             if (axios_client_1.axios.isAxiosError(error)) {
-                console.error('Failed to update state.', (0, logger_1.serializeAxiosError)(error));
+                console.error('Failed to update state', (0, logger_1.serializeAxiosError)(error));
             }
             else {
-                console.error('Failed to update state.', error);
+                console.error('Failed to update state', error);
             }
+            process.exit(1);
         }
     }
     /**
@@ -105,7 +106,7 @@ class State {
                 },
             });
             this.state = JSON.parse(response.data.state);
-            console.log('State fetched successfully. Current state:', (0, logger_1.getPrintableState)(this.state));
+            console.log('State fetched successfully. Current state', (0, logger_1.getPrintableState)(this.state));
             return this.state;
         }
         catch (error) {
@@ -118,7 +119,7 @@ class State {
             }
             else {
                 console.error('Failed to fetch state.', error);
-                return { message: 'Failed to fetch state.' };
+                process.exit(1);
             }
         }
     }

package/dist/tests/test-helpers.js

@@ -31,6 +31,7 @@ function createEvent({ eventType, externalSyncUnits = [], progress, error, delay
                 mode: 'test_mode',
                 request_id: 'test_request_id',
                 snap_in_slug: 'test_snap_in_slug',
+                snap_in_version_id: 'test_snap_in_version_id',
                 sync_run: 'test_sync_run',
                 sync_run_id: 'test_sync_run_id',
                 sync_tier: 'test_sync_tier',

package/dist/types/extraction.d.ts

@@ -131,6 +131,7 @@ export interface EventContext {
     mode: string;
     request_id: string;
     snap_in_slug: string;
+    snap_in_version_id: string;
     sync_run: string;
     sync_run_id: string;
     sync_tier: string;

package/dist/types/workers.d.ts

@@ -1,5 +1,4 @@
 import { Worker } from 'worker_threads';
-import { MessagePort } from 'node:worker_threads';
 import { State } from '../state/state';
 import { WorkerAdapter } from '../workers/worker-adapter';
 import { ExtractorEventType, AirdropEvent } from './extraction';
@@ -10,13 +9,11 @@ import { LoaderEventType } from './loading';
  * @constructor
  * @param {AirdropEvent} event - The event object received from the platform
  * @param {object=} initialState - The initial state of the adapter
- * @param {MessagePort} parentPort - The parent port of the worker thread
  * @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class
  */
 export interface WorkerAdapterInterface<ConnectorState> {
     event: AirdropEvent;
     adapterState: State<ConnectorState>;
-    parentPort: MessagePort;
     options?: WorkerAdapterOptions;
 }
 /**
@@ -139,5 +136,4 @@ export interface WorkerData<ConnectorState> {
 export interface GetWorkerPathInterface {
     event: AirdropEvent;
     connectorWorkerPath?: string | null;
-    options?: WorkerAdapterOptions;
 }

package/dist/uploader/uploader.interfaces.d.ts

@@ -58,7 +58,7 @@ export interface SsorAttachment {
     parent_id: {
         external: string;
     };
-    actor_id: {
+    actor_id?: {
         external: string;
     };
 }

package/dist/uploader/uploader.js

@@ -38,7 +38,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
 const fs_1 = __importStar(require("fs"));
-const axios_devrev_client_1 = require("../http/axios-devrev-client");
+const axios_client_1 = require("../http/axios-client");
 const zlib_1 = __importDefault(require("zlib"));
 const js_jsonl_1 = require("js-jsonl");
 const form_data_1 = __importDefault(require("form-data"));
@@ -67,7 +67,7 @@ class Uploader {
         if (this.isLocalDevelopment) {
             await this.downloadToLocal(itemType, fetchedObjects);
         }
-        // 1. Compress the fetched objects to a gzipped jsonl object
+        // compress the fetched objects to a gzipped jsonl object
         const file = this.compressGzip(js_jsonl_1.jsonl.stringify(fetchedObjects));
         if (!file) {
             return {
@@ -76,21 +76,21 @@ class Uploader {
         }
         const filename = itemType + '.jsonl.gz';
         const fileType = 'application/x-gzip';
-        // 2. Prepare the artifact for uploading
+        // prepare the artifact for uploading
         const preparedArtifact = await this.prepareArtifact(filename, fileType);
         if (!preparedArtifact) {
             return {
                 error: { message: 'Error while preparing artifact.' },
             };
         }
-        // 3. Upload the file to the prepared artifact
+        // upload the file to the prepared artifact
         const uploadedArtifact = await this.uploadToArtifact(preparedArtifact, file);
         if (!uploadedArtifact) {
             return {
                 error: { message: 'Error while uploading artifact.' },
             };
         }
-        // 4. Return the artifact information to the platform
+        // return the artifact information to the platform
         const artifact = {
             id: preparedArtifact.id,
             item_type: itemType,
@@ -108,7 +108,7 @@ class Uploader {
             return response.data;
         }
         catch (error) {
-            if (axios_devrev_client_1.axios.isAxiosError(error)) {
+            if (axios_client_1.axios.isAxiosError(error)) {
                 console.error('Error while preparing artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
@@ -125,13 +125,13 @@ class Uploader {
         }
         formData.append('file', file);
         try {
-            const response = await axios_devrev_client_1.axiosDevRevClient.post(preparedArtifact.url, formData, {
+            const response = await axios_client_1.axiosClient.post(preparedArtifact.url, formData, {
                 headers: Object.assign({}, formData.getHeaders()),
             });
             return response;
         }
         catch (error) {
-            if (axios_devrev_client_1.axios.isAxiosError(error)) {
+            if (axios_client_1.axios.isAxiosError(error)) {
                 console.error('Error while uploading artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
@@ -151,7 +151,7 @@ class Uploader {
             return;
         }
         try {
-            const response = await axios_devrev_client_1.axiosDevRevClient.post(preparedArtifact.url, formData, {
+            const response = await axios_client_1.axiosClient.post(preparedArtifact.url, formData, {
                 headers: Object.assign(Object.assign({}, formData.getHeaders()), (!fileStreamResponse.headers['content-length'] && {
                     'Content-Length': constants_1.MAX_DEVREV_ARTIFACT_SIZE,
                 })),
@@ -159,7 +159,7 @@ class Uploader {
             return response;
         }
         catch (error) {
-            if (axios_devrev_client_1.axios.isAxiosError(error)) {
+            if (axios_client_1.axios.isAxiosError(error)) {
                 console.error('Error while streaming artifact.', (0, logger_1.serializeAxiosError)(error));
             }
             else {
@@ -212,13 +212,13 @@ class Uploader {
     }
     async downloadArtifact(artifactUrl) {
         try {
-            const response = await axios_devrev_client_1.axiosDevRevClient.get(artifactUrl, {
+            const response = await axios_client_1.axiosClient.get(artifactUrl, {
                 responseType: 'arraybuffer',
             });
             return response.data;
         }
         catch (error) {
-            if (axios_devrev_client_1.axios.isAxiosError(error)) {
+            if (axios_client_1.axios.isAxiosError(error)) {
                 console.error('Error while downloading artifact from URL.', (0, logger_1.serializeAxiosError)(error));
             }
             else {

package/dist/workers/create-worker.js

@@ -16,7 +16,7 @@ async function createWorker(workerData) {
                 workerData,
             });
             worker.on(workers_1.WorkerEvent.WorkerError, (error) => {
-                logger.error('Worker error', (0, logger_1.serializeError)(error));
+                logger.error('Worker error', error);
                 reject();
             });
             worker.on(workers_1.WorkerEvent.WorkerOnline, () => {

package/dist/workers/default-workers/load-attachments.js

@@ -6,8 +6,8 @@ const types_1 = require("../../types");
     task: async ({ adapter }) => {
         await adapter.emit(types_1.LoaderEventType.UnknownEventType, {
             error: {
-                message: 'Event type ' + adapter.event.payload.event_type + ' not supported.'
-            }
+                message: 'Event type ' + adapter.event.payload.event_type + ' not supported.',
+            },
         });
     },
     onTimeout: async ({ adapter }) => {

package/dist/workers/process-task.js

@@ -22,7 +22,6 @@ function processTask({ task, onTimeout, }) {
                 const adapter = new worker_adapter_1.WorkerAdapter({
                     event,
                     adapterState,
-                    parentPort: node_worker_threads_1.parentPort,
                     options,
                 });
                 node_worker_threads_1.parentPort.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {

package/dist/workers/spawn.js

@@ -6,9 +6,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Spawn = void 0;
 exports.spawn = spawn;
 const axios_1 = __importDefault(require("axios"));
+const helpers_1 = require("yargs/helpers");
+const yargs_1 = __importDefault(require("yargs"));
 const extraction_1 = require("../types/extraction");
 const control_protocol_1 = require("../common/control-protocol");
-const helpers_1 = require("../common/helpers");
+const helpers_2 = require("../common/helpers");
 const logger_1 = require("../logger/logger");
 const workers_1 = require("../types/workers");
 const create_worker_1 = require("./create-worker");
@@ -74,8 +76,15 @@ async function spawn({ event, initialState, workerPath, options, }) {
     const script = getWorkerPath({
         event,
         connectorWorkerPath: workerPath,
-        options,
     });
+    if (options === null || options === void 0 ? void 0 : options.isLocalDevelopment) {
+        logger.warn('WARN: isLocalDevelopment is deprecated. Please use the -- local flag instead.');
+    }
+    // read the command line arguments to check if the local flag is passed
+    const argv = await (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv)).argv;
+    if (argv._.includes('local')) {
+        options = Object.assign(Object.assign({}, (options || {})), { isLocalDevelopment: true });
+    }
     if (script) {
         try {
             const worker = await (0, create_worker_1.createWorker)({
@@ -182,7 +191,7 @@ class Spawn {
             return;
         }
         this.alreadyEmitted = true;
-        const timeoutEventType = (0, helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
+        const timeoutEventType = (0, helpers_2.getTimeoutErrorEventType)(this.event.payload.event_type);
         if (timeoutEventType) {
             const { eventType } = timeoutEventType;
             try {

package/dist/workers/worker-adapter.d.ts

@@ -6,7 +6,7 @@ import { WorkerAdapterInterface, WorkerAdapterOptions } from '../types/workers';
 import { Repo } from '../repo/repo';
 import { NormalizedAttachment, RepoInterface } from '../repo/repo.interfaces';
 import { ExternalSystemItem, ItemTypesToLoadParams, ItemTypeToLoad, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../types/loading';
-export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
+export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
 /**
  * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
  * as parameter in processTask and onTimeout functions. The class provides
@@ -27,14 +27,13 @@ export declare class WorkerAdapter<ConnectorState> {
     private adapterState;
     private _artifacts;
     private hasWorkerEmitted;
-    private parentPort;
     private isTimeout;
     private repos;
     private loaderReports;
     private _processedFiles;
     private mappers;
     private uploader;
-    constructor({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>);
+    constructor({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>);
     get state(): AdapterState<ConnectorState>;
     set state(value: AdapterState<ConnectorState>);
     get reports(): LoaderReport[];
@@ -64,7 +63,7 @@ export declare class WorkerAdapter<ConnectorState> {
         item: ExternalSystemItem;
         itemTypeToLoad: ItemTypeToLoad;
     }): Promise<LoadItemResponse>;
-    processAttachment: (attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction) => Promise<ProcessAttachmentReturnType>;
+    processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
     loadAttachment({ item, create, }: {
         item: ExternalSystemAttachment;
         create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;

package/dist/workers/worker-adapter.js

@@ -6,6 +6,7 @@ const axios_devrev_client_1 = require("../http/axios-devrev-client");
 const extraction_1 = require("../types/extraction");
 const loading_1 = require("../types/loading");
 const constants_1 = require("../common/constants");
+const node_worker_threads_1 = require("node:worker_threads");
 const control_protocol_1 = require("../common/control-protocol");
 const workers_1 = require("../types/workers");
 const repo_1 = require("../repo/repo");
@@ -14,11 +15,10 @@ const mappers_1 = require("../mappers/mappers");
 const uploader_1 = require("../uploader/uploader");
 const logger_1 = require("../logger/logger");
 const mappers_interface_1 = require("../mappers/mappers.interface");
-function createWorkerAdapter({ event, adapterState, parentPort, options, }) {
+function createWorkerAdapter({ event, adapterState, options, }) {
     return new WorkerAdapter({
         event,
         adapterState,
-        parentPort,
         options,
     });
 }
@@ -37,56 +37,12 @@ function createWorkerAdapter({ event, adapterState, parentPort, options, }) {
  *
  */
 class WorkerAdapter {
-    constructor({ event, adapterState, parentPort, options, }) {
+    constructor({ event, adapterState, options, }) {
         this.repos = [];
-        this.processAttachment = async (attachment, stream) => {
-            var _a, _b;
-            const { httpStream, delay, error } = await stream({
-                item: attachment,
-                event: this.event,
-            });
-            if (error) {
-                console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
-                return { error };
-            }
-            else if (delay) {
-                return { delay };
-            }
-            if (httpStream) {
-                const fileType = ((_a = httpStream.headers) === null || _a === void 0 ? void 0 : _a['content-type']) || 'application/octet-stream';
-                const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
-                if (!preparedArtifact) {
-                    console.warn('Error while preparing artifact for attachment ID ' +
-                        attachment.id +
-                        '. Skipping attachment');
-                    return;
-                }
-                const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
-                if (!uploadedArtifact) {
-                    console.warn('Error while preparing artifact for attachment ID ' + attachment.id);
-                    return;
-                }
-                const ssorAttachment = {
-                    id: {
-                        devrev: preparedArtifact.id,
-                        external: attachment.id,
-                    },
-                    parent_id: {
-                        external: attachment.parent_id,
-                    },
-                    actor_id: {
-                        external: attachment.author_id,
-                    },
-                };
-                await ((_b = this.getRepo('ssor_attachment')) === null || _b === void 0 ? void 0 : _b.push([ssorAttachment]));
-            }
-            return;
-        };
         this.event = event;
         this.options = options;
         this.adapterState = adapterState;
         this._artifacts = [];
-        this.parentPort = parentPort;
         this.hasWorkerEmitted = false;
         this.isTimeout = false;
         // Loader
@@ -156,9 +112,7 @@ class WorkerAdapter {
      */
     async emit(newEventType, data) {
         if (this.hasWorkerEmitted) {
-            console.warn('Unable to emit with event type:' +
-                newEventType +
-                '. Lambda is going to be stopped.');
+            console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
             return;
         }
         // We want to upload all the repos before emitting the event, except for the external sync units done event
@@ -167,15 +121,20 @@ class WorkerAdapter {
         }
         // If the extraction is done, we want to save the timestamp of the last successful sync
         if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
+            console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
             this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
             this.state.lastSyncStarted = '';
         }
         // We want to save the state every time we emit an event, except for the start and delete events
         if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
-            console.log(`Saving state before emitting event with event type: ` +
-                newEventType +
-                '.');
-            await this.adapterState.postState(this.state);
+            console.log(`Saving state before emitting event with event type: ${newEventType}.`);
+            try {
+                await this.adapterState.postState(this.state);
+            }
+            catch (error) {
+                console.error('Error while posting state', error);
+                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+            }
         }
         try {
             await (0, control_protocol_1.emit)({
@@ -190,17 +149,17 @@ class WorkerAdapter {
                 payload: { eventType: newEventType },
             };
             this.artifacts = [];
-            this.parentPort.postMessage(message);
             this.hasWorkerEmitted = true;
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
         }
         catch (error) {
             if (axios_devrev_client_1.axios.isAxiosError(error)) {
-                console.error('Error while emitting event with event type: ' + newEventType + '.', (0, logger_1.serializeAxiosError)(error));
+                console.error(`Error while emitting event with event type: ${newEventType}`, (0, logger_1.serializeAxiosError)(error));
             }
             else {
-                console.error('Error while emitting event with event type: ' + newEventType + '.', error);
+                console.error(`Unknown error while emitting event with event type: ${newEventType}`, error);
             }
-            this.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
         }
     }
     async uploadAllRepos() {
@@ -212,33 +171,40 @@ class WorkerAdapter {
         this.isTimeout = true;
     }
     async loadItemTypes({ itemTypesToLoad, }) {
-        var _a, _b, _c;
+        var _a;
         if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
+            const itemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
+            if (!itemTypes.length) {
+                console.warn('No item types to load, returning.');
+                return {
+                    reports: this.reports,
+                    processed_files: this.processedFiles,
+                };
+            }
+            const filesToLoad = await this.getLoaderBatches({
+                supportedItemTypes: itemTypes,
+            });
             this.adapterState.state.fromDevRev = {
-                filesToLoad: await this.getLoaderBatches({
-                    supportedItemTypes: itemTypesToLoad.map((it) => it.itemType),
-                }),
+                filesToLoad,
             };
         }
-        if (((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
+        if (!this.adapterState.state.fromDevRev ||
+            !this.adapterState.state.fromDevRev.filesToLoad.length) {
             console.warn('No files to load, returning.');
             return {
                 reports: this.reports,
                 processed_files: this.processedFiles,
             };
         }
-        console.log('Files to load in state', (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad);
-        outerloop: for (const fileToLoad of ((_c = this.adapterState.state.fromDevRev) === null || _c === void 0 ? void 0 : _c.filesToLoad) || []) {
+        console.log('Files to load in state', (_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
+        outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev
+            .filesToLoad) {
             const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
             if (!itemTypeToLoad) {
                 console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
-                await (0, control_protocol_1.emit)({
-                    event: this.event,
-                    eventType: loading_1.LoaderEventType.DataLoadingError,
-                    data: {
-                        error: {
-                            message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
-                        },
+                await this.emit(loading_1.LoaderEventType.DataLoadingError, {
+                    error: {
+                        message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
                     },
                 });
                 break;
@@ -249,8 +215,12 @@ class WorkerAdapter {
                     isGzipped: true,
                 }));
                 if (!transformerFile) {
-                    console.error('Transformer file not found.');
-                    break outerloop;
+                    console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
+                    await this.emit(loading_1.LoaderEventType.DataLoadingError, {
+                        error: {
+                            message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
+                        },
+                    });
                 }
                 for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
                     const { report, rateLimit } = await this.loadItem({
@@ -258,14 +228,10 @@ class WorkerAdapter {
                         itemTypeToLoad,
                     });
                     if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
-                        await (0, control_protocol_1.emit)({
-                            event: this.event,
-                            eventType: loading_1.LoaderEventType.DataLoadingDelay,
-                            data: {
-                                delay: rateLimit.delay,
-                                reports: this.reports,
-                                processed_files: this.processedFiles,
-                            },
+                        await this.emit(loading_1.LoaderEventType.DataLoadingDelay, {
+                            delay: rateLimit.delay,
+                            reports: this.reports,
+                            processed_files: this.processedFiles,
                         });
                         break outerloop;
                     }
@@ -315,6 +281,7 @@ class WorkerAdapter {
         }
         if (!this.adapterState.state.fromDevRev ||
             ((_a = this.adapterState.state.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
+            console.log('No files to load, returning.');
             return {
                 reports: this.reports,
                 processed_files: this.processedFiles,
@@ -327,7 +294,7 @@ class WorkerAdapter {
                     isGzipped: true,
                 }));
                 if (!transformerFile) {
-                    console.error('Transformer file not found.');
+                    console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
                     break outerloop;
                 }
                 for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
@@ -336,14 +303,10 @@ class WorkerAdapter {
                         create,
                     });
                     if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
-                        await (0, control_protocol_1.emit)({
-                            event: this.event,
-                            eventType: loading_1.LoaderEventType.DataLoadingDelay,
-                            data: {
-                                delay: rateLimit.delay,
-                                reports: this.reports,
-                                processed_files: this.processedFiles,
-                            },
+                        await this.emit(loading_1.LoaderEventType.DataLoadingDelay, {
+                            delay: rateLimit.delay,
+                            reports: this.reports,
+                            processed_files: this.processedFiles,
                         });
                         break outerloop;
                     }
@@ -374,14 +337,14 @@ class WorkerAdapter {
             });
             const syncMapperRecord = syncMapperRecordResponse.data;
             if (!syncMapperRecord) {
-                console.error('Failed to get sync mapper record from response.');
+                console.warn('Failed to get sync mapper record from response.');
                 return {
                     error: {
                         message: 'Failed to get sync mapper record from response.',
                     },
                 };
             }
-            // Update item
+            // Update item in external system
             const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
                 item,
                 mappers: this.mappers,
@@ -390,7 +353,7 @@ class WorkerAdapter {
             if (id) {
                 if (modifiedDate) {
                     try {
-                        const updateSyncMapperRecordResponse = await this.mappers.update({
+                        await this.mappers.update({
                             id: syncMapperRecord.sync_mapper_record.id,
                             sync_unit: this.event.payload.event_context.sync_unit,
                             status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
@@ -409,11 +372,10 @@ class WorkerAdapter {
                                 add: [devrevId],
                             },
                         });
-                        console.log('Updated sync mapper record', JSON.stringify(updateSyncMapperRecordResponse.data));
                     }
                     catch (error) {
                         if (axios_devrev_client_1.axios.isAxiosError(error)) {
-                            console.error('Failed to update sync mapper record', (0, logger_1.serializeAxiosError)(error));
+                            console.warn('Failed to update sync mapper record', (0, logger_1.serializeAxiosError)(error));
                             return {
                                 error: {
                                     message: error.message,
@@ -421,7 +383,7 @@ class WorkerAdapter {
                             };
                         }
                         else {
-                            console.error('Failed to update sync mapper record', error);
+                            console.warn('Failed to update sync mapper record', error);
                             return {
                                 error: {
                                     message: 'Failed to update sync mapper record' + error,
@@ -438,7 +400,7 @@ class WorkerAdapter {
                 };
             }
             else if (delay) {
-                console.log('Rate limited, delaying for', delay);
+                console.log(`Rate limited while updating item in external system, delaying for ${delay} seconds.`);
                 return {
                     rateLimit: {
                         delay,
@@ -446,7 +408,7 @@ class WorkerAdapter {
                 };
             }
             else {
-                console.error('Failed to update item', error);
+                console.warn('Failed to update item in external system', error);
                 return {
                     report: {
                         item_type: itemTypeToLoad.itemType,
@@ -454,12 +416,12 @@ class WorkerAdapter {
                     },
                 };
             }
-            // Update mapper (optional)
+            // TODO: Update mapper (optional)
         }
         catch (error) {
             if (axios_devrev_client_1.axios.isAxiosError(error)) {
                 if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
-                    // Create item
+                    // Create item in external system if mapper record not found
                     const { id, delay, error } = await itemTypeToLoad.create({
                         item,
                         mappers: this.mappers,
@@ -468,13 +430,12 @@ class WorkerAdapter {
                     if (id) {
                         // Create mapper
                         try {
-                            const createdSyncMapperRecordResponse = await this.mappers.create({
+                            await this.mappers.create({
                                 sync_unit: this.event.payload.event_context.sync_unit,
                                 status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
                                 external_ids: [id],
                                 targets: [devrevId],
                             });
-                            console.log('Created sync mapper record', createdSyncMapperRecordResponse.data.sync_mapper_record.id);
                             return {
                                 report: {
                                     item_type: itemTypeToLoad.itemType,
@@ -484,14 +445,14 @@ class WorkerAdapter {
                         }
                         catch (error) {
                             if (axios_devrev_client_1.axios.isAxiosError(error)) {
-                                console.error('Failed to create sync mapper record', (0, logger_1.serializeAxiosError)(error));
+                                console.warn('Failed to create sync mapper record', (0, logger_1.serializeAxiosError)(error));
                                 return {
                                     error: {
                                         message: error.message,
                                     },
                                 };
                             }
-                            console.error('Failed to create sync mapper record', error);
+                            console.warn('Failed to create sync mapper record', error);
                             return {
                                 error: {
                                     message: 'Failed to create sync mapper record' + error,
@@ -507,7 +468,7 @@ class WorkerAdapter {
                         };
                     }
                     else {
-                        console.error('Failed to create item', error);
+                        console.warn('Failed to create item in external system', error);
                         return {
                             report: {
                                 item_type: itemTypeToLoad.itemType,
@@ -517,7 +478,7 @@ class WorkerAdapter {
                     }
                 }
                 else {
-                    console.error('Failed to get sync mapper record', (0, logger_1.serializeAxiosError)(error));
+                    console.warn('Failed to get sync mapper record', (0, logger_1.serializeAxiosError)(error));
                     return {
                         error: {
                             message: error.message,
@@ -525,7 +486,7 @@ class WorkerAdapter {
                     };
                 }
             }
-            console.error('Failed to get sync mapper record', error);
+            console.warn('Failed to get sync mapper record', error);
             return {
                 error: {
                     message: 'Failed to get sync mapper record' + error,
@@ -533,6 +494,51 @@ class WorkerAdapter {
             };
         }
     }
+    async processAttachment(attachment, stream) {
+        var _a, _b;
+        const { httpStream, delay, error } = await stream({
+            item: attachment,
+            event: this.event,
+        });
+        if (error) {
+            console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
+            return { error };
+        }
+        else if (delay) {
+            return { delay };
+        }
+        if (httpStream) {
+            const fileType = ((_a = httpStream.headers) === null || _a === void 0 ? void 0 : _a['content-type']) || 'application/octet-stream';
+            const preparedArtifact = await this.uploader.prepareArtifact(attachment.file_name, fileType);
+            if (!preparedArtifact) {
+                console.warn('Error while preparing artifact for attachment ID ' +
+                    attachment.id +
+                    '. Skipping attachment');
+                return;
+            }
+            const uploadedArtifact = await this.uploader.streamToArtifact(preparedArtifact, httpStream);
+            if (!uploadedArtifact) {
+                console.warn('Error while preparing artifact for attachment ID ' + attachment.id);
+                return;
+            }
+            const ssorAttachment = {
+                id: {
+                    devrev: preparedArtifact.id,
+                    external: attachment.id,
+                },
+                parent_id: {
+                    external: attachment.parent_id,
+                },
+            };
+            if (attachment.author_id) {
+                ssorAttachment.actor_id = {
+                    external: attachment.author_id,
+                };
+            }
+            await ((_b = this.getRepo('ssor_attachment')) === null || _b === void 0 ? void 0 : _b.push([ssorAttachment]));
+        }
+        return;
+    }
     async loadAttachment({ item, create, }) {
         // Create item
         const { id, delay, error } = await create({
@@ -556,7 +562,7 @@ class WorkerAdapter {
             };
         }
         else {
-            console.error('Failed to create item', error);
+            console.warn('Failed to create attachment in external system', error);
             return {
                 report: {
                     item_type: 'attachment',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "description": "DevRev ADaaS (AirDrop-as-a-Service) Typescript SDK.",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -26,6 +26,7 @@
     "@types/jest": "^29.5.12",
     "@types/lambda-log": "^3.0.3",
     "@types/node": "20.16.11",
+    "@types/yargs": "^17.0.33",
     "@typescript-eslint/eslint-plugin": "^7.12.0",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
@@ -42,7 +43,8 @@
     "form-data": "^4.0.1",
     "js-jsonl": "^1.1.1",
     "lambda-log": "^3.1.0",
-    "ts-node": "^10.9.2"
+    "ts-node": "^10.9.2",
+    "yargs": "^17.7.2"
   },
   "files": [
     "dist"