@devrev/ts-adaas 1.2.6 → 1.3.0

package/dist/common/install-initial-domain-mapping.d.ts

@@ -1,3 +1,3 @@
-import { FunctionInput } from '@devrev/typescript-sdk/dist/snap-ins';
+import { AirdropEvent } from '../types/extraction';
 import { InitialDomainMapping } from '../types/common';
-export declare function installInitialDomainMapping(event: FunctionInput, initialDomainMappingJson: InitialDomainMapping): Promise<void>;
+export declare function installInitialDomainMapping(event: AirdropEvent, initialDomainMappingJson: InitialDomainMapping): Promise<void>;

package/dist/http/axios-client.d.ts

@@ -1,3 +1,26 @@
+/**
+ * Axios client setup with retry capabilities using axios-retry.
+ *
+ * This module exports an Axios client instance (`axiosClient`) that is configured to automatically retry
+ * failed requests under certain conditions.
+ *
+ * Retry Conditions:
+ * 1. Network errors (where no response is received).
+ * 2. Idempotent requests (defaults include GET, HEAD, OPTIONS, PUT).
+ * 3. All 5xx server errors.
+ *
+ * Retry Strategy:
+ * - A maximum of 5 retries are attempted.
+ * - Exponential backoff delay is applied between retries, increasing with each retry attempt.
+ *
+ * Additional Features:
+ * - When the maximum number of retry attempts is reached, sensitive headers (like authorization)
+ *   are removed from error logs for security reasons.
+ *
+ * Exported:
+ * - `axios`: Original axios instance for additional customizations or direct use.
+ * - `axiosClient`: Configured axios instance with retry logic.
+ */
 import axios from 'axios';
 declare const axiosClient: import("axios").AxiosInstance;
 export { axios, axiosClient };

package/dist/http/axios-client.js

@@ -1,4 +1,27 @@
 "use strict";
+/**
+ * Axios client setup with retry capabilities using axios-retry.
+ *
+ * This module exports an Axios client instance (`axiosClient`) that is configured to automatically retry
+ * failed requests under certain conditions.
+ *
+ * Retry Conditions:
+ * 1. Network errors (where no response is received).
+ * 2. Idempotent requests (defaults include GET, HEAD, OPTIONS, PUT).
+ * 3. All 5xx server errors.
+ *
+ * Retry Strategy:
+ * - A maximum of 5 retries are attempted.
+ * - Exponential backoff delay is applied between retries, increasing with each retry attempt.
+ *
+ * Additional Features:
+ * - When the maximum number of retry attempts is reached, sensitive headers (like authorization)
+ *   are removed from error logs for security reasons.
+ *
+ * Exported:
+ * - `axios`: Original axios instance for additional customizations or direct use.
+ * - `axiosClient`: Configured axios instance with retry logic.
+ */
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -19,9 +42,10 @@ exports.axiosClient = axiosClient;
         return delay;
     },
     retryCondition: (error) => {
-        var _a;
-        return (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
-            ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) !== 429);
+        var _a, _b, _c;
+        return ((axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
+            ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) !== 429) ||
+            ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.status) !== null && _c !== void 0 ? _c : 0) >= 500);
     },
     onMaxRetryTimesExceeded(error) {
         var _a, _b, _c, _d;

package/dist/state/state.d.ts

@@ -1,6 +1,6 @@
 import { ErrorRecord } from '../types/common';
 import { AdapterState, StateInterface } from './state.interfaces';
-export declare function createAdapterState<ConnectorState>({ event, initialState, options, }: StateInterface<ConnectorState>): Promise<State<ConnectorState>>;
+export declare function createAdapterState<ConnectorState>({ event, initialState, initialDomainMapping, options, }: StateInterface<ConnectorState>): Promise<State<ConnectorState>>;
 export declare class State<ConnectorState> {
     private _state;
     private initialSdkState;

package/dist/state/state.interfaces.d.ts

@@ -1,14 +1,17 @@
 import { AirdropEvent } from '../types/extraction';
 import { FileToLoad } from '../types/loading';
 import { WorkerAdapterOptions } from '../types/workers';
+import { InitialDomainMapping } from '../types/common';
 export interface SdkState {
     lastSyncStarted?: string;
     lastSuccessfulSyncStarted?: string;
     toDevRev?: ToDevRev;
     fromDevRev?: FromDevRev;
+    snapInVersionId?: string;
 }
 /**
- * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, and attachmentsMetadata.
+ * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor.
+ * It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, snapInVersionId and attachmentsMetadata.
  */
 export type AdapterState<ConnectorState> = ConnectorState & SdkState;
 export interface ToDevRev {
@@ -23,5 +26,6 @@ export interface FromDevRev {
 export interface StateInterface<ConnectorState> {
     event: AirdropEvent;
     initialState: ConnectorState;
+    initialDomainMapping?: InitialDomainMapping;
     options?: WorkerAdapterOptions;
 }

package/dist/state/state.js

@@ -6,16 +6,36 @@ const axios_client_1 = require("../http/axios-client");
 const extraction_1 = require("../types/extraction");
 const constants_1 = require("../common/constants");
 const logger_1 = require("../logger/logger");
+const install_initial_domain_mapping_1 = require("../common/install-initial-domain-mapping");
 const helpers_1 = require("../common/helpers");
-async function createAdapterState({ event, initialState, options, }) {
+async function createAdapterState({ event, initialState, initialDomainMapping, options, }) {
+    var _a;
     const newInitialState = structuredClone(initialState);
     const as = new State({
         event,
         initialState: newInitialState,
+        initialDomainMapping,
         options,
     });
     if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
         await as.fetchState(newInitialState);
+        const snapInVersionId = event.context.snap_in_version_id;
+        const hasSnapInVersionInState = 'snapInVersionId' in as.state;
+        const shouldUpdateIDM = !hasSnapInVersionInState || as.state.snapInVersionId !== snapInVersionId;
+        if (shouldUpdateIDM) {
+            console.log(`Snap-in version in state (${(_a = as.state) === null || _a === void 0 ? void 0 : _a.snapInVersionId}) differs from the version in event context (${snapInVersionId}) - initial domain mapping needs to be updated.`);
+            if (initialDomainMapping) {
+                await (0, install_initial_domain_mapping_1.installInitialDomainMapping)(event, initialDomainMapping);
+                as.state.snapInVersionId = snapInVersionId;
+                console.log('Successfully installed new initial domain mapping.');
+            }
+            else {
+                console.warn('No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.');
+            }
+        }
+        else {
+            console.log(`Snap-in version in state matches the version in event context (${snapInVersionId}). Skipping initial domain mapping installation.`);
+        }
         if (event.payload.event_type === extraction_1.EventType.ExtractionDataStart &&
             !as.state.lastSyncStarted) {
             as.state.lastSyncStarted = new Date().toISOString();
@@ -29,6 +49,7 @@ class State {
         this.initialSdkState =
             (0, helpers_1.getSyncDirection)({ event }) === extraction_1.SyncMode.LOADING
                 ? {
+                    snapInVersionId: '',
                     fromDevRev: {
                         filesToLoad: [],
                     },
@@ -36,6 +57,7 @@ class State {
                 : {
                     lastSyncStarted: '',
                     lastSuccessfulSyncStarted: '',
+                    snapInVersionId: '',
                     toDevRev: {
                         attachmentsMetadata: {
                             artifactIds: [],
@@ -96,7 +118,7 @@ class State {
             this.event.payload.event_context.sync_unit_id +
             '.');
         try {
-            const response = await axios_client_1.axiosClient.post(this.workerUrl + '.get', {}, {
+            const response = await axios_client_1.axiosClient.get(this.workerUrl + '.get', {
                 headers: {
                     Authorization: this.devrevToken,
                 },

package/dist/tests/test-helpers.js

@@ -10,7 +10,7 @@ function createEvent({ eventType, externalSyncUnits = [], progress, error, delay
     return {
         context: Object.assign({ secrets: {
                 service_account_token: 'test_token',
-            }, snap_in_version_id: 'test_snap_in_version_id' }, contextOverrides),
+            }, snap_in_version_id: 'test_snap_in_version_id', snap_in_id: 'test_snap_in_id' }, contextOverrides),
         payload: Object.assign({ connection_data: {
                 org_id: 'test_org_id',
                 org_name: 'test_org_name',

package/dist/types/extraction.d.ts

@@ -197,6 +197,7 @@ export interface AirdropEvent {
             service_account_token: string;
         };
         snap_in_version_id: string;
+        snap_in_id: string;
     };
     payload: AirdropMessage;
     execution_metadata: {

package/dist/types/workers.d.ts

@@ -3,6 +3,7 @@ import { State } from '../state/state';
 import { WorkerAdapter } from '../workers/worker-adapter';
 import { ExtractorEventType, AirdropEvent } from './extraction';
 import { LoaderEventType } from './loading';
+import { InitialDomainMapping } from './common';
 /**
  * WorkerAdapterInterface is an interface for WorkerAdapter class.
  * @interface WorkerAdapterInterface
@@ -52,6 +53,7 @@ export interface SpawnInterface {
  * @param {AirdropEvent} event - The event object received from the platform
  * @param {object=} initialState - The initial state of the adapter
  * @param {string} workerPath - The path to the worker file
+ * @param {string} initialDomainMapping - The initial domain mapping
  * @param {WorkerAdapterOptions} options - The options to create a new instance of Spawn class
  */
 export interface SpawnFactoryInterface<ConnectorState> {
@@ -59,6 +61,7 @@ export interface SpawnFactoryInterface<ConnectorState> {
     initialState: ConnectorState;
     workerPath?: string;
     options?: WorkerAdapterOptions;
+    initialDomainMapping?: InitialDomainMapping;
 }
 /**
  * TaskAdapterInterface is an interface for TaskAdapter class.
@@ -130,6 +133,7 @@ export interface WorkerData<ConnectorState> {
     event: AirdropEvent;
     initialState: ConnectorState;
     workerPath: string;
+    initialDomainMapping?: InitialDomainMapping;
     options?: WorkerAdapterOptions;
 }
 /**

package/dist/workers/process-task.js

@@ -11,11 +11,13 @@ function processTask({ task, onTimeout, }) {
         void (async () => {
             const event = node_worker_threads_1.workerData.event;
             const initialState = node_worker_threads_1.workerData.initialState;
+            const initialDomainMapping = node_worker_threads_1.workerData.initialDomainMapping;
             const options = node_worker_threads_1.workerData.options;
             console = new logger_1.Logger({ event, options });
             const adapterState = await (0, state_1.createAdapterState)({
                 event,
                 initialState,
+                initialDomainMapping,
                 options,
             });
             if (node_worker_threads_1.parentPort && node_worker_threads_1.workerData.event) {

package/dist/workers/spawn.d.ts

@@ -10,7 +10,7 @@ import { SpawnFactoryInterface, SpawnInterface } from '../types/workers';
  * @param {string} workerPath - The path to the worker file
  * @returns {Promise<Spawn>} - A new instance of Spawn class
  */
-export declare function spawn<ConnectorState>({ event, initialState, workerPath, options, }: SpawnFactoryInterface<ConnectorState>): Promise<void>;
+export declare function spawn<ConnectorState>({ event, initialState, workerPath, initialDomainMapping, options, }: SpawnFactoryInterface<ConnectorState>): Promise<void>;
 export declare class Spawn {
     private event;
     private alreadyEmitted;

package/dist/workers/spawn.js

@@ -71,7 +71,7 @@ function getWorkerPath({ event, connectorWorkerPath, }) {
  * @param {string} workerPath - The path to the worker file
  * @returns {Promise<Spawn>} - A new instance of Spawn class
  */
-async function spawn({ event, initialState, workerPath, options, }) {
+async function spawn({ event, initialState, workerPath, initialDomainMapping, options, }) {
     const logger = new logger_1.Logger({ event, options });
     const script = getWorkerPath({
         event,
@@ -91,6 +91,7 @@ async function spawn({ event, initialState, workerPath, options, }) {
                 event,
                 initialState,
                 workerPath: script,
+                initialDomainMapping,
                 options,
             });
             return new Promise((resolve) => {

package/dist/workers/worker-adapter.d.ts

@@ -72,7 +72,7 @@ export declare class WorkerAdapter<ConnectorState> {
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.
      * @param {{ stream, processors }: { stream: ExternalSystemAttachmentStreamingFunction, processors?: ExternalSystemAttachmentProcessors  }} Params - The parameters to stream the attachments
-     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssoAttachment artifact information
+     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssorAttachment artifact information
      * or error information if there was an error
      */
     streamAttachments<NewBatch>({ stream, processors, }: {

package/dist/workers/worker-adapter.js

@@ -117,7 +117,14 @@ class WorkerAdapter {
         }
         // We want to upload all the repos before emitting the event, except for the external sync units done event
         if (newEventType !== extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone) {
-            await this.uploadAllRepos();
+            try {
+                await this.uploadAllRepos();
+            }
+            catch (error) {
+                console.error('Error while uploading repos', error);
+                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+                return;
+            }
         }
         // If the extraction is done, we want to save the timestamp of the last successful sync
         if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
@@ -164,7 +171,10 @@ class WorkerAdapter {
     }
     async uploadAllRepos() {
         for (const repo of this.repos) {
-            await repo.upload();
+            const error = await repo.upload();
+            if (error) {
+                throw error;
+            }
         }
     }
     handleTimeout() {
@@ -501,7 +511,7 @@ class WorkerAdapter {
             event: this.event,
         });
         if (error) {
-            console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
+            console.warn('Error while streaming attachment', error);
             return { error };
         }
         else if (delay) {
@@ -573,27 +583,27 @@ class WorkerAdapter {
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.
      * @param {{ stream, processors }: { stream: ExternalSystemAttachmentStreamingFunction, processors?: ExternalSystemAttachmentProcessors  }} Params - The parameters to stream the attachments
-     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssoAttachment artifact information
+     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssorAttachment artifact information
      * or error information if there was an error
      */
     async streamAttachments({ stream, processors, }) {
-        var _a, _b, _c, _d;
+        var _a, _b;
         const repos = [
             {
                 itemType: 'ssor_attachment',
             },
         ];
         this.initializeRepos(repos);
-        const attachmentsMetadataArtifactIds = (_b = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata) === null || _b === void 0 ? void 0 : _b.artifactIds;
-        if (!attachmentsMetadataArtifactIds ||
-            attachmentsMetadataArtifactIds.length === 0) {
+        if (!((_b = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata) === null || _b === void 0 ? void 0 : _b.artifactIds) ||
+            this.state.toDevRev.attachmentsMetadata.artifactIds.length === 0) {
             console.log(`No attachments metadata artifact IDs found in state.`);
             return;
         }
         else {
-            console.log(`Found ${attachmentsMetadataArtifactIds.length} attachments metadata artifact IDs in state.`);
+            console.log(`Found ${this.state.toDevRev.attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
         }
-        for (const attachmentsMetadataArtifactId of attachmentsMetadataArtifactIds) {
+        while (this.state.toDevRev.attachmentsMetadata.artifactIds.length > 0) {
+            const attachmentsMetadataArtifactId = this.state.toDevRev.attachmentsMetadata.artifactIds[0];
             console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
             const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
                 artifact: attachmentsMetadataArtifactId,
@@ -604,6 +614,9 @@ class WorkerAdapter {
             }
             if (!attachments || attachments.length === 0) {
                 console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                // Remove empty artifact and reset lastProcessed
+                this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
+                this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
                 continue;
             }
             console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
@@ -622,25 +635,24 @@ class WorkerAdapter {
             }
             else {
                 console.log(`Using default processors for attachments.`);
-                const attachmentsToProcess = attachments.slice((_d = (_c = this.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata) === null || _d === void 0 ? void 0 : _d.lastProcessed, attachments.length);
+                const startIndex = this.state.toDevRev.attachmentsMetadata.lastProcessed || 0;
+                const attachmentsToProcess = attachments.slice(startIndex);
                 for (const attachment of attachmentsToProcess) {
                     const response = await this.processAttachment(attachment, stream);
                     if (response === null || response === void 0 ? void 0 : response.delay) {
                         return response;
                     }
                     else if (response === null || response === void 0 ? void 0 : response.error) {
-                        console.warn(`Skipping attachment with ID ${attachment.id} due to error.`);
+                        console.warn('Skipping attachment due to an error while processing', attachment);
                     }
                     if (this.state.toDevRev) {
                         this.state.toDevRev.attachmentsMetadata.lastProcessed += 1;
                     }
                 }
             }
-            if (this.state.toDevRev) {
-                console.log(`Finished processing attachments for artifact ID. Setting last processed to 0 and removing artifact ID from state.`);
-                this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
-                this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
-            }
+            console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+            this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
+            this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
         }
         return;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.2.6",
+  "version": "1.3.0",
   "description": "DevRev ADaaS (AirDrop-as-a-Service) Typescript SDK.",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -37,7 +37,7 @@
     "typescript": "^5.3.3"
   },
   "dependencies": {
-    "@devrev/typescript-sdk": "^1.1.27",
+    "@devrev/typescript-sdk": "^1.1.54",
     "axios": "^1.7.9",
     "axios-retry": "^4.5.0",
     "form-data": "^4.0.1",