@devrev/ts-adaas 0.0.1 → 0.0.3

package/README.md

@@ -1,14 +1,30 @@
 # ADaaS Library
 
-
-# ADaaS Library
-
 Typescript ADaaS Library (@devrev/ts-adaas) provides:
 
 - type definitions for ADaaS control protocol,
 - an adapter for ADaaS control protocol,
 - helpers for uploading artifacts and manage the state for ADaaS snap-in.
 
+## Release Notes
+
+#### v0.0.3
+
+- Support for new recipe management
+
+#### v0.0.2
+
+- Support for the State API
+- HTTP client for API requests
+- Local development environment creates local artifact files
+- Improvements in logging
+
+#### v0.0.1
+
+- Demo implementation of ADaaS snap-in
+- Adapter for ADaaS control protocol with helper functions
+- Uploader for uploading artifacts
+
 ## Usage
 
 Create a new ADaaS adapter on each ADaaS snap-in invocation:
@@ -160,4 +176,4 @@ By managing its own state, the ADaaS snap-in keeps track of the process of extra
 
 ```typescript
     async update({ artifacts, extractor_state}: AdapterUpdateParams)
-```
+```

package/dist/src/adapter/index.d.ts

@@ -1,42 +1,43 @@
-import { Artifact, AirdropEvent, ExtractorEventType, EventData } from '../types';
-import { AdapterUpdateParams } from '../types/common';
+import { AirdropEvent, ExtractorEventType, EventData, AdapterState, Artifact } from '../types';
+import { State } from '../state';
 /**
  * Adapter class is used to interact with Airdrop platform. The class provides
  * utilities to
  *  - emit control events to the platform
  *  - update the state of the extractor
- *  - add artifacts to the list of artifacts to be returned to the platform
- *  - Return the last saved state and artifacts in case timeout
- *  - The event sent in case of timeout for each event type is as follows:
- *     - EXTRACTION_EXTERNAL_SYNC_UNITS_START =>  EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR
- *     - EXTRACTION_METADATA_START => EXTRACTION_METADATA_ERROR
- *     - EXTRACTION_DATA_START => EXTRACTION_DATA_PROGRESS
- *     - EXTRACTION_DATA_CONTINUE => EXTRACTION_DATA_PROGRESS
- *     - EXTRACTION_ATTACHMENTS_START => EXTRACTION_ATTACHMENTS_PROGRESS
- *     - EXTRACTION_ATTACHMENTS_CONTINUE => EXTRACTION_ATTACHMENTS_PROGRESS
+ *  - set the last saved state in case of a timeout
  *
  * @class Adapter
  * @constructor
  * @param {AirdropEvent} event - The event object received from the platform
- * @param {object=} state - Optional state object to be passed to the extractor. If not provided, an empty object is used.
+ * @param {object=} initialState - The initial state of the adapter
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
  */
-export declare class Adapter {
-    private artifacts;
-    /** Adapter level state to return to the platform */
-    private extractorState;
+/**
+ *  Creates an adapter instance.
+ *
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param initialState
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
+ * @return  The adapter instance
+ */
+export declare function createAdapter<ExtractorState>(event: AirdropEvent, initialState: ExtractorState, isLocalDevelopment?: boolean): Promise<Adapter<ExtractorState>>;
+export declare class Adapter<ExtractorState> {
+    private adapterState;
+    private _artifacts;
     private event;
     private callbackUrl;
     private devrevToken;
-    constructor(event: AirdropEvent, state?: object);
-    /**
-     *  Adds artifact to the list of artifacts.
-     *  Overrides extractor state if provided.
-     *
-     * @param {AdapterUpdateParams} params - The parameters to update the adapter
-     * @param {Artifact=} params.artifact - The artifact to be added to the list of artifacts
-     * @param {object=} params.extractor_state - The state object to be updated
-     */
-    update(params: AdapterUpdateParams): Promise<void>;
+    private startTime;
+    private heartBeatFn;
+    private exit;
+    private lambdaTimeout;
+    private heartBeatInterval;
+    constructor(event: AirdropEvent, adapterState: State<ExtractorState>, isLocalDevelopment?: boolean);
+    get state(): AdapterState<ExtractorState>;
+    set state(value: AdapterState<ExtractorState>);
+    get artifacts(): Artifact[];
+    set artifacts(value: Artifact[]);
     /**
      *  Emits an event to the platform.
      *
@@ -45,9 +46,13 @@ export declare class Adapter {
      */
     emit(newEventType: ExtractorEventType, data?: EventData): Promise<void>;
     /**
-     *  Returns the list of artifacts stored in the adapter.
-     *
-     * @return  The list of artifacts
+     * Exit the adapter. This will stop the heartbeat and no
+     * further events will be emitted.
+     */
+    private exitAdapter;
+    /**
+     * Heartbeat function to check if the lambda is about to timeout.
+     * @returns true if 10 minutes have passed since the start of the lambda.
      */
-    getArtifacts(): Artifact[];
+    private heartbeat;
 }

package/dist/src/adapter/index.js

@@ -3,60 +3,73 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Adapter = void 0;
+exports.Adapter = exports.createAdapter = void 0;
 const axios_1 = __importDefault(require("axios"));
-const helpers_1 = require("../adapter/helpers");
+const constants_1 = require("../common/constants");
+const helpers_1 = require("../common/helpers");
+const logging_1 = require("../logging");
+const state_1 = require("../state");
 /**
  * Adapter class is used to interact with Airdrop platform. The class provides
  * utilities to
  *  - emit control events to the platform
  *  - update the state of the extractor
- *  - add artifacts to the list of artifacts to be returned to the platform
- *  - Return the last saved state and artifacts in case timeout
- *  - The event sent in case of timeout for each event type is as follows:
- *     - EXTRACTION_EXTERNAL_SYNC_UNITS_START =>  EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR
- *     - EXTRACTION_METADATA_START => EXTRACTION_METADATA_ERROR
- *     - EXTRACTION_DATA_START => EXTRACTION_DATA_PROGRESS
- *     - EXTRACTION_DATA_CONTINUE => EXTRACTION_DATA_PROGRESS
- *     - EXTRACTION_ATTACHMENTS_START => EXTRACTION_ATTACHMENTS_PROGRESS
- *     - EXTRACTION_ATTACHMENTS_CONTINUE => EXTRACTION_ATTACHMENTS_PROGRESS
+ *  - set the last saved state in case of a timeout
  *
  * @class Adapter
  * @constructor
  * @param {AirdropEvent} event - The event object received from the platform
- * @param {object=} state - Optional state object to be passed to the extractor. If not provided, an empty object is used.
+ * @param {object=} initialState - The initial state of the adapter
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
  */
+/**
+ *  Creates an adapter instance.
+ *
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param initialState
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
+ * @return  The adapter instance
+ */
+async function createAdapter(event, initialState, isLocalDevelopment = false) {
+    const newInitialState = structuredClone(initialState);
+    const adapterState = await (0, state_1.createAdapterState)(event, newInitialState);
+    const a = new Adapter(event, adapterState, isLocalDevelopment);
+    return a;
+}
+exports.createAdapter = createAdapter;
 class Adapter {
-    constructor(event, state) {
+    constructor(event, adapterState, isLocalDevelopment = false) {
+        this.exit = false;
+        this.lambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
+        this.heartBeatInterval = 30 * 1000; // 30 seconds in milliseconds
+        if (!isLocalDevelopment) {
+            logging_1.Logger.init(event);
+        }
+        this.adapterState = adapterState;
+        this._artifacts = [];
         this.event = event;
-        this.artifacts = [];
-        this.extractorState = state || {};
         this.callbackUrl = event.payload.event_context.callback_url;
-        this.devrevToken = event.context.secrets["service_account_token"];
-        // Once lambda is near to timeout, Snap-in needs to submit the information about artifacts
-        // that have been uploaded and state, so next time it is run, can continue where it has left off
-        setTimeout(async () => {
-            const extractorEventType = (0, helpers_1.getTimeoutExtractorEventType)(this.event.payload.event_type);
-            if (extractorEventType) {
-                await this.emit(extractorEventType, { artifacts: this.artifacts });
+        this.devrevToken = event.context.secrets.service_account_token;
+        this.startTime = Date.now();
+        // Run heartbeat every 30 seconds
+        this.heartBeatFn = setInterval(async () => {
+            const b = await this.heartbeat();
+            if (b) {
+                this.exitAdapter();
             }
-        }, 12 * 60 * 1000);
+        }, this.heartBeatInterval);
     }
-    /**
-     *  Adds artifact to the list of artifacts.
-     *  Overrides extractor state if provided.
-     *
-     * @param {AdapterUpdateParams} params - The parameters to update the adapter
-     * @param {Artifact=} params.artifact - The artifact to be added to the list of artifacts
-     * @param {object=} params.extractor_state - The state object to be updated
-     */
-    async update(params) {
-        if (params.artifact) {
-            this.artifacts.push(params.artifact);
-        }
-        if (params.extractor_state) {
-            this.extractorState = params.extractor_state;
-        }
+    get state() {
+        return this.adapterState.state;
+    }
+    set state(value) {
+        this.adapterState.state = value;
+    }
+    get artifacts() {
+        return this._artifacts;
+    }
+    set artifacts(value) {
+        this._artifacts = value;
     }
     /**
      *  Emits an event to the platform.
@@ -65,21 +78,22 @@ class Adapter {
      * @param {EventData=} data - The data to be sent with the event
      */
     async emit(newEventType, data) {
+        if (this.exit) {
+            console.warn('Adapter is already in exit state. No more events can be emitted.');
+            return;
+        }
+        // We want to save the state every time we emit an event, except for the start and delete events
+        if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
+            console.log(`Saving state before emitting event`);
+            await this.adapterState.postState(this.state);
+        }
         const newEvent = {
-            extractor_state: JSON.stringify(this.extractorState),
             event_type: newEventType,
-            event_context: {
-                uuid: this.event.payload.event_context.uuid,
-                sync_run: this.event.payload.event_context.sync_run_id,
-            },
-            event_data: Object.assign(Object.assign({}, data), { artifacts: this.artifacts }),
+            event_context: Object.assign({ uuid: this.event.payload.event_context.uuid, sync_run: this.event.payload.event_context.sync_run_id }, (this.event.payload.event_context.sync_unit_id && {
+                sync_unit: this.event.payload.event_context.sync_unit_id,
+            })),
+            event_data: Object.assign({}, data),
         };
-        // If sync_unit_id is present in the event, add it to the new event
-        if (this.event.payload.event_context.sync_unit_id) {
-            newEvent.event_context.sync_unit =
-                this.event.payload.event_context.sync_unit_id;
-        }
-        console.log('Event that will be emitted: ' + JSON.stringify(newEvent));
         try {
             await axios_1.default.post(this.callbackUrl, Object.assign({}, newEvent), {
                 headers: {
@@ -88,23 +102,46 @@ class Adapter {
                     'Content-Type': 'application/json',
                 },
             });
+            console.log('Successfully emitted event: ' + JSON.stringify(newEvent));
         }
         catch (error) {
             // If this request fails the extraction will be stuck in loop and
             // we need to stop it through UI or think about retrying this request
-            console.log('Emitting failed for this event: ' +
+            console.log('Failed to emit event: ' +
                 JSON.stringify(newEvent) +
                 ', error: ' +
                 error);
         }
+        finally {
+            this.exitAdapter();
+        }
     }
     /**
-     *  Returns the list of artifacts stored in the adapter.
-     *
-     * @return  The list of artifacts
+     * Exit the adapter. This will stop the heartbeat and no
+     * further events will be emitted.
      */
-    getArtifacts() {
-        return this.artifacts;
+    exitAdapter() {
+        this.exit = true;
+        clearInterval(this.heartBeatFn);
+    }
+    /**
+     * Heartbeat function to check if the lambda is about to timeout.
+     * @returns true if 10 minutes have passed since the start of the lambda.
+     */
+    async heartbeat() {
+        if (this.exit) {
+            return true;
+        }
+        if (Date.now() - this.startTime > this.lambdaTimeout) {
+            const timeoutEventType = (0, helpers_1.getTimeoutExtractorEventType)(this.event.payload.event_type);
+            if (timeoutEventType !== null) {
+                const { eventType, isError } = timeoutEventType;
+                const err = isError ? { message: 'Lambda Timeout' } : undefined;
+                await this.emit(eventType, { error: err, artifacts: this._artifacts });
+                return true;
+            }
+        }
+        return false;
     }
 }
 exports.Adapter = Adapter;

package/dist/src/common/constants.d.ts

@@ -0,0 +1,2 @@
+import { EventType } from '../types';
+export declare const STATELESS_EVENT_TYPES: EventType[];

package/dist/src/common/constants.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.STATELESS_EVENT_TYPES = void 0;
+const types_1 = require("../types");
+exports.STATELESS_EVENT_TYPES = [
+    types_1.EventType.ExtractionExternalSyncUnitsStart,
+    types_1.EventType.ExtractionMetadataStart,
+    types_1.EventType.ExtractionDataDelete,
+    types_1.EventType.ExtractionAttachmentsDelete,
+];

package/dist/src/{adapter → common}/helpers.d.ts

@@ -1,4 +1,7 @@
 import { Artifact, EventType, ExtractorEventType } from '../types';
 export declare function createFormData(preparedArtifact: any, fetchedObjects: object[] | object): FormData;
 export declare function createArtifact(preparedArtifact: any, fetchedObjects: object[] | object, entity: string): Artifact;
-export declare function getTimeoutExtractorEventType(eventType: EventType): ExtractorEventType | null;
+export declare function getTimeoutExtractorEventType(eventType: EventType): {
+    eventType: ExtractorEventType;
+    isError: boolean;
+} | null;

package/dist/src/{adapter → common}/helpers.js

@@ -29,15 +29,27 @@ exports.createArtifact = createArtifact;
 function getTimeoutExtractorEventType(eventType) {
     switch (eventType) {
         case types_1.EventType.ExtractionMetadataStart:
-            return types_1.ExtractorEventType.ExtractionMetadataError;
+            return {
+                eventType: types_1.ExtractorEventType.ExtractionMetadataError,
+                isError: true,
+            };
         case types_1.EventType.ExtractionDataStart:
         case types_1.EventType.ExtractionDataContinue:
-            return types_1.ExtractorEventType.ExtractionDataProgress;
+            return {
+                eventType: types_1.ExtractorEventType.ExtractionDataProgress,
+                isError: false,
+            };
         case types_1.EventType.ExtractionAttachmentsStart:
         case types_1.EventType.ExtractionAttachmentsContinue:
-            return types_1.ExtractorEventType.ExtractionAttachmentsProgress;
+            return {
+                eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
+                isError: false,
+            };
         case types_1.EventType.ExtractionExternalSyncUnitsStart:
-            return types_1.ExtractorEventType.ExtractionExternalSyncUnitsError;
+            return {
+                eventType: types_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
+                isError: true,
+            };
         default:
             console.log('Event type not recognized in getTimeoutExtractorEventType function: ' +
                 eventType);

package/dist/src/common/install-initial-domain-mapping.d.ts

@@ -0,0 +1,3 @@
+import { AirdropEvent } from '../types/extraction';
+import { InitialDomainMapping } from '../types/common';
+export declare function installInitialDomainMapping(event: AirdropEvent, initialDomainMappingJson: InitialDomainMapping): Promise<void>;

package/dist/src/common/install-initial-domain-mapping.js

@@ -0,0 +1,60 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.installInitialDomainMapping = void 0;
+const axios_1 = __importDefault(require("axios"));
+async function installInitialDomainMapping(event, initialDomainMappingJson) {
+    const devrevEndpoint = event.execution_metadata.devrev_endpoint;
+    const devrevToken = event.context.secrets.service_account_token;
+    const snapInVersionId = event.context.snap_in_version_id;
+    if (!initialDomainMappingJson) {
+        console.warn('No initial domain mapping found');
+        return;
+    }
+    const snapInVersionResponse = await axios_1.default.get(devrevEndpoint + '/internal/snap-in-versions.get', {
+        headers: {
+            Authorization: devrevToken,
+        },
+        params: {
+            id: snapInVersionId,
+        },
+    });
+    const importSlug = snapInVersionResponse.data.snap_in_version.imports[0].slug;
+    const snapInSlug = snapInVersionResponse.data.snap_in_version.slug;
+    const startingRecipeBlueprint = initialDomainMappingJson === null || initialDomainMappingJson === void 0 ? void 0 : initialDomainMappingJson.starting_recipe_blueprint;
+    let recipeBlueprintId;
+    if (startingRecipeBlueprint &&
+        Object.keys(startingRecipeBlueprint).length !== 0) {
+        try {
+            const recipeBlueprintResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`, Object.assign({}, startingRecipeBlueprint), {
+                headers: {
+                    Authorization: devrevToken,
+                },
+            });
+            recipeBlueprintId = recipeBlueprintResponse.data.recipe_blueprint.id;
+            console.log('Successfully created recipe blueprint with id: ' + recipeBlueprintId);
+        }
+        catch (error) {
+            console.error('Error while creating recipe blueprint', error);
+        }
+    }
+    try {
+        // 2. Install the initial domain mappings
+        const additionalMappings = initialDomainMappingJson.additional_mappings || {};
+        const initialDomainMappingInstallResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, Object.assign(Object.assign({ external_system_type: 'ADaaS', import_slug: importSlug, snap_in_slug: snapInSlug }, (recipeBlueprintId && {
+            starting_recipe_blueprint: recipeBlueprintId,
+        })), additionalMappings), {
+            headers: {
+                Authorization: devrevToken,
+            },
+        });
+        console.log('Successfully installed initial domain mapping', initialDomainMappingInstallResponse.data);
+    }
+    catch (error) {
+        console.error('Error while installing initial domain mapping', error);
+        return;
+    }
+}
+exports.installInitialDomainMapping = installInitialDomainMapping;

package/dist/src/demo-extractor/external_domain_metadata.json

@@ -0,0 +1,38 @@
+{
+    "record_types": {
+        "users": {
+            "fields": {
+                "name": {
+                    "is_required": true,
+                    "type": "text",
+                    "name": "Name",
+                    "text": {
+                        "min_length": 1
+                    }
+                },
+                "email": {
+                    "type": "text",
+                    "name": "Email",
+                    "is_required": true
+                }
+            }
+        },
+        "contacts": {
+            "fields": {
+                "name": {
+                    "is_required": true,
+                    "type": "text",
+                    "name": "Name",
+                    "text": {
+                        "min_length": 1
+                    }
+                },
+                "email": {
+                    "type": "text",
+                    "name": "Email",
+                    "is_required": true
+                }
+            }
+        }
+    }
+}

package/dist/src/demo-extractor/index.d.ts

@@ -1,4 +1,11 @@
 import { AirdropEvent } from '../types';
+import { Adapter } from '../adapter';
+type ExtractorState = object;
 export declare class DemoExtractor {
-    run(event: AirdropEvent): Promise<void>;
+    private event;
+    private adapter;
+    private uploader;
+    constructor(event: AirdropEvent, adapter: Adapter<ExtractorState>);
+    run(): Promise<void>;
 }
+export {};