@devrev/ts-adaas 0.0.1

package/README.md

@@ -0,0 +1,163 @@
+# ADaaS Library
+
+
+# ADaaS Library
+
+Typescript ADaaS Library (@devrev/ts-adaas) provides:
+
+- type definitions for ADaaS control protocol,
+- an adapter for ADaaS control protocol,
+- helpers for uploading artifacts and manage the state for ADaaS snap-in.
+
+## Usage
+
+Create a new ADaaS adapter on each ADaaS snap-in invocation:
+
+```javascript
+const adapter = new Adapter(event: AirdropEvent);
+```
+
+Adapter class provides:
+
+- helper function to emit response,
+- automatic emit event if ADaaS snap-in invocation runs out of time,
+- setter for updating ADaaS snap-in state and adding artifacts to the return ADaaS message.
+
+### Phases of Airdrop Extraction
+
+Each ADaaS snap-in must handle all the phases of ADaaS extraction.
+
+ADaaS library provides type definitions to ensure ADaaS snap-ins are compatible with ADaaS control protocol.
+
+```javascript
+async run() {
+  switch (this.event.payload.event_type) {
+    case EventType.ExtractionExternalSyncUnitsStart: {
+
+      // extract available External Sync Units (projects, organizations, ...)
+
+      await this.adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsDone, {
+        external_sync_units: externalSyncUnits,
+      });
+      break;
+    }
+
+    case EventType.ExtractionMetadataStart: {
+
+      // provide mappings of domain objects by provioding initial_domain_mapping.json file
+      // update ADaaS snap-in state
+
+      await this.adapter.emit(ExtractorEventType.ExtractionMetadataDone);
+      break;
+    }
+
+    case EventType.ExtractionDataStart: {
+
+      // extract Data
+      // upload Data
+      // update ADaaS snap-in state
+      // approximate progress done
+
+      await this.adapter.emit(ExtractorEventType.ExtractionDataContinue, {
+        progress: 10,
+      });
+
+      break;
+    }
+
+    case EventType.ExtractionDataContinue: {
+      await this.processExtractionData();
+
+      // extract Data
+      // upload Data
+      // update ADaaS snap-in state
+      // approximate progress done
+
+      await this.adapter.emit(ExtractorEventType.ExtractionDataDone, {
+        progress: 100,
+      });
+      break;
+    }
+
+    case EventType.ExtractionDataDelete: {
+
+      // if an extraction has any side-effects to 3rd party systems cleanup should be done here.
+
+      await this.adapter.emit(ExtractorEventType.ExtractionDataDeleteDone);
+      break;
+    }
+
+    case EventType.ExtractionAttachmentsStart: {
+
+      // extract Attachments
+      // upload Attachments
+      // update ADaaS snap-in state
+
+      await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsContinue);
+      break;
+    }
+
+    case EventType.ExtractionAttachmentsContinue: {
+
+
+      // extract Attachments
+      // upload Attachments
+      // update ADaaS snap-in state
+
+      await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDone);
+      break;
+    }
+
+    case EventType.ExtractionAttachmentsDelete: {
+
+      // if an extraction has any side-effects to 3rd party systems cleanup should be done here.
+
+      await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDeleteDone);
+      break;
+    }
+
+    default: {
+      console.log('Event not supported' + JSON.stringify(this.event));
+    }
+  }
+}
+```
+
+## Uploading artifacts
+
+Create a new Uploader class for uploading artifacts:
+
+```javascript
+const upload = new Uploader(
+  event.execution_metadata.devrev_endpoint,
+  event.context.secrets.service_account_token
+);
+```
+
+Files with extracted domain objects must be in JSONL (JSON Lines) format. Data files should contain 2000 - 5000 records each.
+
+```javascript
+const entity = 'users';
+const { artifact, error } = await this.uploader.upload(
+  `extractor_${entity}_${i}.jsonl`,
+  entity,
+  data
+);
+if (error) {
+  return error;
+} else {
+  await this.adapter.update({ artifact });
+}
+```
+
+Each uploaded file must be attached to ADaaS adapter as soon as it is uploaded to ensure it is included in the ADaaS response message in case of a lambda timeout.
+
+## Updating ADaaS snap-in state
+
+ADaaS snap-ins keep their own state between sync runs, between the states of a particular sync run and between invocations within a particular state.
+
+By managing its own state, the ADaaS snap-in keeps track of the process of extraction (what items have already been extracted and where to continue), the times of the last successful sync run and keeps record of progress of the extraction.
+
+```typescript
+    async update({ artifacts, extractor_state}: AdapterUpdateParams)
+```

package/dist/src/adapter/helpers.d.ts

@@ -0,0 +1,4 @@
+import { Artifact, EventType, ExtractorEventType } from '../types';
+export declare function createFormData(preparedArtifact: any, fetchedObjects: object[] | object): FormData;
+export declare function createArtifact(preparedArtifact: any, fetchedObjects: object[] | object, entity: string): Artifact;
+export declare function getTimeoutExtractorEventType(eventType: EventType): ExtractorEventType | null;

package/dist/src/adapter/helpers.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getTimeoutExtractorEventType = exports.createArtifact = exports.createFormData = void 0;
+const js_jsonl_1 = require("js-jsonl");
+const types_1 = require("../types");
+function createFormData(
+//eslint-disable-next-line @typescript-eslint/no-explicit-any
+preparedArtifact, fetchedObjects) {
+    const formData = new FormData();
+    for (const item of preparedArtifact.form_data) {
+        formData.append(item.key, item.value);
+    }
+    const output = js_jsonl_1.jsonl.stringify(fetchedObjects);
+    formData.append('file', output);
+    return formData;
+}
+exports.createFormData = createFormData;
+function createArtifact(
+//eslint-disable-next-line @typescript-eslint/no-explicit-any
+preparedArtifact, fetchedObjects, entity) {
+    const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1;
+    return {
+        item_count: itemCount,
+        id: preparedArtifact.id,
+        item_type: entity,
+    };
+}
+exports.createArtifact = createArtifact;
+function getTimeoutExtractorEventType(eventType) {
+    switch (eventType) {
+        case types_1.EventType.ExtractionMetadataStart:
+            return types_1.ExtractorEventType.ExtractionMetadataError;
+        case types_1.EventType.ExtractionDataStart:
+        case types_1.EventType.ExtractionDataContinue:
+            return types_1.ExtractorEventType.ExtractionDataProgress;
+        case types_1.EventType.ExtractionAttachmentsStart:
+        case types_1.EventType.ExtractionAttachmentsContinue:
+            return types_1.ExtractorEventType.ExtractionAttachmentsProgress;
+        case types_1.EventType.ExtractionExternalSyncUnitsStart:
+            return types_1.ExtractorEventType.ExtractionExternalSyncUnitsError;
+        default:
+            console.log('Event type not recognized in getTimeoutExtractorEventType function: ' +
+                eventType);
+            return null;
+    }
+}
+exports.getTimeoutExtractorEventType = getTimeoutExtractorEventType;

package/dist/src/adapter/index.d.ts

@@ -0,0 +1,53 @@
+import { Artifact, AirdropEvent, ExtractorEventType, EventData } from '../types';
+import { AdapterUpdateParams } from '../types/common';
+/**
+ * Adapter class is used to interact with Airdrop platform. The class provides
+ * utilities to
+ *  - emit control events to the platform
+ *  - update the state of the extractor
+ *  - add artifacts to the list of artifacts to be returned to the platform
+ *  - Return the last saved state and artifacts in case timeout
+ *  - The event sent in case of timeout for each event type is as follows:
+ *     - EXTRACTION_EXTERNAL_SYNC_UNITS_START =>  EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR
+ *     - EXTRACTION_METADATA_START => EXTRACTION_METADATA_ERROR
+ *     - EXTRACTION_DATA_START => EXTRACTION_DATA_PROGRESS
+ *     - EXTRACTION_DATA_CONTINUE => EXTRACTION_DATA_PROGRESS
+ *     - EXTRACTION_ATTACHMENTS_START => EXTRACTION_ATTACHMENTS_PROGRESS
+ *     - EXTRACTION_ATTACHMENTS_CONTINUE => EXTRACTION_ATTACHMENTS_PROGRESS
+ *
+ * @class Adapter
+ * @constructor
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object=} state - Optional state object to be passed to the extractor. If not provided, an empty object is used.
+ */
+export declare class Adapter {
+    private artifacts;
+    /** Adapter level state to return to the platform */
+    private extractorState;
+    private event;
+    private callbackUrl;
+    private devrevToken;
+    constructor(event: AirdropEvent, state?: object);
+    /**
+     *  Adds artifact to the list of artifacts.
+     *  Overrides extractor state if provided.
+     *
+     * @param {AdapterUpdateParams} params - The parameters to update the adapter
+     * @param {Artifact=} params.artifact - The artifact to be added to the list of artifacts
+     * @param {object=} params.extractor_state - The state object to be updated
+     */
+    update(params: AdapterUpdateParams): Promise<void>;
+    /**
+     *  Emits an event to the platform.
+     *
+     * @param {ExtractorEventType} newEventType - The event type to be emitted
+     * @param {EventData=} data - The data to be sent with the event
+     */
+    emit(newEventType: ExtractorEventType, data?: EventData): Promise<void>;
+    /**
+     *  Returns the list of artifacts stored in the adapter.
+     *
+     * @return  The list of artifacts
+     */
+    getArtifacts(): Artifact[];
+}

package/dist/src/adapter/index.js

@@ -0,0 +1,110 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Adapter = void 0;
+const axios_1 = __importDefault(require("axios"));
+const helpers_1 = require("../adapter/helpers");
+/**
+ * Adapter class is used to interact with Airdrop platform. The class provides
+ * utilities to
+ *  - emit control events to the platform
+ *  - update the state of the extractor
+ *  - add artifacts to the list of artifacts to be returned to the platform
+ *  - Return the last saved state and artifacts in case timeout
+ *  - The event sent in case of timeout for each event type is as follows:
+ *     - EXTRACTION_EXTERNAL_SYNC_UNITS_START =>  EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR
+ *     - EXTRACTION_METADATA_START => EXTRACTION_METADATA_ERROR
+ *     - EXTRACTION_DATA_START => EXTRACTION_DATA_PROGRESS
+ *     - EXTRACTION_DATA_CONTINUE => EXTRACTION_DATA_PROGRESS
+ *     - EXTRACTION_ATTACHMENTS_START => EXTRACTION_ATTACHMENTS_PROGRESS
+ *     - EXTRACTION_ATTACHMENTS_CONTINUE => EXTRACTION_ATTACHMENTS_PROGRESS
+ *
+ * @class Adapter
+ * @constructor
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object=} state - Optional state object to be passed to the extractor. If not provided, an empty object is used.
+ */
+class Adapter {
+    constructor(event, state) {
+        this.event = event;
+        this.artifacts = [];
+        this.extractorState = state || {};
+        this.callbackUrl = event.payload.event_context.callback_url;
+        this.devrevToken = event.context.secrets["service_account_token"];
+        // Once lambda is near to timeout, Snap-in needs to submit the information about artifacts
+        // that have been uploaded and state, so next time it is run, can continue where it has left off
+        setTimeout(async () => {
+            const extractorEventType = (0, helpers_1.getTimeoutExtractorEventType)(this.event.payload.event_type);
+            if (extractorEventType) {
+                await this.emit(extractorEventType, { artifacts: this.artifacts });
+            }
+        }, 12 * 60 * 1000);
+    }
+    /**
+     *  Adds artifact to the list of artifacts.
+     *  Overrides extractor state if provided.
+     *
+     * @param {AdapterUpdateParams} params - The parameters to update the adapter
+     * @param {Artifact=} params.artifact - The artifact to be added to the list of artifacts
+     * @param {object=} params.extractor_state - The state object to be updated
+     */
+    async update(params) {
+        if (params.artifact) {
+            this.artifacts.push(params.artifact);
+        }
+        if (params.extractor_state) {
+            this.extractorState = params.extractor_state;
+        }
+    }
+    /**
+     *  Emits an event to the platform.
+     *
+     * @param {ExtractorEventType} newEventType - The event type to be emitted
+     * @param {EventData=} data - The data to be sent with the event
+     */
+    async emit(newEventType, data) {
+        const newEvent = {
+            extractor_state: JSON.stringify(this.extractorState),
+            event_type: newEventType,
+            event_context: {
+                uuid: this.event.payload.event_context.uuid,
+                sync_run: this.event.payload.event_context.sync_run_id,
+            },
+            event_data: Object.assign(Object.assign({}, data), { artifacts: this.artifacts }),
+        };
+        // If sync_unit_id is present in the event, add it to the new event
+        if (this.event.payload.event_context.sync_unit_id) {
+            newEvent.event_context.sync_unit =
+                this.event.payload.event_context.sync_unit_id;
+        }
+        console.log('Event that will be emitted: ' + JSON.stringify(newEvent));
+        try {
+            await axios_1.default.post(this.callbackUrl, Object.assign({}, newEvent), {
+                headers: {
+                    Accept: 'application/json, text/plain, */*',
+                    Authorization: this.devrevToken,
+                    'Content-Type': 'application/json',
+                },
+            });
+        }
+        catch (error) {
+            // If this request fails the extraction will be stuck in loop and
+            // we need to stop it through UI or think about retrying this request
+            console.log('Emitting failed for this event: ' +
+                JSON.stringify(newEvent) +
+                ', error: ' +
+                error);
+        }
+    }
+    /**
+     *  Returns the list of artifacts stored in the adapter.
+     *
+     * @return  The list of artifacts
+     */
+    getArtifacts() {
+        return this.artifacts;
+    }
+}
+exports.Adapter = Adapter;

package/dist/src/adapter/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/adapter/index.test.js

@@ -0,0 +1,105 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const types_1 = require("../types");
+const index_1 = require("./index");
+jest.useFakeTimers();
+const defaultEvent = {
+    context: {
+        secrets: {
+            service_account_token: 'mockToken',
+        },
+    },
+    payload: {
+        connection_data: {
+            org_id: 'mockOrgId',
+            org_name: 'mockOrgName',
+            key: 'mockKey',
+            key_type: 'mockKeyType',
+        },
+        event_context: {
+            mode: 'INITIAL',
+            uuid: 'mockUuid',
+            callback_url: 'mockCallbackUrl',
+            dev_org_id: 'DEV-TESTORG',
+            dev_user_id: 'DEV-TESTUSER',
+            external_system_id: 'TESTSYSTEM',
+            sync_run_id: 'mockSyncRunId',
+        },
+        event_type: types_1.EventType.ExtractionExternalSyncUnitsStart,
+    },
+    execution_metadata: {
+        devrev_endpoint: 'http://api.dev.devrev-eng.ai',
+    },
+    input_data: {
+        global_values: {},
+        event_sources: {},
+    },
+};
+describe('Adapter', () => {
+    let adapter;
+    beforeEach(() => {
+        // Initialize the Adapter instance with mock data
+        adapter = new index_1.Adapter(defaultEvent);
+    });
+    afterEach(async () => {
+        // Clear all timers
+        jest.clearAllTimers();
+        // Clear all instances
+        jest.clearAllMocks();
+    });
+    it('should add artifact to the list of artifacts', async () => {
+        const artifact = {
+            id: 'mockId',
+            item_type: 'mockItemType',
+            item_count: 1,
+            // Mock artifact data
+        };
+        adapter.update({ artifact: artifact });
+        const artifacts = adapter.getArtifacts();
+        expect(artifacts).toContain(artifact);
+        // Run all timers
+        jest.runAllTimers();
+    });
+    it('should set extractor state if provided', async () => {
+        const state = {
+            testing: false,
+        };
+        adapter = new index_1.Adapter(Object.assign({}, defaultEvent), state);
+        expect(adapter['extractorState']).toEqual(state);
+        const newState = {
+            testing: true,
+        };
+        adapter.update({ extractor_state: newState });
+        expect(adapter['extractorState']).toEqual(newState);
+        // Run all timers
+        jest.runAllTimers();
+    });
+    it('update both an artifact and extractor state', async () => {
+        const artifact = {
+            id: 'mockId',
+            item_type: 'mockItemType',
+            item_count: 1,
+        };
+        const state = {
+            willUpdateToTrue: false,
+            willAdd4: 0,
+        };
+        adapter.update({ artifact: artifact, extractor_state: state });
+        const artifacts = adapter.getArtifacts();
+        expect(artifacts).toContain(artifact);
+        expect(adapter['extractorState']).toEqual(state);
+        const newState = {
+            willUpdateToTrue: true,
+            willAdd4: 4,
+            newField: 'newField',
+        };
+        const newArtifact = {
+            id: 'newMockId',
+            item_type: 'newMockItemType',
+            item_count: 2,
+        };
+        adapter.update({ artifact: newArtifact, extractor_state: newState });
+        // Run all timers
+        jest.runAllTimers();
+    });
+});

package/dist/src/demo-extractor/index.d.ts

@@ -0,0 +1,4 @@
+import { AirdropEvent } from '../types';
+export declare class DemoExtractor {
+    run(event: AirdropEvent): Promise<void>;
+}

package/dist/src/demo-extractor/index.js

@@ -0,0 +1,149 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DemoExtractor = void 0;
+const types_1 = require("../types");
+const adapter_1 = require("../adapter");
+const uploader_1 = require("../uploader");
+const recipe_json_1 = __importDefault(require("./recipe.json"));
+class DemoExtractor {
+    async run(event) {
+        console.log('Event in DemoExtractor run function: ' + JSON.stringify(event));
+        const adapter = new adapter_1.Adapter(event);
+        const uploader = new uploader_1.Uploader(event.execution_metadata.devrev_endpoint, event.context.secrets["service_account_token"]);
+        switch (event.payload.event_type) {
+            case types_1.EventType.ExtractionExternalSyncUnitsStart: {
+                const externalSyncUnits = [
+                    {
+                        id: 'devrev',
+                        name: 'devrev',
+                        description: 'Loopback for DevRev',
+                        item_count: 0,
+                    },
+                ];
+                await adapter.emit(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
+                    external_sync_units: externalSyncUnits,
+                });
+                break;
+            }
+            case types_1.EventType.ExtractionMetadataStart: {
+                const metadata = [
+                    {
+                        item: 'contacts',
+                        fields: ['name', 'lastName'],
+                    },
+                    {
+                        item: 'users',
+                        fields: ['name', 'lastName'],
+                    },
+                ];
+                const { artifact, error } = await uploader.upload('loopback_metadata_1.jsonl', 'metadata', metadata);
+                if (error) {
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
+                        error,
+                    });
+                }
+                else {
+                    await adapter.update({ artifact });
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataDone);
+                }
+                break;
+            }
+            case types_1.EventType.ExtractionDataStart: {
+                const contacts = [
+                    {
+                        name: 'John',
+                        lastName: 'Doe',
+                    },
+                    {
+                        name: 'Jane',
+                        lastName: 'Doe',
+                    },
+                ];
+                const { artifact, error } = await uploader.upload('loopback_contacts_1.json', 'contacts', contacts);
+                if (error) {
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
+                        error,
+                    });
+                }
+                else {
+                    await adapter.update({ artifact });
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionDataProgress, {
+                        progress: 50,
+                    });
+                }
+                break;
+            }
+            case types_1.EventType.ExtractionDataContinue: {
+                const users = [
+                    {
+                        name: 'John',
+                        lastName: 'Phd',
+                    },
+                    {
+                        name: 'Jane',
+                        lastName: 'Phd',
+                    },
+                ];
+                const { artifact, error } = await uploader.upload('loopback_users_1.json', 'users', users);
+                if (error) {
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
+                        error,
+                    });
+                }
+                else {
+                    await adapter.update({ artifact });
+                    // TODO: Add separated function for uploading recipe.json?
+                    const { artifact: recipe, error } = await uploader.upload('recipe.json', 'initial_domain_mapping', recipe_json_1.default);
+                    await adapter.update({ artifact: recipe });
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone, {
+                        progress: 100,
+                    });
+                }
+                break;
+            }
+            case types_1.EventType.ExtractionDataDelete: {
+                await adapter.emit(types_1.ExtractorEventType.ExtractionDataDeleteDone);
+                break;
+            }
+            case types_1.EventType.ExtractionAttachmentsStart: {
+                const attachment1 = ['This is attachment1.txt content'];
+                const { artifact, error } = await uploader.upload('attachment1.txt', 'attachment', attachment1);
+                if (error) {
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
+                        error,
+                    });
+                }
+                else {
+                    await adapter.update({ artifact });
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsProgress);
+                }
+                break;
+            }
+            case types_1.EventType.ExtractionAttachmentsContinue: {
+                const attachment2 = ['This is attachment2.txt content'];
+                const { artifact, error } = await uploader.upload('attachment2.txt', 'attachment', attachment2);
+                if (error) {
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
+                        error,
+                    });
+                }
+                else {
+                    await adapter.update({ artifact });
+                    await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDone);
+                }
+                break;
+            }
+            case types_1.EventType.ExtractionAttachmentsDelete: {
+                await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
+                break;
+            }
+            default: {
+                console.log('Event in DemoExtractor run not recognized: ' + JSON.stringify(event));
+            }
+        }
+    }
+}
+exports.DemoExtractor = DemoExtractor;

package/dist/src/demo-extractor/recipe.json

@@ -0,0 +1,37 @@
+{
+    "initial_object_mappings": [
+        {
+            "external_type": "contacts",
+            "possible_targets": {
+                "rev_user": {}
+            },
+            "default_target": "rev_user",
+            "allow_item_type_decisions": true
+        },
+        {
+            "external_type": "users",
+            "possible_targets": {
+                "dev_user": {}
+            },
+            "default_target": "dev_user",
+            "allow_item_type_decisions": true
+        },
+        {
+            "external_type": "tickets",
+            "possible_targets": {
+                "work.ticket": {}
+            },
+            "default_target": "work.ticket",
+            "allow_item_type_decisions": true
+        },
+        {
+            "external_type": "conversations",
+            "possible_targets": {
+                "comment": {}
+            },
+            "default_target": "comment",
+            "allow_item_type_decisions": true
+        }
+    ],
+    "external_system_short_name": "source-system"
+}