@devrev/ts-adaas 0.0.1 → 0.0.2

package/dist/src/demo-extractor/initial_domain_mapping.json

@@ -0,0 +1,107 @@
+{
+    "initial_object_mappings": [
+        {
+            "external_type": "contacts",
+            "possible_targets": {
+                "rev_user": {
+                    "initial_field_mappings": [
+                        {
+                            "destination_field": {
+                                "id": "id",
+                                "is_required": true
+                            },
+                            "selected_resolutions": ["id"],
+                            "resolutions": {
+                                "id": {
+                                    "source_field_description": {
+                                        "id": "id",
+                                        "is_required": true
+                                    }
+                                }
+                            },
+                            "is_finished": true
+                        },
+                        {
+                            "destination_field": {
+                                "id": "display_name",
+                                "is_required": true
+                            },
+                            "selected_resolutions": ["name"],
+                            "resolutions": {
+                                "name": {
+                                    "source_field_description": {
+                                        "id": "name",
+                                        "is_required": true
+                                    }
+                                }
+                            },
+                            "is_finished": true
+                        }
+                    ]
+                }
+            },
+            "default_target": "rev_user",
+            "allow_item_type_decisions": true
+        },
+        {
+            "external_type": "users",
+            "possible_targets": {
+                "dev_user": {
+                    "initial_field_mappings": [
+                        {
+                            "destination_field": {
+                                "id": "id",
+                                "is_required": true
+                            },
+                            "selected_resolutions": ["id"],
+                            "resolutions": {
+                                "id": {
+                                    "source_field_description": {
+                                        "id": "id",
+                                        "is_required": true
+                                    }
+                                }
+                            },
+                            "is_finished": true
+                        },
+                        {
+                            "destination_field": {
+                                "id": "display_name",
+                                "is_required": true
+                            },
+                            "selected_resolutions": ["name"],
+                            "resolutions": {
+                                "name": {
+                                    "source_field_description": {
+                                        "id": "name",
+                                        "is_required": true
+                                    }
+                                }
+                            },
+                            "is_finished": true
+                        }
+                    ]
+                }
+            },
+            "default_target": "dev_user",
+            "allow_item_type_decisions": true
+        },
+        {
+            "external_type": "tickets",
+            "possible_targets": {
+                "work.ticket": {}
+            },
+            "default_target": "work.ticket",
+            "allow_item_type_decisions": true
+        },
+        {
+            "external_type": "conversations",
+            "possible_targets": {
+                "comment": {}
+            },
+            "default_target": "comment",
+            "allow_item_type_decisions": true
+        }
+    ],
+    "external_system_short_name": "Freshdesk"
+}

package/dist/src/http/client.d.ts

@@ -1,6 +1,6 @@
 import { HTTPResponse } from './types';
 export declare const defaultResponse: HTTPResponse;
-export default class HTTPClient {
+export declare class HTTPClient {
     private retryAfter;
     private retryAt;
     private axiosInstance;

package/dist/src/http/client.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.defaultResponse = void 0;
+exports.HTTPClient = exports.defaultResponse = void 0;
 const axios_1 = __importStar(require("axios"));
 const constants_1 = require("./constants");
 exports.defaultResponse = {
@@ -144,4 +144,4 @@ class HTTPClient {
         }
     }
 }
-exports.default = HTTPClient;
+exports.HTTPClient = HTTPClient;

package/dist/src/http/index.d.ts

@@ -1,3 +1,2 @@
 export * from './client';
 export * from './types';
-export * from './constants';

package/dist/src/http/index.js

@@ -16,4 +16,3 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./client"), exports);
 __exportStar(require("./types"), exports);
-__exportStar(require("./constants"), exports);

package/dist/src/index.d.ts

@@ -2,3 +2,4 @@ export * from './adapter';
 export * from './demo-extractor';
 export * from './uploader';
 export * from './types';
+export * from './http';

package/dist/src/index.js

@@ -18,3 +18,4 @@ __exportStar(require("./adapter"), exports);
 __exportStar(require("./demo-extractor"), exports);
 __exportStar(require("./uploader"), exports);
 __exportStar(require("./types"), exports);
+__exportStar(require("./http"), exports);

package/dist/src/logging/index.d.ts

@@ -1,8 +1,8 @@
-declare enum LogLevel {
+import { AirdropEvent } from '../types';
+export declare enum LogLevel {
     INFO = "info",
     WARN = "warn",
-    ERROR = "error",
-    DEBUG = "debug"
+    ERROR = "error"
 }
 /**
  * Logger class to log messages based on the log level.
@@ -10,22 +10,9 @@ declare enum LogLevel {
  * - INFO
  * - WARN
  * - ERROR
- * - DEBUG
  *
  * The log tags can be set to any key-value pair.
  */
-declare class Logger {
-    private level;
-    private logTags;
-    private static instance;
-    private constructor();
-    private log;
-    static getInstance(level: LogLevel, logTags?: Record<string, any>): Logger;
-    setTags(tags: Record<string, any>): void;
-    addTags(tags: Record<string, any>): void;
-    info(message: string): void;
-    warn(message: string): void;
-    error(message: string): void;
-    debug(message: string): void;
+export declare class Logger {
+    static init(event: AirdropEvent): void;
 }
-export default Logger;

package/dist/src/logging/index.js

@@ -1,60 +1,39 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.Logger = exports.LogLevel = void 0;
 var LogLevel;
 (function (LogLevel) {
     LogLevel["INFO"] = "info";
     LogLevel["WARN"] = "warn";
     LogLevel["ERROR"] = "error";
-    LogLevel["DEBUG"] = "debug";
-})(LogLevel || (LogLevel = {}));
+})(LogLevel || (exports.LogLevel = LogLevel = {}));
 /**
  * Logger class to log messages based on the log level.
  * The log level can be set to one of the following:
  * - INFO
  * - WARN
  * - ERROR
- * - DEBUG
  *
  * The log tags can be set to any key-value pair.
  */
 class Logger {
-    // Private constructor to prevent instantiation
-    constructor(level, logTags = {}) {
-        this.logTags = {};
-        this.level = level;
-        this.logTags = logTags;
-    }
-    log(level, message) {
-        if (this.level === level) {
-            console.log(`[${level.toUpperCase()}]: ${message}`, this.logTags);
-        }
-    }
-    // Singleton instance
-    static getInstance(level, logTags = {}) {
-        if (!Logger.instance) {
-            Logger.instance = new Logger(level, logTags);
-        }
-        return Logger.instance;
-    }
-    // Set log tags destructively.
-    setTags(tags) {
-        this.logTags = tags;
-    }
-    // Add log tags. If the key already exists, it will be overwritten.
-    addTags(tags) {
-        this.logTags = Object.assign(Object.assign({}, this.logTags), tags);
-    }
-    info(message) {
-        this.log(LogLevel.INFO, message);
-    }
-    warn(message) {
-        this.log(LogLevel.WARN, message);
-    }
-    error(message) {
-        this.log(LogLevel.ERROR, message);
-    }
-    debug(message) {
-        this.log(LogLevel.DEBUG, message);
+    static init(event) {
+        const origLog = console.log;
+        console.log = (message) => {
+            origLog(`[${LogLevel.INFO.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
+        };
+        const origInfo = console.info;
+        console.info = (message) => {
+            origInfo(`[${LogLevel.INFO.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
+        };
+        const origWarn = console.warn;
+        console.warn = (message) => {
+            origWarn(`[${LogLevel.WARN.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
+        };
+        const origError = console.error;
+        console.error = (message) => {
+            origError(`[${LogLevel.ERROR.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
+        };
     }
 }
-exports.default = Logger;
+exports.Logger = Logger;

package/dist/src/state/index.d.ts

@@ -0,0 +1,23 @@
+import { AdapterState, AirdropEvent } from '../types';
+export declare function createAdapterState<ExtractorState>(event: AirdropEvent, initialState: ExtractorState): Promise<State<ExtractorState>>;
+export declare class State<ExtractorState> {
+    private _state;
+    private event;
+    private workerUrl;
+    private devrevToken;
+    constructor(event: AirdropEvent, initialState: ExtractorState);
+    get state(): AdapterState<ExtractorState>;
+    set state(value: AdapterState<ExtractorState>);
+    /**
+     *  Updates the state of the adapter.
+     *
+     * @param {object} state - The state to be updated
+     */
+    postState(state: AdapterState<ExtractorState>): Promise<void>;
+    /**
+     *  Fetches the state of the adapter.
+     *
+     * @return  The state of the adapter
+     */
+    fetchState(initialState: ExtractorState): Promise<AdapterState<ExtractorState> | unknown>;
+}

package/dist/src/state/index.js

@@ -0,0 +1,111 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.State = exports.createAdapterState = void 0;
+const axios_1 = __importStar(require("axios"));
+const constants_1 = require("../common/constants");
+async function createAdapterState(event, initialState) {
+    const newInitialState = structuredClone(initialState);
+    const as = new State(event, newInitialState);
+    if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
+        console.log('Fetching state');
+        await as.fetchState(newInitialState);
+    }
+    return as;
+}
+exports.createAdapterState = createAdapterState;
+class State {
+    constructor(event, initialState) {
+        this._state = Object.assign({ lastSyncStarted: '', lastSuccessfulSyncStarted: '' }, initialState);
+        this.event = event;
+        this.workerUrl = event.payload.event_context.worker_data_url;
+        this.devrevToken = event.context.secrets.service_account_token;
+    }
+    get state() {
+        return this._state;
+    }
+    set state(value) {
+        this._state = value;
+    }
+    /**
+     *  Updates the state of the adapter.
+     *
+     * @param {object} state - The state to be updated
+     */
+    async postState(state) {
+        try {
+            await axios_1.default.post(this.workerUrl + '.update', {
+                state: JSON.stringify(state),
+            }, {
+                headers: {
+                    Authorization: this.devrevToken,
+                },
+                params: {
+                    sync_unit: this.event.payload.event_context.sync_unit_id,
+                },
+            });
+            this.state = state;
+            console.log('State updated successfully');
+        }
+        catch (error) {
+            console.error('Failed to update state, error:' + error);
+        }
+    }
+    /**
+     *  Fetches the state of the adapter.
+     *
+     * @return  The state of the adapter
+     */
+    async fetchState(initialState) {
+        var _a;
+        const state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: '', lastSuccessfulSyncStarted: '' });
+        console.log('Fetching state with sync unit id: ' +
+            this.event.payload.event_context.sync_unit_id);
+        try {
+            const response = await axios_1.default.post(this.workerUrl + '.get', {}, {
+                headers: {
+                    Authorization: this.devrevToken,
+                },
+                params: {
+                    sync_unit: this.event.payload.event_context.sync_unit_id,
+                },
+            });
+            this.state = JSON.parse(response.data.state);
+            console.log('State fetched successfully');
+            return this.state;
+        }
+        catch (error) {
+            if ((0, axios_1.isAxiosError)(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
+                console.log('State not found, returning initial state');
+                this.state = state;
+                this.postState(this.state);
+                return this.state;
+            }
+            console.error('Failed to fetch state, error:' + error);
+            return error;
+        }
+    }
+}
+exports.State = State;

package/dist/src/types/extraction.d.ts

@@ -1,4 +1,4 @@
-import { InputData, Context, ExecutionMetadata } from '@devrev/typescript-sdk/dist/snap-ins';
+import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
 import { Artifact, ErrorRecord } from './common';
 export declare enum EventType {
     ExtractionExternalSyncUnitsStart = "EXTRACTION_EXTERNAL_SYNC_UNITS_START",
@@ -43,7 +43,7 @@ export interface ExternalSyncUnit {
     id: string;
     name: string;
     description: string;
-    item_count: number;
+    item_count?: number;
 }
 export interface EventContextIn {
     mode: string;
@@ -55,6 +55,7 @@ export interface EventContextIn {
     sync_run_id: string;
     external_system_id: string;
     uuid: string;
+    worker_data_url: string;
 }
 export interface EventContextOut {
     uuid: string;
@@ -85,9 +86,15 @@ export interface DomainObjectState {
     count: number;
 }
 export interface AirdropEvent {
-    context: Context;
+    context: {
+        secrets: {
+            service_account_token: string;
+        };
+    };
     payload: AirdropMessage;
-    execution_metadata: ExecutionMetadata;
+    execution_metadata: {
+        devrev_endpoint: string;
+    };
     input_data: InputData;
 }
 export interface AirdropMessage {
@@ -100,6 +107,10 @@ export interface AirdropMessage {
 export interface ExtractorEvent {
     event_type: string;
     event_context: EventContextOut;
-    extractor_state: string;
+    extractor_state?: string;
     event_data?: EventData;
 }
+export type AdapterState<ExtractorState> = ExtractorState & {
+    lastSyncStarted?: string;
+    lastSuccessfulSyncStarted?: string;
+};

package/dist/src/uploader/index.d.ts

@@ -10,11 +10,12 @@ import { UploadResponse } from '../types/common';
  * @constructor
  * @param {string} endpoint - The endpoint of the DevRev platform
  * @param {string} token - The token to authenticate with the DevRev platform
+ * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
  */
 export declare class Uploader {
     private betaDevrevSdk;
-    private publicDevrevSdk;
-    constructor(endpoint: string, token: string);
+    private local;
+    constructor(endpoint: string, token: string, local?: boolean);
     /**
      *
      *  Uploads the file to the DevRev platform. The file is uploaded to the platform
@@ -29,4 +30,5 @@ export declare class Uploader {
     upload(filename: string, entity: string, fetchedObjects: object[] | object, filetype?: string): Promise<UploadResponse>;
     private prepareArtifact;
     private uploadToArtifact;
+    private downloadToLocal;
 }

package/dist/src/uploader/index.js

@@ -1,4 +1,27 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -6,7 +29,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Uploader = void 0;
 const axios_1 = __importDefault(require("axios"));
 const typescript_sdk_1 = require("@devrev/typescript-sdk");
-const helpers_1 = require("../adapter/helpers");
+const fs_1 = __importStar(require("fs"));
+const helpers_1 = require("../common/helpers");
 /**
  * Uploader class is used to upload files to the DevRev platform.
  * The class provides utilities to
@@ -18,14 +42,15 @@ const helpers_1 = require("../adapter/helpers");
  * @constructor
  * @param {string} endpoint - The endpoint of the DevRev platform
  * @param {string} token - The token to authenticate with the DevRev platform
+ * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
  */
 class Uploader {
-    constructor(endpoint, token) {
+    constructor(endpoint, token, local = false) {
         this.betaDevrevSdk = typescript_sdk_1.client.setupBeta({
             endpoint,
             token,
         });
-        this.publicDevrevSdk = typescript_sdk_1.client.setup({ endpoint, token });
+        this.local = local;
     }
     /**
      *
@@ -39,6 +64,9 @@ class Uploader {
      * @returns {Promise<UploadResponse>} - The response object containing the artifact information
      */
     async upload(filename, entity, fetchedObjects, filetype = 'application/jsonl+json') {
+        if (this.local) {
+            this.downloadToLocal(filename, fetchedObjects);
+        }
         const preparedArtifact = await this.prepareArtifact(filename, filetype);
         if (!preparedArtifact) {
             return {
@@ -60,6 +88,7 @@ class Uploader {
             item_type: entity,
             item_count: itemCount,
         };
+        console.log(`Artifact uploaded successfully: ${artifact.id}`);
         return { artifact, error: undefined };
     }
     async prepareArtifact(filename, filetype) {
@@ -71,7 +100,8 @@ class Uploader {
             return response.data;
         }
         catch (error) {
-            throw new Error('Error while fetching upload url: ' + error);
+            console.error('Error while preparing artifact: ' + error);
+            return null;
         }
     }
     async uploadToArtifact(
@@ -91,5 +121,32 @@ class Uploader {
             return null;
         }
     }
+    async downloadToLocal(filePath, fetchedObjects) {
+        console.log(`Uploading ${filePath} to local file system`);
+        try {
+            if (!fs_1.default.existsSync('extracted_files')) {
+                fs_1.default.mkdirSync('extracted_files');
+            }
+            const timestamp = new Date().getTime();
+            const fileHandle = await fs_1.promises.open(`extracted_files/${timestamp}_${filePath}`, 'w');
+            let objArray = [];
+            if (!Array.isArray(fetchedObjects)) {
+                objArray.push(fetchedObjects);
+            }
+            else {
+                objArray = fetchedObjects;
+            }
+            for (const jsonObject of objArray) {
+                const jsonLine = JSON.stringify(jsonObject) + '\n';
+                await fileHandle.write(jsonLine);
+            }
+            await fileHandle.close();
+            console.log('Data successfully written to', filePath);
+        }
+        catch (error) {
+            console.error('Error writing data to file:', error);
+            return Promise.reject(error);
+        }
+    }
 }
 exports.Uploader = Uploader;

package/dist/tests/adapter.helpers.test.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const helpers_1 = require("../src/common/helpers");
+const types_1 = require("../src/types");
+describe('adapter.helpers.ts', () => {
+    it("should create a FormData object with the correct 'key' and 'value'", () => {
+        const preparedArtifact = {
+            form_data: [{ key: 'key', value: 'value' }],
+        };
+        const fetchedObjects = [{ key: 'value' }];
+        const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
+        expect(formData).toBeInstanceOf(FormData);
+        expect(formData.get('key')).toBe('value');
+    });
+    it("should create a FormData object with the correct 'file' key", () => {
+        const preparedArtifact = {
+            form_data: [{ key: 'key', value: 'value' }],
+        };
+        const fetchedObjects = [{ key: 'value' }];
+        const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
+        expect(formData.get('file')).toBeDefined();
+    });
+    it("should create an Artifact object with the correct 'item_count', 'id', and 'item_type'", () => {
+        const preparedArtifact = { id: 'id' };
+        const fetchedObjects = [{ key: 'value' }];
+        const entity = 'entity';
+        const artifact = (0, helpers_1.createArtifact)(preparedArtifact, fetchedObjects, entity);
+        expect(artifact).toEqual({
+            item_count: 1,
+            id: 'id',
+            item_type: 'entity',
+        });
+    });
+    it('should return the correct ExtractorEventType', () => {
+        expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionMetadataStart)).toStrictEqual({
+            eventType: types_1.ExtractorEventType.ExtractionMetadataError,
+            isError: true,
+        });
+        expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionDataStart)).toStrictEqual({
+            eventType: types_1.ExtractorEventType.ExtractionDataProgress,
+            isError: false,
+        });
+        expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionDataContinue)).toStrictEqual({
+            eventType: types_1.ExtractorEventType.ExtractionDataProgress,
+            isError: false,
+        });
+        expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionAttachmentsStart)).toStrictEqual({
+            eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
+            isError: false,
+        });
+        expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionAttachmentsContinue)).toStrictEqual({
+            eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
+            isError: false,
+        });
+        expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionExternalSyncUnitsStart)).toStrictEqual({
+            eventType: types_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
+            isError: true,
+        });
+    });
+});