@devrev/ts-adaas 0.0.1 → 0.0.3

package/dist/src/demo-extractor/index.js

@@ -5,143 +5,149 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DemoExtractor = void 0;
 const types_1 = require("../types");
-const adapter_1 = require("../adapter");
 const uploader_1 = require("../uploader");
-const recipe_json_1 = __importDefault(require("./recipe.json"));
+const external_domain_metadata_json_1 = __importDefault(require("./external_domain_metadata.json"));
 class DemoExtractor {
-    async run(event) {
-        console.log('Event in DemoExtractor run function: ' + JSON.stringify(event));
-        const adapter = new adapter_1.Adapter(event);
-        const uploader = new uploader_1.Uploader(event.execution_metadata.devrev_endpoint, event.context.secrets["service_account_token"]);
-        switch (event.payload.event_type) {
+    constructor(event, adapter) {
+        this.event = event;
+        this.adapter = adapter;
+        this.uploader = new uploader_1.Uploader(this.event.execution_metadata.devrev_endpoint, this.event.context.secrets.service_account_token);
+    }
+    async run() {
+        switch (this.event.payload.event_type) {
             case types_1.EventType.ExtractionExternalSyncUnitsStart: {
                 const externalSyncUnits = [
                     {
                         id: 'devrev',
                         name: 'devrev',
-                        description: 'Loopback for DevRev',
-                        item_count: 0,
+                        description: 'Demo external sync unit',
                     },
                 ];
-                await adapter.emit(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
+                await this.adapter.emit(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
                     external_sync_units: externalSyncUnits,
                 });
                 break;
             }
             case types_1.EventType.ExtractionMetadataStart: {
-                const metadata = [
-                    {
-                        item: 'contacts',
-                        fields: ['name', 'lastName'],
-                    },
-                    {
-                        item: 'users',
-                        fields: ['name', 'lastName'],
-                    },
-                ];
-                const { artifact, error } = await uploader.upload('loopback_metadata_1.jsonl', 'metadata', metadata);
-                if (error) {
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
+                const { artifact, error } = await this.uploader.upload('metadata_1.jsonl', 'external_domain_metadata', external_domain_metadata_json_1.default);
+                if (error || !artifact) {
+                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
                         error,
                     });
+                    return;
                 }
-                else {
-                    await adapter.update({ artifact });
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataDone);
-                }
+                await this.adapter.emit(types_1.ExtractorEventType.ExtractionMetadataDone, {
+                    artifacts: [artifact],
+                });
                 break;
             }
             case types_1.EventType.ExtractionDataStart: {
                 const contacts = [
                     {
-                        name: 'John',
-                        lastName: 'Doe',
+                        id: 'contact-1',
+                        created_date: '1999-12-25T01:00:03+01:00',
+                        modified_date: '1999-12-25T01:00:03+01:00',
+                        data: {
+                            email: 'johnsmith@test.com',
+                            name: 'John Smith',
+                        },
                     },
                     {
-                        name: 'Jane',
-                        lastName: 'Doe',
+                        id: 'contact-2',
+                        created_date: '1999-12-27T15:31:34+01:00',
+                        modified_date: '2002-04-09T01:55:31+02:00',
+                        data: {
+                            email: 'janesmith@test.com',
+                            name: 'Jane Smith',
+                        },
                     },
                 ];
-                const { artifact, error } = await uploader.upload('loopback_contacts_1.json', 'contacts', contacts);
-                if (error) {
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
+                const { artifact, error } = await this.uploader.upload('contacts_1.json', 'contacts', contacts);
+                if (error || !artifact) {
+                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
                         error,
                     });
+                    return;
                 }
-                else {
-                    await adapter.update({ artifact });
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionDataProgress, {
-                        progress: 50,
-                    });
-                }
+                await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataProgress, {
+                    progress: 50,
+                    artifacts: [artifact],
+                });
                 break;
             }
             case types_1.EventType.ExtractionDataContinue: {
                 const users = [
                     {
-                        name: 'John',
-                        lastName: 'Phd',
+                        id: 'user-1',
+                        created_date: '1999-12-25T01:00:03+01:00',
+                        modified_date: '1999-12-25T01:00:03+01:00',
+                        data: {
+                            email: 'johndoe@test.com',
+                            name: 'John Doe',
+                        },
                     },
                     {
-                        name: 'Jane',
-                        lastName: 'Phd',
+                        id: 'user-2',
+                        created_date: '1999-12-27T15:31:34+01:00',
+                        modified_date: '2002-04-09T01:55:31+02:00',
+                        data: {
+                            email: 'janedoe@test.com',
+                            name: 'Jane Doe',
+                        },
                     },
                 ];
-                const { artifact, error } = await uploader.upload('loopback_users_1.json', 'users', users);
-                if (error) {
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
+                const { artifact, error } = await this.uploader.upload('users_1.json', 'users', users);
+                if (error || !artifact) {
+                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
                         error,
                     });
+                    return;
                 }
-                else {
-                    await adapter.update({ artifact });
-                    // TODO: Add separated function for uploading recipe.json?
-                    const { artifact: recipe, error } = await uploader.upload('recipe.json', 'initial_domain_mapping', recipe_json_1.default);
-                    await adapter.update({ artifact: recipe });
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone, {
-                        progress: 100,
-                    });
-                }
+                await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataDone, {
+                    progress: 100,
+                    artifacts: [artifact],
+                });
                 break;
             }
             case types_1.EventType.ExtractionDataDelete: {
-                await adapter.emit(types_1.ExtractorEventType.ExtractionDataDeleteDone);
+                await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataDeleteDone);
                 break;
             }
             case types_1.EventType.ExtractionAttachmentsStart: {
                 const attachment1 = ['This is attachment1.txt content'];
-                const { artifact, error } = await uploader.upload('attachment1.txt', 'attachment', attachment1);
-                if (error) {
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
+                const { artifact, error } = await this.uploader.upload('attachment1.txt', 'attachment', attachment1);
+                if (error || !artifact) {
+                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
                         error,
                     });
+                    return;
                 }
-                else {
-                    await adapter.update({ artifact });
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsProgress);
-                }
+                await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsProgress, {
+                    artifacts: [artifact],
+                });
                 break;
             }
             case types_1.EventType.ExtractionAttachmentsContinue: {
                 const attachment2 = ['This is attachment2.txt content'];
-                const { artifact, error } = await uploader.upload('attachment2.txt', 'attachment', attachment2);
-                if (error) {
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
+                const { artifact, error } = await this.uploader.upload('attachment2.txt', 'attachment', attachment2);
+                if (error || !artifact) {
+                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
                         error,
                     });
+                    return;
                 }
-                else {
-                    await adapter.update({ artifact });
-                    await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDone);
-                }
+                await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDone, {
+                    artifacts: [artifact],
+                });
                 break;
             }
             case types_1.EventType.ExtractionAttachmentsDelete: {
-                await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
+                await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
                 break;
             }
             default: {
-                console.log('Event in DemoExtractor run not recognized: ' + JSON.stringify(event));
+                console.error('Event in DemoExtractor run not recognized: ' +
+                    JSON.stringify(this.event.payload.event_type));
             }
         }
     }

package/dist/src/http/client.d.ts

@@ -1,6 +1,6 @@
 import { HTTPResponse } from './types';
 export declare const defaultResponse: HTTPResponse;
-export default class HTTPClient {
+export declare class HTTPClient {
     private retryAfter;
     private retryAt;
     private axiosInstance;

package/dist/src/http/client.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.defaultResponse = void 0;
+exports.HTTPClient = exports.defaultResponse = void 0;
 const axios_1 = __importStar(require("axios"));
 const constants_1 = require("./constants");
 exports.defaultResponse = {
@@ -144,4 +144,4 @@ class HTTPClient {
         }
     }
 }
-exports.default = HTTPClient;
+exports.HTTPClient = HTTPClient;

package/dist/src/http/index.d.ts

@@ -1,3 +1,2 @@
 export * from './client';
 export * from './types';
-export * from './constants';

package/dist/src/http/index.js

@@ -16,4 +16,3 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./client"), exports);
 __exportStar(require("./types"), exports);
-__exportStar(require("./constants"), exports);

package/dist/src/index.d.ts

@@ -2,3 +2,5 @@ export * from './adapter';
 export * from './demo-extractor';
 export * from './uploader';
 export * from './types';
+export * from './http';
+export * from './common/install-initial-domain-mapping';

package/dist/src/index.js

@@ -18,3 +18,5 @@ __exportStar(require("./adapter"), exports);
 __exportStar(require("./demo-extractor"), exports);
 __exportStar(require("./uploader"), exports);
 __exportStar(require("./types"), exports);
+__exportStar(require("./http"), exports);
+__exportStar(require("./common/install-initial-domain-mapping"), exports);

package/dist/src/logging/index.d.ts

@@ -1,8 +1,8 @@
-declare enum LogLevel {
+import { AirdropEvent } from '../types';
+export declare enum LogLevel {
     INFO = "info",
     WARN = "warn",
-    ERROR = "error",
-    DEBUG = "debug"
+    ERROR = "error"
 }
 /**
  * Logger class to log messages based on the log level.
@@ -10,22 +10,9 @@ declare enum LogLevel {
  * - INFO
  * - WARN
  * - ERROR
- * - DEBUG
  *
  * The log tags can be set to any key-value pair.
  */
-declare class Logger {
-    private level;
-    private logTags;
-    private static instance;
-    private constructor();
-    private log;
-    static getInstance(level: LogLevel, logTags?: Record<string, any>): Logger;
-    setTags(tags: Record<string, any>): void;
-    addTags(tags: Record<string, any>): void;
-    info(message: string): void;
-    warn(message: string): void;
-    error(message: string): void;
-    debug(message: string): void;
+export declare class Logger {
+    static init(event: AirdropEvent): void;
 }
-export default Logger;

package/dist/src/logging/index.js

@@ -1,60 +1,39 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.Logger = exports.LogLevel = void 0;
 var LogLevel;
 (function (LogLevel) {
     LogLevel["INFO"] = "info";
     LogLevel["WARN"] = "warn";
     LogLevel["ERROR"] = "error";
-    LogLevel["DEBUG"] = "debug";
-})(LogLevel || (LogLevel = {}));
+})(LogLevel || (exports.LogLevel = LogLevel = {}));
 /**
  * Logger class to log messages based on the log level.
  * The log level can be set to one of the following:
  * - INFO
  * - WARN
  * - ERROR
- * - DEBUG
  *
  * The log tags can be set to any key-value pair.
  */
 class Logger {
-    // Private constructor to prevent instantiation
-    constructor(level, logTags = {}) {
-        this.logTags = {};
-        this.level = level;
-        this.logTags = logTags;
-    }
-    log(level, message) {
-        if (this.level === level) {
-            console.log(`[${level.toUpperCase()}]: ${message}`, this.logTags);
-        }
-    }
-    // Singleton instance
-    static getInstance(level, logTags = {}) {
-        if (!Logger.instance) {
-            Logger.instance = new Logger(level, logTags);
-        }
-        return Logger.instance;
-    }
-    // Set log tags destructively.
-    setTags(tags) {
-        this.logTags = tags;
-    }
-    // Add log tags. If the key already exists, it will be overwritten.
-    addTags(tags) {
-        this.logTags = Object.assign(Object.assign({}, this.logTags), tags);
-    }
-    info(message) {
-        this.log(LogLevel.INFO, message);
-    }
-    warn(message) {
-        this.log(LogLevel.WARN, message);
-    }
-    error(message) {
-        this.log(LogLevel.ERROR, message);
-    }
-    debug(message) {
-        this.log(LogLevel.DEBUG, message);
+    static init(event) {
+        const origLog = console.log;
+        console.log = (message) => {
+            origLog(`[${LogLevel.INFO.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
+        };
+        const origInfo = console.info;
+        console.info = (message) => {
+            origInfo(`[${LogLevel.INFO.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
+        };
+        const origWarn = console.warn;
+        console.warn = (message) => {
+            origWarn(`[${LogLevel.WARN.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
+        };
+        const origError = console.error;
+        console.error = (message) => {
+            origError(`[${LogLevel.ERROR.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
+        };
     }
 }
-exports.default = Logger;
+exports.Logger = Logger;

package/dist/src/state/index.d.ts

@@ -0,0 +1,23 @@
+import { AdapterState, AirdropEvent } from '../types';
+export declare function createAdapterState<ExtractorState>(event: AirdropEvent, initialState: ExtractorState): Promise<State<ExtractorState>>;
+export declare class State<ExtractorState> {
+    private _state;
+    private event;
+    private workerUrl;
+    private devrevToken;
+    constructor(event: AirdropEvent, initialState: ExtractorState);
+    get state(): AdapterState<ExtractorState>;
+    set state(value: AdapterState<ExtractorState>);
+    /**
+     *  Updates the state of the adapter.
+     *
+     * @param {object} state - The state to be updated
+     */
+    postState(state: AdapterState<ExtractorState>): Promise<void>;
+    /**
+     *  Fetches the state of the adapter.
+     *
+     * @return  The state of the adapter
+     */
+    fetchState(initialState: ExtractorState): Promise<AdapterState<ExtractorState> | unknown>;
+}

package/dist/src/state/index.js

@@ -0,0 +1,111 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.State = exports.createAdapterState = void 0;
+const axios_1 = __importStar(require("axios"));
+const constants_1 = require("../common/constants");
+async function createAdapterState(event, initialState) {
+    const newInitialState = structuredClone(initialState);
+    const as = new State(event, newInitialState);
+    if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
+        console.log('Fetching state');
+        await as.fetchState(newInitialState);
+    }
+    return as;
+}
+exports.createAdapterState = createAdapterState;
+class State {
+    constructor(event, initialState) {
+        this._state = Object.assign({ lastSyncStarted: '', lastSuccessfulSyncStarted: '' }, initialState);
+        this.event = event;
+        this.workerUrl = event.payload.event_context.worker_data_url;
+        this.devrevToken = event.context.secrets.service_account_token;
+    }
+    get state() {
+        return this._state;
+    }
+    set state(value) {
+        this._state = value;
+    }
+    /**
+     *  Updates the state of the adapter.
+     *
+     * @param {object} state - The state to be updated
+     */
+    async postState(state) {
+        try {
+            await axios_1.default.post(this.workerUrl + '.update', {
+                state: JSON.stringify(state),
+            }, {
+                headers: {
+                    Authorization: this.devrevToken,
+                },
+                params: {
+                    sync_unit: this.event.payload.event_context.sync_unit_id,
+                },
+            });
+            this.state = state;
+            console.log('State updated successfully');
+        }
+        catch (error) {
+            console.error('Failed to update state, error:' + error);
+        }
+    }
+    /**
+     *  Fetches the state of the adapter.
+     *
+     * @return  The state of the adapter
+     */
+    async fetchState(initialState) {
+        var _a;
+        const state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: '', lastSuccessfulSyncStarted: '' });
+        console.log('Fetching state with sync unit id: ' +
+            this.event.payload.event_context.sync_unit_id);
+        try {
+            const response = await axios_1.default.post(this.workerUrl + '.get', {}, {
+                headers: {
+                    Authorization: this.devrevToken,
+                },
+                params: {
+                    sync_unit: this.event.payload.event_context.sync_unit_id,
+                },
+            });
+            this.state = JSON.parse(response.data.state);
+            console.log('State fetched successfully');
+            return this.state;
+        }
+        catch (error) {
+            if ((0, axios_1.isAxiosError)(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
+                console.log('State not found, returning initial state');
+                this.state = state;
+                this.postState(this.state);
+                return this.state;
+            }
+            console.error('Failed to fetch state, error:' + error);
+            return error;
+        }
+    }
+}
+exports.State = State;

package/dist/src/types/common.d.ts

@@ -31,3 +31,7 @@ export interface UploadResponse {
     artifact?: Artifact;
     error?: ErrorRecord;
 }
+export interface InitialDomainMapping {
+    starting_recipe_blueprint?: object;
+    additional_mappings?: object;
+}

package/dist/src/types/extraction.d.ts

@@ -1,4 +1,4 @@
-import { InputData, Context, ExecutionMetadata } from '@devrev/typescript-sdk/dist/snap-ins';
+import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
 import { Artifact, ErrorRecord } from './common';
 export declare enum EventType {
     ExtractionExternalSyncUnitsStart = "EXTRACTION_EXTERNAL_SYNC_UNITS_START",
@@ -43,7 +43,7 @@ export interface ExternalSyncUnit {
     id: string;
     name: string;
     description: string;
-    item_count: number;
+    item_count?: number;
 }
 export interface EventContextIn {
     mode: string;
@@ -55,6 +55,7 @@ export interface EventContextIn {
     sync_run_id: string;
     external_system_id: string;
     uuid: string;
+    worker_data_url: string;
 }
 export interface EventContextOut {
     uuid: string;
@@ -85,9 +86,16 @@ export interface DomainObjectState {
     count: number;
 }
 export interface AirdropEvent {
-    context: Context;
+    context: {
+        secrets: {
+            service_account_token: string;
+        };
+        snap_in_version_id: string;
+    };
     payload: AirdropMessage;
-    execution_metadata: ExecutionMetadata;
+    execution_metadata: {
+        devrev_endpoint: string;
+    };
     input_data: InputData;
 }
 export interface AirdropMessage {
@@ -100,6 +108,10 @@ export interface AirdropMessage {
 export interface ExtractorEvent {
     event_type: string;
     event_context: EventContextOut;
-    extractor_state: string;
+    extractor_state?: string;
     event_data?: EventData;
 }
+export type AdapterState<ExtractorState> = ExtractorState & {
+    lastSyncStarted?: string;
+    lastSuccessfulSyncStarted?: string;
+};

package/dist/src/uploader/index.d.ts

@@ -10,11 +10,12 @@ import { UploadResponse } from '../types/common';
  * @constructor
  * @param {string} endpoint - The endpoint of the DevRev platform
  * @param {string} token - The token to authenticate with the DevRev platform
+ * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
  */
 export declare class Uploader {
     private betaDevrevSdk;
-    private publicDevrevSdk;
-    constructor(endpoint: string, token: string);
+    private local;
+    constructor(endpoint: string, token: string, local?: boolean);
     /**
      *
      *  Uploads the file to the DevRev platform. The file is uploaded to the platform
@@ -29,4 +30,5 @@ export declare class Uploader {
     upload(filename: string, entity: string, fetchedObjects: object[] | object, filetype?: string): Promise<UploadResponse>;
     private prepareArtifact;
     private uploadToArtifact;
+    private downloadToLocal;
 }