@devrev/ts-adaas 0.0.2 → 1.0.0

package/dist/workers/default-workers/metadata-extraction.js

@@ -0,0 +1,26 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+const external_domain_metadata_json_1 = __importDefault(require("../dummy-extractor/external_domain_metadata.json"));
+const repos = [
+    {
+        itemType: 'external_domain_metadata',
+    },
+];
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        var _a;
+        adapter.initializeRepos(repos);
+        await ((_a = adapter
+            .getRepo('external_domain_metadata')) === null || _a === void 0 ? void 0 : _a.push([external_domain_metadata_json_1.default]));
+        await adapter.emit(index_1.ExtractorEventType.ExtractionMetadataDone);
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionMetadataError, {
+            error: { message: 'Failed to extract metadata. Lambda timeout.' },
+        });
+    },
+});

package/dist/workers/dummy-extractor/data-normalization.d.ts

@@ -0,0 +1,4 @@
+import { NormalizedItem, NormalizedAttachment } from '../../index';
+export declare function normalizeIssue(item: any): NormalizedItem;
+export declare function normalizeUser(item: any): NormalizedItem;
+export declare function normalizeAttachment(item: any): NormalizedAttachment;

package/dist/workers/dummy-extractor/data-normalization.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeIssue = normalizeIssue;
+exports.normalizeUser = normalizeUser;
+exports.normalizeAttachment = normalizeAttachment;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function normalizeIssue(item) {
+    return {
+        id: item.id,
+        created_date: item.created_date,
+        modified_date: item.modified_date,
+        data: {
+            body: item.body,
+            creator: item.creator,
+            owner: item.owner,
+            title: item.title,
+        },
+    };
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function normalizeUser(item) {
+    return {
+        id: item.id,
+        created_date: item.created_date,
+        modified_date: item.modified_date,
+        data: {
+            email: item.email,
+            name: item.name,
+        },
+    };
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function normalizeAttachment(item) {
+    return {
+        url: item.url,
+        id: item.id,
+        file_name: item.file_name,
+        author_id: item.author_id,
+        parent_id: item.parent_id,
+    };
+}

package/dist/workers/dummy-extractor/external_domain_metadata.json

@@ -0,0 +1,58 @@
+{
+    "record_types": {
+        "issues": {
+            "name": "Issues",
+            "fields": {
+                "title": {
+                    "is_required": true,
+                    "type": "text",
+                    "name": "Title",
+                    "text": {
+                        "min_length": 1
+                    }
+                },
+                "body": {
+                    "type": "rich_text",
+                    "name": "body",
+                    "is_required": true
+                },
+                "owner": {
+                    "is_required": true,
+                    "type": "reference",
+                    "reference": {
+                        "refers_to": {
+                            "#record:users": {}
+                        }
+                    }
+                },
+                "creator": {
+                    "is_required": true,
+                    "type": "reference",
+                    "reference": {
+                        "refers_to": {
+                            "#record:users": {}
+                        }
+                    }
+                }
+            }
+        },
+        "users": {
+            "name": "Users",
+            "fields": {
+                "name": {
+                    "is_required": true,
+                    "type": "text",
+                    "name": "Name",
+                    "text": {
+                        "min_length": 1
+                    }
+                },
+                "email": {
+                    "type": "text",
+                    "name": "Email",
+                    "is_required": true
+                }
+            }
+        }
+    }
+}

package/dist/workers/process-task.d.ts

@@ -0,0 +1,2 @@
+import { ProcessTaskInterface } from '../types/workers';
+export declare function processTask<ConnectorState>({ task, onTimeout, }: ProcessTaskInterface<ConnectorState>): void;

package/dist/workers/process-task.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processTask = processTask;
+const node_worker_threads_1 = require("node:worker_threads");
+const state_1 = require("../state/state");
+const worker_adapter_1 = require("./worker-adapter");
+const workers_1 = require("../types/workers");
+const logger_1 = require("../logger/logger");
+function processTask({ task, onTimeout, }) {
+    if (!node_worker_threads_1.isMainThread) {
+        (async () => {
+            const event = node_worker_threads_1.workerData.event;
+            const initialState = node_worker_threads_1.workerData.initialState;
+            const options = node_worker_threads_1.workerData.options;
+            console = new logger_1.Logger(event);
+            const adapterState = await (0, state_1.createAdapterState)({
+                event,
+                initialState,
+                options,
+            });
+            if (node_worker_threads_1.parentPort && node_worker_threads_1.workerData.event) {
+                const adapter = new worker_adapter_1.WorkerAdapter({
+                    event,
+                    adapterState,
+                    parentPort: node_worker_threads_1.parentPort,
+                    options,
+                });
+                node_worker_threads_1.parentPort.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {
+                    if (message.subject === workers_1.WorkerMessageSubject.WorkerMessageExit) {
+                        adapter.handleTimeout();
+                        await onTimeout({ adapter });
+                        process.exit(0);
+                    }
+                });
+                await task({ adapter });
+                const message = {
+                    subject: workers_1.WorkerMessageSubject.WorkerMessageDone,
+                };
+                node_worker_threads_1.parentPort.postMessage(message);
+                process.exit(0);
+            }
+        })();
+    }
+}

package/dist/workers/spawn.d.ts

@@ -0,0 +1,25 @@
+import { SpawnFactoryInterface, SpawnInterface } from '../types/workers';
+/**
+ * Creates a new instance of Spawn class.
+ * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
+ * The class provides utilities to emit control events to the platform and exit the worker gracefully.
+ * In case of lambda timeout, the class emits a lambda timeout event to the platform.
+ * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object} initialState - The initial state of the adapter
+ * @param {string} workerPath - The path to the worker file
+ * @returns {Promise<Spawn>} - A new instance of Spawn class
+ */
+export declare function spawn<ConnectorState>({ event, initialState, workerPath, options, }: SpawnFactoryInterface<ConnectorState>): Promise<boolean | PromiseLike<boolean>>;
+export declare class Spawn {
+    private event;
+    private hasWorkerEmitted;
+    private defaultLambdaTimeout;
+    private lambdaTimeout;
+    private worker;
+    private resolve;
+    private timer;
+    private logger;
+    constructor({ event, worker, options, resolve }: SpawnInterface);
+    private exitFromMainThread;
+}

package/dist/workers/spawn.js

@@ -0,0 +1,163 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Spawn = void 0;
+exports.spawn = spawn;
+const extraction_1 = require("../types/extraction");
+const control_protocol_1 = require("../common/control-protocol");
+const helpers_1 = require("../common/helpers");
+const logger_1 = require("../logger/logger");
+const constants_1 = require("../common/constants");
+const workers_1 = require("../types/workers");
+const create_worker_1 = require("./create-worker");
+function getWorkerPath({ event, connectorWorkerPath, }) {
+    const logger = new logger_1.Logger(event);
+    if (!constants_1.ALLOWED_EVENT_TYPES.includes(event.payload.event_type)) {
+        return null;
+    }
+    if (connectorWorkerPath)
+        return connectorWorkerPath;
+    let path = null;
+    switch (event.payload.event_type) {
+        case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
+            path = __dirname + '/default-workers/external-sync-units-extraction';
+            break;
+        case extraction_1.EventType.ExtractionMetadataStart:
+            path = __dirname + '/default-workers/metadata-extraction';
+            break;
+        case extraction_1.EventType.ExtractionDataStart:
+        case extraction_1.EventType.ExtractionDataContinue:
+            path = __dirname + '/default-workers/data-extraction';
+            break;
+        case extraction_1.EventType.ExtractionAttachmentsStart:
+        case extraction_1.EventType.ExtractionAttachmentsContinue:
+            path = __dirname + '/default-workers/attachments-extraction';
+            break;
+        case extraction_1.EventType.ExtractionDataDelete:
+            path = __dirname + '/default-workers/data-deletion.js';
+            break;
+        case extraction_1.EventType.ExtractionAttachmentsDelete:
+            path = __dirname + '/default-workers/attachments-deletion';
+            break;
+        default:
+            logger.error('Worker script not found for event type: ' +
+                event.payload.event_type +
+                '.');
+            path = null;
+    }
+    return path;
+}
+/**
+ * Creates a new instance of Spawn class.
+ * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
+ * The class provides utilities to emit control events to the platform and exit the worker gracefully.
+ * In case of lambda timeout, the class emits a lambda timeout event to the platform.
+ * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object} initialState - The initial state of the adapter
+ * @param {string} workerPath - The path to the worker file
+ * @returns {Promise<Spawn>} - A new instance of Spawn class
+ */
+async function spawn({ event, initialState, workerPath, options, }) {
+    const logger = new logger_1.Logger(event);
+    const script = getWorkerPath({
+        event,
+        connectorWorkerPath: workerPath,
+    });
+    if (script) {
+        try {
+            const worker = await (0, create_worker_1.createWorker)({
+                event,
+                initialState,
+                workerPath: script,
+                options,
+            });
+            return new Promise((resolve) => {
+                new Spawn({
+                    event,
+                    worker,
+                    options: options || null,
+                    resolve,
+                });
+            });
+        }
+        catch (error) {
+            logger.error('Worker error while processing task.', error);
+            return false;
+        }
+    }
+    else {
+        throw new Error('Worker script not found.');
+    }
+}
+class Spawn {
+    constructor({ event, worker, options, resolve }) {
+        this.defaultLambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
+        this.hasWorkerEmitted = false;
+        this.event = event;
+        this.lambdaTimeout = (options === null || options === void 0 ? void 0 : options.timeout)
+            ? Math.min(options.timeout, this.defaultLambdaTimeout)
+            : this.defaultLambdaTimeout;
+        this.resolve = resolve;
+        this.timer = setTimeout(async () => {
+            this.logger.log('Lambda timeout reached. Exiting.');
+            if (this.worker) {
+                this.worker.postMessage({
+                    subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
+                });
+            }
+            else {
+                await this.exitFromMainThread();
+            }
+        }, this.lambdaTimeout);
+        this.logger = new logger_1.Logger(event);
+        this.worker = worker;
+        worker.on(workers_1.WorkerEvent.WorkerExit, async (code) => {
+            this.logger.info('Worker exited with exit code: ' + code + '.');
+            if (this.timer) {
+                clearTimeout(this.timer);
+                await this.exitFromMainThread();
+            }
+        });
+        worker.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
+                this.logger.info('Worker has emitted message to ADaaS.');
+                this.hasWorkerEmitted = true;
+            }
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageDone) {
+                this.logger.info('Worker has completed work.');
+                clearTimeout(this.timer);
+                await this.exitFromMainThread();
+            }
+        });
+        worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
+            var _a, _b;
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageLog) {
+                const args = (_a = message.payload) === null || _a === void 0 ? void 0 : _a.args;
+                const level = (_b = message.payload) === null || _b === void 0 ? void 0 : _b.level;
+                this.logger.logFn(args, level);
+            }
+        });
+    }
+    async exitFromMainThread() {
+        if (this.hasWorkerEmitted) {
+            return;
+        }
+        const timeoutEventType = (0, helpers_1.getErrorExtractorEventType)(this.event.payload.event_type);
+        if (timeoutEventType !== null) {
+            const { eventType } = timeoutEventType;
+            await (0, control_protocol_1.emit)({
+                eventType,
+                event: this.event,
+                data: {
+                    error: {
+                        message: 'Worker has not emitted anything. Exited.',
+                    },
+                },
+            }).then(() => {
+                this.logger.error('Worker has not emitted anything. Exited.');
+                this.resolve(true);
+            });
+        }
+    }
+}
+exports.Spawn = Spawn;

package/dist/workers/worker-adapter.d.ts

@@ -0,0 +1,48 @@
+import { AirdropEvent, ExtractorEventType, EventData } from '../types/extraction';
+import { AdapterState } from '../state/state.interfaces';
+import { Artifact } from '../uploader/uploader.interfaces';
+import { WorkerAdapterInterface, WorkerAdapterOptions } from '../types/workers';
+import { Repo } from '../repo/repo';
+import { RepoInterface } from '../repo/repo.interfaces';
+export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
+/**
+ * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
+ * as parameter in processTask and onTimeout functions. The class provides
+ * utilities to emit control events to the platform, update the state of the connector,
+ * and upload artifacts to the platform.
+ * @class WorkerAdapter
+ * @constructor
+ * @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object=} initialState - The initial state of the adapter
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
+ * @param {string} workerPath - The path to the worker file
+ *
+ */
+export declare class WorkerAdapter<ConnectorState> {
+    readonly event: AirdropEvent;
+    readonly options: WorkerAdapterOptions;
+    private adapterState;
+    private _artifacts;
+    private hasWorkerEmitted;
+    private parentPort;
+    private isTimeout;
+    private repos;
+    constructor({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>);
+    get state(): AdapterState<ConnectorState>;
+    set state(value: AdapterState<ConnectorState>);
+    initializeRepos(repos: RepoInterface[]): void;
+    getRepo(itemType: string): Repo | undefined;
+    postState(): Promise<void>;
+    get artifacts(): Artifact[];
+    set artifacts(artifacts: Artifact[]);
+    /**
+     *  Emits an event to the platform.
+     *
+     * @param {ExtractorEventType} newEventType - The event type to be emitted
+     * @param {EventData=} data - The data to be sent with the event
+     */
+    emit(newEventType: ExtractorEventType, data?: EventData): Promise<void>;
+    uploadAllRepos(): Promise<void>;
+    handleTimeout(): void;
+}

package/dist/workers/worker-adapter.js

@@ -0,0 +1,138 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerAdapter = void 0;
+exports.createWorkerAdapter = createWorkerAdapter;
+const extraction_1 = require("../types/extraction");
+const constants_1 = require("../common/constants");
+const control_protocol_1 = require("../common/control-protocol");
+const workers_1 = require("../types/workers");
+const repo_1 = require("../repo/repo");
+function createWorkerAdapter({ event, adapterState, parentPort, options, }) {
+    return new WorkerAdapter({
+        event,
+        adapterState,
+        parentPort,
+        options,
+    });
+}
+/**
+ * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
+ * as parameter in processTask and onTimeout functions. The class provides
+ * utilities to emit control events to the platform, update the state of the connector,
+ * and upload artifacts to the platform.
+ * @class WorkerAdapter
+ * @constructor
+ * @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object=} initialState - The initial state of the adapter
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
+ * @param {string} workerPath - The path to the worker file
+ *
+ */
+class WorkerAdapter {
+    constructor({ event, adapterState, parentPort, options, }) {
+        this.repos = [];
+        this.event = event;
+        this.options = options;
+        this.adapterState = adapterState;
+        this._artifacts = [];
+        this.parentPort = parentPort;
+        this.hasWorkerEmitted = false;
+        this.isTimeout = false;
+    }
+    get state() {
+        return this.adapterState.state;
+    }
+    set state(value) {
+        if (!this.isTimeout) {
+            this.adapterState.state = value;
+        }
+    }
+    initializeRepos(repos) {
+        this.repos = repos.map((repo) => {
+            const shouldNormalize = !Object.values(constants_1.AIRDROP_DEFAULT_ITEM_TYPES).includes(repo.itemType);
+            return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
+                    var _a;
+                    this.artifacts.push(artifact);
+                    // We need to store artifacts ids in state for later use when streaming attachments
+                    if (repo.itemType === constants_1.AIRDROP_DEFAULT_ITEM_TYPES.ATTACHMENTS) {
+                        (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
+                    }
+                }, options: this.options }));
+        });
+    }
+    getRepo(itemType) {
+        const repo = this.repos.find((repo) => repo.itemType === itemType);
+        if (!repo) {
+            console.error(`Repo not found for item type: ${itemType}.`);
+            return;
+        }
+        return repo;
+    }
+    async postState() {
+        await this.adapterState.postState();
+    }
+    get artifacts() {
+        return this._artifacts;
+    }
+    set artifacts(artifacts) {
+        this._artifacts = this._artifacts
+            .concat(artifacts)
+            .filter((value, index, self) => self.indexOf(value) === index);
+    }
+    /**
+     *  Emits an event to the platform.
+     *
+     * @param {ExtractorEventType} newEventType - The event type to be emitted
+     * @param {EventData=} data - The data to be sent with the event
+     */
+    async emit(newEventType, data) {
+        if (this.hasWorkerEmitted) {
+            console.warn('Unable to emit with event type:' +
+                newEventType +
+                '. Lambda is going to be stopped.');
+            return;
+        }
+        // We want to upload all the repos before emitting the event, except for the external sync units done event
+        if (newEventType !== extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone) {
+            await this.uploadAllRepos();
+        }
+        // If the extraction is done, we want to save the timestamp of the last successful sync
+        if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
+            this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
+        }
+        // We want to save the state every time we emit an event, except for the start and delete events
+        if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
+            console.log(`Saving state before emitting event with event type: ` +
+                newEventType +
+                '.');
+            await this.adapterState.postState(this.state);
+        }
+        try {
+            await (0, control_protocol_1.emit)({
+                eventType: newEventType,
+                event: this.event,
+                data: Object.assign(Object.assign({}, data), { artifacts: this.artifacts }),
+            });
+            const message = {
+                subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
+                payload: { eventType: newEventType },
+            };
+            this.artifacts = [];
+            this.parentPort.postMessage(message);
+        }
+        catch (error) {
+            console.error('Error while emitting event with event type: ' + newEventType + '.', error);
+            this.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+        }
+    }
+    async uploadAllRepos() {
+        for (const repo of this.repos) {
+            await repo.upload();
+        }
+    }
+    handleTimeout() {
+        this.isTimeout = true;
+    }
+}
+exports.WorkerAdapter = WorkerAdapter;

package/dist/workers/worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/worker.js

@@ -0,0 +1,6 @@
+"use strict";
+//eslint-disable-next-line @typescript-eslint/no-var-requires
+const { workerData } = require('worker_threads');
+//eslint-disable-next-line @typescript-eslint/no-var-requires
+require('ts-node').register();
+require(workerData.workerPath);

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "0.0.2",
+  "version": "1.0.0",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
-  "main": "./dist/src/index.js",
-  "typings": "./dist/src/index.d.ts",
+  "main": "./dist/index.js",
+  "typings": "./dist/index.d.ts",
   "scripts": {
-    "build": "tsc",
+    "build": "tsc -p ./tsconfig.json",
     "start": "ts-node src/index.ts",
     "lint": "eslint .",
     "lint:fix": "eslint .  --fix",
@@ -17,13 +17,15 @@
     "url": "git+https://github.com/devrev/adaas-sdk.git"
   },
   "publishConfig": {
-    "registry": "https://registry.npmjs.org"
+    "registry": "https://npm.pkg.github.com"
   },
   "keywords": [],
   "author": "devrev",
   "license": "ISC",
   "devDependencies": {
     "@types/jest": "^29.5.12",
+    "@types/lambda-log": "^3.0.3",
+    "@types/node": "20.16.11",
     "@typescript-eslint/eslint-plugin": "^7.12.0",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
@@ -36,7 +38,9 @@
   "dependencies": {
     "@devrev/typescript-sdk": "^1.1.27",
     "axios": "^1.5.1",
+    "form-data": "^4.0.1",
     "js-jsonl": "^1.1.1",
+    "lambda-log": "^3.1.0",
     "ts-node": "^10.9.2"
   },
   "files": [

package/dist/src/common/constants.d.ts

@@ -1,2 +0,0 @@
-import { EventType } from '../types';
-export declare const STATELESS_EVENT_TYPES: EventType[];

package/dist/src/common/constants.js

@@ -1,10 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.STATELESS_EVENT_TYPES = void 0;
-const types_1 = require("../types");
-exports.STATELESS_EVENT_TYPES = [
-    types_1.EventType.ExtractionExternalSyncUnitsStart,
-    types_1.EventType.ExtractionMetadataStart,
-    types_1.EventType.ExtractionDataDelete,
-    types_1.EventType.ExtractionAttachmentsDelete,
-];