@devrev/ts-adaas 0.0.3 → 1.0.0

package/dist/workers/create-worker.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createWorker = createWorker;
+const node_worker_threads_1 = require("node:worker_threads");
+const workers_1 = require("../types/workers");
+const logger_1 = require("../logger/logger");
+async function createWorker(workerData) {
+    return new Promise((resolve, reject) => {
+        if (node_worker_threads_1.isMainThread) {
+            const logger = new logger_1.Logger(workerData.event);
+            const workerFile = __dirname + '/worker.js';
+            const worker = new node_worker_threads_1.Worker(workerFile, {
+                workerData,
+            });
+            worker.on(workers_1.WorkerEvent.WorkerError, (error) => {
+                logger.error('Worker error', error);
+                reject(error);
+            });
+            worker.on(workers_1.WorkerEvent.WorkerOnline, () => {
+                resolve(worker);
+                logger.info('Worker is online. Started processing the task with event type: ' +
+                    workerData.event.payload.event_type +
+                    '.');
+            });
+        }
+        else {
+            reject(new Error('Worker threads can not start more worker threads.'));
+        }
+    });
+}

package/dist/workers/create-worker.test.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_threads_1 = require("worker_threads");
+const test_helpers_1 = require("../tests/test-helpers");
+const extraction_1 = require("../types/extraction");
+const create_worker_1 = require("./create-worker");
+describe('createWorker function', () => {
+    it('should return a Worker instance when a valid worker script is found', async () => {
+        const workerPath = __dirname + '../tests/dummy-worker.ts';
+        const worker = worker_threads_1.isMainThread
+            ? await (0, create_worker_1.createWorker)({
+                event: (0, test_helpers_1.createEvent)({
+                    eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
+                }),
+                initialState: {},
+                workerPath,
+            })
+            : null;
+        expect(worker).not.toBeNull();
+        expect(worker).toBeInstanceOf(worker_threads_1.Worker);
+        if (worker) {
+            await worker.terminate();
+        }
+    });
+});

package/dist/workers/default-workers/attachments-deletion.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/attachments-deletion.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDeleteError, {
+            error: { message: 'Failed to delete attachments. Lambda timeout.' },
+        });
+    },
+});

package/dist/workers/default-workers/attachments-extraction.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/attachments-extraction.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+const uploader_1 = require("../../uploader/uploader");
+const repos = [
+    {
+        itemType: 'ssor_attachment',
+    },
+];
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        var _a, _b, _c, _d, _e;
+        if (!((_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds) ||
+            adapter.state.toDevRev.attachmentsMetadata.artifactIds.length === 0) {
+            console.log('No attachments to extract, skipping.');
+            await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
+            return;
+        }
+        adapter.initializeRepos(repos);
+        const uploader = new uploader_1.Uploader({
+            event: adapter.event,
+            options: adapter.options,
+        });
+        for (const attachmentsMetadataArtifactId of (_b = adapter.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds) {
+            const { ssorAttachments, error } = await uploader.streamAttachments({
+                attachmentsMetadataArtifactId,
+            });
+            if (error || !ssorAttachments) {
+                await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsError, {
+                    error,
+                });
+                return;
+            }
+            await ((_c = adapter.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push(ssorAttachments));
+            (_d = adapter.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata.artifactIds.shift();
+            adapter.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
+            if (((_e = adapter.state.toDevRev) === null || _e === void 0 ? void 0 : _e.attachmentsMetadata.artifactIds.length) === 0) {
+                break;
+            }
+        }
+        await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDone);
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.postState();
+        await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsProgress, {
+            progress: 50,
+        });
+    },
+});

package/dist/workers/default-workers/data-deletion.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/data-deletion.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionDataDeleteDone);
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionDataDeleteError, {
+            error: {
+                message: 'Failed to delete data. Lambda timeout.',
+            },
+        });
+    },
+});

package/dist/workers/default-workers/data-extraction.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/data-extraction.js

@@ -0,0 +1,101 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+const data_normalization_1 = require("../dummy-extractor/data-normalization");
+// Dummy data that originally would be fetched from an external source
+const issues = [
+    {
+        id: 'issue-1',
+        created_date: '1999-12-25T01:00:03+01:00',
+        modified_date: '1999-12-25T01:00:03+01:00',
+        body: '<p>This is issue 1</p>',
+        creator: 'user-1',
+        owner: 'user-1',
+        title: 'Issue 1',
+    },
+    {
+        id: 'issue-2',
+        created_date: '1999-12-27T15:31:34+01:00',
+        modified_date: '2002-04-09T01:55:31+02:00',
+        body: '<p>This is issue 2</p>',
+        creator: 'user-2',
+        owner: 'user-2',
+        title: 'Issue 2',
+    },
+];
+const users = [
+    {
+        id: 'user-1',
+        created_date: '1999-12-25T01:00:03+01:00',
+        modified_date: '1999-12-25T01:00:03+01:00',
+        data: {
+            email: 'johndoe@test.com',
+            name: 'John Doe',
+        },
+    },
+    {
+        id: 'user-2',
+        created_date: '1999-12-27T15:31:34+01:00',
+        modified_date: '2002-04-09T01:55:31+02:00',
+        data: {
+            email: 'janedoe@test.com',
+            name: 'Jane Doe',
+        },
+    },
+];
+const attachments = [
+    {
+        url: 'https://app.dev.devrev-eng.ai/favicon.ico',
+        id: 'attachment-1',
+        file_name: 'dummy.jpg',
+        author_id: 'user-1',
+        parent_id: 'issue-1',
+    },
+    {
+        url: 'https://app.dev.devrev-eng.ai/favicon.ico',
+        id: 'attachment-2',
+        file_name: 'dummy.ico',
+        author_id: 'user-2',
+        parent_id: 'issue-2',
+    },
+];
+const repos = [
+    {
+        itemType: 'issues',
+        normalize: data_normalization_1.normalizeIssue,
+    },
+    {
+        itemType: 'users',
+        normalize: data_normalization_1.normalizeUser,
+    },
+    {
+        itemType: 'attachments',
+        normalize: data_normalization_1.normalizeAttachment,
+    },
+];
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        var _a, _b, _c;
+        console.log('Logging something from worker thread', {});
+        adapter.initializeRepos(repos);
+        if (adapter.event.payload.event_type === index_1.EventType.ExtractionDataStart) {
+            await ((_a = adapter.getRepo('issues')) === null || _a === void 0 ? void 0 : _a.push(issues));
+            await adapter.emit(index_1.ExtractorEventType.ExtractionDataProgress, {
+                progress: 50,
+            });
+        }
+        else {
+            await ((_b = adapter.getRepo('users')) === null || _b === void 0 ? void 0 : _b.push(users));
+            await ((_c = adapter.getRepo('attachments')) === null || _c === void 0 ? void 0 : _c.push(attachments));
+            await adapter.emit(index_1.ExtractorEventType.ExtractionDataDone, {
+                progress: 100,
+            });
+        }
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.postState();
+        await adapter.emit(index_1.ExtractorEventType.ExtractionDataProgress, {
+            progress: 50,
+        });
+    },
+});

package/dist/workers/default-workers/external-sync-units-extraction.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/external-sync-units-extraction.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+// Dummy data that originally would be fetched from an external source
+const externalSyncUnits = [
+    {
+        id: 'devrev',
+        name: 'devrev',
+        description: 'Demo external sync unit',
+        item_count: 2,
+        item_type: 'issues',
+    },
+];
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
+            external_sync_units: externalSyncUnits,
+        });
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionExternalSyncUnitsError, {
+            error: {
+                message: 'Failed to extract external sync units. Lambda timeout.',
+            },
+        });
+    },
+});

package/dist/workers/default-workers/metadata-extraction.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/default-workers/metadata-extraction.js

@@ -0,0 +1,26 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+const external_domain_metadata_json_1 = __importDefault(require("../dummy-extractor/external_domain_metadata.json"));
+const repos = [
+    {
+        itemType: 'external_domain_metadata',
+    },
+];
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        var _a;
+        adapter.initializeRepos(repos);
+        await ((_a = adapter
+            .getRepo('external_domain_metadata')) === null || _a === void 0 ? void 0 : _a.push([external_domain_metadata_json_1.default]));
+        await adapter.emit(index_1.ExtractorEventType.ExtractionMetadataDone);
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionMetadataError, {
+            error: { message: 'Failed to extract metadata. Lambda timeout.' },
+        });
+    },
+});

package/dist/workers/dummy-extractor/data-normalization.d.ts

@@ -0,0 +1,4 @@
+import { NormalizedItem, NormalizedAttachment } from '../../index';
+export declare function normalizeIssue(item: any): NormalizedItem;
+export declare function normalizeUser(item: any): NormalizedItem;
+export declare function normalizeAttachment(item: any): NormalizedAttachment;

package/dist/workers/dummy-extractor/data-normalization.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeIssue = normalizeIssue;
+exports.normalizeUser = normalizeUser;
+exports.normalizeAttachment = normalizeAttachment;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function normalizeIssue(item) {
+    return {
+        id: item.id,
+        created_date: item.created_date,
+        modified_date: item.modified_date,
+        data: {
+            body: item.body,
+            creator: item.creator,
+            owner: item.owner,
+            title: item.title,
+        },
+    };
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function normalizeUser(item) {
+    return {
+        id: item.id,
+        created_date: item.created_date,
+        modified_date: item.modified_date,
+        data: {
+            email: item.email,
+            name: item.name,
+        },
+    };
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function normalizeAttachment(item) {
+    return {
+        url: item.url,
+        id: item.id,
+        file_name: item.file_name,
+        author_id: item.author_id,
+        parent_id: item.parent_id,
+    };
+}

package/dist/workers/dummy-extractor/external_domain_metadata.json

@@ -0,0 +1,58 @@
+{
+    "record_types": {
+        "issues": {
+            "name": "Issues",
+            "fields": {
+                "title": {
+                    "is_required": true,
+                    "type": "text",
+                    "name": "Title",
+                    "text": {
+                        "min_length": 1
+                    }
+                },
+                "body": {
+                    "type": "rich_text",
+                    "name": "body",
+                    "is_required": true
+                },
+                "owner": {
+                    "is_required": true,
+                    "type": "reference",
+                    "reference": {
+                        "refers_to": {
+                            "#record:users": {}
+                        }
+                    }
+                },
+                "creator": {
+                    "is_required": true,
+                    "type": "reference",
+                    "reference": {
+                        "refers_to": {
+                            "#record:users": {}
+                        }
+                    }
+                }
+            }
+        },
+        "users": {
+            "name": "Users",
+            "fields": {
+                "name": {
+                    "is_required": true,
+                    "type": "text",
+                    "name": "Name",
+                    "text": {
+                        "min_length": 1
+                    }
+                },
+                "email": {
+                    "type": "text",
+                    "name": "Email",
+                    "is_required": true
+                }
+            }
+        }
+    }
+}

package/dist/workers/process-task.d.ts

@@ -0,0 +1,2 @@
+import { ProcessTaskInterface } from '../types/workers';
+export declare function processTask<ConnectorState>({ task, onTimeout, }: ProcessTaskInterface<ConnectorState>): void;

package/dist/workers/process-task.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processTask = processTask;
+const node_worker_threads_1 = require("node:worker_threads");
+const state_1 = require("../state/state");
+const worker_adapter_1 = require("./worker-adapter");
+const workers_1 = require("../types/workers");
+const logger_1 = require("../logger/logger");
+function processTask({ task, onTimeout, }) {
+    if (!node_worker_threads_1.isMainThread) {
+        (async () => {
+            const event = node_worker_threads_1.workerData.event;
+            const initialState = node_worker_threads_1.workerData.initialState;
+            const options = node_worker_threads_1.workerData.options;
+            console = new logger_1.Logger(event);
+            const adapterState = await (0, state_1.createAdapterState)({
+                event,
+                initialState,
+                options,
+            });
+            if (node_worker_threads_1.parentPort && node_worker_threads_1.workerData.event) {
+                const adapter = new worker_adapter_1.WorkerAdapter({
+                    event,
+                    adapterState,
+                    parentPort: node_worker_threads_1.parentPort,
+                    options,
+                });
+                node_worker_threads_1.parentPort.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {
+                    if (message.subject === workers_1.WorkerMessageSubject.WorkerMessageExit) {
+                        adapter.handleTimeout();
+                        await onTimeout({ adapter });
+                        process.exit(0);
+                    }
+                });
+                await task({ adapter });
+                const message = {
+                    subject: workers_1.WorkerMessageSubject.WorkerMessageDone,
+                };
+                node_worker_threads_1.parentPort.postMessage(message);
+                process.exit(0);
+            }
+        })();
+    }
+}

package/dist/workers/spawn.d.ts

@@ -0,0 +1,25 @@
+import { SpawnFactoryInterface, SpawnInterface } from '../types/workers';
+/**
+ * Creates a new instance of Spawn class.
+ * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
+ * The class provides utilities to emit control events to the platform and exit the worker gracefully.
+ * In case of lambda timeout, the class emits a lambda timeout event to the platform.
+ * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object} initialState - The initial state of the adapter
+ * @param {string} workerPath - The path to the worker file
+ * @returns {Promise<Spawn>} - A new instance of Spawn class
+ */
+export declare function spawn<ConnectorState>({ event, initialState, workerPath, options, }: SpawnFactoryInterface<ConnectorState>): Promise<boolean | PromiseLike<boolean>>;
+export declare class Spawn {
+    private event;
+    private hasWorkerEmitted;
+    private defaultLambdaTimeout;
+    private lambdaTimeout;
+    private worker;
+    private resolve;
+    private timer;
+    private logger;
+    constructor({ event, worker, options, resolve }: SpawnInterface);
+    private exitFromMainThread;
+}

package/dist/workers/spawn.js

@@ -0,0 +1,163 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Spawn = void 0;
+exports.spawn = spawn;
+const extraction_1 = require("../types/extraction");
+const control_protocol_1 = require("../common/control-protocol");
+const helpers_1 = require("../common/helpers");
+const logger_1 = require("../logger/logger");
+const constants_1 = require("../common/constants");
+const workers_1 = require("../types/workers");
+const create_worker_1 = require("./create-worker");
+function getWorkerPath({ event, connectorWorkerPath, }) {
+    const logger = new logger_1.Logger(event);
+    if (!constants_1.ALLOWED_EVENT_TYPES.includes(event.payload.event_type)) {
+        return null;
+    }
+    if (connectorWorkerPath)
+        return connectorWorkerPath;
+    let path = null;
+    switch (event.payload.event_type) {
+        case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
+            path = __dirname + '/default-workers/external-sync-units-extraction';
+            break;
+        case extraction_1.EventType.ExtractionMetadataStart:
+            path = __dirname + '/default-workers/metadata-extraction';
+            break;
+        case extraction_1.EventType.ExtractionDataStart:
+        case extraction_1.EventType.ExtractionDataContinue:
+            path = __dirname + '/default-workers/data-extraction';
+            break;
+        case extraction_1.EventType.ExtractionAttachmentsStart:
+        case extraction_1.EventType.ExtractionAttachmentsContinue:
+            path = __dirname + '/default-workers/attachments-extraction';
+            break;
+        case extraction_1.EventType.ExtractionDataDelete:
+            path = __dirname + '/default-workers/data-deletion.js';
+            break;
+        case extraction_1.EventType.ExtractionAttachmentsDelete:
+            path = __dirname + '/default-workers/attachments-deletion';
+            break;
+        default:
+            logger.error('Worker script not found for event type: ' +
+                event.payload.event_type +
+                '.');
+            path = null;
+    }
+    return path;
+}
+/**
+ * Creates a new instance of Spawn class.
+ * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
+ * The class provides utilities to emit control events to the platform and exit the worker gracefully.
+ * In case of lambda timeout, the class emits a lambda timeout event to the platform.
+ * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object} initialState - The initial state of the adapter
+ * @param {string} workerPath - The path to the worker file
+ * @returns {Promise<Spawn>} - A new instance of Spawn class
+ */
+async function spawn({ event, initialState, workerPath, options, }) {
+    const logger = new logger_1.Logger(event);
+    const script = getWorkerPath({
+        event,
+        connectorWorkerPath: workerPath,
+    });
+    if (script) {
+        try {
+            const worker = await (0, create_worker_1.createWorker)({
+                event,
+                initialState,
+                workerPath: script,
+                options,
+            });
+            return new Promise((resolve) => {
+                new Spawn({
+                    event,
+                    worker,
+                    options: options || null,
+                    resolve,
+                });
+            });
+        }
+        catch (error) {
+            logger.error('Worker error while processing task.', error);
+            return false;
+        }
+    }
+    else {
+        throw new Error('Worker script not found.');
+    }
+}
+class Spawn {
+    constructor({ event, worker, options, resolve }) {
+        this.defaultLambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
+        this.hasWorkerEmitted = false;
+        this.event = event;
+        this.lambdaTimeout = (options === null || options === void 0 ? void 0 : options.timeout)
+            ? Math.min(options.timeout, this.defaultLambdaTimeout)
+            : this.defaultLambdaTimeout;
+        this.resolve = resolve;
+        this.timer = setTimeout(async () => {
+            this.logger.log('Lambda timeout reached. Exiting.');
+            if (this.worker) {
+                this.worker.postMessage({
+                    subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
+                });
+            }
+            else {
+                await this.exitFromMainThread();
+            }
+        }, this.lambdaTimeout);
+        this.logger = new logger_1.Logger(event);
+        this.worker = worker;
+        worker.on(workers_1.WorkerEvent.WorkerExit, async (code) => {
+            this.logger.info('Worker exited with exit code: ' + code + '.');
+            if (this.timer) {
+                clearTimeout(this.timer);
+                await this.exitFromMainThread();
+            }
+        });
+        worker.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
+                this.logger.info('Worker has emitted message to ADaaS.');
+                this.hasWorkerEmitted = true;
+            }
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageDone) {
+                this.logger.info('Worker has completed work.');
+                clearTimeout(this.timer);
+                await this.exitFromMainThread();
+            }
+        });
+        worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
+            var _a, _b;
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageLog) {
+                const args = (_a = message.payload) === null || _a === void 0 ? void 0 : _a.args;
+                const level = (_b = message.payload) === null || _b === void 0 ? void 0 : _b.level;
+                this.logger.logFn(args, level);
+            }
+        });
+    }
+    async exitFromMainThread() {
+        if (this.hasWorkerEmitted) {
+            return;
+        }
+        const timeoutEventType = (0, helpers_1.getErrorExtractorEventType)(this.event.payload.event_type);
+        if (timeoutEventType !== null) {
+            const { eventType } = timeoutEventType;
+            await (0, control_protocol_1.emit)({
+                eventType,
+                event: this.event,
+                data: {
+                    error: {
+                        message: 'Worker has not emitted anything. Exited.',
+                    },
+                },
+            }).then(() => {
+                this.logger.error('Worker has not emitted anything. Exited.');
+                this.resolve(true);
+            });
+        }
+    }
+}
+exports.Spawn = Spawn;