@devrev/ts-adaas 1.5.0 → 1.6.0

package/dist/tests/timeout-handling/timeout-3b.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("../../index");
+(0, index_1.processTask)({
+    task: async ({ adapter }) => {
+        console.log('Starting intensive computation...');
+        let result = 0;
+        // Intensive loop - calibrated to exceed 10 seconds but show progress
+        for (let i = 1; i <= 30000; i++) {
+            for (let num = 2; num <= 2000; num++) {
+                let isPrime = true;
+                for (let divisor = 2; divisor <= Math.sqrt(num); divisor++) {
+                    if (num % divisor === 0) {
+                        isPrime = false;
+                        break;
+                    }
+                }
+                if (isPrime) {
+                    for (let j = 0; j < 200; j++) {
+                        result +=
+                            Math.sin(num * j) * Math.cos(i * j) * Math.sqrt(num + i + j);
+                        result = Math.abs(result) % 1000000;
+                    }
+                }
+            }
+            // Log every 500 iterations to see progress before timeout
+            if (i % 500 === 0) {
+                await new Promise((resolve) => setTimeout(resolve, 50));
+                console.log(`timeout-3b iteration ${i}, result: ${result}`);
+            }
+        }
+        console.log(`Final computation result: ${result}`);
+        await adapter.emit(index_1.ExtractorEventType.ExtractionDataDone);
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(index_1.ExtractorEventType.ExtractionDataProgress);
+    },
+});

package/dist/tests/timeout-handling/timeout-3b.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/tests/timeout-handling/timeout-3b.test.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const extraction_1 = require("../../types/extraction");
+const test_helpers_1 = require("../test-helpers");
+const extraction_2 = __importDefault(require("./extraction"));
+const mock_server_1 = require("../mock-server");
+jest.setTimeout(15000);
+describe('timeout-3b', () => {
+    let mockServer;
+    beforeAll(async () => {
+        mockServer = new mock_server_1.MockServer(3001);
+        await mockServer.start();
+    });
+    afterAll(async () => {
+        if (mockServer) {
+            await mockServer.stop();
+        }
+    });
+    beforeEach(() => {
+        jest.clearAllMocks();
+        mockServer.clearRequests();
+    });
+    it('should emit progress event when soft timeout is reached', async () => {
+        const baseUrl = mockServer.getBaseUrl();
+        const event = (0, test_helpers_1.createEvent)({
+            eventType: extraction_1.EventType.ExtractionDataStart,
+            eventContextOverrides: {
+                callback_url: `${baseUrl}/internal/airdrop.external-extractor.message`,
+                worker_data_url: `${baseUrl}/internal/airdrop.external-worker`,
+            },
+            executionMetadataOverrides: {
+                devrev_endpoint: `${baseUrl}`,
+            },
+        });
+        await (0, extraction_2.default)([event], __dirname + '/timeout-3b');
+        const requests = mockServer.getRequests();
+        const lastRequest = requests[requests.length - 1];
+        // Expect last request to be emission of progress event
+        expect(lastRequest.url).toContain('airdrop.external-extractor.message');
+        expect(lastRequest.method).toBe('POST');
+        expect(lastRequest.body.event_type).toBe('EXTRACTION_DATA_PROGRESS');
+    });
+});

package/dist/types/workers.d.ts

@@ -97,7 +97,6 @@ export declare enum WorkerEvent {
  */
 export declare enum WorkerMessageSubject {
     WorkerMessageEmitted = "emit",
-    WorkerMessageDone = "done",
     WorkerMessageExit = "exit",
     WorkerMessageLog = "log"
 }
@@ -110,12 +109,6 @@ export interface WorkerMessageEmitted {
         eventType: ExtractorEventType | LoaderEventType;
     };
 }
-/**
- * WorkerMessageDone interface represents the structure of the done worker message.
- */
-export interface WorkerMessageDone {
-    subject: WorkerMessageSubject.WorkerMessageDone;
-}
 /**
  * WorkerMessageExit interface represents the structure of the exit worker message.
  */
@@ -125,7 +118,7 @@ export interface WorkerMessageExit {
 /**
  * WorkerMessage represents the structure of the worker message.
  */
-export type WorkerMessage = WorkerMessageDone | WorkerMessageEmitted | WorkerMessageExit;
+export type WorkerMessage = WorkerMessageEmitted | WorkerMessageExit;
 /**
  * WorkerData represents the structure of the worker data object.
  */

package/dist/types/workers.js

@@ -17,7 +17,6 @@ var WorkerEvent;
 var WorkerMessageSubject;
 (function (WorkerMessageSubject) {
     WorkerMessageSubject["WorkerMessageEmitted"] = "emit";
-    WorkerMessageSubject["WorkerMessageDone"] = "done";
     WorkerMessageSubject["WorkerMessageExit"] = "exit";
     WorkerMessageSubject["WorkerMessageLog"] = "log";
 })(WorkerMessageSubject || (exports.WorkerMessageSubject = WorkerMessageSubject = {}));

package/dist/uploader/uploader.js

@@ -43,6 +43,7 @@ const zlib_1 = __importDefault(require("zlib"));
 const js_jsonl_1 = require("js-jsonl");
 const form_data_1 = __importDefault(require("form-data"));
 const constants_1 = require("../common/constants");
+const helpers_1 = require("../common/helpers");
 const logger_1 = require("../logger/logger");
 class Uploader {
     constructor({ event, options }) {
@@ -114,7 +115,7 @@ class Uploader {
                 params: {
                     request_id: this.requestId,
                     file_type: fileType,
-                    file_name: filename,
+                    file_name: (0, helpers_1.truncateFilename)(filename),
                 },
             });
             return response.data;

package/dist/uploader/uploader.test.js

@@ -30,11 +30,14 @@ const getArtifactUploadUrlMockResponse = {
 describe('Uploader Class Tests', () => {
     const mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart });
     let uploader;
+    let consoleWarnSpy;
     beforeEach(() => {
         uploader = new uploader_1.Uploader({ event: mockEvent });
+        consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
     });
     afterEach(() => {
         jest.clearAllMocks();
+        consoleWarnSpy.mockRestore();
     });
     it('should upload the file to the DevRev platform and return the artifact information', async () => {
         // Mock successful response from getArtifactUploadUrl

package/dist/workers/process-task.js

@@ -34,10 +34,6 @@ function processTask({ task, onTimeout, }) {
                     }
                 });
                 await task({ adapter });
-                const message = {
-                    subject: workers_1.WorkerMessageSubject.WorkerMessageDone,
-                };
-                node_worker_threads_1.parentPort.postMessage(message);
                 process.exit(0);
             }
         })();

package/dist/workers/spawn.d.ts

@@ -16,9 +16,11 @@ export declare class Spawn {
     private alreadyEmitted;
     private defaultLambdaTimeout;
     private lambdaTimeout;
-    private timer;
+    private softTimeoutTimer;
+    private hardTimeoutTimer;
     private logger;
     private resolve;
     constructor({ event, worker, options, resolve }: SpawnInterface);
+    private clearTimeouts;
     private exitFromMainThread;
 }

package/dist/workers/spawn.js

@@ -14,6 +14,7 @@ const helpers_2 = require("../common/helpers");
 const logger_1 = require("../logger/logger");
 const workers_1 = require("../types/workers");
 const create_worker_1 = require("./create-worker");
+const constants_1 = require("../common/constants");
 function getWorkerPath({ event, connectorWorkerPath, }) {
     if (connectorWorkerPath)
         return connectorWorkerPath;
@@ -134,7 +135,7 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
 }
 class Spawn {
     constructor({ event, worker, options, resolve }) {
-        this.defaultLambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
+        this.defaultLambdaTimeout = constants_1.DEFAULT_LAMBDA_TIMEOUT;
         this.alreadyEmitted = false;
         this.event = event;
         this.logger = new logger_1.Logger({ event, options });
@@ -142,78 +143,85 @@ class Spawn {
             ? Math.min(options.timeout, this.defaultLambdaTimeout)
             : this.defaultLambdaTimeout;
         this.resolve = resolve;
-        // if lambda timeout is reached, then send a message to the worker to gracefully exit
-        this.timer = setTimeout(async () => {
-            this.logger.log('Lambda timeout reached. Sending a message to the worker to gracefully exit.');
+        // If soft timeout is reached, send a message to the worker to gracefully exit.
+        this.softTimeoutTimer = setTimeout(async () => {
+            this.logger.log('SOFT TIMEOUT: Sending a message to the worker to gracefully exit.');
             if (worker) {
                 worker.postMessage({
                     subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
                 });
             }
             else {
-                console.log("Worker doesn't exist. Exiting from main thread.");
+                console.log('Worker does not exist. Exiting from main thread.');
                 await this.exitFromMainThread();
             }
         }, this.lambdaTimeout);
-        // if worker exits with process.exit(code) then we need to clear the timer and exit from main thread
+        // If hard timeout is reached, that means the worker did not exit in time. Terminate the worker.
+        this.hardTimeoutTimer = setTimeout(async () => {
+            this.logger.log('HARD TIMEOUT: Worker did not exit in time. Terminating the worker.');
+            if (worker) {
+                worker.terminate();
+            }
+            else {
+                console.log('Worker does not exist. Exiting from main thread.');
+                await this.exitFromMainThread();
+            }
+        }, this.lambdaTimeout * constants_1.HARD_TIMEOUT_MULTIPLIER);
+        // If worker exits with process.exit(code), clear the timeouts and exit from main thread.
         worker.on(workers_1.WorkerEvent.WorkerExit, async (code) => {
             this.logger.info('Worker exited with exit code: ' + code + '.');
-            if (this.timer) {
-                clearTimeout(this.timer);
-            }
+            this.clearTimeouts();
             await this.exitFromMainThread();
         });
         worker.on(workers_1.WorkerEvent.WorkerMessage, async (message) => {
             var _a, _b;
-            // if worker send a log message, then log it from the main thread with logger
+            // Since it is not possible to log from the worker thread, we need to log
+            // from the main thread.
             if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageLog) {
                 const args = (_a = message.payload) === null || _a === void 0 ? void 0 : _a.args;
                 const level = (_b = message.payload) === null || _b === void 0 ? void 0 : _b.level;
                 this.logger.logFn(args, level);
             }
-            // if worker sends a message that it has completed work, then clear the timer and exit from main thread
-            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageDone) {
-                this.logger.info('Worker has completed with executing the task.');
-                if (this.timer) {
-                    clearTimeout(this.timer);
-                }
-                await this.exitFromMainThread();
-            }
-            // if worker sends a message that it has emitted an event, then set alreadyEmitted to true
+            // If worker sends a message that it has emitted an event, then set alreadyEmitted to true.
             if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
                 this.logger.info('Worker has emitted message to ADaaS.');
                 this.alreadyEmitted = true;
             }
         });
     }
+    clearTimeouts() {
+        if (this.softTimeoutTimer) {
+            clearTimeout(this.softTimeoutTimer);
+        }
+        if (this.hardTimeoutTimer) {
+            clearTimeout(this.hardTimeoutTimer);
+        }
+    }
     async exitFromMainThread() {
         if (this.alreadyEmitted) {
             this.resolve();
             return;
         }
         this.alreadyEmitted = true;
-        const timeoutEventType = (0, helpers_2.getTimeoutErrorEventType)(this.event.payload.event_type);
-        if (timeoutEventType) {
-            const { eventType } = timeoutEventType;
-            try {
-                await (0, control_protocol_1.emit)({
-                    eventType,
-                    event: this.event,
-                    data: {
-                        error: {
-                            message: 'Worker has not emitted anything. Exited.',
-                        },
+        const { eventType } = (0, helpers_2.getTimeoutErrorEventType)(this.event.payload.event_type);
+        try {
+            await (0, control_protocol_1.emit)({
+                eventType,
+                event: this.event,
+                data: {
+                    error: {
+                        message: 'Worker has not emitted anything. Exited.',
                     },
-                });
-                this.resolve();
+                },
+            });
+            this.resolve();
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error)) {
+                console.error('Error while emitting event', (0, logger_1.serializeAxiosError)(error));
             }
-            catch (error) {
-                if (axios_1.default.isAxiosError(error)) {
-                    console.error('Error while emitting event', (0, logger_1.serializeAxiosError)(error));
-                }
-                else {
-                    console.error('Error while emitting event', error);
-                }
+            else {
+                console.error('Error while emitting event', error);
             }
         }
     }

package/dist/workers/worker-adapter.js

@@ -15,6 +15,7 @@ const mappers_1 = require("../mappers/mappers");
 const uploader_1 = require("../uploader/uploader");
 const logger_1 = require("../logger/logger");
 const mappers_interface_1 = require("../mappers/mappers.interface");
+const helpers_2 = require("../common/helpers");
 function createWorkerAdapter({ event, adapterState, options, }) {
     return new WorkerAdapter({
         event,
@@ -91,6 +92,10 @@ class WorkerAdapter {
             }
             // Loop through the batches of attachments
             for (let i = lastProcessedBatchIndex; i < reducedAttachments.length; i++) {
+                // Check if we hit timeout 
+                if (adapter.isTimeout) {
+                    await (0, helpers_2.sleep)(constants_1.DEFAULT_SLEEP_DELAY_MS);
+                }
                 const attachmentsBatch = reducedAttachments[i];
                 // Create a list of promises for parallel processing
                 const promises = [];

package/package.json

@@ -1,16 +1,17 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",
   "typings": "./dist/index.d.ts",
   "scripts": {
     "build": "tsc -p ./tsconfig.json",
+    "prepare": "npm run build",
     "start": "ts-node src/index.ts",
     "lint": "eslint .",
     "lint:fix": "eslint .  --fix",
-    "test": "jest --forceExit --coverage"
+    "test": "jest --forceExit --coverage --runInBand"
   },
   "repository": {
     "type": "git",

package/dist/tests/test-worker.js

@@ -1,14 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const process_task_1 = require("../workers/process-task");
-const extraction_1 = require("../types/extraction");
-(0, process_task_1.processTask)({
-    task: async ({ adapter }) => {
-        await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {});
-    },
-    onTimeout: async ({ adapter }) => {
-        await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError, {
-            error: { message: 'External sync unit failed.' },
-        });
-    },
-});