@devrev/ts-adaas 1.12.3-beta.6 → 1.12.3-beta.7

package/dist/workers/create-worker.test.js

@@ -7,7 +7,7 @@ const create_worker_1 = require("./create-worker");
 describe(create_worker_1.createWorker.name, () => {
     it('should create a Worker instance when valid parameters are provided', async () => {
         const workerPath = __dirname + '../tests/dummy-worker.ts';
-        const worker = worker_threads_1.isMainThread
+        const result = worker_threads_1.isMainThread
             ? await (0, create_worker_1.createWorker)({
                 event: (0, test_helpers_1.createEvent)({
                     eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
@@ -16,10 +16,12 @@ describe(create_worker_1.createWorker.name, () => {
                 workerPath,
             })
             : null;
-        expect(worker).not.toBeNull();
-        expect(worker).toBeInstanceOf(worker_threads_1.Worker);
-        if (worker) {
-            await worker.terminate();
+        expect(result).not.toBeNull();
+        expect(result === null || result === void 0 ? void 0 : result.worker).toBeInstanceOf(worker_threads_1.Worker);
+        expect(result === null || result === void 0 ? void 0 : result.memoryConfig).toBeDefined();
+        expect(result === null || result === void 0 ? void 0 : result.resourceLimits).toBeDefined();
+        if (result) {
+            await result.worker.terminate();
         }
     });
     it('should throw error when not in main thread', async () => {
@@ -41,15 +43,15 @@ describe(create_worker_1.createWorker.name, () => {
     it('[edge] should handle worker creation with minimal valid data', async () => {
         const workerPath = __dirname + '../tests/dummy-worker.ts';
         if (worker_threads_1.isMainThread) {
-            const worker = await (0, create_worker_1.createWorker)({
+            const result = await (0, create_worker_1.createWorker)({
                 event: (0, test_helpers_1.createEvent)({
                     eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
                 }),
                 initialState: {},
                 workerPath,
             });
-            expect(worker).toBeInstanceOf(worker_threads_1.Worker);
-            await worker.terminate();
+            expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
+            await result.worker.terminate();
         }
     });
     it('[edge] should handle worker creation with complex initial state', async () => {
@@ -61,29 +63,126 @@ describe(create_worker_1.createWorker.name, () => {
             },
         };
         if (worker_threads_1.isMainThread) {
-            const worker = await (0, create_worker_1.createWorker)({
+            const result = await (0, create_worker_1.createWorker)({
                 event: (0, test_helpers_1.createEvent)({
                     eventType: extraction_1.EventType.ExtractionDataStart,
                 }),
                 initialState: complexState,
                 workerPath,
             });
-            expect(worker).toBeInstanceOf(worker_threads_1.Worker);
-            await worker.terminate();
+            expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
+            await result.worker.terminate();
         }
     });
     it('[edge] should handle different event types', async () => {
         const workerPath = __dirname + '../tests/dummy-worker.ts';
         if (worker_threads_1.isMainThread) {
-            const worker = await (0, create_worker_1.createWorker)({
+            const result = await (0, create_worker_1.createWorker)({
                 event: (0, test_helpers_1.createEvent)({
                     eventType: extraction_1.EventType.ExtractionMetadataStart,
                 }),
                 initialState: {},
                 workerPath,
             });
-            expect(worker).toBeInstanceOf(worker_threads_1.Worker);
-            await worker.terminate();
+            expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
+            await result.worker.terminate();
+        }
+    });
+    it('should include memory configuration in result', async () => {
+        const workerPath = __dirname + '../tests/dummy-worker.ts';
+        if (worker_threads_1.isMainThread) {
+            const result = await (0, create_worker_1.createWorker)({
+                event: (0, test_helpers_1.createEvent)({
+                    eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
+                }),
+                initialState: {},
+                workerPath,
+            });
+            expect(result.memoryConfig).toHaveProperty('maxOldGenerationSizeMb');
+            expect(result.memoryConfig).toHaveProperty('totalAvailableMemoryMb');
+            expect(result.memoryConfig).toHaveProperty('isLambda');
+            expect(result.memoryConfig).toHaveProperty('isLocalDevelopment');
+            expect(result.resourceLimits).toHaveProperty('maxOldGenerationSizeMb');
+            await result.worker.terminate();
+        }
+    });
+    it('should apply testMemoryLimitMb override when provided', async () => {
+        const workerPath = __dirname + '../tests/dummy-worker.ts';
+        const testMemoryLimit = 64;
+        if (worker_threads_1.isMainThread) {
+            const result = await (0, create_worker_1.createWorker)({
+                event: (0, test_helpers_1.createEvent)({
+                    eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
+                }),
+                initialState: {},
+                workerPath,
+                options: {
+                    testMemoryLimitMb: testMemoryLimit,
+                },
+            });
+            expect(result.resourceLimits.maxOldGenerationSizeMb).toBe(testMemoryLimit);
+            await result.worker.terminate();
+        }
+    });
+    it('should create worker without memory limits when enableMemoryLimits is false', async () => {
+        const workerPath = __dirname + '../tests/dummy-worker.ts';
+        if (worker_threads_1.isMainThread) {
+            const result = await (0, create_worker_1.createWorker)({
+                event: (0, test_helpers_1.createEvent)({
+                    eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
+                }),
+                initialState: {},
+                workerPath,
+                options: {
+                    enableMemoryLimits: false,
+                },
+            });
+            // Worker should still be created successfully
+            expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
+            // Memory config should still be calculated (for logging purposes)
+            expect(result.memoryConfig).toBeDefined();
+            expect(result.resourceLimits).toBeDefined();
+            await result.worker.terminate();
+        }
+    });
+    it('should set isLocalDevelopment in memory config when option is provided', async () => {
+        const workerPath = __dirname + '../tests/dummy-worker.ts';
+        if (worker_threads_1.isMainThread) {
+            const result = await (0, create_worker_1.createWorker)({
+                event: (0, test_helpers_1.createEvent)({
+                    eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
+                }),
+                initialState: {},
+                workerPath,
+                options: {
+                    isLocalDevelopment: true,
+                },
+            });
+            expect(result.memoryConfig.isLocalDevelopment).toBe(true);
+            await result.worker.terminate();
+        }
+    });
+    it('[edge] should handle all extraction event types', async () => {
+        const workerPath = __dirname + '../tests/dummy-worker.ts';
+        const eventTypes = [
+            extraction_1.EventType.ExtractionExternalSyncUnitsStart,
+            extraction_1.EventType.ExtractionMetadataStart,
+            extraction_1.EventType.ExtractionDataStart,
+            extraction_1.EventType.ExtractionDataContinue,
+            extraction_1.EventType.ExtractionAttachmentsStart,
+            extraction_1.EventType.ExtractionAttachmentsContinue,
+        ];
+        if (worker_threads_1.isMainThread) {
+            for (const eventType of eventTypes) {
+                const result = await (0, create_worker_1.createWorker)({
+                    event: (0, test_helpers_1.createEvent)({ eventType }),
+                    initialState: {},
+                    workerPath,
+                });
+                expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
+                expect(result.memoryConfig).toBeDefined();
+                await result.worker.terminate();
+            }
         }
     });
 });

package/dist/workers/spawn.d.ts

@@ -22,8 +22,17 @@ export declare class Spawn {
     private resolve;
     private originalConsole;
     private logger;
-    constructor({ event, worker, options, resolve, originalConsole, }: SpawnInterface);
+    private memoryConfig?;
+    private resourceLimits?;
+    private isOOMExit;
+    constructor({ event, worker, options, resolve, originalConsole, memoryConfig, resourceLimits, }: SpawnInterface);
     private clearTimeouts;
     private exitFromMainThread;
+    /**
+     * Handles worker exit due to OOM (Out-Of-Memory) error.
+     * Emits an error event with detailed OOM information.
+     * @param exitCode - The exit code from the worker
+     */
+    private handleOOMExit;
 }
 //# sourceMappingURL=spawn.d.ts.map

package/dist/workers/spawn.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,kBAAkB,CAAC;AAmC1B;;;;;;;;;;GAUG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAuGvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;gBACX,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,GAChB,EAAE,cAAc;IAgGjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;CAiCjC"}
+{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAaA,OAAO,EAGL,qBAAqB,EACrB,cAAc,EAKf,MAAM,kBAAkB,CAAC;AAmC1B;;;;;;;;;;GAUG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA0GvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAC,CAAqB;IAC1C,OAAO,CAAC,cAAc,CAAC,CAAuB;IAC9C,OAAO,CAAC,SAAS,CAAkB;gBAEvB,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,EACf,YAAY,EACZ,cAAc,GACf,EAAE,cAAc;IAwHjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;IAkChC;;;;OAIG;YACW,aAAa;CAyD5B"}

package/dist/workers/spawn.js

@@ -10,6 +10,7 @@ const helpers_1 = require("yargs/helpers");
 const control_protocol_1 = require("../common/control-protocol");
 const event_type_translation_1 = require("../common/event-type-translation");
 const helpers_2 = require("../common/helpers");
+const worker_memory_1 = require("../common/worker-memory");
 const logger_1 = require("../logger/logger");
 const extraction_1 = require("../types/extraction");
 const workers_1 = require("../types/workers");
@@ -86,7 +87,7 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
     }
     if (script) {
         try {
-            const worker = await (0, create_worker_1.createWorker)({
+            const { worker, memoryConfig, resourceLimits } = await (0, create_worker_1.createWorker)({
                 event,
                 initialState,
                 workerPath: script,
@@ -100,6 +101,8 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
                     options,
                     resolve,
                     originalConsole,
+                    memoryConfig,
+                    resourceLimits,
                 });
             });
         }
@@ -136,8 +139,9 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
     }
 }
 class Spawn {
-    constructor({ event, worker, options, resolve, originalConsole, }) {
+    constructor({ event, worker, options, resolve, originalConsole, memoryConfig, resourceLimits, }) {
         this.defaultLambdaTimeout = constants_1.DEFAULT_LAMBDA_TIMEOUT;
+        this.isOOMExit = false;
         this.originalConsole = originalConsole || console;
         this.logger = console;
         this.alreadyEmitted = false;
@@ -146,6 +150,8 @@ class Spawn {
             ? Math.min(options.timeout, this.defaultLambdaTimeout)
             : this.defaultLambdaTimeout;
         this.resolve = resolve;
+        this.memoryConfig = memoryConfig;
+        this.resourceLimits = resourceLimits;
         // If soft timeout is reached, send a message to the worker to gracefully exit.
         this.softTimeoutTimer = setTimeout(() => void (async () => {
             console.log('SOFT TIMEOUT: Sending a message to the worker to gracefully exit.');
@@ -170,12 +176,33 @@ class Spawn {
                 await this.exitFromMainThread();
             }
         })(), this.lambdaTimeout * constants_1.HARD_TIMEOUT_MULTIPLIER);
+        // Listen for worker errors to detect OOM
+        // Node.js worker threads emit ERR_WORKER_OUT_OF_MEMORY error code when OOM occurs
+        // @see https://nodejs.org/api/errors.html#err_worker_out_of_memory
+        worker.on(workers_1.WorkerEvent.WorkerError, (error) => {
+            if ((0, worker_memory_1.isOOMError)(error)) {
+                this.isOOMExit = true;
+                console.error('OOM ERROR DETECTED: Worker ran out of memory.', {
+                    errorCode: error.code,
+                    errorMessage: error === null || error === void 0 ? void 0 : error.message,
+                    memoryConfig: this.memoryConfig,
+                    resourceLimits: this.resourceLimits,
+                });
+            }
+        });
         // If worker exits with process.exit(code), clear the timeouts and exit from
         // main thread.
         worker.on(workers_1.WorkerEvent.WorkerExit, (code) => void (async () => {
             console.info('Worker exited with exit code: ' + code + '.');
             this.clearTimeouts();
-            await this.exitFromMainThread();
+            // Check if this was an OOM exit (non-zero exit code and OOM error detected)
+            // Exit code 134 is common for OOM (SIGABRT), but we also check the error message
+            if (code !== 0 && this.isOOMExit) {
+                await this.handleOOMExit(code);
+            }
+            else {
+                await this.exitFromMainThread();
+            }
         })());
         worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
             var _a, _b;
@@ -248,5 +275,52 @@ class Spawn {
             console.error('Error while emitting event.', (0, logger_1.serializeError)(error));
         }
     }
+    /**
+     * Handles worker exit due to OOM (Out-Of-Memory) error.
+     * Emits an error event with detailed OOM information.
+     * @param exitCode - The exit code from the worker
+     */
+    async handleOOMExit(exitCode) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
+        this.clearTimeouts();
+        // eslint-disable-next-line no-global-assign
+        console = this.originalConsole;
+        if (this.alreadyEmitted) {
+            this.resolve();
+            return;
+        }
+        this.alreadyEmitted = true;
+        const { eventType } = (0, helpers_2.getTimeoutErrorEventType)(this.event.payload.event_type);
+        // Build OOM error info
+        const oomErrorInfo = {
+            type: 'OOM_ERROR',
+            message: `Worker thread ran out of memory and was terminated. ` +
+                `Memory limit: ${(_b = (_a = this.resourceLimits) === null || _a === void 0 ? void 0 : _a.maxOldGenerationSizeMb) !== null && _b !== void 0 ? _b : 'unknown'}MB, ` +
+                `Total available: ${(_e = (_d = (_c = this.memoryConfig) === null || _c === void 0 ? void 0 : _c.totalAvailableMemoryMb) === null || _d === void 0 ? void 0 : _d.toFixed(0)) !== null && _e !== void 0 ? _e : 'unknown'}MB.`,
+            memoryLimitMb: (_g = (_f = this.resourceLimits) === null || _f === void 0 ? void 0 : _f.maxOldGenerationSizeMb) !== null && _g !== void 0 ? _g : 0,
+            totalAvailableMemoryMb: (_j = (_h = this.memoryConfig) === null || _h === void 0 ? void 0 : _h.totalAvailableMemoryMb) !== null && _j !== void 0 ? _j : 0,
+            isLambda: (_l = (_k = this.memoryConfig) === null || _k === void 0 ? void 0 : _k.isLambda) !== null && _l !== void 0 ? _l : false,
+            isLocalDevelopment: (_o = (_m = this.memoryConfig) === null || _m === void 0 ? void 0 : _m.isLocalDevelopment) !== null && _o !== void 0 ? _o : false,
+            exitCode,
+            eventType: this.event.payload.event_type,
+        };
+        console.error('OOM Error Details:', oomErrorInfo);
+        try {
+            await (0, control_protocol_1.emit)({
+                eventType,
+                event: this.event,
+                data: {
+                    error: {
+                        message: oomErrorInfo.message,
+                        oom_error_info: oomErrorInfo,
+                    },
+                },
+            });
+            this.resolve();
+        }
+        catch (error) {
+            console.error('Error while emitting OOM error event.', (0, logger_1.serializeError)(error));
+        }
+    }
 }
 exports.Spawn = Spawn;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.12.3-beta.6",
+  "version": "1.12.3-beta.7",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -12,6 +12,7 @@
     "lint": "eslint .",
     "lint:fix": "eslint .  --fix",
     "test": "jest --forceExit --coverage --runInBand --selectProjects default --",
+    "test:oom": "node --max-old-space-size=4096 ./node_modules/.bin/jest --forceExit --runInBand --selectProjects oom-tests --logHeapUsage --",
     "test:backwards-compatibility": "jest --forceExit --runInBand --selectProjects backwards-compatibility --"
   },
   "repository": {