@devrev/ts-adaas 1.12.3-beta.7 → 1.12.3-beta.8

package/dist/common/worker-memory.d.ts

@@ -1,59 +0,0 @@
-import { WorkerMemoryConfig } from '../types/workers';
-/**
- * Constants for memory limit calculation.
- */
-export declare const MEMORY_CONSTANTS: {
-    /** Percentage of memory to allocate to worker thread (75-80%) */
-    readonly WORKER_MEMORY_PERCENTAGE: 0.75;
-    /** Maximum memory cap for local development in MB */
-    readonly LOCAL_DEV_MAX_TOTAL_MEMORY_MB: 1024;
-    /** Minimum heap size for worker in MB */
-    readonly MIN_WORKER_HEAP_SIZE_MB: 128;
-    /** Default heap size when unable to determine system memory */
-    readonly DEFAULT_WORKER_HEAP_SIZE_MB: 512;
-};
-/**
- * Detects if the current environment is AWS Lambda.
- * @returns true if running in AWS Lambda
- */
-export declare function isLambdaEnvironment(): boolean;
-/**
- * Gets the AWS Lambda memory limit from environment variable.
- * @returns Lambda memory limit in MB, or null if not in Lambda
- */
-export declare function getLambdaMemoryLimitMb(): number | null;
-/**
- * Gets the total available memory based on the environment.
- * For Lambda: Uses AWS_LAMBDA_FUNCTION_MEMORY_SIZE
- * For local development: Caps at LOCAL_DEV_MAX_TOTAL_MEMORY_MB
- * For other environments: Uses system total memory or V8 heap limit
- *
- * @param isLocalDevelopment - Whether running in local development mode
- * @returns Total available memory in MB
- */
-export declare function getTotalAvailableMemoryMb(isLocalDevelopment: boolean): number;
-/**
- * Calculates the worker memory configuration based on the environment.
- *
- * @param isLocalDevelopment - Whether running in local development mode
- * @returns Worker memory configuration
- */
-export declare function calculateWorkerMemoryConfig(isLocalDevelopment?: boolean): WorkerMemoryConfig;
-/**
- * Node.js error code for worker thread OOM.
- * @see https://nodejs.org/api/errors.html#err_worker_out_of_memory
- */
-export declare const ERR_WORKER_OUT_OF_MEMORY = "ERR_WORKER_OUT_OF_MEMORY";
-/**
- * Checks if an error indicates an OOM (Out-Of-Memory) error.
- *
- * This function first checks for the Node.js error code `ERR_WORKER_OUT_OF_MEMORY`
- * which is the standard way to detect worker thread OOM errors.
- * Falls back to regex pattern matching on the error message for other OOM scenarios.
- *
- * @param error - The error to check (can be an Error object or string message)
- * @returns true if the error indicates OOM
- * @see https://nodejs.org/api/errors.html#err_worker_out_of_memory
- */
-export declare function isOOMError(error: Error | string): boolean;
-//# sourceMappingURL=worker-memory.d.ts.map

package/dist/common/worker-memory.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"worker-memory.d.ts","sourceRoot":"","sources":["../../src/common/worker-memory.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD;;GAEG;AACH,eAAO,MAAM,gBAAgB;IAC3B,iEAAiE;;IAEjE,qDAAqD;;IAErD,yCAAyC;;IAEzC,+DAA+D;;CAEvD,CAAC;AAEX;;;GAGG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAM7C;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,GAAG,IAAI,CAStD;AAED;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CAAC,kBAAkB,EAAE,OAAO,GAAG,MAAM,CAsB7E;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,kBAAkB,GAAE,OAAe,GAClC,kBAAkB,CAuBpB;AAED;;;GAGG;AACH,eAAO,MAAM,wBAAwB,6BAA6B,CAAC;AAiBnE;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,CAqBzD"}

package/dist/common/worker-memory.js

@@ -1,177 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ERR_WORKER_OUT_OF_MEMORY = exports.MEMORY_CONSTANTS = void 0;
-exports.isLambdaEnvironment = isLambdaEnvironment;
-exports.getLambdaMemoryLimitMb = getLambdaMemoryLimitMb;
-exports.getTotalAvailableMemoryMb = getTotalAvailableMemoryMb;
-exports.calculateWorkerMemoryConfig = calculateWorkerMemoryConfig;
-exports.isOOMError = isOOMError;
-const os = __importStar(require("os"));
-const v8 = __importStar(require("v8"));
-/**
- * Constants for memory limit calculation.
- */
-exports.MEMORY_CONSTANTS = {
-    /** Percentage of memory to allocate to worker thread (75-80%) */
-    WORKER_MEMORY_PERCENTAGE: 0.75,
-    /** Maximum memory cap for local development in MB */
-    LOCAL_DEV_MAX_TOTAL_MEMORY_MB: 1024,
-    /** Minimum heap size for worker in MB */
-    MIN_WORKER_HEAP_SIZE_MB: 128,
-    /** Default heap size when unable to determine system memory */
-    DEFAULT_WORKER_HEAP_SIZE_MB: 512,
-};
-/**
- * Detects if the current environment is AWS Lambda.
- * @returns true if running in AWS Lambda
- */
-function isLambdaEnvironment() {
-    return !!(process.env.AWS_LAMBDA_FUNCTION_NAME ||
-        process.env.AWS_EXECUTION_ENV ||
-        process.env.LAMBDA_TASK_ROOT);
-}
-/**
- * Gets the AWS Lambda memory limit from environment variable.
- * @returns Lambda memory limit in MB, or null if not in Lambda
- */
-function getLambdaMemoryLimitMb() {
-    const memorySize = process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
-    if (memorySize) {
-        const parsed = parseInt(memorySize, 10);
-        if (!isNaN(parsed) && parsed > 0) {
-            return parsed;
-        }
-    }
-    return null;
-}
-/**
- * Gets the total available memory based on the environment.
- * For Lambda: Uses AWS_LAMBDA_FUNCTION_MEMORY_SIZE
- * For local development: Caps at LOCAL_DEV_MAX_TOTAL_MEMORY_MB
- * For other environments: Uses system total memory or V8 heap limit
- *
- * @param isLocalDevelopment - Whether running in local development mode
- * @returns Total available memory in MB
- */
-function getTotalAvailableMemoryMb(isLocalDevelopment) {
-    // For Lambda, use the configured memory limit
-    const lambdaMemory = getLambdaMemoryLimitMb();
-    if (lambdaMemory !== null) {
-        return lambdaMemory;
-    }
-    // For local development, cap at LOCAL_DEV_MAX_TOTAL_MEMORY_MB
-    if (isLocalDevelopment) {
-        const systemMemoryMb = os.totalmem() / (1024 * 1024);
-        return Math.min(systemMemoryMb, exports.MEMORY_CONSTANTS.LOCAL_DEV_MAX_TOTAL_MEMORY_MB);
-    }
-    // For other environments, use the smaller of system memory or V8 heap limit
-    const systemMemoryMb = os.totalmem() / (1024 * 1024);
-    const heapStats = v8.getHeapStatistics();
-    const heapLimitMb = heapStats.heap_size_limit / (1024 * 1024);
-    return Math.min(systemMemoryMb, heapLimitMb);
-}
-/**
- * Calculates the worker memory configuration based on the environment.
- *
- * @param isLocalDevelopment - Whether running in local development mode
- * @returns Worker memory configuration
- */
-function calculateWorkerMemoryConfig(isLocalDevelopment = false) {
-    const isLambda = isLambdaEnvironment();
-    const totalAvailableMemoryMb = getTotalAvailableMemoryMb(isLocalDevelopment);
-    const workerMemoryPercentage = exports.MEMORY_CONSTANTS.WORKER_MEMORY_PERCENTAGE;
-    // Calculate worker heap size (75-80% of total available)
-    let maxOldGenerationSizeMb = Math.floor(totalAvailableMemoryMb * workerMemoryPercentage);
-    // Ensure minimum heap size
-    maxOldGenerationSizeMb = Math.max(maxOldGenerationSizeMb, exports.MEMORY_CONSTANTS.MIN_WORKER_HEAP_SIZE_MB);
-    return {
-        maxOldGenerationSizeMb,
-        totalAvailableMemoryMb,
-        isLambda,
-        isLocalDevelopment,
-        workerMemoryPercentage,
-    };
-}
-/**
- * Node.js error code for worker thread OOM.
- * @see https://nodejs.org/api/errors.html#err_worker_out_of_memory
- */
-exports.ERR_WORKER_OUT_OF_MEMORY = 'ERR_WORKER_OUT_OF_MEMORY';
-/**
- * Regex patterns to detect OOM errors from error messages.
- * Used as a fallback when the error code is not available.
- */
-const OOM_MESSAGE_PATTERNS = [
-    /JavaScript heap out of memory/i,
-    /JS heap out of memory/i,
-    /FATAL ERROR: .* out of memory/i,
-    /Allocation failed - JavaScript heap out of memory/i,
-    /FATAL ERROR: Reached heap limit/i,
-    /FATAL ERROR: CALL_AND_RETRY_LAST/i,
-    /memory allocation failed/i,
-    /Worker terminated due to reaching memory limit/i,
-];
-/**
- * Checks if an error indicates an OOM (Out-Of-Memory) error.
- *
- * This function first checks for the Node.js error code `ERR_WORKER_OUT_OF_MEMORY`
- * which is the standard way to detect worker thread OOM errors.
- * Falls back to regex pattern matching on the error message for other OOM scenarios.
- *
- * @param error - The error to check (can be an Error object or string message)
- * @returns true if the error indicates OOM
- * @see https://nodejs.org/api/errors.html#err_worker_out_of_memory
- */
-function isOOMError(error) {
-    // If it's an Error object, check the code property first
-    if (error instanceof Error) {
-        // Node.js worker thread OOM errors have this code
-        if (error.code === exports.ERR_WORKER_OUT_OF_MEMORY) {
-            return true;
-        }
-        // Fall back to checking the message
-        return OOM_MESSAGE_PATTERNS.some((pattern) => pattern.test(error.message));
-    }
-    // If it's a string, check against patterns (including the error code as a string)
-    if (typeof error === 'string') {
-        // Check if the string contains the error code
-        if (error.includes(exports.ERR_WORKER_OUT_OF_MEMORY)) {
-            return true;
-        }
-        return OOM_MESSAGE_PATTERNS.some((pattern) => pattern.test(error));
-    }
-    return false;
-}

package/dist/tests/oom-handling/extraction.d.ts

@@ -1,11 +0,0 @@
-import { AirdropEvent } from '../../index';
-/**
- * Run extraction with OOM-specific options.
- * Uses a very small memory limit to trigger OOM quickly.
- */
-declare const run: (events: AirdropEvent[], workerPath: string, options?: {
-    enableMemoryLimits?: boolean;
-    testMemoryLimitMb?: number;
-}) => Promise<void>;
-export default run;
-//# sourceMappingURL=extraction.d.ts.map

package/dist/tests/oom-handling/extraction.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../../src/tests/oom-handling/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAS,MAAM,aAAa,CAAC;AAOlD;;;GAGG;AACH,QAAA,MAAM,GAAG,GACP,QAAQ,YAAY,EAAE,EACtB,YAAY,MAAM,EAClB,UAAU;IACR,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,kBAkBF,CAAC;AAEF,eAAe,GAAG,CAAC"}

package/dist/tests/oom-handling/extraction.js

@@ -1,29 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const index_1 = require("../../index");
-const initialState = {};
-const initialDomainMapping = {};
-/**
- * Run extraction with OOM-specific options.
- * Uses a very small memory limit to trigger OOM quickly.
- */
-const run = async (events, workerPath, options) => {
-    var _a, _b;
-    for (const event of events) {
-        await (0, index_1.spawn)({
-            event,
-            initialState,
-            workerPath,
-            initialDomainMapping,
-            options: {
-                batchSize: 1000,
-                timeout: 60 * 1000, // 60 seconds - enough time for OOM to occur
-                isLocalDevelopment: true,
-                enableMemoryLimits: (_a = options === null || options === void 0 ? void 0 : options.enableMemoryLimits) !== null && _a !== void 0 ? _a : true,
-                // Use a small memory limit for testing (64MB) to trigger OOM quickly
-                testMemoryLimitMb: (_b = options === null || options === void 0 ? void 0 : options.testMemoryLimitMb) !== null && _b !== void 0 ? _b : 64,
-            },
-        });
-    }
-};
-exports.default = run;

package/dist/tests/oom-handling/jest.setup.d.ts

@@ -1,7 +0,0 @@
-/**
- * Jest setup file for OOM handling tests.
- *
- * These tests intentionally trigger out-of-memory conditions to test
- * OOM detection and handling. This setup ensures proper cleanup between tests.
- */
-//# sourceMappingURL=jest.setup.d.ts.map

package/dist/tests/oom-handling/jest.setup.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"jest.setup.d.ts","sourceRoot":"","sources":["../../../src/tests/oom-handling/jest.setup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/tests/oom-handling/jest.setup.js

@@ -1,26 +0,0 @@
-"use strict";
-/**
- * Jest setup file for OOM handling tests.
- *
- * These tests intentionally trigger out-of-memory conditions to test
- * OOM detection and handling. This setup ensures proper cleanup between tests.
- */
-// Force garbage collection between tests if available
-// This helps prevent memory accumulation across test cases
-afterEach(() => {
-    // Clear any module caches that might retain memory
-    jest.clearAllMocks();
-    jest.resetModules();
-    // Force garbage collection if available (Node.js must be run with --expose-gc)
-    if (global.gc) {
-        global.gc();
-    }
-});
-afterAll(() => {
-    // Final cleanup after all OOM tests complete
-    jest.clearAllMocks();
-    jest.resetModules();
-    if (global.gc) {
-        global.gc();
-    }
-});

package/dist/tests/oom-handling/oom-after-emit-worker.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=oom-after-emit-worker.d.ts.map

package/dist/tests/oom-handling/oom-after-emit-worker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"oom-after-emit-worker.d.ts","sourceRoot":"","sources":["../../../src/tests/oom-handling/oom-after-emit-worker.ts"],"names":[],"mappings":""}

package/dist/tests/oom-handling/oom-after-emit-worker.js

@@ -1,35 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const index_1 = require("../../index");
-/**
- * Worker that emits an event before causing an OOM error.
- * This tests the edge case where alreadyEmitted is true when OOM occurs.
- */
-(0, index_1.processTask)({
-    task: async ({ adapter }) => {
-        console.log('OOM after emit worker - emitting progress first');
-        // Emit a progress event BEFORE causing OOM
-        await adapter.emit(index_1.ExtractorEventType.DataExtractionProgress);
-        console.log('Progress emitted successfully, waiting a moment...');
-        // Wait a bit to ensure the emit was fully processed
-        await new Promise((resolve) => setTimeout(resolve, 100));
-        console.log('Now causing OOM...');
-        // Array to hold references to prevent garbage collection
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const memoryHog = [];
-        // Allocate memory until OOM
-        while (true) {
-            const chunk = [];
-            for (let i = 0; i < 10000; i++) {
-                chunk.push({
-                    data: 'x'.repeat(100),
-                    index: i,
-                });
-            }
-            memoryHog.push(chunk);
-        }
-    },
-    onTimeout: async ({ adapter }) => {
-        await adapter.emit(index_1.ExtractorEventType.DataExtractionProgress);
-    },
-});

package/dist/tests/oom-handling/oom-attachments-worker.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=oom-attachments-worker.d.ts.map

package/dist/tests/oom-handling/oom-attachments-worker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"oom-attachments-worker.d.ts","sourceRoot":"","sources":["../../../src/tests/oom-handling/oom-attachments-worker.ts"],"names":[],"mappings":""}

package/dist/tests/oom-handling/oom-attachments-worker.js

@@ -1,29 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const index_1 = require("../../index");
-/* eslint-disable @typescript-eslint/require-await */
-/* eslint-disable @typescript-eslint/no-unused-vars */
-/**
- * Worker that causes OOM during attachments extraction.
- * Tests OOM handling for EventType.StartExtractingAttachments.
- */
-(0, index_1.processTask)({
-    task: async ({ adapter }) => {
-        console.log('OOM attachments worker starting');
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const memoryHog = [];
-        while (true) {
-            const chunk = [];
-            for (let i = 0; i < 10000; i++) {
-                chunk.push({
-                    data: 'attachment'.repeat(10),
-                    index: i,
-                });
-            }
-            memoryHog.push(chunk);
-        }
-    },
-    onTimeout: async ({ adapter }) => {
-        await adapter.emit(index_1.ExtractorEventType.AttachmentExtractionProgress);
-    },
-});

package/dist/tests/oom-handling/oom-external-sync-units-worker.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=oom-external-sync-units-worker.d.ts.map

package/dist/tests/oom-handling/oom-external-sync-units-worker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"oom-external-sync-units-worker.d.ts","sourceRoot":"","sources":["../../../src/tests/oom-handling/oom-external-sync-units-worker.ts"],"names":[],"mappings":""}

package/dist/tests/oom-handling/oom-external-sync-units-worker.js

@@ -1,29 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const index_1 = require("../../index");
-/* eslint-disable @typescript-eslint/require-await */
-/* eslint-disable @typescript-eslint/no-unused-vars */
-/**
- * Worker that causes OOM during external sync units extraction.
- * Tests OOM handling for EventType.StartExtractingExternalSyncUnits.
- */
-(0, index_1.processTask)({
-    task: async ({ adapter }) => {
-        console.log('OOM external sync units worker starting');
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const memoryHog = [];
-        while (true) {
-            const chunk = [];
-            for (let i = 0; i < 10000; i++) {
-                chunk.push({
-                    data: 'sync_unit'.repeat(11),
-                    index: i,
-                });
-            }
-            memoryHog.push(chunk);
-        }
-    },
-    onTimeout: async ({ adapter }) => {
-        await adapter.emit(index_1.ExtractorEventType.ExternalSyncUnitExtractionDone);
-    },
-});

package/dist/tests/oom-handling/oom-gradual-worker.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=oom-gradual-worker.d.ts.map

package/dist/tests/oom-handling/oom-gradual-worker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"oom-gradual-worker.d.ts","sourceRoot":"","sources":["../../../src/tests/oom-handling/oom-gradual-worker.ts"],"names":[],"mappings":""}

package/dist/tests/oom-handling/oom-gradual-worker.js

@@ -1,47 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const index_1 = require("../../index");
-/**
- * Worker that gradually consumes memory to simulate a memory leak.
- * This is a more realistic OOM scenario than rapid allocation.
- */
-(0, index_1.processTask)({
-    task: async ({ adapter }) => {
-        console.log('Gradual OOM worker starting - simulating memory leak');
-        // Array to hold references to prevent garbage collection
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const memoryHog = [];
-        let totalAllocated = 0;
-        try {
-            // Allocate memory gradually with delays to simulate real processing
-            while (true) {
-                // Simulate some work
-                await new Promise((resolve) => setTimeout(resolve, 10));
-                // Create smaller chunks to simulate gradual memory growth
-                const chunk = [];
-                for (let i = 0; i < 2000; i++) {
-                    chunk.push({
-                        data: 'leak'.repeat(50), // ~200 bytes per string
-                        timestamp: Date.now(),
-                        index: i,
-                    });
-                }
-                memoryHog.push(chunk);
-                totalAllocated++;
-                if (totalAllocated % 5 === 0) {
-                    console.log(`Gradual allocation: ~${totalAllocated * 0.4}MB consumed`);
-                }
-            }
-        }
-        catch (error) {
-            console.error('Error during gradual memory allocation:', error);
-            await adapter.emit(index_1.ExtractorEventType.DataExtractionError, {
-                error: { message: 'Gradual memory allocation failed' },
-            });
-        }
-    },
-    onTimeout: async ({ adapter }) => {
-        console.log('Gradual OOM worker timeout handler called');
-        await adapter.emit(index_1.ExtractorEventType.DataExtractionProgress);
-    },
-});

package/dist/tests/oom-handling/oom-handling.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=oom-handling.test.d.ts.map

package/dist/tests/oom-handling/oom-handling.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"oom-handling.test.d.ts","sourceRoot":"","sources":["../../../src/tests/oom-handling/oom-handling.test.ts"],"names":[],"mappings":""}