@devrev/ts-adaas 1.12.3-beta.7 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constants.d.ts +1 -1
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +1 -1
- package/dist/types/common.d.ts +0 -3
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/workers.d.ts +0 -58
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.js +4 -7
- package/dist/workers/create-worker.d.ts +2 -20
- package/dist/workers/create-worker.d.ts.map +1 -1
- package/dist/workers/create-worker.js +3 -38
- package/dist/workers/create-worker.test.js +14 -113
- package/dist/workers/spawn.d.ts +1 -10
- package/dist/workers/spawn.d.ts.map +1 -1
- package/dist/workers/spawn.js +3 -77
- package/package.json +1 -2
- package/dist/common/worker-memory.d.ts +0 -59
- package/dist/common/worker-memory.d.ts.map +0 -1
- package/dist/common/worker-memory.js +0 -177
- package/dist/tests/oom-handling/extraction.d.ts +0 -11
- package/dist/tests/oom-handling/extraction.d.ts.map +0 -1
- package/dist/tests/oom-handling/extraction.js +0 -29
- package/dist/tests/oom-handling/jest.setup.d.ts +0 -7
- package/dist/tests/oom-handling/jest.setup.d.ts.map +0 -1
- package/dist/tests/oom-handling/jest.setup.js +0 -26
- package/dist/tests/oom-handling/oom-after-emit-worker.d.ts +0 -2
- package/dist/tests/oom-handling/oom-after-emit-worker.d.ts.map +0 -1
- package/dist/tests/oom-handling/oom-after-emit-worker.js +0 -35
- package/dist/tests/oom-handling/oom-attachments-worker.d.ts +0 -2
- package/dist/tests/oom-handling/oom-attachments-worker.d.ts.map +0 -1
- package/dist/tests/oom-handling/oom-attachments-worker.js +0 -29
- package/dist/tests/oom-handling/oom-external-sync-units-worker.d.ts +0 -2
- package/dist/tests/oom-handling/oom-external-sync-units-worker.d.ts.map +0 -1
- package/dist/tests/oom-handling/oom-external-sync-units-worker.js +0 -29
- package/dist/tests/oom-handling/oom-gradual-worker.d.ts +0 -2
- package/dist/tests/oom-handling/oom-gradual-worker.d.ts.map +0 -1
- package/dist/tests/oom-handling/oom-gradual-worker.js +0 -47
- package/dist/tests/oom-handling/oom-handling.test.d.ts +0 -2
- package/dist/tests/oom-handling/oom-handling.test.d.ts.map +0 -1
- package/dist/tests/oom-handling/oom-handling.test.js +0 -407
- package/dist/tests/oom-handling/oom-metadata-worker.d.ts +0 -2
- package/dist/tests/oom-handling/oom-metadata-worker.d.ts.map +0 -1
- package/dist/tests/oom-handling/oom-metadata-worker.js +0 -29
- package/dist/tests/oom-handling/oom-worker.d.ts +0 -2
- package/dist/tests/oom-handling/oom-worker.d.ts.map +0 -1
- package/dist/tests/oom-handling/oom-worker.js +0 -55
- package/dist/tests/oom-handling/worker-memory.test.d.ts +0 -2
- package/dist/tests/oom-handling/worker-memory.test.d.ts.map +0 -1
- package/dist/tests/oom-handling/worker-memory.test.js +0 -306
|
@@ -9,7 +9,7 @@ export declare const STATEFUL_EXTRACTION_EVENT_TYPES: EventType[];
|
|
|
9
9
|
export declare const STATEFUL_LOADING_EVENT_TYPES: EventType[];
|
|
10
10
|
export declare const STATEFUL_EVENT_TYPES: EventType[];
|
|
11
11
|
export declare const ARTIFACT_BATCH_SIZE = 2000;
|
|
12
|
-
export declare const MAX_DEVREV_ARTIFACT_SIZE
|
|
12
|
+
export declare const MAX_DEVREV_ARTIFACT_SIZE: number;
|
|
13
13
|
export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
14
14
|
export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
15
15
|
export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAEvD,eAAO,MAAM,0BAA0B;;;;CAItC,CAAC;AAEF,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -43,7 +43,7 @@ exports.STATEFUL_EVENT_TYPES = [
|
|
|
43
43
|
...exports.STATEFUL_LOADING_EVENT_TYPES,
|
|
44
44
|
];
|
|
45
45
|
exports.ARTIFACT_BATCH_SIZE = 2000;
|
|
46
|
-
exports.MAX_DEVREV_ARTIFACT_SIZE =
|
|
46
|
+
exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
|
|
47
47
|
exports.MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
48
48
|
exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20; // 20 characters for the file extension
|
|
49
49
|
exports.AIRDROP_DEFAULT_ITEM_TYPES = {
|
package/dist/types/common.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Artifact } from '../uploader/uploader.interfaces';
|
|
2
|
-
import { OOMErrorInfo } from './workers';
|
|
3
2
|
/**
|
|
4
3
|
* ErrorLevel is an enum that represents the level of an error.
|
|
5
4
|
* @deprecated
|
|
@@ -14,8 +13,6 @@ export declare enum ErrorLevel {
|
|
|
14
13
|
*/
|
|
15
14
|
export interface ErrorRecord {
|
|
16
15
|
message: string;
|
|
17
|
-
/** OOM error information, present when worker died due to out-of-memory */
|
|
18
|
-
oom_error_info?: OOMErrorInfo;
|
|
19
16
|
}
|
|
20
17
|
/**
|
|
21
18
|
* LogRecord is an interface that defines the structure of a log record.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/types/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/types/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D;;;GAGG;AACH,oBAAY,UAAU;IACpB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,IAAI,SAAS;CACd;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;GAKG;AACH,oBAAY,QAAQ;IAClB,OAAO,YAAY;IACnB,WAAW,gBAAgB;IAC3B,OAAO,YAAY;CACpB"}
|
package/dist/types/workers.d.ts
CHANGED
|
@@ -25,17 +25,12 @@ export interface WorkerAdapterInterface<ConnectorState> {
|
|
|
25
25
|
* @param {number=} timeout - The timeout for the worker thread
|
|
26
26
|
* @param {number=} batchSize - Maximum number of extracted items in a batch
|
|
27
27
|
* @param {Record<EventType, string>=} workerPathOverrides - A map of event types to custom worker paths to override default worker paths
|
|
28
|
-
* @param {boolean=} enableMemoryLimits - Whether to enable automatic memory limits for worker threads (default: true)
|
|
29
|
-
* @param {number=} testMemoryLimitMb - Override memory limit for testing purposes (only used when enableMemoryLimits is true)
|
|
30
28
|
*/
|
|
31
29
|
export interface WorkerAdapterOptions {
|
|
32
30
|
isLocalDevelopment?: boolean;
|
|
33
31
|
timeout?: number;
|
|
34
32
|
batchSize?: number;
|
|
35
33
|
workerPathOverrides?: WorkerPathOverrides;
|
|
36
|
-
enableMemoryLimits?: boolean;
|
|
37
|
-
/** Override memory limit for testing purposes (only used when enableMemoryLimits is true) */
|
|
38
|
-
testMemoryLimitMb?: number;
|
|
39
34
|
}
|
|
40
35
|
/**
|
|
41
36
|
* SpawnInterface is an interface for Spawn class.
|
|
@@ -43,8 +38,6 @@ export interface WorkerAdapterOptions {
|
|
|
43
38
|
* @constructor
|
|
44
39
|
* @param {AirdropEvent} event - The event object received from the platform
|
|
45
40
|
* @param {Worker} worker - The worker thread
|
|
46
|
-
* @param {WorkerMemoryConfig} memoryConfig - Memory configuration for the worker
|
|
47
|
-
* @param {WorkerResourceLimits} resourceLimits - Resource limits applied to the worker
|
|
48
41
|
*/
|
|
49
42
|
export interface SpawnInterface {
|
|
50
43
|
event: AirdropEvent;
|
|
@@ -52,23 +45,6 @@ export interface SpawnInterface {
|
|
|
52
45
|
options?: WorkerAdapterOptions;
|
|
53
46
|
resolve: (value: void | PromiseLike<void>) => void;
|
|
54
47
|
originalConsole?: Console;
|
|
55
|
-
memoryConfig?: WorkerMemoryConfig;
|
|
56
|
-
resourceLimits?: WorkerResourceLimits;
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Memory configuration for worker threads.
|
|
60
|
-
*/
|
|
61
|
-
export interface WorkerMemoryConfig {
|
|
62
|
-
/** Maximum heap size in MB for the worker thread */
|
|
63
|
-
maxOldGenerationSizeMb: number;
|
|
64
|
-
/** Total available memory in MB (for logging/debugging) */
|
|
65
|
-
totalAvailableMemoryMb: number;
|
|
66
|
-
/** Whether running in Lambda environment */
|
|
67
|
-
isLambda: boolean;
|
|
68
|
-
/** Whether running in local development mode */
|
|
69
|
-
isLocalDevelopment: boolean;
|
|
70
|
-
/** The percentage of memory allocated to the worker (0-1) */
|
|
71
|
-
workerMemoryPercentage: number;
|
|
72
48
|
}
|
|
73
49
|
/**
|
|
74
50
|
* SpawnFactoryInterface is an interface for Spawn class factory.
|
|
@@ -169,38 +145,4 @@ export interface GetWorkerPathInterface {
|
|
|
169
145
|
* WorkerPathOverrides represents a mapping of event types to custom worker paths.
|
|
170
146
|
*/
|
|
171
147
|
export type WorkerPathOverrides = Partial<Record<EventType, string>>;
|
|
172
|
-
/**
|
|
173
|
-
* WorkerResourceLimits represents the resource limits applied to a worker thread.
|
|
174
|
-
*/
|
|
175
|
-
export interface WorkerResourceLimits {
|
|
176
|
-
/** Maximum size of the main (old generation) heap in MB */
|
|
177
|
-
maxOldGenerationSizeMb: number;
|
|
178
|
-
/** Maximum size of the young generation heap in MB (optional) */
|
|
179
|
-
maxYoungGenerationSizeMb?: number;
|
|
180
|
-
/** Size of pre-allocated memory range for generated code in MB (optional) */
|
|
181
|
-
codeRangeSizeMb?: number;
|
|
182
|
-
/** Maximum stack size in MB (optional) */
|
|
183
|
-
stackSizeMb?: number;
|
|
184
|
-
}
|
|
185
|
-
/**
|
|
186
|
-
* OOMErrorInfo represents detailed information about an OOM (Out-Of-Memory) error.
|
|
187
|
-
*/
|
|
188
|
-
export interface OOMErrorInfo {
|
|
189
|
-
/** Error type identifier */
|
|
190
|
-
type: 'OOM_ERROR';
|
|
191
|
-
/** Human-readable error message */
|
|
192
|
-
message: string;
|
|
193
|
-
/** Memory limit that was configured for the worker in MB */
|
|
194
|
-
memoryLimitMb: number;
|
|
195
|
-
/** Total available memory at spawn time in MB */
|
|
196
|
-
totalAvailableMemoryMb: number;
|
|
197
|
-
/** Whether running in Lambda environment */
|
|
198
|
-
isLambda: boolean;
|
|
199
|
-
/** Whether running in local development mode */
|
|
200
|
-
isLocalDevelopment: boolean;
|
|
201
|
-
/** Exit code from the worker (if available) */
|
|
202
|
-
exitCode?: number;
|
|
203
|
-
/** The event type being processed when OOM occurred */
|
|
204
|
-
eventType?: string;
|
|
205
|
-
}
|
|
206
148
|
//# sourceMappingURL=workers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED
|
|
1
|
+
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACxC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,SAAS,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED;;GAEG;AACH,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,oBAAoB,GAAG,iBAAiB,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACzB,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;;;;OAQG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwDpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAuB7B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmB1B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACzB,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;;;;OAQG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwDpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAuB7B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmB1B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA+B1B,qBAAqB,CACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAwBhC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAef,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAqCY,sBAAsB;YAyBtB,gBAAgB;IAe9B,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,UAAU;IASZ,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC;YAyBvB,eAAe;CAgC9B"}
|
|
@@ -146,11 +146,6 @@ class Uploader {
|
|
|
146
146
|
formData.append(field, artifact.form_data[field]);
|
|
147
147
|
}
|
|
148
148
|
formData.append('file', fileStream.data);
|
|
149
|
-
if (fileStream.headers['content-length'] > constants_1.MAX_DEVREV_ARTIFACT_SIZE) {
|
|
150
|
-
console.warn(`File size exceeds the maximum limit of ${constants_1.MAX_DEVREV_ARTIFACT_SIZE} bytes.`);
|
|
151
|
-
this.destroyStream(fileStream);
|
|
152
|
-
return;
|
|
153
|
-
}
|
|
154
149
|
try {
|
|
155
150
|
const response = await axios_client_internal_1.axiosClient.post(artifact.upload_url, formData, {
|
|
156
151
|
headers: Object.assign(Object.assign({}, formData.getHeaders()), (!fileStream.headers['content-length']
|
|
@@ -158,8 +153,10 @@ class Uploader {
|
|
|
158
153
|
'Content-Length': constants_1.MAX_DEVREV_ARTIFACT_SIZE,
|
|
159
154
|
}
|
|
160
155
|
: {})),
|
|
161
|
-
|
|
162
|
-
|
|
156
|
+
// Prevents buffering of the response in the memory
|
|
157
|
+
maxRedirects: 0,
|
|
158
|
+
// Allow 2xx and 3xx (redirects) to be considered successful, 4xx and 5xx will throw errors and be caught in the catch block
|
|
159
|
+
validateStatus: (status) => status >= 200 && status < 400,
|
|
163
160
|
});
|
|
164
161
|
this.destroyStream(fileStream);
|
|
165
162
|
return response;
|
|
@@ -1,23 +1,5 @@
|
|
|
1
1
|
import { Worker } from 'node:worker_threads';
|
|
2
|
-
import { WorkerData
|
|
3
|
-
|
|
4
|
-
* Result of creating a worker, including memory configuration.
|
|
5
|
-
*/
|
|
6
|
-
export interface CreateWorkerResult {
|
|
7
|
-
worker: Worker;
|
|
8
|
-
memoryConfig: WorkerMemoryConfig;
|
|
9
|
-
resourceLimits: WorkerResourceLimits;
|
|
10
|
-
}
|
|
11
|
-
/**
|
|
12
|
-
* Creates a worker thread with optional memory limits.
|
|
13
|
-
* Memory limits are calculated based on the environment:
|
|
14
|
-
* - For Lambda: Uses AWS_LAMBDA_FUNCTION_MEMORY_SIZE (75% for worker)
|
|
15
|
-
* - For local development: Caps total at 1024MB (75% for worker = ~768MB)
|
|
16
|
-
* - For other environments: Uses system memory or V8 heap limit
|
|
17
|
-
*
|
|
18
|
-
* @param workerData - Worker data including event, state, and options
|
|
19
|
-
* @returns Promise resolving to CreateWorkerResult with worker and memory config
|
|
20
|
-
*/
|
|
21
|
-
declare function createWorker<ConnectorState>(workerData: WorkerData<ConnectorState>): Promise<CreateWorkerResult>;
|
|
2
|
+
import { WorkerData } from '../types/workers';
|
|
3
|
+
declare function createWorker<ConnectorState>(workerData: WorkerData<ConnectorState>): Promise<Worker>;
|
|
22
4
|
export { createWorker };
|
|
23
5
|
//# sourceMappingURL=create-worker.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-worker.d.ts","sourceRoot":"","sources":["../../src/workers/create-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,MAAM,
|
|
1
|
+
{"version":3,"file":"create-worker.d.ts","sourceRoot":"","sources":["../../src/workers/create-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAe,MAAM,kBAAkB,CAAC;AAE3D,iBAAe,YAAY,CAAC,cAAc,EACxC,UAAU,EAAE,UAAU,CAAC,cAAc,CAAC,GACrC,OAAO,CAAC,MAAM,CAAC,CAyBjB;AAED,OAAO,EAAE,YAAY,EAAE,CAAC"}
|
|
@@ -2,58 +2,23 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createWorker = createWorker;
|
|
4
4
|
const node_worker_threads_1 = require("node:worker_threads");
|
|
5
|
-
const worker_memory_1 = require("../common/worker-memory");
|
|
6
5
|
const workers_1 = require("../types/workers");
|
|
7
|
-
/**
|
|
8
|
-
* Creates a worker thread with optional memory limits.
|
|
9
|
-
* Memory limits are calculated based on the environment:
|
|
10
|
-
* - For Lambda: Uses AWS_LAMBDA_FUNCTION_MEMORY_SIZE (75% for worker)
|
|
11
|
-
* - For local development: Caps total at 1024MB (75% for worker = ~768MB)
|
|
12
|
-
* - For other environments: Uses system memory or V8 heap limit
|
|
13
|
-
*
|
|
14
|
-
* @param workerData - Worker data including event, state, and options
|
|
15
|
-
* @returns Promise resolving to CreateWorkerResult with worker and memory config
|
|
16
|
-
*/
|
|
17
6
|
async function createWorker(workerData) {
|
|
18
7
|
return new Promise((resolve, reject) => {
|
|
19
|
-
var _a, _b, _c, _d;
|
|
20
8
|
if (node_worker_threads_1.isMainThread) {
|
|
21
9
|
const workerFile = __dirname + '/worker.js';
|
|
22
|
-
|
|
23
|
-
const isLocalDevelopment = (_b = (_a = workerData.options) === null || _a === void 0 ? void 0 : _a.isLocalDevelopment) !== null && _b !== void 0 ? _b : false;
|
|
24
|
-
const enableMemoryLimits = ((_c = workerData.options) === null || _c === void 0 ? void 0 : _c.enableMemoryLimits) !== false;
|
|
25
|
-
const testMemoryLimitMb = (_d = workerData.options) === null || _d === void 0 ? void 0 : _d.testMemoryLimitMb;
|
|
26
|
-
const memoryConfig = (0, worker_memory_1.calculateWorkerMemoryConfig)(isLocalDevelopment);
|
|
27
|
-
// Allow test override for memory limit
|
|
28
|
-
const effectiveMemoryLimitMb = testMemoryLimitMb !== null && testMemoryLimitMb !== void 0 ? testMemoryLimitMb : memoryConfig.maxOldGenerationSizeMb;
|
|
29
|
-
const resourceLimits = {
|
|
30
|
-
maxOldGenerationSizeMb: effectiveMemoryLimitMb,
|
|
31
|
-
};
|
|
32
|
-
// Build worker options
|
|
33
|
-
const workerOptions = {
|
|
10
|
+
const worker = new node_worker_threads_1.Worker(workerFile, {
|
|
34
11
|
workerData,
|
|
35
|
-
};
|
|
36
|
-
// Apply resource limits if enabled
|
|
37
|
-
if (enableMemoryLimits) {
|
|
38
|
-
workerOptions.resourceLimits = {
|
|
39
|
-
maxOldGenerationSizeMb: resourceLimits.maxOldGenerationSizeMb,
|
|
40
|
-
};
|
|
41
|
-
console.info(`Worker memory limits configured: ` +
|
|
42
|
-
`maxOldGenerationSizeMb=${resourceLimits.maxOldGenerationSizeMb}MB, ` +
|
|
43
|
-
`totalAvailable=${memoryConfig.totalAvailableMemoryMb.toFixed(0)}MB, ` +
|
|
44
|
-
`isLambda=${memoryConfig.isLambda}, ` +
|
|
45
|
-
`isLocalDevelopment=${memoryConfig.isLocalDevelopment}`);
|
|
46
|
-
}
|
|
47
|
-
const worker = new node_worker_threads_1.Worker(workerFile, workerOptions);
|
|
12
|
+
});
|
|
48
13
|
worker.on(workers_1.WorkerEvent.WorkerError, (error) => {
|
|
49
14
|
console.error('Worker error', error);
|
|
50
15
|
reject(error);
|
|
51
16
|
});
|
|
52
17
|
worker.on(workers_1.WorkerEvent.WorkerOnline, () => {
|
|
18
|
+
resolve(worker);
|
|
53
19
|
console.info('Worker is online. Started processing the task with event type: ' +
|
|
54
20
|
workerData.event.payload.event_type +
|
|
55
21
|
'.');
|
|
56
|
-
resolve({ worker, memoryConfig, resourceLimits });
|
|
57
22
|
});
|
|
58
23
|
}
|
|
59
24
|
else {
|
|
@@ -7,7 +7,7 @@ const create_worker_1 = require("./create-worker");
|
|
|
7
7
|
describe(create_worker_1.createWorker.name, () => {
|
|
8
8
|
it('should create a Worker instance when valid parameters are provided', async () => {
|
|
9
9
|
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
10
|
-
const
|
|
10
|
+
const worker = worker_threads_1.isMainThread
|
|
11
11
|
? await (0, create_worker_1.createWorker)({
|
|
12
12
|
event: (0, test_helpers_1.createEvent)({
|
|
13
13
|
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
@@ -16,12 +16,10 @@ describe(create_worker_1.createWorker.name, () => {
|
|
|
16
16
|
workerPath,
|
|
17
17
|
})
|
|
18
18
|
: null;
|
|
19
|
-
expect(
|
|
20
|
-
expect(
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
if (result) {
|
|
24
|
-
await result.worker.terminate();
|
|
19
|
+
expect(worker).not.toBeNull();
|
|
20
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
21
|
+
if (worker) {
|
|
22
|
+
await worker.terminate();
|
|
25
23
|
}
|
|
26
24
|
});
|
|
27
25
|
it('should throw error when not in main thread', async () => {
|
|
@@ -43,15 +41,15 @@ describe(create_worker_1.createWorker.name, () => {
|
|
|
43
41
|
it('[edge] should handle worker creation with minimal valid data', async () => {
|
|
44
42
|
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
45
43
|
if (worker_threads_1.isMainThread) {
|
|
46
|
-
const
|
|
44
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
47
45
|
event: (0, test_helpers_1.createEvent)({
|
|
48
46
|
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
49
47
|
}),
|
|
50
48
|
initialState: {},
|
|
51
49
|
workerPath,
|
|
52
50
|
});
|
|
53
|
-
expect(
|
|
54
|
-
await
|
|
51
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
52
|
+
await worker.terminate();
|
|
55
53
|
}
|
|
56
54
|
});
|
|
57
55
|
it('[edge] should handle worker creation with complex initial state', async () => {
|
|
@@ -63,126 +61,29 @@ describe(create_worker_1.createWorker.name, () => {
|
|
|
63
61
|
},
|
|
64
62
|
};
|
|
65
63
|
if (worker_threads_1.isMainThread) {
|
|
66
|
-
const
|
|
64
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
67
65
|
event: (0, test_helpers_1.createEvent)({
|
|
68
66
|
eventType: extraction_1.EventType.ExtractionDataStart,
|
|
69
67
|
}),
|
|
70
68
|
initialState: complexState,
|
|
71
69
|
workerPath,
|
|
72
70
|
});
|
|
73
|
-
expect(
|
|
74
|
-
await
|
|
71
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
72
|
+
await worker.terminate();
|
|
75
73
|
}
|
|
76
74
|
});
|
|
77
75
|
it('[edge] should handle different event types', async () => {
|
|
78
76
|
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
79
77
|
if (worker_threads_1.isMainThread) {
|
|
80
|
-
const
|
|
78
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
81
79
|
event: (0, test_helpers_1.createEvent)({
|
|
82
80
|
eventType: extraction_1.EventType.ExtractionMetadataStart,
|
|
83
81
|
}),
|
|
84
82
|
initialState: {},
|
|
85
83
|
workerPath,
|
|
86
84
|
});
|
|
87
|
-
expect(
|
|
88
|
-
await
|
|
89
|
-
}
|
|
90
|
-
});
|
|
91
|
-
it('should include memory configuration in result', async () => {
|
|
92
|
-
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
93
|
-
if (worker_threads_1.isMainThread) {
|
|
94
|
-
const result = await (0, create_worker_1.createWorker)({
|
|
95
|
-
event: (0, test_helpers_1.createEvent)({
|
|
96
|
-
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
97
|
-
}),
|
|
98
|
-
initialState: {},
|
|
99
|
-
workerPath,
|
|
100
|
-
});
|
|
101
|
-
expect(result.memoryConfig).toHaveProperty('maxOldGenerationSizeMb');
|
|
102
|
-
expect(result.memoryConfig).toHaveProperty('totalAvailableMemoryMb');
|
|
103
|
-
expect(result.memoryConfig).toHaveProperty('isLambda');
|
|
104
|
-
expect(result.memoryConfig).toHaveProperty('isLocalDevelopment');
|
|
105
|
-
expect(result.resourceLimits).toHaveProperty('maxOldGenerationSizeMb');
|
|
106
|
-
await result.worker.terminate();
|
|
107
|
-
}
|
|
108
|
-
});
|
|
109
|
-
it('should apply testMemoryLimitMb override when provided', async () => {
|
|
110
|
-
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
111
|
-
const testMemoryLimit = 64;
|
|
112
|
-
if (worker_threads_1.isMainThread) {
|
|
113
|
-
const result = await (0, create_worker_1.createWorker)({
|
|
114
|
-
event: (0, test_helpers_1.createEvent)({
|
|
115
|
-
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
116
|
-
}),
|
|
117
|
-
initialState: {},
|
|
118
|
-
workerPath,
|
|
119
|
-
options: {
|
|
120
|
-
testMemoryLimitMb: testMemoryLimit,
|
|
121
|
-
},
|
|
122
|
-
});
|
|
123
|
-
expect(result.resourceLimits.maxOldGenerationSizeMb).toBe(testMemoryLimit);
|
|
124
|
-
await result.worker.terminate();
|
|
125
|
-
}
|
|
126
|
-
});
|
|
127
|
-
it('should create worker without memory limits when enableMemoryLimits is false', async () => {
|
|
128
|
-
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
129
|
-
if (worker_threads_1.isMainThread) {
|
|
130
|
-
const result = await (0, create_worker_1.createWorker)({
|
|
131
|
-
event: (0, test_helpers_1.createEvent)({
|
|
132
|
-
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
133
|
-
}),
|
|
134
|
-
initialState: {},
|
|
135
|
-
workerPath,
|
|
136
|
-
options: {
|
|
137
|
-
enableMemoryLimits: false,
|
|
138
|
-
},
|
|
139
|
-
});
|
|
140
|
-
// Worker should still be created successfully
|
|
141
|
-
expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
142
|
-
// Memory config should still be calculated (for logging purposes)
|
|
143
|
-
expect(result.memoryConfig).toBeDefined();
|
|
144
|
-
expect(result.resourceLimits).toBeDefined();
|
|
145
|
-
await result.worker.terminate();
|
|
146
|
-
}
|
|
147
|
-
});
|
|
148
|
-
it('should set isLocalDevelopment in memory config when option is provided', async () => {
|
|
149
|
-
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
150
|
-
if (worker_threads_1.isMainThread) {
|
|
151
|
-
const result = await (0, create_worker_1.createWorker)({
|
|
152
|
-
event: (0, test_helpers_1.createEvent)({
|
|
153
|
-
eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
154
|
-
}),
|
|
155
|
-
initialState: {},
|
|
156
|
-
workerPath,
|
|
157
|
-
options: {
|
|
158
|
-
isLocalDevelopment: true,
|
|
159
|
-
},
|
|
160
|
-
});
|
|
161
|
-
expect(result.memoryConfig.isLocalDevelopment).toBe(true);
|
|
162
|
-
await result.worker.terminate();
|
|
163
|
-
}
|
|
164
|
-
});
|
|
165
|
-
it('[edge] should handle all extraction event types', async () => {
|
|
166
|
-
const workerPath = __dirname + '../tests/dummy-worker.ts';
|
|
167
|
-
const eventTypes = [
|
|
168
|
-
extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
169
|
-
extraction_1.EventType.ExtractionMetadataStart,
|
|
170
|
-
extraction_1.EventType.ExtractionDataStart,
|
|
171
|
-
extraction_1.EventType.ExtractionDataContinue,
|
|
172
|
-
extraction_1.EventType.ExtractionAttachmentsStart,
|
|
173
|
-
extraction_1.EventType.ExtractionAttachmentsContinue,
|
|
174
|
-
];
|
|
175
|
-
if (worker_threads_1.isMainThread) {
|
|
176
|
-
for (const eventType of eventTypes) {
|
|
177
|
-
const result = await (0, create_worker_1.createWorker)({
|
|
178
|
-
event: (0, test_helpers_1.createEvent)({ eventType }),
|
|
179
|
-
initialState: {},
|
|
180
|
-
workerPath,
|
|
181
|
-
});
|
|
182
|
-
expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
183
|
-
expect(result.memoryConfig).toBeDefined();
|
|
184
|
-
await result.worker.terminate();
|
|
185
|
-
}
|
|
85
|
+
expect(worker).toBeInstanceOf(worker_threads_1.Worker);
|
|
86
|
+
await worker.terminate();
|
|
186
87
|
}
|
|
187
88
|
});
|
|
188
89
|
});
|
package/dist/workers/spawn.d.ts
CHANGED
|
@@ -22,17 +22,8 @@ export declare class Spawn {
|
|
|
22
22
|
private resolve;
|
|
23
23
|
private originalConsole;
|
|
24
24
|
private logger;
|
|
25
|
-
|
|
26
|
-
private resourceLimits?;
|
|
27
|
-
private isOOMExit;
|
|
28
|
-
constructor({ event, worker, options, resolve, originalConsole, memoryConfig, resourceLimits, }: SpawnInterface);
|
|
25
|
+
constructor({ event, worker, options, resolve, originalConsole, }: SpawnInterface);
|
|
29
26
|
private clearTimeouts;
|
|
30
27
|
private exitFromMainThread;
|
|
31
|
-
/**
|
|
32
|
-
* Handles worker exit due to OOM (Out-Of-Memory) error.
|
|
33
|
-
* Emits an error event with detailed OOM information.
|
|
34
|
-
* @param exitCode - The exit code from the worker
|
|
35
|
-
*/
|
|
36
|
-
private handleOOMExit;
|
|
37
28
|
}
|
|
38
29
|
//# sourceMappingURL=spawn.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,kBAAkB,CAAC;AAmC1B;;;;;;;;;;GAUG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAuGvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;gBACX,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,GAChB,EAAE,cAAc;IAgGjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;CAiCjC"}
|
package/dist/workers/spawn.js
CHANGED
|
@@ -10,7 +10,6 @@ const helpers_1 = require("yargs/helpers");
|
|
|
10
10
|
const control_protocol_1 = require("../common/control-protocol");
|
|
11
11
|
const event_type_translation_1 = require("../common/event-type-translation");
|
|
12
12
|
const helpers_2 = require("../common/helpers");
|
|
13
|
-
const worker_memory_1 = require("../common/worker-memory");
|
|
14
13
|
const logger_1 = require("../logger/logger");
|
|
15
14
|
const extraction_1 = require("../types/extraction");
|
|
16
15
|
const workers_1 = require("../types/workers");
|
|
@@ -87,7 +86,7 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
|
|
|
87
86
|
}
|
|
88
87
|
if (script) {
|
|
89
88
|
try {
|
|
90
|
-
const
|
|
89
|
+
const worker = await (0, create_worker_1.createWorker)({
|
|
91
90
|
event,
|
|
92
91
|
initialState,
|
|
93
92
|
workerPath: script,
|
|
@@ -101,8 +100,6 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
|
|
|
101
100
|
options,
|
|
102
101
|
resolve,
|
|
103
102
|
originalConsole,
|
|
104
|
-
memoryConfig,
|
|
105
|
-
resourceLimits,
|
|
106
103
|
});
|
|
107
104
|
});
|
|
108
105
|
}
|
|
@@ -139,9 +136,8 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
|
|
|
139
136
|
}
|
|
140
137
|
}
|
|
141
138
|
class Spawn {
|
|
142
|
-
constructor({ event, worker, options, resolve, originalConsole,
|
|
139
|
+
constructor({ event, worker, options, resolve, originalConsole, }) {
|
|
143
140
|
this.defaultLambdaTimeout = constants_1.DEFAULT_LAMBDA_TIMEOUT;
|
|
144
|
-
this.isOOMExit = false;
|
|
145
141
|
this.originalConsole = originalConsole || console;
|
|
146
142
|
this.logger = console;
|
|
147
143
|
this.alreadyEmitted = false;
|
|
@@ -150,8 +146,6 @@ class Spawn {
|
|
|
150
146
|
? Math.min(options.timeout, this.defaultLambdaTimeout)
|
|
151
147
|
: this.defaultLambdaTimeout;
|
|
152
148
|
this.resolve = resolve;
|
|
153
|
-
this.memoryConfig = memoryConfig;
|
|
154
|
-
this.resourceLimits = resourceLimits;
|
|
155
149
|
// If soft timeout is reached, send a message to the worker to gracefully exit.
|
|
156
150
|
this.softTimeoutTimer = setTimeout(() => void (async () => {
|
|
157
151
|
console.log('SOFT TIMEOUT: Sending a message to the worker to gracefully exit.');
|
|
@@ -176,33 +170,12 @@ class Spawn {
|
|
|
176
170
|
await this.exitFromMainThread();
|
|
177
171
|
}
|
|
178
172
|
})(), this.lambdaTimeout * constants_1.HARD_TIMEOUT_MULTIPLIER);
|
|
179
|
-
// Listen for worker errors to detect OOM
|
|
180
|
-
// Node.js worker threads emit ERR_WORKER_OUT_OF_MEMORY error code when OOM occurs
|
|
181
|
-
// @see https://nodejs.org/api/errors.html#err_worker_out_of_memory
|
|
182
|
-
worker.on(workers_1.WorkerEvent.WorkerError, (error) => {
|
|
183
|
-
if ((0, worker_memory_1.isOOMError)(error)) {
|
|
184
|
-
this.isOOMExit = true;
|
|
185
|
-
console.error('OOM ERROR DETECTED: Worker ran out of memory.', {
|
|
186
|
-
errorCode: error.code,
|
|
187
|
-
errorMessage: error === null || error === void 0 ? void 0 : error.message,
|
|
188
|
-
memoryConfig: this.memoryConfig,
|
|
189
|
-
resourceLimits: this.resourceLimits,
|
|
190
|
-
});
|
|
191
|
-
}
|
|
192
|
-
});
|
|
193
173
|
// If worker exits with process.exit(code), clear the timeouts and exit from
|
|
194
174
|
// main thread.
|
|
195
175
|
worker.on(workers_1.WorkerEvent.WorkerExit, (code) => void (async () => {
|
|
196
176
|
console.info('Worker exited with exit code: ' + code + '.');
|
|
197
177
|
this.clearTimeouts();
|
|
198
|
-
|
|
199
|
-
// Exit code 134 is common for OOM (SIGABRT), but we also check the error message
|
|
200
|
-
if (code !== 0 && this.isOOMExit) {
|
|
201
|
-
await this.handleOOMExit(code);
|
|
202
|
-
}
|
|
203
|
-
else {
|
|
204
|
-
await this.exitFromMainThread();
|
|
205
|
-
}
|
|
178
|
+
await this.exitFromMainThread();
|
|
206
179
|
})());
|
|
207
180
|
worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
|
|
208
181
|
var _a, _b;
|
|
@@ -275,52 +248,5 @@ class Spawn {
|
|
|
275
248
|
console.error('Error while emitting event.', (0, logger_1.serializeError)(error));
|
|
276
249
|
}
|
|
277
250
|
}
|
|
278
|
-
/**
|
|
279
|
-
* Handles worker exit due to OOM (Out-Of-Memory) error.
|
|
280
|
-
* Emits an error event with detailed OOM information.
|
|
281
|
-
* @param exitCode - The exit code from the worker
|
|
282
|
-
*/
|
|
283
|
-
async handleOOMExit(exitCode) {
|
|
284
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
285
|
-
this.clearTimeouts();
|
|
286
|
-
// eslint-disable-next-line no-global-assign
|
|
287
|
-
console = this.originalConsole;
|
|
288
|
-
if (this.alreadyEmitted) {
|
|
289
|
-
this.resolve();
|
|
290
|
-
return;
|
|
291
|
-
}
|
|
292
|
-
this.alreadyEmitted = true;
|
|
293
|
-
const { eventType } = (0, helpers_2.getTimeoutErrorEventType)(this.event.payload.event_type);
|
|
294
|
-
// Build OOM error info
|
|
295
|
-
const oomErrorInfo = {
|
|
296
|
-
type: 'OOM_ERROR',
|
|
297
|
-
message: `Worker thread ran out of memory and was terminated. ` +
|
|
298
|
-
`Memory limit: ${(_b = (_a = this.resourceLimits) === null || _a === void 0 ? void 0 : _a.maxOldGenerationSizeMb) !== null && _b !== void 0 ? _b : 'unknown'}MB, ` +
|
|
299
|
-
`Total available: ${(_e = (_d = (_c = this.memoryConfig) === null || _c === void 0 ? void 0 : _c.totalAvailableMemoryMb) === null || _d === void 0 ? void 0 : _d.toFixed(0)) !== null && _e !== void 0 ? _e : 'unknown'}MB.`,
|
|
300
|
-
memoryLimitMb: (_g = (_f = this.resourceLimits) === null || _f === void 0 ? void 0 : _f.maxOldGenerationSizeMb) !== null && _g !== void 0 ? _g : 0,
|
|
301
|
-
totalAvailableMemoryMb: (_j = (_h = this.memoryConfig) === null || _h === void 0 ? void 0 : _h.totalAvailableMemoryMb) !== null && _j !== void 0 ? _j : 0,
|
|
302
|
-
isLambda: (_l = (_k = this.memoryConfig) === null || _k === void 0 ? void 0 : _k.isLambda) !== null && _l !== void 0 ? _l : false,
|
|
303
|
-
isLocalDevelopment: (_o = (_m = this.memoryConfig) === null || _m === void 0 ? void 0 : _m.isLocalDevelopment) !== null && _o !== void 0 ? _o : false,
|
|
304
|
-
exitCode,
|
|
305
|
-
eventType: this.event.payload.event_type,
|
|
306
|
-
};
|
|
307
|
-
console.error('OOM Error Details:', oomErrorInfo);
|
|
308
|
-
try {
|
|
309
|
-
await (0, control_protocol_1.emit)({
|
|
310
|
-
eventType,
|
|
311
|
-
event: this.event,
|
|
312
|
-
data: {
|
|
313
|
-
error: {
|
|
314
|
-
message: oomErrorInfo.message,
|
|
315
|
-
oom_error_info: oomErrorInfo,
|
|
316
|
-
},
|
|
317
|
-
},
|
|
318
|
-
});
|
|
319
|
-
this.resolve();
|
|
320
|
-
}
|
|
321
|
-
catch (error) {
|
|
322
|
-
console.error('Error while emitting OOM error event.', (0, logger_1.serializeError)(error));
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
251
|
}
|
|
326
252
|
exports.Spawn = Spawn;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devrev/ts-adaas",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.13.0",
|
|
4
4
|
"description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
"lint": "eslint .",
|
|
13
13
|
"lint:fix": "eslint . --fix",
|
|
14
14
|
"test": "jest --forceExit --coverage --runInBand --selectProjects default --",
|
|
15
|
-
"test:oom": "node --max-old-space-size=4096 ./node_modules/.bin/jest --forceExit --runInBand --selectProjects oom-tests --logHeapUsage --",
|
|
16
15
|
"test:backwards-compatibility": "jest --forceExit --runInBand --selectProjects backwards-compatibility --"
|
|
17
16
|
},
|
|
18
17
|
"repository": {
|