@devrev/ts-adaas 1.12.3-beta.7 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/common/constants.d.ts +1 -1
  2. package/dist/common/constants.d.ts.map +1 -1
  3. package/dist/common/constants.js +1 -1
  4. package/dist/types/common.d.ts +0 -3
  5. package/dist/types/common.d.ts.map +1 -1
  6. package/dist/types/workers.d.ts +0 -58
  7. package/dist/types/workers.d.ts.map +1 -1
  8. package/dist/uploader/uploader.d.ts.map +1 -1
  9. package/dist/uploader/uploader.js +4 -7
  10. package/dist/workers/create-worker.d.ts +2 -20
  11. package/dist/workers/create-worker.d.ts.map +1 -1
  12. package/dist/workers/create-worker.js +3 -38
  13. package/dist/workers/create-worker.test.js +14 -113
  14. package/dist/workers/spawn.d.ts +1 -10
  15. package/dist/workers/spawn.d.ts.map +1 -1
  16. package/dist/workers/spawn.js +3 -77
  17. package/package.json +1 -2
  18. package/dist/common/worker-memory.d.ts +0 -59
  19. package/dist/common/worker-memory.d.ts.map +0 -1
  20. package/dist/common/worker-memory.js +0 -177
  21. package/dist/tests/oom-handling/extraction.d.ts +0 -11
  22. package/dist/tests/oom-handling/extraction.d.ts.map +0 -1
  23. package/dist/tests/oom-handling/extraction.js +0 -29
  24. package/dist/tests/oom-handling/jest.setup.d.ts +0 -7
  25. package/dist/tests/oom-handling/jest.setup.d.ts.map +0 -1
  26. package/dist/tests/oom-handling/jest.setup.js +0 -26
  27. package/dist/tests/oom-handling/oom-after-emit-worker.d.ts +0 -2
  28. package/dist/tests/oom-handling/oom-after-emit-worker.d.ts.map +0 -1
  29. package/dist/tests/oom-handling/oom-after-emit-worker.js +0 -35
  30. package/dist/tests/oom-handling/oom-attachments-worker.d.ts +0 -2
  31. package/dist/tests/oom-handling/oom-attachments-worker.d.ts.map +0 -1
  32. package/dist/tests/oom-handling/oom-attachments-worker.js +0 -29
  33. package/dist/tests/oom-handling/oom-external-sync-units-worker.d.ts +0 -2
  34. package/dist/tests/oom-handling/oom-external-sync-units-worker.d.ts.map +0 -1
  35. package/dist/tests/oom-handling/oom-external-sync-units-worker.js +0 -29
  36. package/dist/tests/oom-handling/oom-gradual-worker.d.ts +0 -2
  37. package/dist/tests/oom-handling/oom-gradual-worker.d.ts.map +0 -1
  38. package/dist/tests/oom-handling/oom-gradual-worker.js +0 -47
  39. package/dist/tests/oom-handling/oom-handling.test.d.ts +0 -2
  40. package/dist/tests/oom-handling/oom-handling.test.d.ts.map +0 -1
  41. package/dist/tests/oom-handling/oom-handling.test.js +0 -407
  42. package/dist/tests/oom-handling/oom-metadata-worker.d.ts +0 -2
  43. package/dist/tests/oom-handling/oom-metadata-worker.d.ts.map +0 -1
  44. package/dist/tests/oom-handling/oom-metadata-worker.js +0 -29
  45. package/dist/tests/oom-handling/oom-worker.d.ts +0 -2
  46. package/dist/tests/oom-handling/oom-worker.d.ts.map +0 -1
  47. package/dist/tests/oom-handling/oom-worker.js +0 -55
  48. package/dist/tests/oom-handling/worker-memory.test.d.ts +0 -2
  49. package/dist/tests/oom-handling/worker-memory.test.d.ts.map +0 -1
  50. package/dist/tests/oom-handling/worker-memory.test.js +0 -306
@@ -9,7 +9,7 @@ export declare const STATEFUL_EXTRACTION_EVENT_TYPES: EventType[];
9
9
  export declare const STATEFUL_LOADING_EVENT_TYPES: EventType[];
10
10
  export declare const STATEFUL_EVENT_TYPES: EventType[];
11
11
  export declare const ARTIFACT_BATCH_SIZE = 2000;
12
- export declare const MAX_DEVREV_ARTIFACT_SIZE = 262144000;
12
+ export declare const MAX_DEVREV_ARTIFACT_SIZE: number;
13
13
  export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
14
14
  export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
15
15
  export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
@@ -1 +1 @@
1
- {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,YAAY,CAAC;AAClD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAEvD,eAAO,MAAM,0BAA0B;;;;CAItC,CAAC;AAEF,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC"}
1
+ {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAEvD,eAAO,MAAM,0BAA0B;;;;CAItC,CAAC;AAEF,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC"}
@@ -43,7 +43,7 @@ exports.STATEFUL_EVENT_TYPES = [
43
43
  ...exports.STATEFUL_LOADING_EVENT_TYPES,
44
44
  ];
45
45
  exports.ARTIFACT_BATCH_SIZE = 2000;
46
- exports.MAX_DEVREV_ARTIFACT_SIZE = 262144000; // 250MB
46
+ exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
47
47
  exports.MAX_DEVREV_FILENAME_LENGTH = 256;
48
48
  exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20; // 20 characters for the file extension
49
49
  exports.AIRDROP_DEFAULT_ITEM_TYPES = {
@@ -1,5 +1,4 @@
1
1
  import { Artifact } from '../uploader/uploader.interfaces';
2
- import { OOMErrorInfo } from './workers';
3
2
  /**
4
3
  * ErrorLevel is an enum that represents the level of an error.
5
4
  * @deprecated
@@ -14,8 +13,6 @@ export declare enum ErrorLevel {
14
13
  */
15
14
  export interface ErrorRecord {
16
15
  message: string;
17
- /** OOM error information, present when worker died due to out-of-memory */
18
- oom_error_info?: OOMErrorInfo;
19
16
  }
20
17
  /**
21
18
  * LogRecord is an interface that defines the structure of a log record.
@@ -1 +1 @@
1
- {"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/types/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC;;;GAGG;AACH,oBAAY,UAAU;IACpB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,IAAI,SAAS;CACd;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,2EAA2E;IAC3E,cAAc,CAAC,EAAE,YAAY,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;GAKG;AACH,oBAAY,QAAQ;IAClB,OAAO,YAAY;IACnB,WAAW,gBAAgB;IAC3B,OAAO,YAAY;CACpB"}
1
+ {"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/types/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D;;;GAGG;AACH,oBAAY,UAAU;IACpB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,IAAI,SAAS;CACd;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;GAKG;AACH,oBAAY,QAAQ;IAClB,OAAO,YAAY;IACnB,WAAW,gBAAgB;IAC3B,OAAO,YAAY;CACpB"}
@@ -25,17 +25,12 @@ export interface WorkerAdapterInterface<ConnectorState> {
25
25
  * @param {number=} timeout - The timeout for the worker thread
26
26
  * @param {number=} batchSize - Maximum number of extracted items in a batch
27
27
  * @param {Record<EventType, string>=} workerPathOverrides - A map of event types to custom worker paths to override default worker paths
28
- * @param {boolean=} enableMemoryLimits - Whether to enable automatic memory limits for worker threads (default: true)
29
- * @param {number=} testMemoryLimitMb - Override memory limit for testing purposes (only used when enableMemoryLimits is true)
30
28
  */
31
29
  export interface WorkerAdapterOptions {
32
30
  isLocalDevelopment?: boolean;
33
31
  timeout?: number;
34
32
  batchSize?: number;
35
33
  workerPathOverrides?: WorkerPathOverrides;
36
- enableMemoryLimits?: boolean;
37
- /** Override memory limit for testing purposes (only used when enableMemoryLimits is true) */
38
- testMemoryLimitMb?: number;
39
34
  }
40
35
  /**
41
36
  * SpawnInterface is an interface for Spawn class.
@@ -43,8 +38,6 @@ export interface WorkerAdapterOptions {
43
38
  * @constructor
44
39
  * @param {AirdropEvent} event - The event object received from the platform
45
40
  * @param {Worker} worker - The worker thread
46
- * @param {WorkerMemoryConfig} memoryConfig - Memory configuration for the worker
47
- * @param {WorkerResourceLimits} resourceLimits - Resource limits applied to the worker
48
41
  */
49
42
  export interface SpawnInterface {
50
43
  event: AirdropEvent;
@@ -52,23 +45,6 @@ export interface SpawnInterface {
52
45
  options?: WorkerAdapterOptions;
53
46
  resolve: (value: void | PromiseLike<void>) => void;
54
47
  originalConsole?: Console;
55
- memoryConfig?: WorkerMemoryConfig;
56
- resourceLimits?: WorkerResourceLimits;
57
- }
58
- /**
59
- * Memory configuration for worker threads.
60
- */
61
- export interface WorkerMemoryConfig {
62
- /** Maximum heap size in MB for the worker thread */
63
- maxOldGenerationSizeMb: number;
64
- /** Total available memory in MB (for logging/debugging) */
65
- totalAvailableMemoryMb: number;
66
- /** Whether running in Lambda environment */
67
- isLambda: boolean;
68
- /** Whether running in local development mode */
69
- isLocalDevelopment: boolean;
70
- /** The percentage of memory allocated to the worker (0-1) */
71
- workerMemoryPercentage: number;
72
48
  }
73
49
  /**
74
50
  * SpawnFactoryInterface is an interface for Spawn class factory.
@@ -169,38 +145,4 @@ export interface GetWorkerPathInterface {
169
145
  * WorkerPathOverrides represents a mapping of event types to custom worker paths.
170
146
  */
171
147
  export type WorkerPathOverrides = Partial<Record<EventType, string>>;
172
- /**
173
- * WorkerResourceLimits represents the resource limits applied to a worker thread.
174
- */
175
- export interface WorkerResourceLimits {
176
- /** Maximum size of the main (old generation) heap in MB */
177
- maxOldGenerationSizeMb: number;
178
- /** Maximum size of the young generation heap in MB (optional) */
179
- maxYoungGenerationSizeMb?: number;
180
- /** Size of pre-allocated memory range for generated code in MB (optional) */
181
- codeRangeSizeMb?: number;
182
- /** Maximum stack size in MB (optional) */
183
- stackSizeMb?: number;
184
- }
185
- /**
186
- * OOMErrorInfo represents detailed information about an OOM (Out-Of-Memory) error.
187
- */
188
- export interface OOMErrorInfo {
189
- /** Error type identifier */
190
- type: 'OOM_ERROR';
191
- /** Human-readable error message */
192
- message: string;
193
- /** Memory limit that was configured for the worker in MB */
194
- memoryLimitMb: number;
195
- /** Total available memory at spawn time in MB */
196
- totalAvailableMemoryMb: number;
197
- /** Whether running in Lambda environment */
198
- isLambda: boolean;
199
- /** Whether running in local development mode */
200
- isLocalDevelopment: boolean;
201
- /** Exit code from the worker (if available) */
202
- exitCode?: number;
203
- /** The event type being processed when OOM occurred */
204
- eventType?: string;
205
- }
206
148
  //# sourceMappingURL=workers.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,6FAA6F;IAC7F,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,oDAAoD;IACpD,sBAAsB,EAAE,MAAM,CAAC;IAC/B,2DAA2D;IAC3D,sBAAsB,EAAE,MAAM,CAAC;IAC/B,4CAA4C;IAC5C,QAAQ,EAAE,OAAO,CAAC;IAClB,gDAAgD;IAChD,kBAAkB,EAAE,OAAO,CAAC;IAC5B,6DAA6D;IAC7D,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACxC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,SAAS,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED;;GAEG;AACH,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,oBAAoB,GAAG,iBAAiB,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,2DAA2D;IAC3D,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iEAAiE;IACjE,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,6EAA6E;IAC7E,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,4BAA4B;IAC5B,IAAI,EAAE,WAAW,CAAC;IAClB,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,aAAa,EAAE,MAAM,CAAC;IACtB,iDAAiD;IACjD,sBAAsB,EAAE,MAAM,CAAC;IAC/B,4CAA4C;IAC5C,QAAQ,EAAE,OAAO,CAAC;IAClB,gDAAgD;IAChD,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}
1
+ {"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACxC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,SAAS,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED;;GAEG;AACH,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,oBAAoB,GAAG,iBAAiB,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACzB,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;;;;OAQG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwDpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAuB7B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmB1B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAqC1B,qBAAqB,CACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAwBhC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAef,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAqCY,sBAAsB;YAyBtB,gBAAgB;IAe9B,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,UAAU;IASZ,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC;YAyBvB,eAAe;CAgC9B"}
1
+ {"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACzB,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;;;;OAQG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwDpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAuB7B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmB1B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA+B1B,qBAAqB,CACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAwBhC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAef,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAqCY,sBAAsB;YAyBtB,gBAAgB;IAe9B,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,UAAU;IASZ,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC;YAyBvB,eAAe;CAgC9B"}
@@ -146,11 +146,6 @@ class Uploader {
146
146
  formData.append(field, artifact.form_data[field]);
147
147
  }
148
148
  formData.append('file', fileStream.data);
149
- if (fileStream.headers['content-length'] > constants_1.MAX_DEVREV_ARTIFACT_SIZE) {
150
- console.warn(`File size exceeds the maximum limit of ${constants_1.MAX_DEVREV_ARTIFACT_SIZE} bytes.`);
151
- this.destroyStream(fileStream);
152
- return;
153
- }
154
149
  try {
155
150
  const response = await axios_client_internal_1.axiosClient.post(artifact.upload_url, formData, {
156
151
  headers: Object.assign(Object.assign({}, formData.getHeaders()), (!fileStream.headers['content-length']
@@ -158,8 +153,10 @@ class Uploader {
158
153
  'Content-Length': constants_1.MAX_DEVREV_ARTIFACT_SIZE,
159
154
  }
160
155
  : {})),
161
- maxRedirects: 0, // Prevents buffering
162
- validateStatus: () => true, // Prevents errors on redirects
156
+ // Prevents buffering of the response in the memory
157
+ maxRedirects: 0,
158
+ // Allow 2xx and 3xx (redirects) to be considered successful, 4xx and 5xx will throw errors and be caught in the catch block
159
+ validateStatus: (status) => status >= 200 && status < 400,
163
160
  });
164
161
  this.destroyStream(fileStream);
165
162
  return response;
@@ -1,23 +1,5 @@
1
1
  import { Worker } from 'node:worker_threads';
2
- import { WorkerData, WorkerMemoryConfig, WorkerResourceLimits } from '../types/workers';
3
- /**
4
- * Result of creating a worker, including memory configuration.
5
- */
6
- export interface CreateWorkerResult {
7
- worker: Worker;
8
- memoryConfig: WorkerMemoryConfig;
9
- resourceLimits: WorkerResourceLimits;
10
- }
11
- /**
12
- * Creates a worker thread with optional memory limits.
13
- * Memory limits are calculated based on the environment:
14
- * - For Lambda: Uses AWS_LAMBDA_FUNCTION_MEMORY_SIZE (75% for worker)
15
- * - For local development: Caps total at 1024MB (75% for worker = ~768MB)
16
- * - For other environments: Uses system memory or V8 heap limit
17
- *
18
- * @param workerData - Worker data including event, state, and options
19
- * @returns Promise resolving to CreateWorkerResult with worker and memory config
20
- */
21
- declare function createWorker<ConnectorState>(workerData: WorkerData<ConnectorState>): Promise<CreateWorkerResult>;
2
+ import { WorkerData } from '../types/workers';
3
+ declare function createWorker<ConnectorState>(workerData: WorkerData<ConnectorState>): Promise<Worker>;
22
4
  export { createWorker };
23
5
  //# sourceMappingURL=create-worker.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"create-worker.d.ts","sourceRoot":"","sources":["../../src/workers/create-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,MAAM,EAAiB,MAAM,qBAAqB,CAAC;AAG1E,OAAO,EACL,UAAU,EAEV,kBAAkB,EAClB,oBAAoB,EACrB,MAAM,kBAAkB,CAAC;AAE1B;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,kBAAkB,CAAC;IACjC,cAAc,EAAE,oBAAoB,CAAC;CACtC;AAED;;;;;;;;;GASG;AACH,iBAAe,YAAY,CAAC,cAAc,EACxC,UAAU,EAAE,UAAU,CAAC,cAAc,CAAC,GACrC,OAAO,CAAC,kBAAkB,CAAC,CA8D7B;AAED,OAAO,EAAE,YAAY,EAAE,CAAC"}
1
+ {"version":3,"file":"create-worker.d.ts","sourceRoot":"","sources":["../../src/workers/create-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAe,MAAM,kBAAkB,CAAC;AAE3D,iBAAe,YAAY,CAAC,cAAc,EACxC,UAAU,EAAE,UAAU,CAAC,cAAc,CAAC,GACrC,OAAO,CAAC,MAAM,CAAC,CAyBjB;AAED,OAAO,EAAE,YAAY,EAAE,CAAC"}
@@ -2,58 +2,23 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.createWorker = createWorker;
4
4
  const node_worker_threads_1 = require("node:worker_threads");
5
- const worker_memory_1 = require("../common/worker-memory");
6
5
  const workers_1 = require("../types/workers");
7
- /**
8
- * Creates a worker thread with optional memory limits.
9
- * Memory limits are calculated based on the environment:
10
- * - For Lambda: Uses AWS_LAMBDA_FUNCTION_MEMORY_SIZE (75% for worker)
11
- * - For local development: Caps total at 1024MB (75% for worker = ~768MB)
12
- * - For other environments: Uses system memory or V8 heap limit
13
- *
14
- * @param workerData - Worker data including event, state, and options
15
- * @returns Promise resolving to CreateWorkerResult with worker and memory config
16
- */
17
6
  async function createWorker(workerData) {
18
7
  return new Promise((resolve, reject) => {
19
- var _a, _b, _c, _d;
20
8
  if (node_worker_threads_1.isMainThread) {
21
9
  const workerFile = __dirname + '/worker.js';
22
- // Calculate memory configuration based on environment
23
- const isLocalDevelopment = (_b = (_a = workerData.options) === null || _a === void 0 ? void 0 : _a.isLocalDevelopment) !== null && _b !== void 0 ? _b : false;
24
- const enableMemoryLimits = ((_c = workerData.options) === null || _c === void 0 ? void 0 : _c.enableMemoryLimits) !== false;
25
- const testMemoryLimitMb = (_d = workerData.options) === null || _d === void 0 ? void 0 : _d.testMemoryLimitMb;
26
- const memoryConfig = (0, worker_memory_1.calculateWorkerMemoryConfig)(isLocalDevelopment);
27
- // Allow test override for memory limit
28
- const effectiveMemoryLimitMb = testMemoryLimitMb !== null && testMemoryLimitMb !== void 0 ? testMemoryLimitMb : memoryConfig.maxOldGenerationSizeMb;
29
- const resourceLimits = {
30
- maxOldGenerationSizeMb: effectiveMemoryLimitMb,
31
- };
32
- // Build worker options
33
- const workerOptions = {
10
+ const worker = new node_worker_threads_1.Worker(workerFile, {
34
11
  workerData,
35
- };
36
- // Apply resource limits if enabled
37
- if (enableMemoryLimits) {
38
- workerOptions.resourceLimits = {
39
- maxOldGenerationSizeMb: resourceLimits.maxOldGenerationSizeMb,
40
- };
41
- console.info(`Worker memory limits configured: ` +
42
- `maxOldGenerationSizeMb=${resourceLimits.maxOldGenerationSizeMb}MB, ` +
43
- `totalAvailable=${memoryConfig.totalAvailableMemoryMb.toFixed(0)}MB, ` +
44
- `isLambda=${memoryConfig.isLambda}, ` +
45
- `isLocalDevelopment=${memoryConfig.isLocalDevelopment}`);
46
- }
47
- const worker = new node_worker_threads_1.Worker(workerFile, workerOptions);
12
+ });
48
13
  worker.on(workers_1.WorkerEvent.WorkerError, (error) => {
49
14
  console.error('Worker error', error);
50
15
  reject(error);
51
16
  });
52
17
  worker.on(workers_1.WorkerEvent.WorkerOnline, () => {
18
+ resolve(worker);
53
19
  console.info('Worker is online. Started processing the task with event type: ' +
54
20
  workerData.event.payload.event_type +
55
21
  '.');
56
- resolve({ worker, memoryConfig, resourceLimits });
57
22
  });
58
23
  }
59
24
  else {
@@ -7,7 +7,7 @@ const create_worker_1 = require("./create-worker");
7
7
  describe(create_worker_1.createWorker.name, () => {
8
8
  it('should create a Worker instance when valid parameters are provided', async () => {
9
9
  const workerPath = __dirname + '../tests/dummy-worker.ts';
10
- const result = worker_threads_1.isMainThread
10
+ const worker = worker_threads_1.isMainThread
11
11
  ? await (0, create_worker_1.createWorker)({
12
12
  event: (0, test_helpers_1.createEvent)({
13
13
  eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
@@ -16,12 +16,10 @@ describe(create_worker_1.createWorker.name, () => {
16
16
  workerPath,
17
17
  })
18
18
  : null;
19
- expect(result).not.toBeNull();
20
- expect(result === null || result === void 0 ? void 0 : result.worker).toBeInstanceOf(worker_threads_1.Worker);
21
- expect(result === null || result === void 0 ? void 0 : result.memoryConfig).toBeDefined();
22
- expect(result === null || result === void 0 ? void 0 : result.resourceLimits).toBeDefined();
23
- if (result) {
24
- await result.worker.terminate();
19
+ expect(worker).not.toBeNull();
20
+ expect(worker).toBeInstanceOf(worker_threads_1.Worker);
21
+ if (worker) {
22
+ await worker.terminate();
25
23
  }
26
24
  });
27
25
  it('should throw error when not in main thread', async () => {
@@ -43,15 +41,15 @@ describe(create_worker_1.createWorker.name, () => {
43
41
  it('[edge] should handle worker creation with minimal valid data', async () => {
44
42
  const workerPath = __dirname + '../tests/dummy-worker.ts';
45
43
  if (worker_threads_1.isMainThread) {
46
- const result = await (0, create_worker_1.createWorker)({
44
+ const worker = await (0, create_worker_1.createWorker)({
47
45
  event: (0, test_helpers_1.createEvent)({
48
46
  eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
49
47
  }),
50
48
  initialState: {},
51
49
  workerPath,
52
50
  });
53
- expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
54
- await result.worker.terminate();
51
+ expect(worker).toBeInstanceOf(worker_threads_1.Worker);
52
+ await worker.terminate();
55
53
  }
56
54
  });
57
55
  it('[edge] should handle worker creation with complex initial state', async () => {
@@ -63,126 +61,29 @@ describe(create_worker_1.createWorker.name, () => {
63
61
  },
64
62
  };
65
63
  if (worker_threads_1.isMainThread) {
66
- const result = await (0, create_worker_1.createWorker)({
64
+ const worker = await (0, create_worker_1.createWorker)({
67
65
  event: (0, test_helpers_1.createEvent)({
68
66
  eventType: extraction_1.EventType.ExtractionDataStart,
69
67
  }),
70
68
  initialState: complexState,
71
69
  workerPath,
72
70
  });
73
- expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
74
- await result.worker.terminate();
71
+ expect(worker).toBeInstanceOf(worker_threads_1.Worker);
72
+ await worker.terminate();
75
73
  }
76
74
  });
77
75
  it('[edge] should handle different event types', async () => {
78
76
  const workerPath = __dirname + '../tests/dummy-worker.ts';
79
77
  if (worker_threads_1.isMainThread) {
80
- const result = await (0, create_worker_1.createWorker)({
78
+ const worker = await (0, create_worker_1.createWorker)({
81
79
  event: (0, test_helpers_1.createEvent)({
82
80
  eventType: extraction_1.EventType.ExtractionMetadataStart,
83
81
  }),
84
82
  initialState: {},
85
83
  workerPath,
86
84
  });
87
- expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
88
- await result.worker.terminate();
89
- }
90
- });
91
- it('should include memory configuration in result', async () => {
92
- const workerPath = __dirname + '../tests/dummy-worker.ts';
93
- if (worker_threads_1.isMainThread) {
94
- const result = await (0, create_worker_1.createWorker)({
95
- event: (0, test_helpers_1.createEvent)({
96
- eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
97
- }),
98
- initialState: {},
99
- workerPath,
100
- });
101
- expect(result.memoryConfig).toHaveProperty('maxOldGenerationSizeMb');
102
- expect(result.memoryConfig).toHaveProperty('totalAvailableMemoryMb');
103
- expect(result.memoryConfig).toHaveProperty('isLambda');
104
- expect(result.memoryConfig).toHaveProperty('isLocalDevelopment');
105
- expect(result.resourceLimits).toHaveProperty('maxOldGenerationSizeMb');
106
- await result.worker.terminate();
107
- }
108
- });
109
- it('should apply testMemoryLimitMb override when provided', async () => {
110
- const workerPath = __dirname + '../tests/dummy-worker.ts';
111
- const testMemoryLimit = 64;
112
- if (worker_threads_1.isMainThread) {
113
- const result = await (0, create_worker_1.createWorker)({
114
- event: (0, test_helpers_1.createEvent)({
115
- eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
116
- }),
117
- initialState: {},
118
- workerPath,
119
- options: {
120
- testMemoryLimitMb: testMemoryLimit,
121
- },
122
- });
123
- expect(result.resourceLimits.maxOldGenerationSizeMb).toBe(testMemoryLimit);
124
- await result.worker.terminate();
125
- }
126
- });
127
- it('should create worker without memory limits when enableMemoryLimits is false', async () => {
128
- const workerPath = __dirname + '../tests/dummy-worker.ts';
129
- if (worker_threads_1.isMainThread) {
130
- const result = await (0, create_worker_1.createWorker)({
131
- event: (0, test_helpers_1.createEvent)({
132
- eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
133
- }),
134
- initialState: {},
135
- workerPath,
136
- options: {
137
- enableMemoryLimits: false,
138
- },
139
- });
140
- // Worker should still be created successfully
141
- expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
142
- // Memory config should still be calculated (for logging purposes)
143
- expect(result.memoryConfig).toBeDefined();
144
- expect(result.resourceLimits).toBeDefined();
145
- await result.worker.terminate();
146
- }
147
- });
148
- it('should set isLocalDevelopment in memory config when option is provided', async () => {
149
- const workerPath = __dirname + '../tests/dummy-worker.ts';
150
- if (worker_threads_1.isMainThread) {
151
- const result = await (0, create_worker_1.createWorker)({
152
- event: (0, test_helpers_1.createEvent)({
153
- eventType: extraction_1.EventType.ExtractionExternalSyncUnitsStart,
154
- }),
155
- initialState: {},
156
- workerPath,
157
- options: {
158
- isLocalDevelopment: true,
159
- },
160
- });
161
- expect(result.memoryConfig.isLocalDevelopment).toBe(true);
162
- await result.worker.terminate();
163
- }
164
- });
165
- it('[edge] should handle all extraction event types', async () => {
166
- const workerPath = __dirname + '../tests/dummy-worker.ts';
167
- const eventTypes = [
168
- extraction_1.EventType.ExtractionExternalSyncUnitsStart,
169
- extraction_1.EventType.ExtractionMetadataStart,
170
- extraction_1.EventType.ExtractionDataStart,
171
- extraction_1.EventType.ExtractionDataContinue,
172
- extraction_1.EventType.ExtractionAttachmentsStart,
173
- extraction_1.EventType.ExtractionAttachmentsContinue,
174
- ];
175
- if (worker_threads_1.isMainThread) {
176
- for (const eventType of eventTypes) {
177
- const result = await (0, create_worker_1.createWorker)({
178
- event: (0, test_helpers_1.createEvent)({ eventType }),
179
- initialState: {},
180
- workerPath,
181
- });
182
- expect(result.worker).toBeInstanceOf(worker_threads_1.Worker);
183
- expect(result.memoryConfig).toBeDefined();
184
- await result.worker.terminate();
185
- }
85
+ expect(worker).toBeInstanceOf(worker_threads_1.Worker);
86
+ await worker.terminate();
186
87
  }
187
88
  });
188
89
  });
@@ -22,17 +22,8 @@ export declare class Spawn {
22
22
  private resolve;
23
23
  private originalConsole;
24
24
  private logger;
25
- private memoryConfig?;
26
- private resourceLimits?;
27
- private isOOMExit;
28
- constructor({ event, worker, options, resolve, originalConsole, memoryConfig, resourceLimits, }: SpawnInterface);
25
+ constructor({ event, worker, options, resolve, originalConsole, }: SpawnInterface);
29
26
  private clearTimeouts;
30
27
  private exitFromMainThread;
31
- /**
32
- * Handles worker exit due to OOM (Out-Of-Memory) error.
33
- * Emits an error event with detailed OOM information.
34
- * @param exitCode - The exit code from the worker
35
- */
36
- private handleOOMExit;
37
28
  }
38
29
  //# sourceMappingURL=spawn.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAaA,OAAO,EAGL,qBAAqB,EACrB,cAAc,EAKf,MAAM,kBAAkB,CAAC;AAmC1B;;;;;;;;;;GAUG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA0GvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAC,CAAqB;IAC1C,OAAO,CAAC,cAAc,CAAC,CAAuB;IAC9C,OAAO,CAAC,SAAS,CAAkB;gBAEvB,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,EACf,YAAY,EACZ,cAAc,GACf,EAAE,cAAc;IAwHjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;IAkChC;;;;OAIG;YACW,aAAa;CAyD5B"}
1
+ {"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,kBAAkB,CAAC;AAmC1B;;;;;;;;;;GAUG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAuGvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;gBACX,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,GAChB,EAAE,cAAc;IAgGjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;CAiCjC"}
@@ -10,7 +10,6 @@ const helpers_1 = require("yargs/helpers");
10
10
  const control_protocol_1 = require("../common/control-protocol");
11
11
  const event_type_translation_1 = require("../common/event-type-translation");
12
12
  const helpers_2 = require("../common/helpers");
13
- const worker_memory_1 = require("../common/worker-memory");
14
13
  const logger_1 = require("../logger/logger");
15
14
  const extraction_1 = require("../types/extraction");
16
15
  const workers_1 = require("../types/workers");
@@ -87,7 +86,7 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
87
86
  }
88
87
  if (script) {
89
88
  try {
90
- const { worker, memoryConfig, resourceLimits } = await (0, create_worker_1.createWorker)({
89
+ const worker = await (0, create_worker_1.createWorker)({
91
90
  event,
92
91
  initialState,
93
92
  workerPath: script,
@@ -101,8 +100,6 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
101
100
  options,
102
101
  resolve,
103
102
  originalConsole,
104
- memoryConfig,
105
- resourceLimits,
106
103
  });
107
104
  });
108
105
  }
@@ -139,9 +136,8 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
139
136
  }
140
137
  }
141
138
  class Spawn {
142
- constructor({ event, worker, options, resolve, originalConsole, memoryConfig, resourceLimits, }) {
139
+ constructor({ event, worker, options, resolve, originalConsole, }) {
143
140
  this.defaultLambdaTimeout = constants_1.DEFAULT_LAMBDA_TIMEOUT;
144
- this.isOOMExit = false;
145
141
  this.originalConsole = originalConsole || console;
146
142
  this.logger = console;
147
143
  this.alreadyEmitted = false;
@@ -150,8 +146,6 @@ class Spawn {
150
146
  ? Math.min(options.timeout, this.defaultLambdaTimeout)
151
147
  : this.defaultLambdaTimeout;
152
148
  this.resolve = resolve;
153
- this.memoryConfig = memoryConfig;
154
- this.resourceLimits = resourceLimits;
155
149
  // If soft timeout is reached, send a message to the worker to gracefully exit.
156
150
  this.softTimeoutTimer = setTimeout(() => void (async () => {
157
151
  console.log('SOFT TIMEOUT: Sending a message to the worker to gracefully exit.');
@@ -176,33 +170,12 @@ class Spawn {
176
170
  await this.exitFromMainThread();
177
171
  }
178
172
  })(), this.lambdaTimeout * constants_1.HARD_TIMEOUT_MULTIPLIER);
179
- // Listen for worker errors to detect OOM
180
- // Node.js worker threads emit ERR_WORKER_OUT_OF_MEMORY error code when OOM occurs
181
- // @see https://nodejs.org/api/errors.html#err_worker_out_of_memory
182
- worker.on(workers_1.WorkerEvent.WorkerError, (error) => {
183
- if ((0, worker_memory_1.isOOMError)(error)) {
184
- this.isOOMExit = true;
185
- console.error('OOM ERROR DETECTED: Worker ran out of memory.', {
186
- errorCode: error.code,
187
- errorMessage: error === null || error === void 0 ? void 0 : error.message,
188
- memoryConfig: this.memoryConfig,
189
- resourceLimits: this.resourceLimits,
190
- });
191
- }
192
- });
193
173
  // If worker exits with process.exit(code), clear the timeouts and exit from
194
174
  // main thread.
195
175
  worker.on(workers_1.WorkerEvent.WorkerExit, (code) => void (async () => {
196
176
  console.info('Worker exited with exit code: ' + code + '.');
197
177
  this.clearTimeouts();
198
- // Check if this was an OOM exit (non-zero exit code and OOM error detected)
199
- // Exit code 134 is common for OOM (SIGABRT), but we also check the error message
200
- if (code !== 0 && this.isOOMExit) {
201
- await this.handleOOMExit(code);
202
- }
203
- else {
204
- await this.exitFromMainThread();
205
- }
178
+ await this.exitFromMainThread();
206
179
  })());
207
180
  worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
208
181
  var _a, _b;
@@ -275,52 +248,5 @@ class Spawn {
275
248
  console.error('Error while emitting event.', (0, logger_1.serializeError)(error));
276
249
  }
277
250
  }
278
- /**
279
- * Handles worker exit due to OOM (Out-Of-Memory) error.
280
- * Emits an error event with detailed OOM information.
281
- * @param exitCode - The exit code from the worker
282
- */
283
- async handleOOMExit(exitCode) {
284
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
285
- this.clearTimeouts();
286
- // eslint-disable-next-line no-global-assign
287
- console = this.originalConsole;
288
- if (this.alreadyEmitted) {
289
- this.resolve();
290
- return;
291
- }
292
- this.alreadyEmitted = true;
293
- const { eventType } = (0, helpers_2.getTimeoutErrorEventType)(this.event.payload.event_type);
294
- // Build OOM error info
295
- const oomErrorInfo = {
296
- type: 'OOM_ERROR',
297
- message: `Worker thread ran out of memory and was terminated. ` +
298
- `Memory limit: ${(_b = (_a = this.resourceLimits) === null || _a === void 0 ? void 0 : _a.maxOldGenerationSizeMb) !== null && _b !== void 0 ? _b : 'unknown'}MB, ` +
299
- `Total available: ${(_e = (_d = (_c = this.memoryConfig) === null || _c === void 0 ? void 0 : _c.totalAvailableMemoryMb) === null || _d === void 0 ? void 0 : _d.toFixed(0)) !== null && _e !== void 0 ? _e : 'unknown'}MB.`,
300
- memoryLimitMb: (_g = (_f = this.resourceLimits) === null || _f === void 0 ? void 0 : _f.maxOldGenerationSizeMb) !== null && _g !== void 0 ? _g : 0,
301
- totalAvailableMemoryMb: (_j = (_h = this.memoryConfig) === null || _h === void 0 ? void 0 : _h.totalAvailableMemoryMb) !== null && _j !== void 0 ? _j : 0,
302
- isLambda: (_l = (_k = this.memoryConfig) === null || _k === void 0 ? void 0 : _k.isLambda) !== null && _l !== void 0 ? _l : false,
303
- isLocalDevelopment: (_o = (_m = this.memoryConfig) === null || _m === void 0 ? void 0 : _m.isLocalDevelopment) !== null && _o !== void 0 ? _o : false,
304
- exitCode,
305
- eventType: this.event.payload.event_type,
306
- };
307
- console.error('OOM Error Details:', oomErrorInfo);
308
- try {
309
- await (0, control_protocol_1.emit)({
310
- eventType,
311
- event: this.event,
312
- data: {
313
- error: {
314
- message: oomErrorInfo.message,
315
- oom_error_info: oomErrorInfo,
316
- },
317
- },
318
- });
319
- this.resolve();
320
- }
321
- catch (error) {
322
- console.error('Error while emitting OOM error event.', (0, logger_1.serializeError)(error));
323
- }
324
- }
325
251
  }
326
252
  exports.Spawn = Spawn;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@devrev/ts-adaas",
3
- "version": "1.12.3-beta.7",
3
+ "version": "1.13.0",
4
4
  "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
5
5
  "type": "commonjs",
6
6
  "main": "./dist/index.js",
@@ -12,7 +12,6 @@
12
12
  "lint": "eslint .",
13
13
  "lint:fix": "eslint . --fix",
14
14
  "test": "jest --forceExit --coverage --runInBand --selectProjects default --",
15
- "test:oom": "node --max-old-space-size=4096 ./node_modules/.bin/jest --forceExit --runInBand --selectProjects oom-tests --logHeapUsage --",
16
15
  "test:backwards-compatibility": "jest --forceExit --runInBand --selectProjects backwards-compatibility --"
17
16
  },
18
17
  "repository": {