@devrev/ts-adaas 1.12.3-beta.4 → 1.12.3-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constants.d.ts +0 -1
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +1 -2
- package/dist/tests/backwards-compatibility/backwards-compatibility.test.d.ts.map +1 -1
- package/dist/tests/backwards-compatibility/backwards-compatibility.test.js +27 -1
- package/dist/types/workers.d.ts +2 -2
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.js +5 -13
- package/dist/workers/spawn.d.ts +1 -1
- package/dist/workers/spawn.d.ts.map +1 -1
- package/dist/workers/spawn.js +7 -6
- package/package.json +1 -1
|
@@ -9,7 +9,6 @@ export declare const STATEFUL_EXTRACTION_EVENT_TYPES: EventType[];
|
|
|
9
9
|
export declare const STATEFUL_LOADING_EVENT_TYPES: EventType[];
|
|
10
10
|
export declare const STATEFUL_EVENT_TYPES: EventType[];
|
|
11
11
|
export declare const ARTIFACT_BATCH_SIZE = 2000;
|
|
12
|
-
export declare const MAX_DEVREV_ARTIFACT_SIZE = 262144000;
|
|
13
12
|
export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
14
13
|
export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
|
|
15
14
|
export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAEvD,eAAO,MAAM,0BAA0B;;;;CAItC,CAAC;AAEF,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.
|
|
3
|
+
exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
|
|
4
4
|
const extraction_1 = require("../types/extraction");
|
|
5
5
|
const helpers_1 = require("./helpers");
|
|
6
6
|
exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
|
|
@@ -43,7 +43,6 @@ exports.STATEFUL_EVENT_TYPES = [
|
|
|
43
43
|
...exports.STATEFUL_LOADING_EVENT_TYPES,
|
|
44
44
|
];
|
|
45
45
|
exports.ARTIFACT_BATCH_SIZE = 2000;
|
|
46
|
-
exports.MAX_DEVREV_ARTIFACT_SIZE = 262144000; // 250MB
|
|
47
46
|
exports.MAX_DEVREV_FILENAME_LENGTH = 256;
|
|
48
47
|
exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20; // 20 characters for the file extension
|
|
49
48
|
exports.AIRDROP_DEFAULT_ITEM_TYPES = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backwards-compatibility.test.d.ts","sourceRoot":"","sources":["../../../src/tests/backwards-compatibility/backwards-compatibility.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,cAAc,EAGd,WAAW,EAGX,kBAAkB,EAKnB,MAAM,gCAAgC,CAAC;AAgBxC,wBAAgB,0BAA0B,CACxC,WAAW,EAAE,WAAW,GAAG,cAAc,GAAG,kBAAkB,EAC9D,eAAe,EAAE,WAAW,GAAG,cAAc,GAAG,kBAAkB,
|
|
1
|
+
{"version":3,"file":"backwards-compatibility.test.d.ts","sourceRoot":"","sources":["../../../src/tests/backwards-compatibility/backwards-compatibility.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,cAAc,EAGd,WAAW,EAGX,kBAAkB,EAKnB,MAAM,gCAAgC,CAAC;AAgBxC,wBAAgB,0BAA0B,CACxC,WAAW,EAAE,WAAW,GAAG,cAAc,GAAG,kBAAkB,EAC9D,eAAe,EAAE,WAAW,GAAG,cAAc,GAAG,kBAAkB,QAuHnE"}
|
|
@@ -14,7 +14,29 @@ function checkFunctionCompatibility(newFunction, currentFunction) {
|
|
|
14
14
|
.slice(0, lengthOfPreviousParameters)
|
|
15
15
|
.map((p) => p.name);
|
|
16
16
|
const currentFunctionParamNames = currentFunction.parameters.map((p) => p.name);
|
|
17
|
-
|
|
17
|
+
// Check each parameter position for destructured parameter compatibility
|
|
18
|
+
for (let i = 0; i <
|
|
19
|
+
Math.min(newFunctionParamNames.length, currentFunctionParamNames.length); i++) {
|
|
20
|
+
const newParam = newFunctionParamNames[i];
|
|
21
|
+
const currentParam = currentFunctionParamNames[i];
|
|
22
|
+
// If both are destructured parameters (contain '{')
|
|
23
|
+
if (newParam.includes('{') && currentParam.includes('{')) {
|
|
24
|
+
// Extract field names from destructured parameters
|
|
25
|
+
const extractFields = (param) => param
|
|
26
|
+
.replace(/[{}\s]/g, '')
|
|
27
|
+
.split(',')
|
|
28
|
+
.filter((f) => f.length > 0);
|
|
29
|
+
const newFields = extractFields(newParam);
|
|
30
|
+
const currentFields = extractFields(currentParam);
|
|
31
|
+
// Check that all current fields are present in new fields
|
|
32
|
+
const missingFields = currentFields.filter((field) => !newFields.includes(field));
|
|
33
|
+
expect(missingFields).toEqual([]);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
// For non-destructured parameters, they must match exactly
|
|
37
|
+
expect(newParam).toEqual(currentParam);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
18
40
|
});
|
|
19
41
|
it(`Function ${newFunction.displayName} should have compatible parameter types with the current function`, () => {
|
|
20
42
|
const newFunctionParamTypes = newFunction.parameters
|
|
@@ -53,6 +75,10 @@ function checkFunctionCompatibility(newFunction, currentFunction) {
|
|
|
53
75
|
if (currentParam.isOptional && !newParam.isOptional) {
|
|
54
76
|
throw new Error(`Parameter ${newParam.name} became required but was optional`);
|
|
55
77
|
}
|
|
78
|
+
// Skip interface compatibility check for destructured parameters
|
|
79
|
+
if (newParam.name.includes('{') || currentParam.name.includes('{')) {
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
56
82
|
}
|
|
57
83
|
});
|
|
58
84
|
}
|
package/dist/types/workers.d.ts
CHANGED
|
@@ -24,14 +24,12 @@ export interface WorkerAdapterInterface<ConnectorState> {
|
|
|
24
24
|
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
25
25
|
* @param {number=} timeout - The timeout for the worker thread
|
|
26
26
|
* @param {number=} batchSize - Maximum number of extracted items in a batch
|
|
27
|
-
* @param {string=} baseWorkerPath - The base path for the worker files, usually `__dirname`
|
|
28
27
|
* @param {Record<EventType, string>=} workerPathOverrides - A map of event types to custom worker paths to override default worker paths
|
|
29
28
|
*/
|
|
30
29
|
export interface WorkerAdapterOptions {
|
|
31
30
|
isLocalDevelopment?: boolean;
|
|
32
31
|
timeout?: number;
|
|
33
32
|
batchSize?: number;
|
|
34
|
-
baseWorkerPath?: string;
|
|
35
33
|
workerPathOverrides?: WorkerPathOverrides;
|
|
36
34
|
}
|
|
37
35
|
/**
|
|
@@ -60,6 +58,7 @@ export interface SpawnInterface {
|
|
|
60
58
|
* @param {string} workerPath - The path to the worker file
|
|
61
59
|
* @param {string} initialDomainMapping - The initial domain mapping
|
|
62
60
|
* @param {WorkerAdapterOptions} options - The options to create a new instance of Spawn class
|
|
61
|
+
* @param {string=} baseWorkerPath - The base path for the worker files, usually `__dirname`
|
|
63
62
|
*/
|
|
64
63
|
export interface SpawnFactoryInterface<ConnectorState> {
|
|
65
64
|
event: AirdropEvent;
|
|
@@ -67,6 +66,7 @@ export interface SpawnFactoryInterface<ConnectorState> {
|
|
|
67
66
|
workerPath?: string;
|
|
68
67
|
options?: WorkerAdapterOptions;
|
|
69
68
|
initialDomainMapping?: InitialDomainMapping;
|
|
69
|
+
baseWorkerPath?: string;
|
|
70
70
|
}
|
|
71
71
|
/**
|
|
72
72
|
* TaskAdapterInterface is an interface for TaskAdapter class.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED
|
|
1
|
+
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACxC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,SAAS,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED;;GAEG;AACH,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,oBAAoB,GAAG,iBAAiB,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACzB,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;;;;OAQG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwDpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAuB7B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmB1B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA0B1B,qBAAqB,CACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAwBhC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAef,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAqCY,sBAAsB;YAyBtB,gBAAgB;IAe9B,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,UAAU;IASZ,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC;YAyBvB,eAAe;CAgC9B"}
|
|
@@ -42,7 +42,6 @@ const fs_1 = __importStar(require("fs"));
|
|
|
42
42
|
const js_jsonl_1 = require("js-jsonl");
|
|
43
43
|
const zlib_1 = __importDefault(require("zlib"));
|
|
44
44
|
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
45
|
-
const constants_1 = require("../common/constants");
|
|
46
45
|
const helpers_1 = require("../common/helpers");
|
|
47
46
|
const logger_1 = require("../logger/logger");
|
|
48
47
|
class Uploader {
|
|
@@ -146,20 +145,13 @@ class Uploader {
|
|
|
146
145
|
formData.append(field, artifact.form_data[field]);
|
|
147
146
|
}
|
|
148
147
|
formData.append('file', fileStream.data);
|
|
149
|
-
if (fileStream.headers['content-length'] > constants_1.MAX_DEVREV_ARTIFACT_SIZE) {
|
|
150
|
-
console.warn(`File size exceeds the maximum limit of ${constants_1.MAX_DEVREV_ARTIFACT_SIZE} bytes.`);
|
|
151
|
-
this.destroyStream(fileStream);
|
|
152
|
-
return;
|
|
153
|
-
}
|
|
154
148
|
try {
|
|
155
149
|
const response = await axios_client_internal_1.axiosClient.post(artifact.upload_url, formData, {
|
|
156
|
-
headers: Object.assign(
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
maxRedirects: 0, // Prevents buffering
|
|
162
|
-
validateStatus: () => true, // Prevents errors on redirects
|
|
150
|
+
headers: Object.assign({}, formData.getHeaders()),
|
|
151
|
+
// Prevents buffering of the response in the memory
|
|
152
|
+
maxRedirects: 0,
|
|
153
|
+
// Allow 2xx and 3xx (redirects) to be considered successful, 4xx and 5xx will throw errors and be caught in the catch block
|
|
154
|
+
validateStatus: (status) => status >= 200 && status < 400,
|
|
163
155
|
});
|
|
164
156
|
this.destroyStream(fileStream);
|
|
165
157
|
return response;
|
package/dist/workers/spawn.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { SpawnFactoryInterface, SpawnInterface } from '../types/workers';
|
|
|
10
10
|
* @param {string} options.workerPath - The path to the worker file
|
|
11
11
|
* @returns {Promise<Spawn>} - A new instance of Spawn class
|
|
12
12
|
*/
|
|
13
|
-
export declare function spawn<ConnectorState>({ event, initialState, workerPath, initialDomainMapping, options, }: SpawnFactoryInterface<ConnectorState>): Promise<void>;
|
|
13
|
+
export declare function spawn<ConnectorState>({ event, initialState, workerPath, initialDomainMapping, options, baseWorkerPath, }: SpawnFactoryInterface<ConnectorState>): Promise<void>;
|
|
14
14
|
export declare class Spawn {
|
|
15
15
|
private event;
|
|
16
16
|
private alreadyEmitted;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,kBAAkB,CAAC;AAmC1B;;;;;;;;;;GAUG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,
|
|
1
|
+
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../src/workers/spawn.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,kBAAkB,CAAC;AAmC1B;;;;;;;;;;GAUG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAuGvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;gBACX,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,GAChB,EAAE,cAAc;IAgGjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;CAiCjC"}
|
package/dist/workers/spawn.js
CHANGED
|
@@ -15,7 +15,7 @@ const extraction_1 = require("../types/extraction");
|
|
|
15
15
|
const workers_1 = require("../types/workers");
|
|
16
16
|
const constants_1 = require("../common/constants");
|
|
17
17
|
const create_worker_1 = require("./create-worker");
|
|
18
|
-
function getWorkerPath({ event, workerBasePath }) {
|
|
18
|
+
function getWorkerPath({ event, workerBasePath, }) {
|
|
19
19
|
let path = null;
|
|
20
20
|
switch (event.payload.event_type) {
|
|
21
21
|
case extraction_1.EventType.StartExtractingExternalSyncUnits:
|
|
@@ -46,8 +46,7 @@ function getWorkerPath({ event, workerBasePath }) {
|
|
|
46
46
|
* @param {string} options.workerPath - The path to the worker file
|
|
47
47
|
* @returns {Promise<Spawn>} - A new instance of Spawn class
|
|
48
48
|
*/
|
|
49
|
-
async function spawn({ event, initialState, workerPath, initialDomainMapping, options, }) {
|
|
50
|
-
var _a;
|
|
49
|
+
async function spawn({ event, initialState, workerPath, initialDomainMapping, options, baseWorkerPath, }) {
|
|
51
50
|
// Translates incoming event type for backwards compatibility
|
|
52
51
|
// This allows the SDK to accept both old and new event type formats
|
|
53
52
|
const originalEventType = event.payload.event_type;
|
|
@@ -72,15 +71,17 @@ async function spawn({ event, initialState, workerPath, initialDomainMapping, op
|
|
|
72
71
|
if (workerPath != null) {
|
|
73
72
|
script = workerPath;
|
|
74
73
|
}
|
|
75
|
-
else if (
|
|
74
|
+
else if (baseWorkerPath != null &&
|
|
76
75
|
(options === null || options === void 0 ? void 0 : options.workerPathOverrides) != null &&
|
|
77
76
|
options.workerPathOverrides[translatedEventType] != null) {
|
|
78
|
-
script =
|
|
77
|
+
script =
|
|
78
|
+
baseWorkerPath +
|
|
79
|
+
options.workerPathOverrides[translatedEventType];
|
|
79
80
|
}
|
|
80
81
|
else {
|
|
81
82
|
script = getWorkerPath({
|
|
82
83
|
event,
|
|
83
|
-
workerBasePath:
|
|
84
|
+
workerBasePath: baseWorkerPath !== null && baseWorkerPath !== void 0 ? baseWorkerPath : __dirname,
|
|
84
85
|
});
|
|
85
86
|
}
|
|
86
87
|
if (script) {
|
package/package.json
CHANGED