@devrev/ts-adaas 1.20.2-beta.5 → 1.20.2-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +1 -0
- package/dist/common/event-type-translation.d.ts.map +1 -1
- package/dist/common/event-type-translation.js +1 -0
- package/dist/common/event-type-translation.test.js +1 -0
- package/dist/http/axios-client.d.ts +6 -2
- package/dist/http/axios-client.d.ts.map +1 -1
- package/dist/http/axios-client.js +7 -4
- package/dist/mock-server/mock-server.d.ts +4 -0
- package/dist/mock-server/mock-server.d.ts.map +1 -1
- package/dist/mock-server/mock-server.interfaces.d.ts +9 -0
- package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -1
- package/dist/mock-server/mock-server.js +39 -6
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +21 -6
- package/dist/multithreading/process-task.test.js +28 -1
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +9 -0
- package/dist/multithreading/spawn/spawn.helpers.test.js +34 -0
- package/dist/multithreading/spawn/spawn.js +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +13 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +51 -5
- package/dist/multithreading/worker-adapter/worker-adapter.js +131 -79
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.js +135 -0
- package/dist/repo/repo.d.ts +1 -2
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.js +6 -12
- package/dist/repo/repo.test.js +32 -1
- package/dist/state/state.js +1 -1
- package/dist/state/state.time-value-resolution.test.js +3 -3
- package/dist/types/extraction.d.ts +6 -3
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +3 -0
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAU1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AAEzD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC;AAEpD;;;;;;GAMG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
|
package/dist/common/constants.js
CHANGED
|
@@ -6,6 +6,7 @@ const helpers_1 = require("./helpers");
|
|
|
6
6
|
exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
|
|
7
7
|
extraction_1.EventType.StartExtractingExternalSyncUnits,
|
|
8
8
|
extraction_1.EventType.StartExtractingMetadata,
|
|
9
|
+
extraction_1.EventType.ContinueExtractingMetadata,
|
|
9
10
|
extraction_1.EventType.StartExtractingData,
|
|
10
11
|
extraction_1.EventType.ContinueExtractingData,
|
|
11
12
|
extraction_1.EventType.StartDeletingExtractorState,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event-type-translation.d.ts","sourceRoot":"","sources":["../../src/common/event-type-translation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,eAAe,EAAE,MAAM,GAAG,SAAS,
|
|
1
|
+
{"version":3,"file":"event-type-translation.d.ts","sourceRoot":"","sources":["../../src/common/event-type-translation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,eAAe,EAAE,MAAM,GAAG,SAAS,CA0D7E;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,SAAS,EAAE,kBAAkB,GAC5B,kBAAkB,CA0CpB;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,eAAe,GACzB,eAAe,CAmBjB;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,kBAAkB,GAAG,eAAe,GAC9C,kBAAkB,GAAG,eAAe,CAatC"}
|
|
@@ -27,6 +27,7 @@ function translateIncomingEventType(eventTypeString) {
|
|
|
27
27
|
// New extraction event types (already correct, map to new enum members)
|
|
28
28
|
[extraction_1.EventType.StartExtractingExternalSyncUnits]: extraction_1.EventType.StartExtractingExternalSyncUnits,
|
|
29
29
|
[extraction_1.EventType.StartExtractingMetadata]: extraction_1.EventType.StartExtractingMetadata,
|
|
30
|
+
[extraction_1.EventType.ContinueExtractingMetadata]: extraction_1.EventType.ContinueExtractingMetadata,
|
|
30
31
|
[extraction_1.EventType.StartExtractingData]: extraction_1.EventType.StartExtractingData,
|
|
31
32
|
[extraction_1.EventType.ContinueExtractingData]: extraction_1.EventType.ContinueExtractingData,
|
|
32
33
|
[extraction_1.EventType.StartDeletingExtractorState]: extraction_1.EventType.StartDeletingExtractorState,
|
|
@@ -31,6 +31,7 @@ describe(event_type_translation_1.translateIncomingEventType.name, () => {
|
|
|
31
31
|
it.each([
|
|
32
32
|
[extraction_1.EventType.StartExtractingExternalSyncUnits],
|
|
33
33
|
[extraction_1.EventType.StartExtractingMetadata],
|
|
34
|
+
[extraction_1.EventType.ContinueExtractingMetadata],
|
|
34
35
|
[extraction_1.EventType.StartExtractingData],
|
|
35
36
|
[extraction_1.EventType.ContinueExtractingData],
|
|
36
37
|
[extraction_1.EventType.StartDeletingExtractorState],
|
|
@@ -22,6 +22,10 @@
|
|
|
22
22
|
* - `axiosClient`: Configured axios instance with retry logic.
|
|
23
23
|
*/
|
|
24
24
|
import axios from 'axios';
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
/**
|
|
26
|
+
* @deprecated This client has been deprecated because each use requires a
|
|
27
|
+
* specific configuration. It is intended for SDK-internal use only.
|
|
28
|
+
*/
|
|
29
|
+
export declare const axiosClient: import("axios").AxiosInstance;
|
|
30
|
+
export { axios };
|
|
27
31
|
//# sourceMappingURL=axios-client.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"axios-client.d.ts","sourceRoot":"","sources":["../../src/http/axios-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAqB,MAAM,OAAO,CAAC;AAI1C,
|
|
1
|
+
{"version":3,"file":"axios-client.d.ts","sourceRoot":"","sources":["../../src/http/axios-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAqB,MAAM,OAAO,CAAC;AAI1C;;;GAGG;AACH,eAAO,MAAM,WAAW,+BAAiB,CAAC;AAuC1C,OAAO,EAAE,KAAK,EAAE,CAAC"}
|
|
@@ -26,14 +26,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.
|
|
29
|
+
exports.axios = exports.axiosClient = void 0;
|
|
30
30
|
const axios_1 = __importDefault(require("axios"));
|
|
31
31
|
exports.axios = axios_1.default;
|
|
32
32
|
const axios_retry_1 = __importDefault(require("axios-retry"));
|
|
33
33
|
const logger_context_1 = require("../logger/logger.context");
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
/**
|
|
35
|
+
* @deprecated This client has been deprecated because each use requires a
|
|
36
|
+
* specific configuration. It is intended for SDK-internal use only.
|
|
37
|
+
*/
|
|
38
|
+
exports.axiosClient = axios_1.default.create();
|
|
39
|
+
(0, axios_retry_1.default)(exports.axiosClient, {
|
|
37
40
|
retries: 5,
|
|
38
41
|
retryDelay: (retryCount, error) => {
|
|
39
42
|
// exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
|
|
@@ -31,6 +31,10 @@ export declare class MockServer {
|
|
|
31
31
|
* Also clears request tracking data.
|
|
32
32
|
*/
|
|
33
33
|
resetRoutes(): void;
|
|
34
|
+
/**
|
|
35
|
+
* Returns all POST requests to the platform callback URL.
|
|
36
|
+
*/
|
|
37
|
+
getCallbackRequests(): RequestInfo[];
|
|
34
38
|
/**
|
|
35
39
|
* Returns the most recent request or undefined if no requests exist.
|
|
36
40
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mock-server.d.ts","sourceRoot":"","sources":["../../src/mock-server/mock-server.ts"],"names":[],"mappings":"AAEA,OAAO,EAKL,WAAW,EACX,WAAW,EAEZ,MAAM,0BAA0B,CAAC;AA4ElC;;;;GAIG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,YAAY,CAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,aAAa,CAA4B;IACjD,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,aAAa,CAA4B;gBAErC,IAAI,GAAE,MAAiC;IAMtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBpB,aAAa;IA+B3B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6B3B,OAAO,CAAC,WAAW;IAInB;;OAEG;IACI,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"mock-server.d.ts","sourceRoot":"","sources":["../../src/mock-server/mock-server.ts"],"names":[],"mappings":"AAEA,OAAO,EAKL,WAAW,EACX,WAAW,EAEZ,MAAM,0BAA0B,CAAC;AA4ElC;;;;GAIG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,YAAY,CAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,aAAa,CAA4B;IACjD,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,aAAa,CAA4B;gBAErC,IAAI,GAAE,MAAiC;IAMtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBpB,aAAa;IA+B3B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6B3B,OAAO,CAAC,WAAW;IAInB;;OAEG;IACI,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IA+G1C;;;OAGG;IACI,WAAW,IAAI,IAAI;IAM1B;;OAEG;IACI,mBAAmB,IAAI,WAAW,EAAE;IAO3C;;OAEG;IACI,cAAc,IAAI,WAAW,GAAG,SAAS;IAOhD;;OAEG;IACI,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAI5D;;OAEG;IACI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE;CAQhE"}
|
|
@@ -51,6 +51,15 @@ export interface RouteConfig {
|
|
|
51
51
|
headers?: Record<string, string>;
|
|
52
52
|
/** Optional retry configuration for simulating failures before success */
|
|
53
53
|
retry?: RetryConfig;
|
|
54
|
+
/**
|
|
55
|
+
* Respond successfully for the first N requests, then return an error.
|
|
56
|
+
* Useful for partial-upload scenarios (first batch succeeds, second fails).
|
|
57
|
+
*/
|
|
58
|
+
succeedThenFail?: {
|
|
59
|
+
successCount: number;
|
|
60
|
+
errorStatus?: number;
|
|
61
|
+
errorBody?: unknown;
|
|
62
|
+
};
|
|
54
63
|
/** Optional delay in milliseconds before sending the response */
|
|
55
64
|
delay?: number;
|
|
56
65
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mock-server.interfaces.d.ts","sourceRoot":"","sources":["../../src/mock-server/mock-server.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAC;AAEvD,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAE7C,MAAM,WAAW,aAAc,SAAQ,eAAe;IACpD,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,gCAAgC;IAChC,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,YAAa,SAAQ,cAAc;IAClD,yCAAyC;IACzC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,YAAY,CAAC;IACnD,+BAA+B;IAC/B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAC;IACnC,2BAA2B;IAC3B,IAAI,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAAC;IAC1B,0CAA0C;IAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,6BAA6B;IAC7B,IAAI,IAAI,IAAI,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,0EAA0E;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,4EAA4E;IAC5E,IAAI,EAAE,MAAM,CAAC;IACb,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,6CAA6C;IAC7C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,2FAA2F;IAC3F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,0EAA0E;IAC1E,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,YAAY,KAAK,OAAO,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,oDAAoD;IACpD,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"mock-server.interfaces.d.ts","sourceRoot":"","sources":["../../src/mock-server/mock-server.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,MAAM,CAAC;AAEvD,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAE7C,MAAM,WAAW,aAAc,SAAQ,eAAe;IACpD,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,gCAAgC;IAChC,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,YAAa,SAAQ,cAAc;IAClD,yCAAyC;IACzC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,YAAY,CAAC;IACnD,+BAA+B;IAC/B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAC;IACnC,2BAA2B;IAC3B,IAAI,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAAC;IAC1B,0CAA0C;IAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,6BAA6B;IAC7B,IAAI,IAAI,IAAI,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,0EAA0E;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,4EAA4E;IAC5E,IAAI,EAAE,MAAM,CAAC;IACb,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,6CAA6C;IAC7C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,2FAA2F;IAC3F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,0EAA0E;IAC1E,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;IACF,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,YAAY,KAAK,OAAO,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,oDAAoD;IACpD,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC"}
|
|
@@ -178,19 +178,46 @@ class MockServer {
|
|
|
178
178
|
* Configures a route to return a specific status code and optional response body.
|
|
179
179
|
*/
|
|
180
180
|
setRoute(config) {
|
|
181
|
-
const { path, method, status, body, bodyBuffer, retry, headers, delay } = config;
|
|
181
|
+
const { path, method, status, body, bodyBuffer, retry, succeedThenFail, headers, delay, } = config;
|
|
182
182
|
const key = this.getRouteKey(method, path);
|
|
183
|
-
if (retry) {
|
|
183
|
+
if (retry || succeedThenFail) {
|
|
184
184
|
this.requestCounts.set(key, 0);
|
|
185
185
|
}
|
|
186
186
|
this.routeHandlers.set(key, (req, res) => {
|
|
187
187
|
const sendResponse = (responseDelay) => {
|
|
188
188
|
const send = () => {
|
|
189
|
-
var _a, _b;
|
|
190
|
-
if (
|
|
189
|
+
var _a, _b, _c;
|
|
190
|
+
if (succeedThenFail) {
|
|
191
191
|
const currentCount = this.requestCounts.get(key) || 0;
|
|
192
|
-
|
|
193
|
-
|
|
192
|
+
this.requestCounts.set(key, currentCount + 1);
|
|
193
|
+
if (currentCount < succeedThenFail.successCount) {
|
|
194
|
+
if (headers) {
|
|
195
|
+
res.set(headers);
|
|
196
|
+
}
|
|
197
|
+
if (bodyBuffer !== undefined) {
|
|
198
|
+
res.status(status).buffer(bodyBuffer);
|
|
199
|
+
}
|
|
200
|
+
else if (body !== undefined) {
|
|
201
|
+
res.status(status).json(body);
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
this.defaultRouteHandler(req, res);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
const errorStatus = (_a = succeedThenFail.errorStatus) !== null && _a !== void 0 ? _a : 400;
|
|
209
|
+
if (succeedThenFail.errorBody !== undefined) {
|
|
210
|
+
res.status(errorStatus).json(succeedThenFail.errorBody);
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
res.status(errorStatus).send();
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
else if (retry) {
|
|
218
|
+
const currentCount = this.requestCounts.get(key) || 0;
|
|
219
|
+
const failureCount = (_b = retry.failureCount) !== null && _b !== void 0 ? _b : 4;
|
|
220
|
+
const errorStatus = (_c = retry.errorStatus) !== null && _c !== void 0 ? _c : 500;
|
|
194
221
|
if (currentCount < failureCount) {
|
|
195
222
|
this.requestCounts.set(key, currentCount + 1);
|
|
196
223
|
const sendFailure = () => {
|
|
@@ -261,6 +288,12 @@ class MockServer {
|
|
|
261
288
|
this.requestCounts.clear();
|
|
262
289
|
this.requests = [];
|
|
263
290
|
}
|
|
291
|
+
/**
|
|
292
|
+
* Returns all POST requests to the platform callback URL.
|
|
293
|
+
*/
|
|
294
|
+
getCallbackRequests() {
|
|
295
|
+
return this.requests.filter((req) => req.method.toUpperCase() === 'POST' && req.url.includes('callback_url'));
|
|
296
|
+
}
|
|
264
297
|
/**
|
|
265
298
|
* Returns the most recent request or undefined if no requests exist.
|
|
266
299
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAQA,OAAO,EACL,oBAAoB,EAGrB,MAAM,kBAAkB,CAAC;AAG1B,wBAAgB,WAAW,CAAC,cAAc,EAAE,EAC1C,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,cAAc,CAAC,
|
|
1
|
+
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAQA,OAAO,EACL,oBAAoB,EAGrB,MAAM,kBAAkB,CAAC;AAG1B,wBAAgB,WAAW,CAAC,cAAc,EAAE,EAC1C,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,cAAc,CAAC,QA0FtC"}
|
|
@@ -14,6 +14,7 @@ function processTask({ task, onTimeout, }) {
|
|
|
14
14
|
}
|
|
15
15
|
void (async () => {
|
|
16
16
|
await (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
17
|
+
let adapter;
|
|
17
18
|
try {
|
|
18
19
|
const event = node_worker_threads_1.workerData.event;
|
|
19
20
|
// TODO: Remove when the old types are completely phased out
|
|
@@ -29,26 +30,40 @@ function processTask({ task, onTimeout, }) {
|
|
|
29
30
|
initialDomainMapping,
|
|
30
31
|
options,
|
|
31
32
|
});
|
|
32
|
-
const
|
|
33
|
+
const workerAdapter = new worker_adapter_1.WorkerAdapter({
|
|
33
34
|
event,
|
|
34
35
|
adapterState,
|
|
35
36
|
options,
|
|
36
37
|
});
|
|
38
|
+
adapter = workerAdapter;
|
|
37
39
|
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
|
|
38
40
|
if (message.subject !== workers_1.WorkerMessageSubject.WorkerMessageExit) {
|
|
39
41
|
return;
|
|
40
42
|
}
|
|
41
43
|
console.log('Timeout received. Waiting for the task to finish.');
|
|
42
|
-
|
|
44
|
+
workerAdapter.isTimeout = true;
|
|
43
45
|
});
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
console.log('Event passsed from SDK to the connector:', workerAdapter.event);
|
|
47
|
+
await (0, logger_context_1.runWithUserLogContext)(async () => task({ adapter: workerAdapter }));
|
|
48
|
+
if (workerAdapter.isTimeout && !workerAdapter.hasWorkerEmitted) {
|
|
49
|
+
await (0, logger_context_1.runWithUserLogContext)(async () => onTimeout({ adapter: workerAdapter }));
|
|
47
50
|
}
|
|
48
51
|
process.exit(0);
|
|
49
52
|
}
|
|
50
53
|
catch (error) {
|
|
51
|
-
(0, logger_context_1.runWithUserLogContext)(() => {
|
|
54
|
+
await (0, logger_context_1.runWithUserLogContext)(async () => {
|
|
55
|
+
if (adapter && !adapter.hasWorkerEmitted) {
|
|
56
|
+
try {
|
|
57
|
+
await adapter.emitError(error);
|
|
58
|
+
}
|
|
59
|
+
catch (failureError) {
|
|
60
|
+
console.error('Error while emitting task failure.', (0, logger_1.serializeError)(failureError));
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (adapter === null || adapter === void 0 ? void 0 : adapter.hasWorkerEmitted) {
|
|
64
|
+
process.exit(1);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
52
67
|
const errorMessage = `Error while processing task. ${(0, logger_1.serializeError)(error)}`;
|
|
53
68
|
console.error(errorMessage);
|
|
54
69
|
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
@@ -56,6 +56,7 @@ jest.mock('./worker-adapter/worker-adapter', () => ({
|
|
|
56
56
|
WorkerAdapter: jest.fn().mockImplementation(() => ({
|
|
57
57
|
isTimeout: false,
|
|
58
58
|
hasWorkerEmitted: false,
|
|
59
|
+
emitError: jest.fn(),
|
|
59
60
|
})),
|
|
60
61
|
}));
|
|
61
62
|
const process_task_1 = require("./process-task");
|
|
@@ -124,7 +125,11 @@ describe(process_task_1.processTask.name, () => {
|
|
|
124
125
|
// Arrange
|
|
125
126
|
const event = makeEvent();
|
|
126
127
|
setWorkerData({ event, initialState: {}, options: {} });
|
|
127
|
-
const mockAdapter = {
|
|
128
|
+
const mockAdapter = {
|
|
129
|
+
isTimeout: false,
|
|
130
|
+
hasWorkerEmitted: false,
|
|
131
|
+
emitError: jest.fn(),
|
|
132
|
+
};
|
|
128
133
|
worker_adapter_1.WorkerAdapter.mockImplementation(() => mockAdapter);
|
|
129
134
|
const task = jest.fn().mockResolvedValue(undefined);
|
|
130
135
|
const onTimeout = jest.fn().mockResolvedValue(undefined);
|
|
@@ -163,4 +168,26 @@ describe(process_task_1.processTask.name, () => {
|
|
|
163
168
|
expect(processExitSpy).toHaveBeenCalledWith(1);
|
|
164
169
|
expect(onTimeout).not.toHaveBeenCalled();
|
|
165
170
|
});
|
|
171
|
+
it('should emit a failure event through the adapter before falling back', async () => {
|
|
172
|
+
const event = makeEvent();
|
|
173
|
+
setWorkerData({ event, initialState: {}, options: {} });
|
|
174
|
+
const mockAdapter = {
|
|
175
|
+
isTimeout: false,
|
|
176
|
+
hasWorkerEmitted: false,
|
|
177
|
+
emitError: jest.fn().mockImplementation(() => {
|
|
178
|
+
mockAdapter.hasWorkerEmitted = true;
|
|
179
|
+
}),
|
|
180
|
+
};
|
|
181
|
+
worker_adapter_1.WorkerAdapter.mockImplementation(() => mockAdapter);
|
|
182
|
+
const taskError = new Error('task upload failed');
|
|
183
|
+
const task = jest.fn().mockRejectedValue(taskError);
|
|
184
|
+
const onTimeout = jest.fn().mockResolvedValue(undefined);
|
|
185
|
+
(0, process_task_1.processTask)({ task, onTimeout });
|
|
186
|
+
await flush();
|
|
187
|
+
expect(mockAdapter.emitError).toHaveBeenCalledWith(taskError);
|
|
188
|
+
expect(mockParentPortPostMessage).not.toHaveBeenCalledWith(expect.objectContaining({
|
|
189
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
|
|
190
|
+
}));
|
|
191
|
+
expect(processExitSpy).toHaveBeenCalledWith(1);
|
|
192
|
+
});
|
|
166
193
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../../src/multithreading/spawn/spawn.ts"],"names":[],"mappings":"AAQA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../../src/multithreading/spawn/spawn.ts"],"names":[],"mappings":"AAQA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,qBAAqB,CAAC;AAgD7B;;;;;;;;;;;GAWG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAwFvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,mBAAmB,CAAqB;gBACpC,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,GAChB,EAAE,cAAc;IAiHjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;CAoCjC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"spawn.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/spawn/spawn.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,SAAS,GAAG;IAC9D,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;CACjD,
|
|
1
|
+
{"version":3,"file":"spawn.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/spawn/spawn.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,SAAS,GAAG;IAC9D,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;CACjD,CAoFA;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,SAAS;;;;EAmCxD"}
|
|
@@ -13,6 +13,7 @@ function getTimeoutErrorEventType(eventType) {
|
|
|
13
13
|
switch (eventType) {
|
|
14
14
|
// Metadata extraction (handles both old and new enum members)
|
|
15
15
|
case extraction_1.EventType.StartExtractingMetadata:
|
|
16
|
+
case extraction_1.EventType.ContinueExtractingMetadata:
|
|
16
17
|
case extraction_1.EventType.ExtractionMetadataStart:
|
|
17
18
|
return {
|
|
18
19
|
eventType: extraction_1.ExtractorEventType.MetadataExtractionError,
|
|
@@ -88,6 +89,14 @@ function getTimeoutErrorEventType(eventType) {
|
|
|
88
89
|
*/
|
|
89
90
|
function getNoScriptEventType(eventType) {
|
|
90
91
|
switch (eventType) {
|
|
92
|
+
// A missing metadata worker is a misconfigured connector, so report it as a
|
|
93
|
+
// phase error rather than an unknown event type.
|
|
94
|
+
case extraction_1.EventType.StartExtractingMetadata:
|
|
95
|
+
case extraction_1.EventType.ContinueExtractingMetadata:
|
|
96
|
+
case extraction_1.EventType.ExtractionMetadataStart:
|
|
97
|
+
return {
|
|
98
|
+
eventType: extraction_1.ExtractorEventType.MetadataExtractionError,
|
|
99
|
+
};
|
|
91
100
|
case extraction_1.EventType.StartDeletingExtractorState:
|
|
92
101
|
return {
|
|
93
102
|
eventType: extraction_1.ExtractorEventType.ExtractorStateDeletionDone,
|
|
@@ -13,6 +13,14 @@ describe(spawn_helpers_1.getTimeoutErrorEventType.name, () => {
|
|
|
13
13
|
// Assert
|
|
14
14
|
expect(result.eventType).toBe(extraction_1.ExtractorEventType.MetadataExtractionError);
|
|
15
15
|
});
|
|
16
|
+
it('should return MetadataExtractionError for ContinueExtractingMetadata', () => {
|
|
17
|
+
// Arrange
|
|
18
|
+
const eventType = extraction_1.EventType.ContinueExtractingMetadata;
|
|
19
|
+
// Act
|
|
20
|
+
const result = (0, spawn_helpers_1.getTimeoutErrorEventType)(eventType);
|
|
21
|
+
// Assert
|
|
22
|
+
expect(result.eventType).toBe(extraction_1.ExtractorEventType.MetadataExtractionError);
|
|
23
|
+
});
|
|
16
24
|
it('should return MetadataExtractionError for deprecated ExtractionMetadataStart', () => {
|
|
17
25
|
// Arrange
|
|
18
26
|
const eventType = extraction_1.EventType.ExtractionMetadataStart;
|
|
@@ -219,6 +227,32 @@ describe(spawn_helpers_1.getTimeoutErrorEventType.name, () => {
|
|
|
219
227
|
});
|
|
220
228
|
});
|
|
221
229
|
describe(spawn_helpers_1.getNoScriptEventType.name, () => {
|
|
230
|
+
describe('metadata extraction', () => {
|
|
231
|
+
it('should return MetadataExtractionError for StartExtractingMetadata', () => {
|
|
232
|
+
// Arrange
|
|
233
|
+
const eventType = extraction_1.EventType.StartExtractingMetadata;
|
|
234
|
+
// Act
|
|
235
|
+
const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
|
|
236
|
+
// Assert
|
|
237
|
+
expect(result.eventType).toBe(extraction_1.ExtractorEventType.MetadataExtractionError);
|
|
238
|
+
});
|
|
239
|
+
it('should return MetadataExtractionError for ContinueExtractingMetadata', () => {
|
|
240
|
+
// Arrange
|
|
241
|
+
const eventType = extraction_1.EventType.ContinueExtractingMetadata;
|
|
242
|
+
// Act
|
|
243
|
+
const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
|
|
244
|
+
// Assert
|
|
245
|
+
expect(result.eventType).toBe(extraction_1.ExtractorEventType.MetadataExtractionError);
|
|
246
|
+
});
|
|
247
|
+
it('should return MetadataExtractionError for deprecated ExtractionMetadataStart', () => {
|
|
248
|
+
// Arrange
|
|
249
|
+
const eventType = extraction_1.EventType.ExtractionMetadataStart;
|
|
250
|
+
// Act
|
|
251
|
+
const result = (0, spawn_helpers_1.getNoScriptEventType)(eventType);
|
|
252
|
+
// Assert
|
|
253
|
+
expect(result.eventType).toBe(extraction_1.ExtractorEventType.MetadataExtractionError);
|
|
254
|
+
});
|
|
255
|
+
});
|
|
222
256
|
describe('extractor state deletion', () => {
|
|
223
257
|
it('should return ExtractorStateDeletionDone for StartDeletingExtractorState', () => {
|
|
224
258
|
// Arrange
|
|
@@ -23,6 +23,7 @@ function getWorkerPath({ event, workerBasePath, }) {
|
|
|
23
23
|
path = '/workers/external-sync-units-extraction';
|
|
24
24
|
break;
|
|
25
25
|
case extraction_1.EventType.StartExtractingMetadata:
|
|
26
|
+
case extraction_1.EventType.ContinueExtractingMetadata:
|
|
26
27
|
path = '/workers/metadata-extraction';
|
|
27
28
|
break;
|
|
28
29
|
case extraction_1.EventType.StartExtractingData:
|
|
@@ -64,7 +64,20 @@ export declare class WorkerAdapter<ConnectorState> {
|
|
|
64
64
|
* @param data - The data to be sent with the event
|
|
65
65
|
*/
|
|
66
66
|
emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
|
|
67
|
+
/**
|
|
68
|
+
* Emits a phase-specific error after task or pre-emit execution fails. This
|
|
69
|
+
* path avoids flushing repos again, so artifacts uploaded before the failure
|
|
70
|
+
* are still reported without retrying the failed batch.
|
|
71
|
+
*/
|
|
72
|
+
emitError(error: unknown): Promise<void>;
|
|
67
73
|
uploadAllRepos(): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* Pre-emit hook: uploads all repos and updates extraction boundaries/state.
|
|
76
|
+
* Throws if uploading repos or persisting state fails. The caller emits a
|
|
77
|
+
* compensating error event with whatever artifacts did upload.
|
|
78
|
+
*/
|
|
79
|
+
private beforeEmit;
|
|
80
|
+
private buildEmitPayload;
|
|
68
81
|
loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
|
|
69
82
|
getLoaderBatches({ supportedItemTypes, }: {
|
|
70
83
|
supportedItemTypes: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;
|
|
1
|
+
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAK9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAC1C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,CAElC;IAEH,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAoBzC,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,EAK3B;IAED,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAyCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAiHhB;;;;OAIG;IACG,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAyCxC,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAOrC;;;;OAIG;YACW,UAAU;IA4ExB,OAAO,CAAC,gBAAgB;IAwBlB,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAoHvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAuIzC"}
|
|
@@ -114,19 +114,43 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.emit`, () => {
|
|
|
114
114
|
// Assert
|
|
115
115
|
expect(mockPostMessage).toHaveBeenCalledTimes(1);
|
|
116
116
|
});
|
|
117
|
-
it('should
|
|
117
|
+
it('should log state size and current state before posting state', async () => {
|
|
118
118
|
// Arrange
|
|
119
|
+
const logSpy = jest.spyOn(console, 'log').mockImplementation();
|
|
119
120
|
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
120
|
-
adapter.uploadAllRepos = jest
|
|
121
|
-
.fn()
|
|
122
|
-
.mockRejectedValue(new Error('uploadAllRepos error'));
|
|
121
|
+
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
123
122
|
// Act
|
|
124
123
|
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
125
124
|
reports: [],
|
|
126
125
|
processed_files: [],
|
|
127
126
|
});
|
|
128
127
|
// Assert
|
|
129
|
-
|
|
128
|
+
const stateLogCall = logSpy.mock.calls.find(([message]) => typeof message === 'string' &&
|
|
129
|
+
message.includes('Saving ') &&
|
|
130
|
+
message.includes('Current state'));
|
|
131
|
+
expect(stateLogCall).toBeDefined();
|
|
132
|
+
expect(stateLogCall === null || stateLogCall === void 0 ? void 0 : stateLogCall[0]).toContain('KB state before emitting event with event type: METADATA_EXTRACTION_ERROR. Current state');
|
|
133
|
+
expect(stateLogCall === null || stateLogCall === void 0 ? void 0 : stateLogCall[1]).toEqual(expect.objectContaining({ attachments: { completed: false } }));
|
|
134
|
+
});
|
|
135
|
+
it('should emit phase extraction error when uploadAllRepos fails', async () => {
|
|
136
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
137
|
+
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
138
|
+
adapter.uploadAllRepos = jest
|
|
139
|
+
.fn()
|
|
140
|
+
.mockRejectedValue(new Error('uploadAllRepos error'));
|
|
141
|
+
await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
|
|
142
|
+
expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
|
|
143
|
+
eventType: types_1.ExtractorEventType.DataExtractionError,
|
|
144
|
+
data: expect.objectContaining({
|
|
145
|
+
error: expect.objectContaining({
|
|
146
|
+
message: expect.stringContaining('uploadAllRepos error'),
|
|
147
|
+
}),
|
|
148
|
+
}),
|
|
149
|
+
}));
|
|
150
|
+
expect(mockPostMessage).toHaveBeenCalledWith(expect.objectContaining({
|
|
151
|
+
subject: 'emit',
|
|
152
|
+
payload: { eventType: types_1.ExtractorEventType.DataExtractionError },
|
|
153
|
+
}));
|
|
130
154
|
});
|
|
131
155
|
it('should include artifacts in data for extraction events', async () => {
|
|
132
156
|
// Arrange
|
|
@@ -405,6 +429,28 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.emit — ExternalSyncUnitExtrac
|
|
|
405
429
|
const emittedData = mockEmit.mock.calls[0][0].data;
|
|
406
430
|
expect(emittedData).not.toHaveProperty('external_sync_units');
|
|
407
431
|
});
|
|
432
|
+
it('should emit an error when external sync unit upload fails', async () => {
|
|
433
|
+
const { adapter } = makeAdapter(types_1.EventType.StartExtractingExternalSyncUnits);
|
|
434
|
+
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
435
|
+
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
436
|
+
const pushMock = jest
|
|
437
|
+
.fn()
|
|
438
|
+
.mockRejectedValue(new Error('external sync unit upload failed'));
|
|
439
|
+
jest.spyOn(adapter, 'initializeRepos').mockImplementation(() => undefined);
|
|
440
|
+
jest.spyOn(adapter, 'getRepo').mockReturnValue({ push: pushMock });
|
|
441
|
+
await adapter.emit(types_1.ExtractorEventType.ExternalSyncUnitExtractionDone, {
|
|
442
|
+
external_sync_units: [{ id: 'esu-1' }],
|
|
443
|
+
});
|
|
444
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
445
|
+
expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
|
|
446
|
+
eventType: types_1.ExtractorEventType.ExternalSyncUnitExtractionError,
|
|
447
|
+
data: expect.objectContaining({
|
|
448
|
+
error: expect.objectContaining({
|
|
449
|
+
message: expect.stringContaining('external sync unit upload failed'),
|
|
450
|
+
}),
|
|
451
|
+
}),
|
|
452
|
+
}));
|
|
453
|
+
});
|
|
408
454
|
});
|
|
409
455
|
describe('WorkerAdapter — workersOldest / workersNewest boundary updates', () => {
|
|
410
456
|
let adapter;
|
|
@@ -22,6 +22,7 @@ const workers_1 = require("../../types/workers");
|
|
|
22
22
|
const uploader_1 = require("../../uploader/uploader");
|
|
23
23
|
const event_type_translation_1 = require("../../common/event-type-translation");
|
|
24
24
|
const helpers_1 = require("../../common/helpers");
|
|
25
|
+
const spawn_helpers_1 = require("../spawn/spawn.helpers");
|
|
25
26
|
function createWorkerAdapter({ event, adapterState, options, }) {
|
|
26
27
|
return new WorkerAdapter({
|
|
27
28
|
event,
|
|
@@ -159,93 +160,49 @@ class WorkerAdapter {
|
|
|
159
160
|
*/
|
|
160
161
|
async emit(newEventType, data) {
|
|
161
162
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
162
|
-
var _a
|
|
163
|
+
var _a;
|
|
163
164
|
newEventType = (0, event_type_translation_1.translateOutgoingEventType)(newEventType);
|
|
164
165
|
if (this.hasWorkerEmitted) {
|
|
165
166
|
console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
|
|
166
167
|
return;
|
|
167
168
|
}
|
|
168
|
-
|
|
169
|
-
// TODO: Remove in v2.0.0
|
|
170
|
-
if (newEventType === extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone &&
|
|
171
|
-
(data === null || data === void 0 ? void 0 : data.external_sync_units) &&
|
|
172
|
-
data.external_sync_units.length > 0) {
|
|
173
|
-
console.log(`Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.`);
|
|
174
|
-
this.initializeRepos([
|
|
175
|
-
{
|
|
176
|
-
itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
|
|
177
|
-
overridenOptions: {
|
|
178
|
-
batchSize: 25000,
|
|
179
|
-
skipConfirmation: true,
|
|
180
|
-
},
|
|
181
|
-
},
|
|
182
|
-
]);
|
|
183
|
-
await ((_a = this.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)) === null || _a === void 0 ? void 0 : _a.push(data.external_sync_units));
|
|
184
|
-
// Remove inline external_sync_units from data to avoid SQS size issues
|
|
185
|
-
delete data.external_sync_units;
|
|
186
|
-
}
|
|
187
|
-
// Upload all repos before emitting the event
|
|
188
|
-
console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
|
|
169
|
+
let eventPayload;
|
|
189
170
|
try {
|
|
190
|
-
|
|
171
|
+
// If the event is ExternalSyncUnitExtractionDone, upload external sync units via a Repo before emitting.
|
|
172
|
+
// TODO: Remove in v2.0.0
|
|
173
|
+
if (newEventType === extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone &&
|
|
174
|
+
(data === null || data === void 0 ? void 0 : data.external_sync_units) &&
|
|
175
|
+
data.external_sync_units.length > 0) {
|
|
176
|
+
console.log(`Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.`);
|
|
177
|
+
this.initializeRepos([
|
|
178
|
+
{
|
|
179
|
+
itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
|
|
180
|
+
overridenOptions: {
|
|
181
|
+
batchSize: 25000,
|
|
182
|
+
skipConfirmation: true,
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
]);
|
|
186
|
+
await ((_a = this.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)) === null || _a === void 0 ? void 0 : _a.push(data.external_sync_units));
|
|
187
|
+
// Remove inline external_sync_units from data to avoid SQS size issues
|
|
188
|
+
delete data.external_sync_units;
|
|
189
|
+
}
|
|
190
|
+
await this.beforeEmit(newEventType);
|
|
191
|
+
eventPayload = this.buildEmitPayload(newEventType, data);
|
|
191
192
|
}
|
|
192
193
|
catch (error) {
|
|
193
|
-
console.error('Error while
|
|
194
|
-
|
|
195
|
-
this.hasWorkerEmitted = true;
|
|
194
|
+
console.error('Error while preparing event for emission.', (0, logger_1.serializeError)(error));
|
|
195
|
+
await this.emitError(error);
|
|
196
196
|
return;
|
|
197
197
|
}
|
|
198
|
-
// If the extraction is done, we want to save the timestamp of the last successful sync
|
|
199
|
-
if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
|
|
200
|
-
console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
|
|
201
|
-
this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
|
|
202
|
-
this.state.lastSyncStarted = '';
|
|
203
|
-
// Clear pending extraction boundaries now that the cycle is complete
|
|
204
|
-
this.state.pendingWorkersOldest = '';
|
|
205
|
-
this.state.pendingWorkersNewest = '';
|
|
206
|
-
// Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
|
|
207
|
-
// Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
|
|
208
|
-
const extractionStart = this.event.payload.event_context.extract_from;
|
|
209
|
-
const extractionEnd = this.event.payload.event_context.extract_to;
|
|
210
|
-
if (extractionStart &&
|
|
211
|
-
(!this.state.workersOldest ||
|
|
212
|
-
extractionStart < this.state.workersOldest)) {
|
|
213
|
-
console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
|
|
214
|
-
this.state.workersOldest = extractionStart;
|
|
215
|
-
}
|
|
216
|
-
if (extractionEnd &&
|
|
217
|
-
(!this.state.workersNewest ||
|
|
218
|
-
extractionEnd > this.state.workersNewest)) {
|
|
219
|
-
console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
|
|
220
|
-
this.state.workersNewest = extractionEnd;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
// We want to save the state every time we emit an event, except for the start and delete events
|
|
224
|
-
if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
|
|
225
|
-
console.log(`Saving state before emitting event with event type: ${newEventType}.`);
|
|
226
|
-
try {
|
|
227
|
-
await this.adapterState.postState(this.state);
|
|
228
|
-
}
|
|
229
|
-
catch (error) {
|
|
230
|
-
console.error('Error while posting state', error);
|
|
231
|
-
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
232
|
-
this.hasWorkerEmitted = true;
|
|
233
|
-
return;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
198
|
try {
|
|
237
|
-
// Always prune error messages to make them shorter before emit
|
|
238
|
-
if ((_b = data === null || data === void 0 ? void 0 : data.error) === null || _b === void 0 ? void 0 : _b.message) {
|
|
239
|
-
data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
|
|
240
|
-
}
|
|
241
199
|
const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
|
|
242
|
-
const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
|
|
243
200
|
const progressData = {};
|
|
244
201
|
if (isExtractionEvent &&
|
|
245
|
-
(newEventType
|
|
246
|
-
newEventType
|
|
247
|
-
newEventType
|
|
248
|
-
newEventType
|
|
202
|
+
(newEventType === extraction_1.ExtractorEventType.DataExtractionDone ||
|
|
203
|
+
newEventType === extraction_1.ExtractorEventType.DataExtractionProgress ||
|
|
204
|
+
newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone ||
|
|
205
|
+
newEventType === extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
|
|
249
206
|
const repo = this.lastExtractedItemType
|
|
250
207
|
? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
|
|
251
208
|
: undefined;
|
|
@@ -260,9 +217,7 @@ class WorkerAdapter {
|
|
|
260
217
|
await (0, control_protocol_1.emit)({
|
|
261
218
|
eventType: newEventType,
|
|
262
219
|
event: this.event,
|
|
263
|
-
data:
|
|
264
|
-
? { reports: this.reports, processed_files: this.processedFiles }
|
|
265
|
-
: {})),
|
|
220
|
+
data: eventPayload,
|
|
266
221
|
worker_metadata: Object.assign({}, progressData),
|
|
267
222
|
});
|
|
268
223
|
const message = {
|
|
@@ -280,14 +235,111 @@ class WorkerAdapter {
|
|
|
280
235
|
}
|
|
281
236
|
});
|
|
282
237
|
}
|
|
238
|
+
/**
|
|
239
|
+
* Emits a phase-specific error after task or pre-emit execution fails. This
|
|
240
|
+
* path avoids flushing repos again, so artifacts uploaded before the failure
|
|
241
|
+
* are still reported without retrying the failed batch.
|
|
242
|
+
*/
|
|
243
|
+
async emitError(error) {
|
|
244
|
+
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
245
|
+
if (this.hasWorkerEmitted) {
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
// Include artifacts uploaded before a failure so they are not orphaned
|
|
249
|
+
// and can be associated with the emitted phase error.
|
|
250
|
+
for (const repo of this.repos) {
|
|
251
|
+
this.artifacts = [...this.artifacts, ...repo.uploadedArtifacts];
|
|
252
|
+
}
|
|
253
|
+
const { eventType } = (0, spawn_helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
|
|
254
|
+
try {
|
|
255
|
+
await (0, control_protocol_1.emit)({
|
|
256
|
+
eventType,
|
|
257
|
+
event: this.event,
|
|
258
|
+
data: this.buildEmitPayload(eventType, {
|
|
259
|
+
error: { message: (0, logger_1.serializeError)(error) },
|
|
260
|
+
}),
|
|
261
|
+
});
|
|
262
|
+
const message = {
|
|
263
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
|
|
264
|
+
payload: { eventType },
|
|
265
|
+
};
|
|
266
|
+
this.artifacts = [];
|
|
267
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
|
|
268
|
+
this.hasWorkerEmitted = true;
|
|
269
|
+
}
|
|
270
|
+
catch (emitError) {
|
|
271
|
+
console.error(`Error while emitting error event with event type: ${eventType}.`, (0, logger_1.serializeError)(emitError));
|
|
272
|
+
}
|
|
273
|
+
});
|
|
274
|
+
}
|
|
283
275
|
async uploadAllRepos() {
|
|
284
276
|
for (const repo of this.repos) {
|
|
285
|
-
|
|
277
|
+
await repo.upload();
|
|
286
278
|
this.artifacts.push(...repo.uploadedArtifacts);
|
|
287
|
-
|
|
288
|
-
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Pre-emit hook: uploads all repos and updates extraction boundaries/state.
|
|
283
|
+
* Throws if uploading repos or persisting state fails. The caller emits a
|
|
284
|
+
* compensating error event with whatever artifacts did upload.
|
|
285
|
+
*/
|
|
286
|
+
async beforeEmit(eventType) {
|
|
287
|
+
// Upload all repos before emitting the event.
|
|
288
|
+
console.log(`Uploading all repos before emitting event with event type: ${eventType}.`);
|
|
289
|
+
await this.uploadAllRepos();
|
|
290
|
+
// If the extraction is done, save the timestamp of the last successful sync.
|
|
291
|
+
if (eventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
|
|
292
|
+
console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
|
|
293
|
+
this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
|
|
294
|
+
this.state.lastSyncStarted = '';
|
|
295
|
+
// Clear pending extraction boundaries now that the cycle is complete.
|
|
296
|
+
this.state.pendingWorkersOldest = '';
|
|
297
|
+
this.state.pendingWorkersNewest = '';
|
|
298
|
+
// Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
|
|
299
|
+
const extractionStart = this.event.payload.event_context.extract_from;
|
|
300
|
+
const extractionEnd = this.event.payload.event_context.extract_to;
|
|
301
|
+
if (extractionStart &&
|
|
302
|
+
(!this.state.workersOldest ||
|
|
303
|
+
extractionStart < this.state.workersOldest)) {
|
|
304
|
+
console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
|
|
305
|
+
this.state.workersOldest = extractionStart;
|
|
306
|
+
}
|
|
307
|
+
if (extractionEnd &&
|
|
308
|
+
(!this.state.workersNewest || extractionEnd > this.state.workersNewest)) {
|
|
309
|
+
console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
|
|
310
|
+
this.state.workersNewest = extractionEnd;
|
|
289
311
|
}
|
|
290
312
|
}
|
|
313
|
+
if (constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
let stateSizeKb = 0;
|
|
317
|
+
try {
|
|
318
|
+
stateSizeKb =
|
|
319
|
+
Buffer.byteLength(JSON.stringify(this.state), 'utf8') / 1024;
|
|
320
|
+
}
|
|
321
|
+
catch (_a) {
|
|
322
|
+
stateSizeKb = 0;
|
|
323
|
+
}
|
|
324
|
+
console.log(`Saving ${stateSizeKb.toFixed(2)} KB state before emitting event with event type: ${eventType}. Current state`, (0, logger_1.getPrintableState)(this.state));
|
|
325
|
+
try {
|
|
326
|
+
await this.adapterState.postState(this.state);
|
|
327
|
+
}
|
|
328
|
+
catch (error) {
|
|
329
|
+
console.error('Error while posting state.', (0, logger_1.serializeError)(error));
|
|
330
|
+
throw error;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
buildEmitPayload(eventType, data) {
|
|
334
|
+
var _a;
|
|
335
|
+
if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
|
|
336
|
+
data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
|
|
337
|
+
}
|
|
338
|
+
const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(eventType);
|
|
339
|
+
const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(eventType);
|
|
340
|
+
return Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
|
|
341
|
+
? { reports: this.reports, processed_files: this.processedFiles }
|
|
342
|
+
: {}));
|
|
291
343
|
}
|
|
292
344
|
async loadItemTypes({ itemTypesToLoad, }) {
|
|
293
345
|
return (0, logger_context_1.runWithSdkLogContext)(async () => {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-adapter.upload-failure.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.upload-failure.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const state_1 = require("../../state/state");
|
|
4
|
+
const jest_setup_1 = require("../../tests/jest.setup");
|
|
5
|
+
const test_helpers_1 = require("../../tests/test-helpers");
|
|
6
|
+
const test_utils_1 = require("../../common/test-utils");
|
|
7
|
+
const types_1 = require("../../types");
|
|
8
|
+
const worker_adapter_1 = require("./worker-adapter");
|
|
9
|
+
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
10
|
+
jest.mock('../../common/control-protocol', () => ({
|
|
11
|
+
emit: jest.fn().mockResolvedValue({}),
|
|
12
|
+
}));
|
|
13
|
+
jest.mock('../../mappers/mappers');
|
|
14
|
+
jest.mock('node:worker_threads', () => ({
|
|
15
|
+
parentPort: { postMessage: jest.fn() },
|
|
16
|
+
}));
|
|
17
|
+
jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({
|
|
18
|
+
AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({
|
|
19
|
+
streamAll: jest.fn().mockResolvedValue(undefined),
|
|
20
|
+
})),
|
|
21
|
+
}));
|
|
22
|
+
const UPLOAD_URL_PATH = '/internal/airdrop.artifacts.upload-url';
|
|
23
|
+
function makeAdapter() {
|
|
24
|
+
const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
25
|
+
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
26
|
+
});
|
|
27
|
+
const initialState = {
|
|
28
|
+
attachments: { completed: false },
|
|
29
|
+
lastSyncStarted: '',
|
|
30
|
+
lastSuccessfulSyncStarted: '',
|
|
31
|
+
snapInVersionId: '',
|
|
32
|
+
toDevRev: {
|
|
33
|
+
attachmentsMetadata: {
|
|
34
|
+
artifactIds: [],
|
|
35
|
+
lastProcessed: 0,
|
|
36
|
+
lastProcessedAttachmentsIdsList: [],
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
const adapterState = new state_1.State({ event, initialState });
|
|
41
|
+
const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
|
|
42
|
+
const workerThreads = require('node:worker_threads');
|
|
43
|
+
const mockPostMessage = jest.fn();
|
|
44
|
+
if (workerThreads.parentPort) {
|
|
45
|
+
jest
|
|
46
|
+
.spyOn(workerThreads.parentPort, 'postMessage')
|
|
47
|
+
.mockImplementation(mockPostMessage);
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
workerThreads.parentPort = { postMessage: mockPostMessage };
|
|
51
|
+
}
|
|
52
|
+
return { adapter, mockPostMessage };
|
|
53
|
+
}
|
|
54
|
+
describe(`${worker_adapter_1.WorkerAdapter.name} upload failure (near-integration)`, () => {
|
|
55
|
+
let adapter;
|
|
56
|
+
let mockPostMessage;
|
|
57
|
+
beforeEach(() => {
|
|
58
|
+
jest.clearAllMocks();
|
|
59
|
+
jest_setup_1.mockServer.resetRoutes();
|
|
60
|
+
({ adapter, mockPostMessage } = makeAdapter());
|
|
61
|
+
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
62
|
+
});
|
|
63
|
+
afterEach(() => {
|
|
64
|
+
jest.restoreAllMocks();
|
|
65
|
+
});
|
|
66
|
+
it('should throw from repo.push when upload-url returns 400 during batch upload', async () => {
|
|
67
|
+
var _a, _b;
|
|
68
|
+
jest_setup_1.mockServer.setRoute({
|
|
69
|
+
path: UPLOAD_URL_PATH,
|
|
70
|
+
method: 'GET',
|
|
71
|
+
status: 400,
|
|
72
|
+
});
|
|
73
|
+
adapter.initializeRepos([
|
|
74
|
+
{
|
|
75
|
+
itemType: 'tasks',
|
|
76
|
+
overridenOptions: { batchSize: 10 },
|
|
77
|
+
},
|
|
78
|
+
]);
|
|
79
|
+
await expect((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(20))).rejects.toThrow('artifact upload URL');
|
|
80
|
+
expect((_b = adapter.getRepo('tasks')) === null || _b === void 0 ? void 0 : _b.getItems().length).toBe(20);
|
|
81
|
+
});
|
|
82
|
+
it('should emit DataExtractionError when uploadAllRepos fails', async () => {
|
|
83
|
+
var _a;
|
|
84
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
85
|
+
jest_setup_1.mockServer.setRoute({
|
|
86
|
+
path: UPLOAD_URL_PATH,
|
|
87
|
+
method: 'GET',
|
|
88
|
+
status: 400,
|
|
89
|
+
});
|
|
90
|
+
adapter.initializeRepos([{ itemType: 'tasks' }]);
|
|
91
|
+
await ((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
|
|
92
|
+
await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
|
|
93
|
+
expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
|
|
94
|
+
eventType: types_1.ExtractorEventType.DataExtractionError,
|
|
95
|
+
data: expect.objectContaining({
|
|
96
|
+
error: expect.objectContaining({
|
|
97
|
+
message: expect.stringContaining('artifact upload URL'),
|
|
98
|
+
}),
|
|
99
|
+
}),
|
|
100
|
+
}));
|
|
101
|
+
expect(mockPostMessage).toHaveBeenCalledWith(expect.objectContaining({
|
|
102
|
+
subject: 'emit',
|
|
103
|
+
payload: { eventType: types_1.ExtractorEventType.DataExtractionError },
|
|
104
|
+
}));
|
|
105
|
+
});
|
|
106
|
+
it('should include partial artifacts when push batch succeeded before uploadAllRepos fails', async () => {
|
|
107
|
+
var _a;
|
|
108
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
109
|
+
jest_setup_1.mockServer.setRoute({
|
|
110
|
+
path: UPLOAD_URL_PATH,
|
|
111
|
+
method: 'GET',
|
|
112
|
+
status: 200,
|
|
113
|
+
succeedThenFail: {
|
|
114
|
+
successCount: 1,
|
|
115
|
+
errorStatus: 400,
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
adapter.initializeRepos([
|
|
119
|
+
{
|
|
120
|
+
itemType: 'tasks',
|
|
121
|
+
overridenOptions: { batchSize: 10 },
|
|
122
|
+
},
|
|
123
|
+
]);
|
|
124
|
+
await ((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(15)));
|
|
125
|
+
await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
|
|
126
|
+
expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
|
|
127
|
+
eventType: types_1.ExtractorEventType.DataExtractionError,
|
|
128
|
+
data: expect.objectContaining({
|
|
129
|
+
artifacts: expect.arrayContaining([
|
|
130
|
+
expect.objectContaining({ item_count: 10, item_type: 'tasks' }),
|
|
131
|
+
]),
|
|
132
|
+
}),
|
|
133
|
+
}));
|
|
134
|
+
});
|
|
135
|
+
});
|
package/dist/repo/repo.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Item } from '../repo/repo.interfaces';
|
|
2
|
-
import { ErrorRecord } from '../types/common';
|
|
3
2
|
import { Artifact } from '../uploader/uploader.interfaces';
|
|
4
3
|
import { NormalizedAttachment, NormalizedItem, RepoFactoryInterface } from './repo.interfaces';
|
|
5
4
|
export declare class Repo {
|
|
@@ -22,7 +21,7 @@ export declare class Repo {
|
|
|
22
21
|
};
|
|
23
22
|
constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
|
|
24
23
|
getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
|
|
25
|
-
upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void
|
|
24
|
+
upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
|
|
26
25
|
push(items: Item[]): Promise<boolean>;
|
|
27
26
|
}
|
|
28
27
|
//# sourceMappingURL=repo.d.ts.map
|
package/dist/repo/repo.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAG3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IA4DV,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAkC5C"}
|
package/dist/repo/repo.js
CHANGED
|
@@ -23,6 +23,7 @@ class Repo {
|
|
|
23
23
|
return this.items;
|
|
24
24
|
}
|
|
25
25
|
async upload(batch) {
|
|
26
|
+
var _a;
|
|
26
27
|
const itemsToUpload = batch || this.items;
|
|
27
28
|
if (itemsToUpload.length > 0) {
|
|
28
29
|
for (const item of itemsToUpload) {
|
|
@@ -44,8 +45,8 @@ class Repo {
|
|
|
44
45
|
console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
|
|
45
46
|
const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
|
|
46
47
|
if (error || !artifact) {
|
|
47
|
-
console.error(
|
|
48
|
-
|
|
48
|
+
console.error(`Error while uploading batch of ${itemsToUpload.length} items of type ${this.itemType}.`, error);
|
|
49
|
+
throw new Error((_a = error === null || error === void 0 ? void 0 : error.message) !== null && _a !== void 0 ? _a : `Upload failed for item type "${this.itemType}" without artifact.`);
|
|
49
50
|
}
|
|
50
51
|
this.onUpload(artifact);
|
|
51
52
|
this.uploadedArtifacts.push(artifact);
|
|
@@ -80,16 +81,9 @@ class Repo {
|
|
|
80
81
|
// Upload in batches while the number of items exceeds the batch size
|
|
81
82
|
const batchSize = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.batchSize) || constants_1.ARTIFACT_BATCH_SIZE;
|
|
82
83
|
while (this.items.length >= batchSize) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
// Upload the batch
|
|
87
|
-
await this.upload(batch);
|
|
88
|
-
}
|
|
89
|
-
catch (error) {
|
|
90
|
-
console.error('Error while uploading batch', error);
|
|
91
|
-
return false;
|
|
92
|
-
}
|
|
84
|
+
const batch = this.items.slice(0, batchSize);
|
|
85
|
+
await this.upload(batch);
|
|
86
|
+
this.items.splice(0, batchSize);
|
|
93
87
|
}
|
|
94
88
|
return true;
|
|
95
89
|
}
|
package/dist/repo/repo.test.js
CHANGED
|
@@ -313,7 +313,7 @@ describe(repo_1.Repo.name, () => {
|
|
|
313
313
|
error: new Error('fail'),
|
|
314
314
|
artifact: null,
|
|
315
315
|
});
|
|
316
|
-
await repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')]);
|
|
316
|
+
await expect(repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')])).rejects.toThrow('fail');
|
|
317
317
|
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
318
318
|
expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
319
319
|
});
|
|
@@ -343,4 +343,35 @@ describe(repo_1.Repo.name, () => {
|
|
|
343
343
|
expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
|
|
344
344
|
});
|
|
345
345
|
});
|
|
346
|
+
it('should throw when upload fails', async () => {
|
|
347
|
+
mockUploadFn.mockResolvedValueOnce({
|
|
348
|
+
error: { message: 'upload failed' },
|
|
349
|
+
});
|
|
350
|
+
await expect(repo.upload((0, test_helpers_1.createItems)(1))).rejects.toThrow('upload failed');
|
|
351
|
+
});
|
|
352
|
+
it('should retain items in repo when batch upload fails during push', async () => {
|
|
353
|
+
repo = new repo_1.Repo({
|
|
354
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
355
|
+
payload: { event_type: types_1.EventType.ExtractionDataStart },
|
|
356
|
+
}),
|
|
357
|
+
itemType: 'test_item_type',
|
|
358
|
+
normalize,
|
|
359
|
+
onUpload: jest.fn(),
|
|
360
|
+
options: { batchSize: 10 },
|
|
361
|
+
});
|
|
362
|
+
const items = (0, test_helpers_1.createItems)(20);
|
|
363
|
+
mockUploadFn
|
|
364
|
+
.mockResolvedValueOnce({
|
|
365
|
+
artifact: {
|
|
366
|
+
id: 'artifact-1',
|
|
367
|
+
item_type: 'test_item_type',
|
|
368
|
+
item_count: 10,
|
|
369
|
+
},
|
|
370
|
+
})
|
|
371
|
+
.mockResolvedValueOnce({
|
|
372
|
+
error: { message: 'second batch failed' },
|
|
373
|
+
});
|
|
374
|
+
await expect(repo.push(items)).rejects.toThrow('second batch failed');
|
|
375
|
+
expect(repo.getItems().length).toBe(10);
|
|
376
|
+
});
|
|
346
377
|
});
|
package/dist/state/state.js
CHANGED
|
@@ -67,7 +67,7 @@ async function createAdapterState({ event, initialState, initialDomainMapping, o
|
|
|
67
67
|
const eventContext = event.payload.event_context;
|
|
68
68
|
const overrideExtractFrom = (value) => {
|
|
69
69
|
if (eventContext.extract_from !== undefined) {
|
|
70
|
-
eventContext.
|
|
70
|
+
eventContext.original_extract_from = eventContext.extract_from;
|
|
71
71
|
}
|
|
72
72
|
eventContext.extract_from = value;
|
|
73
73
|
};
|
|
@@ -102,7 +102,7 @@ describe('State — TimeValue resolution', () => {
|
|
|
102
102
|
// Assert: the successful sync boundary takes precedence and is retained
|
|
103
103
|
// for later phases.
|
|
104
104
|
expect(event.payload.event_context.extract_from).toBe('2025-05-01T00:00:00.000Z');
|
|
105
|
-
expect(event.payload.event_context.
|
|
105
|
+
expect(event.payload.event_context.original_extract_from).toBe(platformExtractFrom);
|
|
106
106
|
expect(state.state.pendingWorkersOldest).toBe('2025-05-01T00:00:00.000Z');
|
|
107
107
|
});
|
|
108
108
|
it('should preserve the platform extract_from when resolving a CPv2 start time', async () => {
|
|
@@ -134,7 +134,7 @@ describe('State — TimeValue resolution', () => {
|
|
|
134
134
|
initialDomainMapping: {},
|
|
135
135
|
});
|
|
136
136
|
expect(event.payload.event_context.extract_from).toBe(resolvedExtractFrom);
|
|
137
|
-
expect(event.payload.event_context.
|
|
137
|
+
expect(event.payload.event_context.original_extract_from).toBe(platformExtractFrom);
|
|
138
138
|
});
|
|
139
139
|
it('should use the platform extract_from when lastSuccessfulSyncStarted is not set', async () => {
|
|
140
140
|
// Arrange: CPv1.1 sends a concrete extract_from instead of a TimeValue.
|
|
@@ -255,7 +255,7 @@ describe('State — TimeValue resolution', () => {
|
|
|
255
255
|
initialDomainMapping: {},
|
|
256
256
|
});
|
|
257
257
|
expect(event.payload.event_context.extract_from).toBe(lastSuccessfulSyncStarted);
|
|
258
|
-
expect(event.payload.event_context.
|
|
258
|
+
expect(event.payload.event_context.original_extract_from).toBe(platformExtractFrom);
|
|
259
259
|
});
|
|
260
260
|
it('should skip resolution when extraction_end_time has no type', async () => {
|
|
261
261
|
// Arrange: platform sends extraction_end_time without a type field
|
|
@@ -56,7 +56,8 @@ export declare enum EventType {
|
|
|
56
56
|
StartDeletingExtractorState = "START_DELETING_EXTRACTOR_STATE",
|
|
57
57
|
StartExtractingAttachments = "START_EXTRACTING_ATTACHMENTS",
|
|
58
58
|
ContinueExtractingAttachments = "CONTINUE_EXTRACTING_ATTACHMENTS",
|
|
59
|
-
StartDeletingExtractorAttachmentsState = "START_DELETING_EXTRACTOR_ATTACHMENTS_STATE"
|
|
59
|
+
StartDeletingExtractorAttachmentsState = "START_DELETING_EXTRACTOR_ATTACHMENTS_STATE",
|
|
60
|
+
ContinueExtractingMetadata = "CONTINUE_EXTRACTING_METADATA"
|
|
60
61
|
}
|
|
61
62
|
/**
|
|
62
63
|
* ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
|
|
@@ -143,7 +144,9 @@ export declare enum ExtractorEventType {
|
|
|
143
144
|
AttachmentExtractionDone = "ATTACHMENT_EXTRACTION_DONE",
|
|
144
145
|
AttachmentExtractionError = "ATTACHMENT_EXTRACTION_ERROR",
|
|
145
146
|
ExtractorAttachmentsStateDeletionDone = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE",
|
|
146
|
-
ExtractorAttachmentsStateDeletionError = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR"
|
|
147
|
+
ExtractorAttachmentsStateDeletionError = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR",
|
|
148
|
+
MetadataExtractionProgress = "METADATA_EXTRACTION_PROGRESS",
|
|
149
|
+
MetadataExtractionDelayed = "METADATA_EXTRACTION_DELAYED"
|
|
147
150
|
}
|
|
148
151
|
/**
|
|
149
152
|
* @deprecated
|
|
@@ -303,7 +306,7 @@ export interface EventContext {
|
|
|
303
306
|
* Original extraction start timestamp supplied by the platform before the SDK
|
|
304
307
|
* overrides extract_from.
|
|
305
308
|
*/
|
|
306
|
-
|
|
309
|
+
original_extract_from?: string;
|
|
307
310
|
import_slug: string;
|
|
308
311
|
initial_sync_scope?: InitialSyncScope;
|
|
309
312
|
mode: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;IACrF,0BAA0B,iCAAiC;CAC5D;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;IACrF,0BAA0B,iCAAiC;IAC3D,yBAAyB,gCAAgC;CAC1D;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,oBAAY,aAAa;IACvB,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,aAAa,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,0BAA0B,CAAC,EAAE,aAAa,EAAE,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAI9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,GAAG;QACpB,oFAAoF;QACpF,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACH,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
package/dist/types/extraction.js
CHANGED
|
@@ -58,6 +58,7 @@ var EventType;
|
|
|
58
58
|
EventType["StartExtractingAttachments"] = "START_EXTRACTING_ATTACHMENTS";
|
|
59
59
|
EventType["ContinueExtractingAttachments"] = "CONTINUE_EXTRACTING_ATTACHMENTS";
|
|
60
60
|
EventType["StartDeletingExtractorAttachmentsState"] = "START_DELETING_EXTRACTOR_ATTACHMENTS_STATE";
|
|
61
|
+
EventType["ContinueExtractingMetadata"] = "CONTINUE_EXTRACTING_METADATA";
|
|
61
62
|
})(EventType || (exports.EventType = EventType = {}));
|
|
62
63
|
/**
|
|
63
64
|
* ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
|
|
@@ -149,6 +150,8 @@ var ExtractorEventType;
|
|
|
149
150
|
ExtractorEventType["AttachmentExtractionError"] = "ATTACHMENT_EXTRACTION_ERROR";
|
|
150
151
|
ExtractorEventType["ExtractorAttachmentsStateDeletionDone"] = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE";
|
|
151
152
|
ExtractorEventType["ExtractorAttachmentsStateDeletionError"] = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR";
|
|
153
|
+
ExtractorEventType["MetadataExtractionProgress"] = "METADATA_EXTRACTION_PROGRESS";
|
|
154
|
+
ExtractorEventType["MetadataExtractionDelayed"] = "METADATA_EXTRACTION_DELAYED";
|
|
152
155
|
})(ExtractorEventType || (exports.ExtractorEventType = ExtractorEventType = {}));
|
|
153
156
|
/**
|
|
154
157
|
* @deprecated
|
package/package.json
CHANGED