@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -11
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +2 -10
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +15 -43
- package/dist/common/constants.d.ts +1 -0
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +2 -1
- package/dist/common/helpers.d.ts +0 -14
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +0 -36
- package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
- package/dist/http/client.d.ts.map +1 -0
- package/dist/index.d.ts +22 -11
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +46 -28
- package/dist/logger/logger.d.ts +14 -0
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.js +55 -8
- package/dist/mappers/mappers.d.ts +9 -10
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/{mappers.interface.d.ts → mappers.interfaces.d.ts} +5 -5
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
- package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +3 -3
- package/dist/mappers/mappers.js +13 -9
- package/dist/multithreading/adapters/base-adapter.d.ts +23 -8
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +41 -8
- package/dist/multithreading/adapters/extraction-adapter.d.ts +8 -9
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +24 -26
- package/dist/multithreading/adapters/loading-adapter.d.ts +7 -7
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.js +53 -83
- package/dist/{common/control-protocol.d.ts → multithreading/emit.d.ts} +1 -1
- package/dist/multithreading/emit.d.ts.map +1 -0
- package/dist/{common/control-protocol.js → multithreading/emit.js} +3 -3
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +26 -9
- package/dist/multithreading/spawn/spawn.helpers.d.ts +23 -3
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +152 -2
- package/dist/multithreading/spawn/spawn.js +7 -7
- package/dist/multithreading/worker.js +1 -2
- package/dist/repo/repo.js +2 -2
- package/dist/state/base-state.d.ts +5 -5
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +7 -7
- package/dist/state/extraction-state.d.ts +9 -9
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +10 -10
- package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
- package/dist/{common → state}/install-initial-domain-mapping.js +4 -4
- package/dist/state/loading-state.d.ts +7 -8
- package/dist/state/loading-state.d.ts.map +1 -1
- package/dist/state/loading-state.js +6 -7
- package/dist/state/state.d.ts +4 -0
- package/dist/state/state.d.ts.map +1 -0
- package/dist/state/state.interfaces.d.ts +28 -18
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +13 -6
- package/dist/state/state.js +11 -0
- package/dist/{common → state}/time-value-resolver.d.ts +6 -7
- package/dist/state/time-value-resolver.d.ts.map +1 -0
- package/dist/{common → state}/time-value-resolver.js +9 -8
- package/dist/{common/test-utils.d.ts → testing/mock-event.d.ts} +1 -2
- package/dist/testing/mock-event.d.ts.map +1 -0
- package/dist/{common/test-utils.js → testing/mock-event.js} +14 -3
- package/dist/{mock-server → testing}/mock-server.d.ts +5 -0
- package/dist/testing/mock-server.d.ts.map +1 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
- package/dist/{mock-server → testing}/mock-server.js +6 -1
- package/dist/types/errors.d.ts +9 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/extraction.d.ts +70 -15
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +4 -6
- package/dist/types/loading.d.ts +1 -2
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +0 -1
- package/dist/types/workers.d.ts +66 -23
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +5 -4
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.d.ts +1 -1
- package/dist/uploader/uploader.js +10 -10
- package/package.json +3 -3
- package/dist/airsync-sdk.d.ts +0 -2059
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
- package/dist/common/control-protocol.d.ts.map +0 -1
- package/dist/common/errors.d.ts +0 -6
- package/dist/common/errors.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
- package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.js +0 -211
- package/dist/common/test-utils.d.ts.map +0 -1
- package/dist/common/time-value-resolver.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.d.ts +0 -2
- package/dist/common/time-value-resolver.test.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.js +0 -268
- package/dist/http/axios-client-internal.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.d.ts +0 -2
- package/dist/http/axios-client-internal.test.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.js +0 -175
- package/dist/http/axios-client.d.ts +0 -27
- package/dist/http/axios-client.d.ts.map +0 -1
- package/dist/http/axios-client.js +0 -60
- package/dist/http/constants.d.ts +0 -4
- package/dist/http/constants.d.ts.map +0 -1
- package/dist/http/constants.js +0 -6
- package/dist/http/index.d.ts +0 -2
- package/dist/http/index.d.ts.map +0 -1
- package/dist/http/index.js +0 -17
- package/dist/logger/logger.constants.d.ts +0 -6
- package/dist/logger/logger.constants.d.ts.map +0 -1
- package/dist/logger/logger.constants.js +0 -13
- package/dist/logger/logger.context.d.ts +0 -58
- package/dist/logger/logger.context.d.ts.map +0 -1
- package/dist/logger/logger.context.js +0 -86
- package/dist/logger/logger.test.d.ts +0 -2
- package/dist/logger/logger.test.d.ts.map +0 -1
- package/dist/logger/logger.test.js +0 -537
- package/dist/mappers/mappers.interface.d.ts.map +0 -1
- package/dist/mappers/mappers.test.d.ts +0 -2
- package/dist/mappers/mappers.test.d.ts.map +0 -1
- package/dist/mappers/mappers.test.js +0 -105
- package/dist/mock-server/mock-server.d.ts.map +0 -1
- package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
- package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
- package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
- package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/serialization.test.js +0 -71
- package/dist/multithreading/create-worker.test.d.ts +0 -2
- package/dist/multithreading/create-worker.test.d.ts.map +0 -1
- package/dist/multithreading/create-worker.test.js +0 -108
- package/dist/multithreading/process-task.test.d.ts +0 -2
- package/dist/multithreading/process-task.test.d.ts.map +0 -1
- package/dist/multithreading/process-task.test.js +0 -146
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
- package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.test.js +0 -232
- package/dist/repo/repo.test.d.ts +0 -2
- package/dist/repo/repo.test.d.ts.map +0 -1
- package/dist/repo/repo.test.js +0 -184
- package/dist/state/state.extract-window.test.d.ts +0 -2
- package/dist/state/state.extract-window.test.d.ts.map +0 -1
- package/dist/state/state.extract-window.test.js +0 -163
- package/dist/state/state.pending-boundaries.test.d.ts +0 -2
- package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
- package/dist/state/state.pending-boundaries.test.js +0 -189
- package/dist/state/state.post-state.test.d.ts +0 -2
- package/dist/state/state.post-state.test.d.ts.map +0 -1
- package/dist/state/state.post-state.test.js +0 -78
- package/dist/state/state.test.d.ts +0 -2
- package/dist/state/state.test.d.ts.map +0 -1
- package/dist/state/state.test.js +0 -271
- package/dist/state/state.time-value-resolution.test.d.ts +0 -2
- package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
- package/dist/state/state.time-value-resolution.test.js +0 -175
- package/dist/tsdoc-metadata.json +0 -11
- package/dist/types/extraction.test.d.ts +0 -2
- package/dist/types/extraction.test.d.ts.map +0 -1
- package/dist/types/extraction.test.js +0 -96
- package/dist/types/index.d.ts +0 -10
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -21
- package/dist/uploader/uploader.helpers.test.d.ts +0 -2
- package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
- package/dist/uploader/uploader.helpers.test.js +0 -256
- package/dist/uploader/uploader.test.d.ts +0 -2
- package/dist/uploader/uploader.test.d.ts.map +0 -1
- package/dist/uploader/uploader.test.js +0 -601
- /package/dist/http/{axios-client-internal.js → client.js} +0 -0
- /package/dist/{common → state}/install-initial-domain-mapping.d.ts +0 -0
- /package/dist/{mock-server → testing}/mock-server.interfaces.d.ts +0 -0
- /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
- /package/dist/{common → types}/errors.js +0 -0
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { MappersCreateParams, MappersCreateResponse, MappersFactoryInterface, MappersGetByExternalIdParams, MappersGetByExternalIdResponse, MappersGetByTargetIdParams, MappersGetByTargetIdResponse, MappersUpdateParams, MappersUpdateResponse } from './mappers.interface';
|
|
1
|
+
import { MappersCreateParams, MappersCreateResponse, MappersFactoryInterface, MappersGetByExternalIdParams, MappersGetByExternalIdResponse, MappersGetByTargetIdParams, MappersGetByTargetIdResponse, MappersUpdateParams, MappersUpdateResponse } from './mappers.interfaces';
|
|
3
2
|
/**
|
|
4
3
|
* Manages sync mapper records that link external system items to DevRev items.
|
|
5
4
|
*
|
|
@@ -15,18 +14,18 @@ export declare class Mappers {
|
|
|
15
14
|
* Used to find the mapping when you know the DevRev ID and want to find the external system ID.
|
|
16
15
|
*
|
|
17
16
|
* @param params - Query parameters of type MappersGetByTargetIdParams
|
|
18
|
-
* @returns Promise
|
|
17
|
+
* @returns Promise resolving to the sync mapper record
|
|
19
18
|
*/
|
|
20
|
-
getByTargetId(params: MappersGetByTargetIdParams): Promise<
|
|
19
|
+
getByTargetId(params: MappersGetByTargetIdParams): Promise<MappersGetByTargetIdResponse>;
|
|
21
20
|
/**
|
|
22
21
|
* Retrieves a sync mapper record by external system ID.
|
|
23
22
|
*
|
|
24
23
|
* Used to find the mapping when you know the external system ID and want to find the DevRev ID.
|
|
25
24
|
*
|
|
26
25
|
* @param params - Query parameters of type MappersGetByExternalIdParams
|
|
27
|
-
* @returns Promise
|
|
26
|
+
* @returns Promise resolving to the sync mapper record
|
|
28
27
|
*/
|
|
29
|
-
getByExternalId(params: MappersGetByExternalIdParams): Promise<
|
|
28
|
+
getByExternalId(params: MappersGetByExternalIdParams): Promise<MappersGetByExternalIdResponse>;
|
|
30
29
|
/**
|
|
31
30
|
* Creates a new sync mapper record to establish a relationship between external system
|
|
32
31
|
* entities and DevRev entities.
|
|
@@ -35,9 +34,9 @@ export declare class Mappers {
|
|
|
35
34
|
* the mapping for future synchronization operations.
|
|
36
35
|
*
|
|
37
36
|
* @param params - Creation parameters of type MappersCreateParams
|
|
38
|
-
* @returns Promise
|
|
37
|
+
* @returns Promise resolving to the created sync mapper record
|
|
39
38
|
*/
|
|
40
|
-
create(params: MappersCreateParams): Promise<
|
|
39
|
+
create(params: MappersCreateParams): Promise<MappersCreateResponse>;
|
|
41
40
|
/**
|
|
42
41
|
* Updates an existing sync mapper record.
|
|
43
42
|
*
|
|
@@ -45,8 +44,8 @@ export declare class Mappers {
|
|
|
45
44
|
* additional DevRev entities need to be associated.
|
|
46
45
|
*
|
|
47
46
|
* @param params - Update parameters of type MappersUpdateParams
|
|
48
|
-
* @returns Promise
|
|
47
|
+
* @returns Promise resolving to the updated sync mapper record
|
|
49
48
|
*/
|
|
50
|
-
update(params: MappersUpdateParams): Promise<
|
|
49
|
+
update(params: MappersUpdateParams): Promise<MappersUpdateResponse>;
|
|
51
50
|
}
|
|
52
51
|
//# sourceMappingURL=mappers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../../src/mappers/mappers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../../src/mappers/mappers.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,uBAAuB,EACvB,4BAA4B,EAC5B,8BAA8B,EAC9B,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,qBAAqB,EACtB,MAAM,sBAAsB,CAAC;AAE9B;;;;GAIG;AACH,qBAAa,OAAO;IAClB,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;gBAEnB,EAAE,KAAK,EAAE,EAAE,uBAAuB;IAK9C;;;;;;;OAOG;IACG,aAAa,CACjB,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC,4BAA4B,CAAC;IAcxC;;;;;;;OAOG;IACG,eAAe,CACnB,MAAM,EAAE,4BAA4B,GACnC,OAAO,CAAC,8BAA8B,CAAC;IAc1C;;;;;;;;;OASG;IACG,MAAM,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAazE;;;;;;;;OAQG;IACG,MAAM,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC;CAY1E"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AirSyncEvent } from '../types';
|
|
1
|
+
import { AirSyncEvent } from '../types/extraction';
|
|
2
2
|
import { DonV2 } from '../types/loading';
|
|
3
3
|
import { WorkerAdapterOptions } from '../types/workers';
|
|
4
4
|
/**
|
|
@@ -262,9 +262,9 @@ export interface MappersGetByExternalIdParams {
|
|
|
262
262
|
export declare enum SyncMapperRecordTargetType {
|
|
263
263
|
ACCESS_CONTROL_ENTRY = "access_control_entry",
|
|
264
264
|
ACCOUNT = "account",
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
265
|
+
AIRDROP_AUTHORIZATION_POLICY = "airdrop_authorization_policy",
|
|
266
|
+
AIRDROP_FIELD_AUTHORIZATION_POLICY = "airdrop_field_authorization_policy",
|
|
267
|
+
AIRDROP_PLATFORM_GROUP = "airdrop_platform_group",
|
|
268
268
|
ARTICLE = "article",
|
|
269
269
|
ARTIFACT = "artifact",
|
|
270
270
|
CHAT = "chat",
|
|
@@ -291,4 +291,4 @@ export declare enum SyncMapperRecordTargetType {
|
|
|
291
291
|
export interface MappersGetByExternalIdResponse {
|
|
292
292
|
sync_mapper_record: SyncMapperRecord;
|
|
293
293
|
}
|
|
294
|
-
//# sourceMappingURL=mappers.
|
|
294
|
+
//# sourceMappingURL=mappers.interfaces.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mappers.interfaces.d.ts","sourceRoot":"","sources":["../../src/mappers/mappers.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,iCAAiC;IACjC,YAAY,EAAE;QACZ,GAAG,EAAE,MAAM,EAAE,CAAC;KACf,CAAC;IACF;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,+BAA+B;IAC/B,OAAO,EAAE;QACP,GAAG,EAAE,KAAK,EAAE,CAAC;KACd,CAAC;IACF,MAAM,EAAE,sBAAsB,CAAC;IAC/B;;;;OAIG;IACH,WAAW,CAAC,EAAE;QACZ,GAAG,EAAE,MAAM,EAAE,CAAC;KACf,CAAC;IACF;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE;QAClB,GAAG,EAAE,+BAA+B,EAAE,CAAC;KACxC,CAAC;IACF;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,KAAK,CAAC;IACV,sEAAsE;IACtE,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,wDAAwD;IACxD,OAAO,EAAE,KAAK,EAAE,CAAC;IACjB,MAAM,EAAE,sBAAsB,CAAC;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,+BAA+B,EAAE,CAAC;IACtD;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,+DAA+D;IAC/D,SAAS,EAAE,KAAK,CAAC;IACjB,sCAAsC;IACtC,MAAM,EAAE,KAAK,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,kBAAkB,EAAE,gBAAgB,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,+DAA+D;IAC/D,SAAS,EAAE,KAAK,CAAC;IACjB,2CAA2C;IAC3C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,wDAAwD;IACxD,OAAO,EAAE,KAAK,EAAE,CAAC;IACjB,MAAM,EAAE,sBAAsB,CAAC;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,+BAA+B,EAAE,CAAC;IACtD;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,kBAAkB,EAAE,gBAAgB,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,0DAA0D;IAC1D,EAAE,EAAE,KAAK,CAAC;IACV,+DAA+D;IAC/D,SAAS,EAAE,KAAK,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE;QACZ,GAAG,EAAE,MAAM,EAAE,CAAC;KACf,CAAC;IACF;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,uDAAuD;IACvD,OAAO,EAAE;QACP,GAAG,EAAE,KAAK,EAAE,CAAC;KACd,CAAC;IACF,MAAM,EAAE,sBAAsB,CAAC;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE;QACZ,GAAG,EAAE,MAAM,EAAE,CAAC;KACf,CAAC;IACF;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE;QAClB,GAAG,EAAE,+BAA+B,EAAE,CAAC;KACxC,CAAC;IACF;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,kBAAkB,EAAE,gBAAgB,CAAC;CACtC;AAED;;GAEG;AACH,oBAAY,sBAAsB;IAChC,sDAAsD;IACtD,WAAW,gBAAgB;IAC3B,2DAA2D;IAC3D,QAAQ,aAAa;IACrB;;;OAGG;IACH,OAAO,YAAY;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,+BAA+B;IAC9C,sEAAsE;IACtE,cAAc,EAAE,MAAM,CAAC;IACvB,mFAAmF;IACnF,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,+DAA+D;IAC/D,SAAS,EAAE,KAAK,CAAC;IACjB,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,WAAW,EAAE,0BAA0B,CAAC;CACzC;AAED;;GAEG;AACH,oBAAY,0BAA0B;IACpC,oBAAoB,yBAAyB;IAC7C,OAAO,YAAY;IACnB,4BAA4B,iCAAiC;IAC7D,kCAAkC,uCAAuC;IACzE,sBAAsB,2BAA2B;IACjD,OAAO,YAAY;IACnB,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,aAAa,kBAAkB;IAC/B,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,GAAG,QAAQ;IACX,gBAAgB,qBAAqB;IACrC,IAAI,SAAS;IACb,IAAI,SAAS;CACd;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C,kBAAkB,EAAE,gBAAgB,CAAC;CACtC"}
|
|
@@ -23,9 +23,9 @@ var SyncMapperRecordTargetType;
|
|
|
23
23
|
(function (SyncMapperRecordTargetType) {
|
|
24
24
|
SyncMapperRecordTargetType["ACCESS_CONTROL_ENTRY"] = "access_control_entry";
|
|
25
25
|
SyncMapperRecordTargetType["ACCOUNT"] = "account";
|
|
26
|
-
SyncMapperRecordTargetType["
|
|
27
|
-
SyncMapperRecordTargetType["
|
|
28
|
-
SyncMapperRecordTargetType["
|
|
26
|
+
SyncMapperRecordTargetType["AIRDROP_AUTHORIZATION_POLICY"] = "airdrop_authorization_policy";
|
|
27
|
+
SyncMapperRecordTargetType["AIRDROP_FIELD_AUTHORIZATION_POLICY"] = "airdrop_field_authorization_policy";
|
|
28
|
+
SyncMapperRecordTargetType["AIRDROP_PLATFORM_GROUP"] = "airdrop_platform_group";
|
|
29
29
|
SyncMapperRecordTargetType["ARTICLE"] = "article";
|
|
30
30
|
SyncMapperRecordTargetType["ARTIFACT"] = "artifact";
|
|
31
31
|
SyncMapperRecordTargetType["CHAT"] = "chat";
|
package/dist/mappers/mappers.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Mappers = void 0;
|
|
4
|
-
const
|
|
4
|
+
const client_1 = require("../http/client");
|
|
5
5
|
/**
|
|
6
6
|
* Manages sync mapper records that link external system items to DevRev items.
|
|
7
7
|
*
|
|
@@ -18,16 +18,17 @@ class Mappers {
|
|
|
18
18
|
* Used to find the mapping when you know the DevRev ID and want to find the external system ID.
|
|
19
19
|
*
|
|
20
20
|
* @param params - Query parameters of type MappersGetByTargetIdParams
|
|
21
|
-
* @returns Promise
|
|
21
|
+
* @returns Promise resolving to the sync mapper record
|
|
22
22
|
*/
|
|
23
23
|
async getByTargetId(params) {
|
|
24
24
|
const { sync_unit, target } = params;
|
|
25
|
-
|
|
25
|
+
const response = await client_1.axiosClient.get(`${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-target`, {
|
|
26
26
|
headers: {
|
|
27
27
|
Authorization: this.devrevApiToken,
|
|
28
28
|
},
|
|
29
29
|
params: { sync_unit, target },
|
|
30
30
|
});
|
|
31
|
+
return response.data;
|
|
31
32
|
}
|
|
32
33
|
/**
|
|
33
34
|
* Retrieves a sync mapper record by external system ID.
|
|
@@ -35,16 +36,17 @@ class Mappers {
|
|
|
35
36
|
* Used to find the mapping when you know the external system ID and want to find the DevRev ID.
|
|
36
37
|
*
|
|
37
38
|
* @param params - Query parameters of type MappersGetByExternalIdParams
|
|
38
|
-
* @returns Promise
|
|
39
|
+
* @returns Promise resolving to the sync mapper record
|
|
39
40
|
*/
|
|
40
41
|
async getByExternalId(params) {
|
|
41
42
|
const { sync_unit, external_id, target_type } = params;
|
|
42
|
-
|
|
43
|
+
const response = await client_1.axiosClient.get(`${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-external-id`, {
|
|
43
44
|
headers: {
|
|
44
45
|
Authorization: this.devrevApiToken,
|
|
45
46
|
},
|
|
46
47
|
params: { sync_unit, external_id, target_type },
|
|
47
48
|
});
|
|
49
|
+
return response.data;
|
|
48
50
|
}
|
|
49
51
|
/**
|
|
50
52
|
* Creates a new sync mapper record to establish a relationship between external system
|
|
@@ -54,14 +56,15 @@ class Mappers {
|
|
|
54
56
|
* the mapping for future synchronization operations.
|
|
55
57
|
*
|
|
56
58
|
* @param params - Creation parameters of type MappersCreateParams
|
|
57
|
-
* @returns Promise
|
|
59
|
+
* @returns Promise resolving to the created sync mapper record
|
|
58
60
|
*/
|
|
59
61
|
async create(params) {
|
|
60
|
-
|
|
62
|
+
const response = await client_1.axiosClient.post(`${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.create`, params, {
|
|
61
63
|
headers: {
|
|
62
64
|
Authorization: this.devrevApiToken,
|
|
63
65
|
},
|
|
64
66
|
});
|
|
67
|
+
return response.data;
|
|
65
68
|
}
|
|
66
69
|
/**
|
|
67
70
|
* Updates an existing sync mapper record.
|
|
@@ -70,14 +73,15 @@ class Mappers {
|
|
|
70
73
|
* additional DevRev entities need to be associated.
|
|
71
74
|
*
|
|
72
75
|
* @param params - Update parameters of type MappersUpdateParams
|
|
73
|
-
* @returns Promise
|
|
76
|
+
* @returns Promise resolving to the updated sync mapper record
|
|
74
77
|
*/
|
|
75
78
|
async update(params) {
|
|
76
|
-
|
|
79
|
+
const response = await client_1.axiosClient.post(`${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.update`, params, {
|
|
77
80
|
headers: {
|
|
78
81
|
Authorization: this.devrevApiToken,
|
|
79
82
|
},
|
|
80
83
|
});
|
|
84
|
+
return response.data;
|
|
81
85
|
}
|
|
82
86
|
}
|
|
83
87
|
exports.Mappers = Mappers;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { BaseState } from '../../state/
|
|
2
|
-
import {
|
|
1
|
+
import { BaseState } from '../../state/state';
|
|
2
|
+
import { SdkState } from '../../state/state.interfaces';
|
|
3
3
|
import { AirSyncEvent, EventData, ExtractorEventType } from '../../types/extraction';
|
|
4
4
|
import { LoaderEventType } from '../../types/loading';
|
|
5
|
-
import { WorkerAdapterOptions } from '../../types/workers';
|
|
5
|
+
import { TaskResult, WorkerAdapterOptions } from '../../types/workers';
|
|
6
6
|
import { Uploader } from '../../uploader/uploader';
|
|
7
7
|
/**
|
|
8
8
|
* BaseAdapter holds the state and behavior shared by both sync modes and owns
|
|
@@ -11,22 +11,25 @@ import { Uploader } from '../../uploader/uploader';
|
|
|
11
11
|
* their own pre-emit work and event payload shaping.
|
|
12
12
|
*
|
|
13
13
|
* @typeParam ConnectorState - the connector-owned state shape
|
|
14
|
-
* @typeParam StateClass - the mode's `BaseState` subclass
|
|
15
14
|
*/
|
|
16
|
-
export declare abstract class BaseAdapter<ConnectorState
|
|
15
|
+
export declare abstract class BaseAdapter<ConnectorState> {
|
|
17
16
|
readonly event: AirSyncEvent;
|
|
18
17
|
readonly options?: WorkerAdapterOptions;
|
|
19
18
|
isTimeout: boolean;
|
|
20
19
|
hasWorkerEmitted: boolean;
|
|
21
|
-
protected adapterState:
|
|
20
|
+
protected adapterState: BaseState<ConnectorState>;
|
|
22
21
|
protected uploader: Uploader;
|
|
23
22
|
constructor({ event, adapterState, options, }: {
|
|
24
23
|
event: AirSyncEvent;
|
|
25
|
-
adapterState:
|
|
24
|
+
adapterState: BaseState<ConnectorState>;
|
|
26
25
|
options?: WorkerAdapterOptions;
|
|
27
26
|
});
|
|
27
|
+
/** Connector-owned state exposed to snap-in code. */
|
|
28
28
|
get state(): ConnectorState;
|
|
29
29
|
set state(value: ConnectorState);
|
|
30
|
+
/** SDK-internal bookkeeping state. Used by SDK internals; not for connector use. */
|
|
31
|
+
get sdkState(): SdkState;
|
|
32
|
+
get extractionScope(): import("../../types/workers").ExtractionScope;
|
|
30
33
|
postState(): Promise<void>;
|
|
31
34
|
/**
|
|
32
35
|
* Pre-emit hook run before any state is persisted or the event is sent.
|
|
@@ -45,12 +48,24 @@ export declare abstract class BaseAdapter<ConnectorState, StateClass extends Bas
|
|
|
45
48
|
* clears its accumulated artifacts here; loading has nothing to do.
|
|
46
49
|
*/
|
|
47
50
|
protected abstract afterEmit(newEventType: ExtractorEventType | LoaderEventType): void;
|
|
51
|
+
/**
|
|
52
|
+
* Maps a {@link TaskResult} returned by a worker's task/onTimeout callback to
|
|
53
|
+
* the phase-appropriate platform event and emits it exactly once.
|
|
54
|
+
*
|
|
55
|
+
* This is the SDK-internal bridge between the return-based connector contract
|
|
56
|
+
* and the control protocol; it is invoked by the worker driver, not by
|
|
57
|
+
* connectors. Connectors signal outcomes by returning a `TaskResult`, never by
|
|
58
|
+
* calling `emit` directly.
|
|
59
|
+
*
|
|
60
|
+
* @param result - The status the worker reported for the current phase.
|
|
61
|
+
*/
|
|
62
|
+
emitFromResult(result: TaskResult): Promise<void>;
|
|
48
63
|
/**
|
|
49
64
|
* Emits an event to the platform.
|
|
50
65
|
*
|
|
51
66
|
* @param newEventType - The event type to be emitted
|
|
52
67
|
* @param data - The data to be sent with the event
|
|
53
68
|
*/
|
|
54
|
-
emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
|
|
69
|
+
protected emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
|
|
55
70
|
}
|
|
56
71
|
//# sourceMappingURL=base-adapter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EACL,YAAY,EACZ,SAAS,EACT,kBAAkB,EACnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EACL,UAAU,EACV,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAGnD;;;;;;;GAOG;AACH,8BAAsB,WAAW,CAAC,cAAc;IAC9C,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IACnB,gBAAgB,EAAE,OAAO,CAAC;IAE1B,SAAS,CAAC,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC;gBAEjB,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE;QACD,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAYD,qDAAqD;IACrD,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,oFAAoF;IACpF,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,eAAe,kDAElB;IAEK,SAAS;IAMf;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,CAAC,UAAU,CAC3B,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,OAAO,CAAC,IAAI,CAAC;IAEhB;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CACjC,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAEZ;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,SAAS,CAC1B,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,IAAI;IAEP;;;;;;;;;;OAUG;IACG,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBvD;;;;;OAKG;cACa,IAAI,CAClB,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;CAkEjB"}
|
|
@@ -3,12 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.BaseAdapter = void 0;
|
|
4
4
|
const node_worker_threads_1 = require("node:worker_threads");
|
|
5
5
|
const constants_1 = require("../../common/constants");
|
|
6
|
-
const
|
|
7
|
-
const helpers_1 = require("../../common/helpers");
|
|
6
|
+
const emit_1 = require("../emit");
|
|
8
7
|
const logger_1 = require("../../logger/logger");
|
|
9
|
-
const logger_context_1 = require("../../logger/logger.context");
|
|
10
8
|
const workers_1 = require("../../types/workers");
|
|
11
9
|
const uploader_1 = require("../../uploader/uploader");
|
|
10
|
+
const spawn_helpers_1 = require("../spawn/spawn.helpers");
|
|
12
11
|
/**
|
|
13
12
|
* BaseAdapter holds the state and behavior shared by both sync modes and owns
|
|
14
13
|
* the `emit` control-protocol flow as a template method. Mode-specific adapters
|
|
@@ -16,7 +15,6 @@ const uploader_1 = require("../../uploader/uploader");
|
|
|
16
15
|
* their own pre-emit work and event payload shaping.
|
|
17
16
|
*
|
|
18
17
|
* @typeParam ConnectorState - the connector-owned state shape
|
|
19
|
-
* @typeParam StateClass - the mode's `BaseState` subclass
|
|
20
18
|
*/
|
|
21
19
|
class BaseAdapter {
|
|
22
20
|
constructor({ event, adapterState, options, }) {
|
|
@@ -30,17 +28,52 @@ class BaseAdapter {
|
|
|
30
28
|
options,
|
|
31
29
|
});
|
|
32
30
|
}
|
|
31
|
+
/** Connector-owned state exposed to snap-in code. */
|
|
33
32
|
get state() {
|
|
34
33
|
return this.adapterState.state;
|
|
35
34
|
}
|
|
36
35
|
set state(value) {
|
|
37
36
|
this.adapterState.state = value;
|
|
38
37
|
}
|
|
38
|
+
/** SDK-internal bookkeeping state. Used by SDK internals; not for connector use. */
|
|
39
|
+
get sdkState() {
|
|
40
|
+
return this.adapterState.sdkState;
|
|
41
|
+
}
|
|
42
|
+
get extractionScope() {
|
|
43
|
+
return this.adapterState.extractionScope;
|
|
44
|
+
}
|
|
39
45
|
async postState() {
|
|
40
|
-
return (0,
|
|
46
|
+
return (0, logger_1.runWithSdkLogContext)(async () => {
|
|
41
47
|
await this.adapterState.postState();
|
|
42
48
|
});
|
|
43
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* Maps a {@link TaskResult} returned by a worker's task/onTimeout callback to
|
|
52
|
+
* the phase-appropriate platform event and emits it exactly once.
|
|
53
|
+
*
|
|
54
|
+
* This is the SDK-internal bridge between the return-based connector contract
|
|
55
|
+
* and the control protocol; it is invoked by the worker driver, not by
|
|
56
|
+
* connectors. Connectors signal outcomes by returning a `TaskResult`, never by
|
|
57
|
+
* calling `emit` directly.
|
|
58
|
+
*
|
|
59
|
+
* @param result - The status the worker reported for the current phase.
|
|
60
|
+
*/
|
|
61
|
+
async emitFromResult(result) {
|
|
62
|
+
const { eventType, illegal } = (0, spawn_helpers_1.getEventTypeForResult)(this.event.payload.event_type, result.status);
|
|
63
|
+
const data = {};
|
|
64
|
+
if (result.status === 'delay') {
|
|
65
|
+
data.delay = result.delaySeconds;
|
|
66
|
+
}
|
|
67
|
+
else if (result.status === 'error') {
|
|
68
|
+
data.error = result.error;
|
|
69
|
+
}
|
|
70
|
+
else if (illegal) {
|
|
71
|
+
data.error = {
|
|
72
|
+
message: `Worker returned status '${result.status}' for a non-resumable phase (${this.event.payload.event_type}), which is not allowed. Emitting an error event instead.`,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
await this.emit(eventType, data);
|
|
76
|
+
}
|
|
44
77
|
/**
|
|
45
78
|
* Emits an event to the platform.
|
|
46
79
|
*
|
|
@@ -48,7 +81,7 @@ class BaseAdapter {
|
|
|
48
81
|
* @param data - The data to be sent with the event
|
|
49
82
|
*/
|
|
50
83
|
async emit(newEventType, data) {
|
|
51
|
-
return (0,
|
|
84
|
+
return (0, logger_1.runWithSdkLogContext)(async () => {
|
|
52
85
|
var _a;
|
|
53
86
|
if (this.hasWorkerEmitted) {
|
|
54
87
|
console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
|
|
@@ -79,9 +112,9 @@ class BaseAdapter {
|
|
|
79
112
|
try {
|
|
80
113
|
// Always prune error messages to make them shorter before emit
|
|
81
114
|
if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
|
|
82
|
-
data.error.message = (0,
|
|
115
|
+
data.error.message = (0, logger_1.truncateMessage)(data.error.message);
|
|
83
116
|
}
|
|
84
|
-
await (0,
|
|
117
|
+
await (0, emit_1.emit)({
|
|
85
118
|
eventType: newEventType,
|
|
86
119
|
event: this.event,
|
|
87
120
|
data: Object.assign(Object.assign({}, data), this.buildEmitPayload(newEventType)),
|
|
@@ -1,28 +1,27 @@
|
|
|
1
1
|
import { Repo } from '../../repo/repo';
|
|
2
2
|
import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
|
|
3
|
-
import {
|
|
4
|
-
import { AirSyncEvent, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, EventData, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../../types/extraction';
|
|
3
|
+
import { AirSyncEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType } from '../../types/extraction';
|
|
5
4
|
import { LoaderEventType } from '../../types/loading';
|
|
6
|
-
import {
|
|
5
|
+
import { BaseState } from '../../state/state';
|
|
6
|
+
import { TaskResult, WorkerAdapterOptions } from '../../types/workers';
|
|
7
7
|
import { Artifact } from '../../uploader/uploader.interfaces';
|
|
8
8
|
import { BaseAdapter } from './base-adapter';
|
|
9
9
|
/**
|
|
10
10
|
* ExtractionAdapter is the adapter passed to extraction tasks. It exposes the
|
|
11
|
-
* extraction surface (repos, artifacts, attachment streaming) and
|
|
12
|
-
* extraction boundaries before emitting.
|
|
11
|
+
* extraction surface (repos, artifacts, attachment streaming) and uploads
|
|
12
|
+
* pending repos and updates the extraction boundaries before emitting.
|
|
13
13
|
*
|
|
14
14
|
* @public
|
|
15
15
|
*/
|
|
16
|
-
export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<ConnectorState
|
|
16
|
+
export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
|
|
17
17
|
private _artifacts;
|
|
18
18
|
private repos;
|
|
19
19
|
private currentEventDataLength;
|
|
20
20
|
constructor(params: {
|
|
21
21
|
event: AirSyncEvent;
|
|
22
|
-
adapterState:
|
|
22
|
+
adapterState: BaseState<ConnectorState>;
|
|
23
23
|
options?: WorkerAdapterOptions;
|
|
24
24
|
});
|
|
25
|
-
get extractionScope(): import("../../types/workers").ExtractionScope;
|
|
26
25
|
/**
|
|
27
26
|
* Returns whether the given item type should be extracted.
|
|
28
27
|
* Defaults to true if the scope is empty or the item type is not listed.
|
|
@@ -53,6 +52,6 @@ export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<Conne
|
|
|
53
52
|
stream: ExternalSystemAttachmentStreamingFunction;
|
|
54
53
|
processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
|
|
55
54
|
batchSize?: number;
|
|
56
|
-
}): Promise<
|
|
55
|
+
}): Promise<TaskResult>;
|
|
57
56
|
}
|
|
58
57
|
//# sourceMappingURL=extraction-adapter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAElB,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C;;;;;;GAMG;AACH,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,sBAAsB,CAAa;gBAE/B,MAAM,EAAE;QAClB,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAKD;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAa3C,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;cAEe,UAAU,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,OAAO,CAAC,IAAI,CAAC;IA2ChB,SAAS,CAAC,gBAAgB,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAOZ,SAAS,CAAC,SAAS,IAAI,IAAI;IAIrB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA+GvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAczB;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAyIxB"}
|
|
@@ -4,14 +4,13 @@ exports.ExtractionAdapter = void 0;
|
|
|
4
4
|
const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
|
|
5
5
|
const constants_1 = require("../../common/constants");
|
|
6
6
|
const logger_1 = require("../../logger/logger");
|
|
7
|
-
const logger_context_1 = require("../../logger/logger.context");
|
|
8
7
|
const repo_1 = require("../../repo/repo");
|
|
9
8
|
const extraction_1 = require("../../types/extraction");
|
|
10
9
|
const base_adapter_1 = require("./base-adapter");
|
|
11
10
|
/**
|
|
12
11
|
* ExtractionAdapter is the adapter passed to extraction tasks. It exposes the
|
|
13
|
-
* extraction surface (repos, artifacts, attachment streaming) and
|
|
14
|
-
* extraction boundaries before emitting.
|
|
12
|
+
* extraction surface (repos, artifacts, attachment streaming) and uploads
|
|
13
|
+
* pending repos and updates the extraction boundaries before emitting.
|
|
15
14
|
*
|
|
16
15
|
* @public
|
|
17
16
|
*/
|
|
@@ -22,9 +21,6 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
22
21
|
this.currentEventDataLength = 0;
|
|
23
22
|
this._artifacts = [];
|
|
24
23
|
}
|
|
25
|
-
get extractionScope() {
|
|
26
|
-
return this.adapterState.extractionScope;
|
|
27
|
-
}
|
|
28
24
|
/**
|
|
29
25
|
* Returns whether the given item type should be extracted.
|
|
30
26
|
* Defaults to true if the scope is empty or the item type is not listed.
|
|
@@ -45,7 +41,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
45
41
|
var _a;
|
|
46
42
|
// We need to store artifacts ids in state for later use when streaming attachments
|
|
47
43
|
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
48
|
-
(_a = this.
|
|
44
|
+
(_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
49
45
|
}
|
|
50
46
|
// Calculate size of the entire artifact object that goes in the SQS message
|
|
51
47
|
this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
|
|
@@ -57,7 +53,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
57
53
|
});
|
|
58
54
|
}
|
|
59
55
|
getRepo(itemType) {
|
|
60
|
-
return (0,
|
|
56
|
+
return (0, logger_1.runWithSdkLogContext)(() => {
|
|
61
57
|
const repo = this.repos.find((repo) => repo.itemType === itemType);
|
|
62
58
|
if (!repo) {
|
|
63
59
|
console.error(`Repo for item type ${itemType} not found.`);
|
|
@@ -78,8 +74,10 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
78
74
|
// Upload all repos before emitting the event
|
|
79
75
|
console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
|
|
80
76
|
await this.uploadAllRepos();
|
|
77
|
+
// When the full extraction cycle completes, commit the extraction window
|
|
78
|
+
// boundaries so subsequent incremental syncs can resume from them.
|
|
81
79
|
if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
|
|
82
|
-
const sdkState = this.
|
|
80
|
+
const sdkState = this.sdkState;
|
|
83
81
|
// Clear pending extraction boundaries now that the cycle is complete
|
|
84
82
|
sdkState.pendingWorkersOldest = '';
|
|
85
83
|
sdkState.pendingWorkersNewest = '';
|
|
@@ -116,9 +114,9 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
116
114
|
}
|
|
117
115
|
}
|
|
118
116
|
async processAttachment(attachment, stream) {
|
|
119
|
-
return (0,
|
|
117
|
+
return (0, logger_1.runWithSdkLogContext)(async () => {
|
|
120
118
|
var _a, _b, _c;
|
|
121
|
-
const { httpStream, delay, error } = await (0,
|
|
119
|
+
const { httpStream, delay, error } = await (0, logger_1.runWithUserLogContext)(async () => stream({
|
|
122
120
|
item: attachment,
|
|
123
121
|
event: this.event,
|
|
124
122
|
}));
|
|
@@ -228,7 +226,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
228
226
|
*/
|
|
229
227
|
async streamAttachments({ stream, processors, batchSize = 1, // By default, we want to stream one attachment at a time
|
|
230
228
|
}) {
|
|
231
|
-
return (0,
|
|
229
|
+
return (0, logger_1.runWithSdkLogContext)(async () => {
|
|
232
230
|
var _a, _b;
|
|
233
231
|
if (batchSize <= 0) {
|
|
234
232
|
console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
|
|
@@ -244,11 +242,11 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
244
242
|
},
|
|
245
243
|
];
|
|
246
244
|
this.initializeRepos(repos);
|
|
247
|
-
const attachmentsMetadata = (_a = this.
|
|
245
|
+
const attachmentsMetadata = (_a = this.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
|
|
248
246
|
// If there are no attachments metadata artifact IDs in state, finish here
|
|
249
247
|
if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
|
|
250
248
|
console.log(`No attachments metadata artifact IDs found in state.`);
|
|
251
|
-
return;
|
|
249
|
+
return { status: 'success' };
|
|
252
250
|
}
|
|
253
251
|
else {
|
|
254
252
|
console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
|
|
@@ -262,7 +260,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
262
260
|
});
|
|
263
261
|
if (error) {
|
|
264
262
|
console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
265
|
-
return { error };
|
|
263
|
+
return { status: 'error', error };
|
|
266
264
|
}
|
|
267
265
|
if (!attachments || attachments.length === 0) {
|
|
268
266
|
console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
@@ -277,12 +275,12 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
277
275
|
console.log(`Using custom processors for attachments.`);
|
|
278
276
|
const reducer = processors.reducer;
|
|
279
277
|
const iterator = processors.iterator;
|
|
280
|
-
const reducedAttachments = (0,
|
|
278
|
+
const reducedAttachments = (0, logger_1.runWithUserLogContext)(() => reducer({
|
|
281
279
|
attachments,
|
|
282
280
|
adapter: this,
|
|
283
281
|
batchSize,
|
|
284
282
|
}));
|
|
285
|
-
response = await (0,
|
|
283
|
+
response = await (0, logger_1.runWithUserLogContext)(async () => {
|
|
286
284
|
return await iterator({
|
|
287
285
|
reducedAttachments,
|
|
288
286
|
adapter: this,
|
|
@@ -295,21 +293,21 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
295
293
|
const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
|
|
296
294
|
adapter: this,
|
|
297
295
|
attachments,
|
|
298
|
-
attachmentsMetadata,
|
|
299
296
|
batchSize,
|
|
300
297
|
stream,
|
|
301
298
|
});
|
|
302
299
|
response = await attachmentsPool.streamAll();
|
|
303
300
|
}
|
|
304
|
-
if (
|
|
305
|
-
return response;
|
|
301
|
+
if (response === null || response === void 0 ? void 0 : response.error) {
|
|
302
|
+
return { status: 'error', error: response.error };
|
|
303
|
+
}
|
|
304
|
+
if (response === null || response === void 0 ? void 0 : response.delay) {
|
|
305
|
+
return { status: 'delay', delaySeconds: response.delay };
|
|
306
306
|
}
|
|
307
|
-
// On timeout,
|
|
307
|
+
// On timeout, return progress to allow continuation in a fresh invocation.
|
|
308
308
|
if (this.isTimeout) {
|
|
309
|
-
console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}.
|
|
310
|
-
|
|
311
|
-
process.exit(0);
|
|
312
|
-
return;
|
|
309
|
+
console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
|
|
310
|
+
return { status: 'progress' };
|
|
313
311
|
}
|
|
314
312
|
console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
|
|
315
313
|
attachmentsMetadata.artifactIds.shift();
|
|
@@ -318,7 +316,7 @@ class ExtractionAdapter extends base_adapter_1.BaseAdapter {
|
|
|
318
316
|
attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
|
|
319
317
|
}
|
|
320
318
|
}
|
|
321
|
-
return;
|
|
319
|
+
return { status: 'success' };
|
|
322
320
|
});
|
|
323
321
|
}
|
|
324
322
|
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Mappers } from '../../mappers/mappers';
|
|
2
|
-
import {
|
|
2
|
+
import { BaseState } from '../../state/state';
|
|
3
3
|
import { AirSyncEvent, EventData, ExtractorEventType } from '../../types/extraction';
|
|
4
|
-
import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse
|
|
5
|
-
import { WorkerAdapterOptions } from '../../types/workers';
|
|
4
|
+
import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse } from '../../types/loading';
|
|
5
|
+
import { TaskResult, WorkerAdapterOptions } from '../../types/workers';
|
|
6
6
|
import { BaseAdapter } from './base-adapter';
|
|
7
7
|
/**
|
|
8
8
|
* LoadingAdapter is the adapter passed to loading tasks. It exposes the loading
|
|
@@ -10,13 +10,13 @@ import { BaseAdapter } from './base-adapter';
|
|
|
10
10
|
*
|
|
11
11
|
* @public
|
|
12
12
|
*/
|
|
13
|
-
export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState
|
|
13
|
+
export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
|
|
14
14
|
private loaderReports;
|
|
15
15
|
private _processedFiles;
|
|
16
16
|
private _mappers;
|
|
17
17
|
constructor(params: {
|
|
18
18
|
event: AirSyncEvent;
|
|
19
|
-
adapterState:
|
|
19
|
+
adapterState: BaseState<ConnectorState>;
|
|
20
20
|
options?: WorkerAdapterOptions;
|
|
21
21
|
});
|
|
22
22
|
get reports(): LoaderReport[];
|
|
@@ -25,13 +25,13 @@ export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<Connecto
|
|
|
25
25
|
protected beforeEmit(): Promise<void>;
|
|
26
26
|
protected buildEmitPayload(newEventType: ExtractorEventType | LoaderEventType): EventData;
|
|
27
27
|
protected afterEmit(): void;
|
|
28
|
-
loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<
|
|
28
|
+
loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<TaskResult>;
|
|
29
29
|
getLoaderBatches({ supportedItemTypes, }: {
|
|
30
30
|
supportedItemTypes: string[];
|
|
31
31
|
}): Promise<FileToLoad[]>;
|
|
32
32
|
loadAttachments({ create, }: {
|
|
33
33
|
create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
|
|
34
|
-
}): Promise<
|
|
34
|
+
}): Promise<TaskResult>;
|
|
35
35
|
loadItem({ item, itemTypeToLoad, }: {
|
|
36
36
|
item: ExternalSystemItem;
|
|
37
37
|
itemTypeToLoad: ItemTypeToLoad;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EACnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAEjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C;;;;;GAKG;AACH,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;gBAEd,MAAM,EAAE;QAClB,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAUD,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;cAEe,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3C,SAAS,CAAC,gBAAgB,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAYZ,SAAS,CAAC,SAAS,IAAI,IAAI;IAIrB,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IAsHxC,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IA4FjB,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAsMvB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsD9B"}
|