@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +1028 -0
- package/README.md +5 -11
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -9
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +54 -56
- package/dist/common/constants.d.ts +4 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +6 -7
- package/dist/common/helpers.d.ts +3 -35
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +5 -50
- package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
- package/dist/http/client.d.ts.map +1 -0
- package/dist/http/{axios-client-internal.js → client.js} +4 -8
- package/dist/index.d.ts +24 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +43 -32
- package/dist/logger/logger.d.ts +12 -53
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +0 -1
- package/dist/logger/logger.interfaces.d.ts.map +1 -1
- package/dist/logger/logger.js +36 -68
- package/dist/mappers/mappers.d.ts +7 -46
- package/dist/mappers/mappers.d.ts.map +1 -1
- package/dist/mappers/mappers.interfaces.d.ts +175 -0
- package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
- package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +4 -13
- package/dist/mappers/mappers.js +11 -45
- package/dist/multithreading/adapters/base-adapter.d.ts +30 -32
- package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/base-adapter.js +97 -62
- package/dist/multithreading/adapters/extraction-adapter.d.ts +16 -29
- package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/extraction-adapter.js +203 -206
- package/dist/multithreading/adapters/loading-adapter.d.ts +8 -13
- package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
- package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
- package/dist/multithreading/adapters/loading-adapter.js +294 -346
- package/dist/multithreading/emit.d.ts +11 -0
- package/dist/multithreading/emit.d.ts.map +1 -0
- package/dist/{common/control-protocol.js → multithreading/emit.js} +5 -7
- package/dist/multithreading/process-task.d.ts +2 -12
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +46 -43
- package/dist/multithreading/spawn/spawn.d.ts +3 -9
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +128 -20
- package/dist/multithreading/spawn/spawn.js +24 -39
- package/dist/multithreading/worker.js +1 -4
- package/dist/repo/repo.d.ts +10 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.helpers.d.ts +6 -0
- package/dist/repo/repo.helpers.d.ts.map +1 -0
- package/dist/repo/repo.helpers.js +16 -0
- package/dist/repo/repo.interfaces.d.ts +4 -16
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +23 -8
- package/dist/state/base-state.d.ts +16 -38
- package/dist/state/base-state.d.ts.map +1 -1
- package/dist/state/base-state.js +19 -41
- package/dist/state/extraction-state.d.ts +6 -21
- package/dist/state/extraction-state.d.ts.map +1 -1
- package/dist/state/extraction-state.js +8 -25
- package/dist/{common → state}/install-initial-domain-mapping.d.ts +1 -1
- package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
- package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
- package/dist/state/loading-state.d.ts +3 -14
- package/dist/state/loading-state.d.ts.map +1 -1
- package/dist/state/loading-state.js +3 -14
- package/dist/state/state.d.ts +4 -0
- package/dist/state/state.d.ts.map +1 -0
- package/dist/state/state.interfaces.d.ts +33 -37
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +16 -11
- package/dist/state/state.js +11 -0
- package/dist/state/time-value-resolver.d.ts +17 -0
- package/dist/state/time-value-resolver.d.ts.map +1 -0
- package/dist/{common → state}/time-value-resolver.js +9 -41
- package/dist/testing/mock-event.d.ts +10 -0
- package/dist/testing/mock-event.d.ts.map +1 -0
- package/dist/{common/test-utils.js → testing/mock-event.js} +17 -14
- package/dist/testing/mock-server.d.ts +29 -0
- package/dist/testing/mock-server.d.ts.map +1 -0
- package/dist/testing/mock-server.interfaces.d.ts +48 -0
- package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
- package/dist/{mock-server → testing}/mock-server.js +21 -33
- package/dist/types/common.d.ts +1 -12
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -6
- package/dist/types/errors.d.ts +9 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/external-domain-metadata.d.ts +30 -171
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.d.ts +64 -94
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +6 -34
- package/dist/types/loading.d.ts +1 -2
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +0 -1
- package/dist/types/workers.d.ts +39 -94
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +0 -6
- package/dist/uploader/uploader.d.ts +8 -59
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +4 -27
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +64 -30
- package/dist/uploader/uploader.interfaces.d.ts +15 -25
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +8 -0
- package/dist/uploader/uploader.js +23 -89
- package/package.json +8 -6
- package/dist/airsync-sdk.d.ts +0 -2059
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
- package/dist/common/control-protocol.d.ts +0 -10
- package/dist/common/control-protocol.d.ts.map +0 -1
- package/dist/common/errors.d.ts +0 -6
- package/dist/common/errors.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
- package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
- package/dist/common/install-initial-domain-mapping.test.js +0 -211
- package/dist/common/test-utils.d.ts +0 -19
- package/dist/common/test-utils.d.ts.map +0 -1
- package/dist/common/time-value-resolver.d.ts +0 -47
- package/dist/common/time-value-resolver.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.d.ts +0 -2
- package/dist/common/time-value-resolver.test.d.ts.map +0 -1
- package/dist/common/time-value-resolver.test.js +0 -268
- package/dist/http/axios-client-internal.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.d.ts +0 -2
- package/dist/http/axios-client-internal.test.d.ts.map +0 -1
- package/dist/http/axios-client-internal.test.js +0 -175
- package/dist/http/axios-client.d.ts +0 -27
- package/dist/http/axios-client.d.ts.map +0 -1
- package/dist/http/axios-client.js +0 -60
- package/dist/http/constants.d.ts +0 -4
- package/dist/http/constants.d.ts.map +0 -1
- package/dist/http/constants.js +0 -6
- package/dist/http/index.d.ts +0 -2
- package/dist/http/index.d.ts.map +0 -1
- package/dist/http/index.js +0 -17
- package/dist/logger/logger.constants.d.ts +0 -6
- package/dist/logger/logger.constants.d.ts.map +0 -1
- package/dist/logger/logger.constants.js +0 -13
- package/dist/logger/logger.context.d.ts +0 -58
- package/dist/logger/logger.context.d.ts.map +0 -1
- package/dist/logger/logger.context.js +0 -86
- package/dist/logger/logger.test.d.ts +0 -2
- package/dist/logger/logger.test.d.ts.map +0 -1
- package/dist/logger/logger.test.js +0 -537
- package/dist/mappers/mappers.interface.d.ts +0 -294
- package/dist/mappers/mappers.interface.d.ts.map +0 -1
- package/dist/mappers/mappers.test.d.ts +0 -2
- package/dist/mappers/mappers.test.d.ts.map +0 -1
- package/dist/mappers/mappers.test.js +0 -105
- package/dist/mock-server/mock-server.d.ts +0 -47
- package/dist/mock-server/mock-server.d.ts.map +0 -1
- package/dist/mock-server/mock-server.interfaces.d.ts +0 -73
- package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
- package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
- package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
- package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
- package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
- package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
- package/dist/multithreading/adapters/serialization.test.js +0 -71
- package/dist/multithreading/create-worker.test.d.ts +0 -2
- package/dist/multithreading/create-worker.test.d.ts.map +0 -1
- package/dist/multithreading/create-worker.test.js +0 -108
- package/dist/multithreading/process-task.test.d.ts +0 -2
- package/dist/multithreading/process-task.test.d.ts.map +0 -1
- package/dist/multithreading/process-task.test.js +0 -146
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
- package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
- package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
- package/dist/multithreading/spawn/spawn.test.js +0 -232
- package/dist/repo/repo.test.d.ts +0 -2
- package/dist/repo/repo.test.d.ts.map +0 -1
- package/dist/repo/repo.test.js +0 -184
- package/dist/state/state.extract-window.test.d.ts +0 -2
- package/dist/state/state.extract-window.test.d.ts.map +0 -1
- package/dist/state/state.extract-window.test.js +0 -163
- package/dist/state/state.pending-boundaries.test.d.ts +0 -2
- package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
- package/dist/state/state.pending-boundaries.test.js +0 -189
- package/dist/state/state.post-state.test.d.ts +0 -2
- package/dist/state/state.post-state.test.d.ts.map +0 -1
- package/dist/state/state.post-state.test.js +0 -78
- package/dist/state/state.test.d.ts +0 -2
- package/dist/state/state.test.d.ts.map +0 -1
- package/dist/state/state.test.js +0 -271
- package/dist/state/state.time-value-resolution.test.d.ts +0 -2
- package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
- package/dist/state/state.time-value-resolution.test.js +0 -175
- package/dist/tsdoc-metadata.json +0 -11
- package/dist/types/extraction.test.d.ts +0 -2
- package/dist/types/extraction.test.d.ts.map +0 -1
- package/dist/types/extraction.test.js +0 -96
- package/dist/types/index.d.ts +0 -10
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -21
- package/dist/uploader/uploader.helpers.test.d.ts +0 -2
- package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
- package/dist/uploader/uploader.helpers.test.js +0 -256
- package/dist/uploader/uploader.test.d.ts +0 -2
- package/dist/uploader/uploader.test.d.ts.map +0 -1
- package/dist/uploader/uploader.test.js +0 -601
- /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
- /package/dist/{common → types}/errors.js +0 -0
package/dist/repo/repo.d.ts
CHANGED
|
@@ -10,6 +10,16 @@ export declare class Repo {
|
|
|
10
10
|
private onUpload;
|
|
11
11
|
private options?;
|
|
12
12
|
uploadedArtifacts: Artifact[];
|
|
13
|
+
dateRanges: {
|
|
14
|
+
creationDate: {
|
|
15
|
+
oldest?: number;
|
|
16
|
+
newest?: number;
|
|
17
|
+
};
|
|
18
|
+
modifiedDate: {
|
|
19
|
+
oldest?: number;
|
|
20
|
+
newest?: number;
|
|
21
|
+
};
|
|
22
|
+
};
|
|
13
23
|
constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
|
|
14
24
|
getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
|
|
15
25
|
upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
|
package/dist/repo/repo.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAG3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAsDxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAkC5C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"repo.helpers.d.ts","sourceRoot":"","sources":["../../src/repo/repo.helpers.ts"],"names":[],"mappings":"AAAA,wBAAgB,WAAW,CACzB,KAAK,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,EAC3C,EAAE,EAAE,MAAM,GACT,IAAI,CAON;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGlE"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.updateRange = updateRange;
|
|
4
|
+
exports.toValidTimestamp = toValidTimestamp;
|
|
5
|
+
function updateRange(range, ms) {
|
|
6
|
+
if (range.oldest === undefined || ms < range.oldest) {
|
|
7
|
+
range.oldest = ms;
|
|
8
|
+
}
|
|
9
|
+
if (range.newest === undefined || ms > range.newest) {
|
|
10
|
+
range.newest = ms;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function toValidTimestamp(value) {
|
|
14
|
+
const ms = new Date(value).getTime();
|
|
15
|
+
return Number.isFinite(ms) ? ms : undefined;
|
|
16
|
+
}
|
|
@@ -1,17 +1,12 @@
|
|
|
1
|
-
import { Artifact } from '../uploader/uploader.interfaces';
|
|
2
1
|
import { AirSyncEvent } from '../types/extraction';
|
|
3
2
|
import { WorkerAdapterOptions } from '../types/workers';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
*/
|
|
3
|
+
import { Artifact } from '../uploader/uploader.interfaces';
|
|
4
|
+
/** Stores and uploads extracted data. */
|
|
7
5
|
export interface RepoInterface {
|
|
8
6
|
itemType: string;
|
|
9
7
|
normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
|
|
10
8
|
overridenOptions?: WorkerAdapterOptions;
|
|
11
9
|
}
|
|
12
|
-
/**
|
|
13
|
-
* RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo.
|
|
14
|
-
*/
|
|
15
10
|
export interface RepoFactoryInterface {
|
|
16
11
|
event: AirSyncEvent;
|
|
17
12
|
itemType: string;
|
|
@@ -19,18 +14,12 @@ export interface RepoFactoryInterface {
|
|
|
19
14
|
onUpload: (artifact: Artifact) => void;
|
|
20
15
|
options?: WorkerAdapterOptions;
|
|
21
16
|
}
|
|
22
|
-
/**
|
|
23
|
-
* NormalizedItem is an interface of item after normalization.
|
|
24
|
-
*/
|
|
25
17
|
export interface NormalizedItem {
|
|
26
18
|
id: string;
|
|
27
19
|
created_date: string;
|
|
28
20
|
modified_date: string;
|
|
29
21
|
data: object;
|
|
30
22
|
}
|
|
31
|
-
/**
|
|
32
|
-
* NormalizedAttachment is an interface of attachment after normalization.
|
|
33
|
-
*/
|
|
34
23
|
export interface NormalizedAttachment {
|
|
35
24
|
url: string;
|
|
36
25
|
id: string;
|
|
@@ -39,10 +28,9 @@ export interface NormalizedAttachment {
|
|
|
39
28
|
author_id?: string;
|
|
40
29
|
inline?: boolean;
|
|
41
30
|
content_type?: string;
|
|
31
|
+
created_date?: string;
|
|
32
|
+
modified_date?: string;
|
|
42
33
|
grand_parent_id?: number | string;
|
|
43
34
|
}
|
|
44
|
-
/**
|
|
45
|
-
* Item is an interface that defines the structure of an item.
|
|
46
|
-
*/
|
|
47
35
|
export type Item = Record<string, any>;
|
|
48
36
|
//# sourceMappingURL=repo.interfaces.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,yCAAyC;AACzC,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAGD,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
|
package/dist/repo/repo.js
CHANGED
|
@@ -3,9 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Repo = void 0;
|
|
4
4
|
const constants_1 = require("../common/constants");
|
|
5
5
|
const uploader_1 = require("../uploader/uploader");
|
|
6
|
-
const
|
|
6
|
+
const repo_helpers_1 = require("./repo.helpers");
|
|
7
7
|
class Repo {
|
|
8
8
|
constructor({ event, itemType, normalize, onUpload, options, }) {
|
|
9
|
+
this.dateRanges = {
|
|
10
|
+
creationDate: {},
|
|
11
|
+
modifiedDate: {},
|
|
12
|
+
};
|
|
9
13
|
this.items = [];
|
|
10
14
|
this.itemType = itemType;
|
|
11
15
|
this.normalize = normalize;
|
|
@@ -20,6 +24,22 @@ class Repo {
|
|
|
20
24
|
async upload(batch) {
|
|
21
25
|
const itemsToUpload = batch || this.items;
|
|
22
26
|
if (itemsToUpload.length > 0) {
|
|
27
|
+
for (const item of itemsToUpload) {
|
|
28
|
+
const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
|
|
29
|
+
if (createdDate != null) {
|
|
30
|
+
const createdMs = (0, repo_helpers_1.toValidTimestamp)(createdDate);
|
|
31
|
+
if (createdMs !== undefined) {
|
|
32
|
+
(0, repo_helpers_1.updateRange)(this.dateRanges.creationDate, createdMs);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
const modifiedDate = item === null || item === void 0 ? void 0 : item.modified_date;
|
|
36
|
+
if (modifiedDate != null && modifiedDate !== '') {
|
|
37
|
+
const modifiedMs = (0, repo_helpers_1.toValidTimestamp)(modifiedDate);
|
|
38
|
+
if (modifiedMs !== undefined) {
|
|
39
|
+
(0, repo_helpers_1.updateRange)(this.dateRanges.modifiedDate, modifiedMs);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
23
43
|
console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
|
|
24
44
|
const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
|
|
25
45
|
if (error || !artifact) {
|
|
@@ -28,7 +48,7 @@ class Repo {
|
|
|
28
48
|
}
|
|
29
49
|
this.onUpload(artifact);
|
|
30
50
|
this.uploadedArtifacts.push(artifact);
|
|
31
|
-
//
|
|
51
|
+
// An explicit batch was already spliced out of this.items by the caller
|
|
32
52
|
if (!batch) {
|
|
33
53
|
this.items = [];
|
|
34
54
|
}
|
|
@@ -45,24 +65,19 @@ class Repo {
|
|
|
45
65
|
console.log(`No items to push for type ${this.itemType}. Skipping push.`);
|
|
46
66
|
return true;
|
|
47
67
|
}
|
|
48
|
-
// Normalize items if needed
|
|
49
68
|
if (this.normalize &&
|
|
50
69
|
this.itemType != constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
|
|
51
70
|
this.itemType != constants_1.SSOR_ATTACHMENT) {
|
|
52
|
-
recordsToPush =
|
|
71
|
+
recordsToPush = items.map((item) => this.normalize(item));
|
|
53
72
|
}
|
|
54
73
|
else {
|
|
55
74
|
recordsToPush = items;
|
|
56
75
|
}
|
|
57
|
-
// Add the new records to the items array
|
|
58
76
|
this.items.push(...recordsToPush);
|
|
59
|
-
// Upload in batches while the number of items exceeds the batch size
|
|
60
77
|
const batchSize = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.batchSize) || constants_1.ARTIFACT_BATCH_SIZE;
|
|
61
78
|
while (this.items.length >= batchSize) {
|
|
62
|
-
// Slice out a batch of batchSize items to upload
|
|
63
79
|
const batch = this.items.splice(0, batchSize);
|
|
64
80
|
try {
|
|
65
|
-
// Upload the batch
|
|
66
81
|
await this.upload(batch);
|
|
67
82
|
}
|
|
68
83
|
catch (error) {
|
|
@@ -1,19 +1,13 @@
|
|
|
1
1
|
import { InitialDomainMapping } from '../types/common';
|
|
2
2
|
import { AirSyncEvent } from '../types/extraction';
|
|
3
3
|
import { ExtractionScope } from '../types/workers';
|
|
4
|
-
import {
|
|
4
|
+
import { SdkState, StateInterface } from './state.interfaces';
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* Mode-specific subclasses (`ExtractionState`, `LoadingState`) narrow `sdkState`
|
|
11
|
-
* to their own SDK state shape and add mode-specific setup in their factories.
|
|
12
|
-
*
|
|
13
|
-
* @typeParam ConnectorState - the connector-owned state shape
|
|
14
|
-
* @typeParam SdkState - the SDK bookkeeping shape for this mode
|
|
6
|
+
* State lifecycle shared by all sync modes: connector/SDK state separation,
|
|
7
|
+
* fetch/init/post, the v1->v2 migration shim, and the snap-in-version-gated
|
|
8
|
+
* initial domain mapping install. Subclasses seed the SDK-owned state.
|
|
15
9
|
*/
|
|
16
|
-
export declare abstract class BaseState<ConnectorState
|
|
10
|
+
export declare abstract class BaseState<ConnectorState> {
|
|
17
11
|
protected _connectorState: ConnectorState;
|
|
18
12
|
protected _sdkState: SdkState;
|
|
19
13
|
protected _extractionScope: ExtractionScope;
|
|
@@ -24,7 +18,7 @@ export declare abstract class BaseState<ConnectorState, SdkState extends BaseSdk
|
|
|
24
18
|
private syncUnitId;
|
|
25
19
|
private requestId;
|
|
26
20
|
constructor({ event, initialState }: StateInterface<ConnectorState>, initialSdkState: SdkState);
|
|
27
|
-
/** Connector-owned state
|
|
21
|
+
/** Connector-owned state; what `adapter.state` exposes to snap-in code. */
|
|
28
22
|
get state(): ConnectorState;
|
|
29
23
|
set state(value: ConnectorState);
|
|
30
24
|
/** SDK-internal bookkeeping state. Never exposed to connector code. */
|
|
@@ -32,41 +26,25 @@ export declare abstract class BaseState<ConnectorState, SdkState extends BaseSdk
|
|
|
32
26
|
set sdkState(value: SdkState);
|
|
33
27
|
get extractionScope(): ExtractionScope;
|
|
34
28
|
/**
|
|
35
|
-
* Installs the initial domain mapping when the snap-in version in state
|
|
36
|
-
*
|
|
37
|
-
*
|
|
29
|
+
* Installs the initial domain mapping when the snap-in version in state
|
|
30
|
+
* differs from the event context. Shared by all modes so a loading run still
|
|
31
|
+
* installs the mapping if extraction has not done so.
|
|
38
32
|
*/
|
|
39
33
|
installInitialDomainMappingIfNeeded(initialDomainMapping?: InitialDomainMapping): Promise<void>;
|
|
40
34
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat
|
|
45
|
-
* v1 blob (connector keys merged with SDK keys), migrating the latter on read.
|
|
46
|
-
* Always persists the v2 envelope going forward.
|
|
47
|
-
* @param initialState The initial connector state provided by the spawn function
|
|
35
|
+
* Fetches state from API, or creates the initial state on 404. Reads both
|
|
36
|
+
* the v2 envelope and a legacy flat v1 blob (migrated on read); always
|
|
37
|
+
* persists the v2 envelope going forward.
|
|
48
38
|
*/
|
|
49
39
|
init(initialState: ConnectorState): Promise<void>;
|
|
50
40
|
/**
|
|
51
|
-
* Normalizes
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* - v2 envelope (`{ connectorState, sdkState }`): used as-is.
|
|
55
|
-
* - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`,
|
|
56
|
-
* everything else becomes connector state.
|
|
57
|
-
* - Malformed envelope (one side present, the other missing) fails loud.
|
|
41
|
+
* Normalizes fetched state into the `{ connectorState, sdkState }` envelope.
|
|
42
|
+
* A flat v1 blob is split by `V1_SDK_STATE_KEYS`; an envelope with only one
|
|
43
|
+
* side present fails loud.
|
|
58
44
|
*/
|
|
59
45
|
private normalizeFetchedState;
|
|
60
|
-
/**
|
|
61
|
-
* Updates the state of the adapter by posting to API.
|
|
62
|
-
* Persists the v2 `{ connectorState, sdkState }` envelope.
|
|
63
|
-
* @param {object} state - The connector state to be updated
|
|
64
|
-
*/
|
|
46
|
+
/** Posts the v2 `{ connectorState, sdkState }` envelope to the API. */
|
|
65
47
|
postState(state?: ConnectorState): Promise<void>;
|
|
66
|
-
/**
|
|
67
|
-
* Fetches the state of the adapter from API.
|
|
68
|
-
* @return The raw state data from API
|
|
69
|
-
*/
|
|
70
48
|
fetchState(): Promise<{
|
|
71
49
|
state: string;
|
|
72
50
|
objects?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"base-state.d.ts","sourceRoot":"","sources":["../../src/state/base-state.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAGnD,OAAO,EAEL,QAAQ,EACR,cAAc,EAEf,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,8BAAsB,SAAS,CAAC,cAAc;IAC5C,SAAS,CAAC,eAAe,EAAE,cAAc,CAAC;IAC1C,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC9B,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAM;IACjD,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IACvC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAGxB,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC,EACvD,eAAe,EAAE,QAAQ;IAY3B,2EAA2E;IAC3E,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,uEAAuE;IACvE,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,mCAAmC,CACvC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC;IAwChB;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDvD;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAyC7B,uEAAuE;IACjE,SAAS,CAAC,KAAK,CAAC,EAAE,cAAc;IA4DhC,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}
|
package/dist/state/base-state.js
CHANGED
|
@@ -4,23 +4,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.BaseState = void 0;
|
|
7
|
-
const axios_1 = __importDefault(require("axios"));
|
|
8
7
|
const node_worker_threads_1 = require("node:worker_threads");
|
|
9
|
-
const
|
|
10
|
-
const
|
|
8
|
+
const axios_1 = __importDefault(require("axios"));
|
|
9
|
+
const client_1 = require("../http/client");
|
|
11
10
|
const logger_1 = require("../logger/logger");
|
|
12
11
|
const workers_1 = require("../types/workers");
|
|
12
|
+
const install_initial_domain_mapping_1 = require("./install-initial-domain-mapping");
|
|
13
13
|
const state_interfaces_1 = require("./state.interfaces");
|
|
14
14
|
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* Mode-specific subclasses (`ExtractionState`, `LoadingState`) narrow `sdkState`
|
|
20
|
-
* to their own SDK state shape and add mode-specific setup in their factories.
|
|
21
|
-
*
|
|
22
|
-
* @typeParam ConnectorState - the connector-owned state shape
|
|
23
|
-
* @typeParam SdkState - the SDK bookkeeping shape for this mode
|
|
15
|
+
* State lifecycle shared by all sync modes: connector/SDK state separation,
|
|
16
|
+
* fetch/init/post, the v1->v2 migration shim, and the snap-in-version-gated
|
|
17
|
+
* initial domain mapping install. Subclasses seed the SDK-owned state.
|
|
24
18
|
*/
|
|
25
19
|
class BaseState {
|
|
26
20
|
constructor({ event, initialState }, initialSdkState) {
|
|
@@ -34,7 +28,7 @@ class BaseState {
|
|
|
34
28
|
this.syncUnitId = event.payload.event_context.sync_unit_id;
|
|
35
29
|
this.requestId = event.payload.event_context.request_id_adaas;
|
|
36
30
|
}
|
|
37
|
-
/** Connector-owned state
|
|
31
|
+
/** Connector-owned state; what `adapter.state` exposes to snap-in code. */
|
|
38
32
|
get state() {
|
|
39
33
|
return this._connectorState;
|
|
40
34
|
}
|
|
@@ -52,9 +46,9 @@ class BaseState {
|
|
|
52
46
|
return this._extractionScope;
|
|
53
47
|
}
|
|
54
48
|
/**
|
|
55
|
-
* Installs the initial domain mapping when the snap-in version in state
|
|
56
|
-
*
|
|
57
|
-
*
|
|
49
|
+
* Installs the initial domain mapping when the snap-in version in state
|
|
50
|
+
* differs from the event context. Shared by all modes so a loading run still
|
|
51
|
+
* installs the mapping if extraction has not done so.
|
|
58
52
|
*/
|
|
59
53
|
async installInitialDomainMappingIfNeeded(initialDomainMapping) {
|
|
60
54
|
const snapInVersionId = this.event.context.snap_in_version_id;
|
|
@@ -86,13 +80,9 @@ class BaseState {
|
|
|
86
80
|
}
|
|
87
81
|
}
|
|
88
82
|
/**
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
* Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat
|
|
93
|
-
* v1 blob (connector keys merged with SDK keys), migrating the latter on read.
|
|
94
|
-
* Always persists the v2 envelope going forward.
|
|
95
|
-
* @param initialState The initial connector state provided by the spawn function
|
|
83
|
+
* Fetches state from API, or creates the initial state on 404. Reads both
|
|
84
|
+
* the v2 envelope and a legacy flat v1 blob (migrated on read); always
|
|
85
|
+
* persists the v2 envelope going forward.
|
|
96
86
|
*/
|
|
97
87
|
async init(initialState) {
|
|
98
88
|
var _a;
|
|
@@ -143,13 +133,9 @@ class BaseState {
|
|
|
143
133
|
}
|
|
144
134
|
}
|
|
145
135
|
/**
|
|
146
|
-
* Normalizes
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
* - v2 envelope (`{ connectorState, sdkState }`): used as-is.
|
|
150
|
-
* - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`,
|
|
151
|
-
* everything else becomes connector state.
|
|
152
|
-
* - Malformed envelope (one side present, the other missing) fails loud.
|
|
136
|
+
* Normalizes fetched state into the `{ connectorState, sdkState }` envelope.
|
|
137
|
+
* A flat v1 blob is split by `V1_SDK_STATE_KEYS`; an envelope with only one
|
|
138
|
+
* side present fails loud.
|
|
153
139
|
*/
|
|
154
140
|
normalizeFetchedState(parsed) {
|
|
155
141
|
if (parsed === null || typeof parsed !== 'object') {
|
|
@@ -183,11 +169,7 @@ class BaseState {
|
|
|
183
169
|
sdkState: Object.assign(Object.assign({}, this.initialSdkState), sdkState),
|
|
184
170
|
};
|
|
185
171
|
}
|
|
186
|
-
/**
|
|
187
|
-
* Updates the state of the adapter by posting to API.
|
|
188
|
-
* Persists the v2 `{ connectorState, sdkState }` envelope.
|
|
189
|
-
* @param {object} state - The connector state to be updated
|
|
190
|
-
*/
|
|
172
|
+
/** Posts the v2 `{ connectorState, sdkState }` envelope to the API. */
|
|
191
173
|
async postState(state) {
|
|
192
174
|
const url = this.workerUrl + '.update';
|
|
193
175
|
this.state = state || this.state;
|
|
@@ -209,7 +191,7 @@ class BaseState {
|
|
|
209
191
|
process.exit(1);
|
|
210
192
|
}
|
|
211
193
|
try {
|
|
212
|
-
await
|
|
194
|
+
await client_1.axiosClient.post(url, {
|
|
213
195
|
state: stringifiedState,
|
|
214
196
|
}, {
|
|
215
197
|
headers: {
|
|
@@ -235,15 +217,11 @@ class BaseState {
|
|
|
235
217
|
process.exit(1);
|
|
236
218
|
}
|
|
237
219
|
}
|
|
238
|
-
/**
|
|
239
|
-
* Fetches the state of the adapter from API.
|
|
240
|
-
* @return The raw state data from API
|
|
241
|
-
*/
|
|
242
220
|
async fetchState() {
|
|
243
221
|
var _a, _b;
|
|
244
222
|
console.log(`Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.`);
|
|
245
223
|
const url = this.workerUrl + '.get';
|
|
246
|
-
const response = await
|
|
224
|
+
const response = await client_1.axiosClient.get(url, {
|
|
247
225
|
headers: {
|
|
248
226
|
Authorization: this.devrevToken,
|
|
249
227
|
},
|
|
@@ -1,29 +1,14 @@
|
|
|
1
1
|
import { BaseState } from './base-state';
|
|
2
|
-
import {
|
|
3
|
-
/**
|
|
4
|
-
|
|
5
|
-
* `sdkState` to `ExtractionSdkState` (extraction boundaries + attachments
|
|
6
|
-
* bookkeeping) on top of the shared lifecycle provided by `BaseState`.
|
|
7
|
-
*/
|
|
8
|
-
export declare class ExtractionState<ConnectorState> extends BaseState<ConnectorState, ExtractionSdkState> {
|
|
2
|
+
import { StateInterface } from './state.interfaces';
|
|
3
|
+
/** State for extraction workers: seeds extraction SDK state and adds extraction-window resolution. */
|
|
4
|
+
export declare class ExtractionState<ConnectorState> extends BaseState<ConnectorState> {
|
|
9
5
|
constructor(params: StateInterface<ConnectorState>);
|
|
10
6
|
/**
|
|
11
|
-
* Resolves the extraction window onto the event context.
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* event context and cache them as pending boundaries (always overwrite).
|
|
15
|
-
* On all other events: reuse the pending boundaries cached during
|
|
16
|
-
* StartExtractingMetadata. Finally, validate that extract_from < extract_to.
|
|
7
|
+
* Resolves the extraction window onto the event context. StartExtractingMetadata
|
|
8
|
+
* resolves fresh from the event's TimeValues and overwrites the pending boundaries;
|
|
9
|
+
* all other events reuse those cached boundaries. Validates extract_from < extract_to.
|
|
17
10
|
*/
|
|
18
11
|
resolveExtractionWindow(): void;
|
|
19
12
|
}
|
|
20
|
-
/**
|
|
21
|
-
* Creates and initializes an `ExtractionState` for an extraction worker.
|
|
22
|
-
*
|
|
23
|
-
* For non-stateless events this fetches/migrates persisted state, installs the
|
|
24
|
-
* initial domain mapping if the snap-in version changed, then resolves the
|
|
25
|
-
* extraction window (time-value resolution + pending boundary reuse) and
|
|
26
|
-
* validates it.
|
|
27
|
-
*/
|
|
28
13
|
export declare function createExtractionState<ConnectorState>({ event, initialState, initialDomainMapping, options, }: StateInterface<ConnectorState>): Promise<ExtractionState<ConnectorState>>;
|
|
29
14
|
//# sourceMappingURL=extraction-state.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extraction-state.d.ts","sourceRoot":"","sources":["../../src/state/extraction-state.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAsB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGxE,sGAAsG;AACtG,qBAAa,eAAe,CAAC,cAAc,CAAE,SAAQ,SAAS,CAAC,cAAc,CAAC;gBAChE,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;IAIlD;;;;OAIG;IACH,uBAAuB,IAAI,IAAI;CA6EhC;AAED,wBAAsB,qBAAqB,CAAC,cAAc,EAAE,EAC1D,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,CAkB3E"}
|
|
@@ -4,32 +4,25 @@ exports.ExtractionState = void 0;
|
|
|
4
4
|
exports.createExtractionState = createExtractionState;
|
|
5
5
|
const node_worker_threads_1 = require("node:worker_threads");
|
|
6
6
|
const constants_1 = require("../common/constants");
|
|
7
|
-
const time_value_resolver_1 = require("../common/time-value-resolver");
|
|
8
7
|
const logger_1 = require("../logger/logger");
|
|
9
8
|
const extraction_1 = require("../types/extraction");
|
|
10
9
|
const workers_1 = require("../types/workers");
|
|
11
10
|
const base_state_1 = require("./base-state");
|
|
12
11
|
const state_interfaces_1 = require("./state.interfaces");
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
* `sdkState` to `ExtractionSdkState` (extraction boundaries + attachments
|
|
16
|
-
* bookkeeping) on top of the shared lifecycle provided by `BaseState`.
|
|
17
|
-
*/
|
|
12
|
+
const time_value_resolver_1 = require("./time-value-resolver");
|
|
13
|
+
/** State for extraction workers: seeds extraction SDK state and adds extraction-window resolution. */
|
|
18
14
|
class ExtractionState extends base_state_1.BaseState {
|
|
19
15
|
constructor(params) {
|
|
20
|
-
super(params, state_interfaces_1.
|
|
16
|
+
super(params, state_interfaces_1.extractionSdkState);
|
|
21
17
|
}
|
|
22
18
|
/**
|
|
23
|
-
* Resolves the extraction window onto the event context.
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* event context and cache them as pending boundaries (always overwrite).
|
|
27
|
-
* On all other events: reuse the pending boundaries cached during
|
|
28
|
-
* StartExtractingMetadata. Finally, validate that extract_from < extract_to.
|
|
19
|
+
* Resolves the extraction window onto the event context. StartExtractingMetadata
|
|
20
|
+
* resolves fresh from the event's TimeValues and overwrites the pending boundaries;
|
|
21
|
+
* all other events reuse those cached boundaries. Validates extract_from < extract_to.
|
|
29
22
|
*/
|
|
30
23
|
resolveExtractionWindow() {
|
|
31
|
-
const eventContext = this.event.payload.event_context;
|
|
32
24
|
const sdkState = this.sdkState;
|
|
25
|
+
const eventContext = this.event.payload.event_context;
|
|
33
26
|
if (this.event.payload.event_type === extraction_1.EventType.StartExtractingMetadata) {
|
|
34
27
|
const timeFields = [
|
|
35
28
|
{
|
|
@@ -65,7 +58,6 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
65
58
|
}
|
|
66
59
|
}
|
|
67
60
|
else {
|
|
68
|
-
// Non-StartExtractingMetadata events: reuse pending values from state
|
|
69
61
|
if (sdkState.pendingWorkersOldest) {
|
|
70
62
|
eventContext.extract_from = sdkState.pendingWorkersOldest;
|
|
71
63
|
console.log(`Reusing pendingWorkersOldest as extract_from: ${sdkState.pendingWorkersOldest}.`);
|
|
@@ -81,7 +73,6 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
81
73
|
console.log('pendingWorkersNewest is not set in state. extract_to will not be populated for this invocation.');
|
|
82
74
|
}
|
|
83
75
|
}
|
|
84
|
-
// Validate that extract_from is before extract_to
|
|
85
76
|
if (eventContext.extract_from && eventContext.extract_to) {
|
|
86
77
|
if (eventContext.extract_from >= eventContext.extract_to) {
|
|
87
78
|
const errorMessage = `Invalid extraction window: extract_from (${eventContext.extract_from}) must be older than extract_to (${eventContext.extract_to}). This indicates an error in the platform.`;
|
|
@@ -96,16 +87,8 @@ class ExtractionState extends base_state_1.BaseState {
|
|
|
96
87
|
}
|
|
97
88
|
}
|
|
98
89
|
exports.ExtractionState = ExtractionState;
|
|
99
|
-
/**
|
|
100
|
-
* Creates and initializes an `ExtractionState` for an extraction worker.
|
|
101
|
-
*
|
|
102
|
-
* For non-stateless events this fetches/migrates persisted state, installs the
|
|
103
|
-
* initial domain mapping if the snap-in version changed, then resolves the
|
|
104
|
-
* extraction window (time-value resolution + pending boundary reuse) and
|
|
105
|
-
* validates it.
|
|
106
|
-
*/
|
|
107
90
|
async function createExtractionState({ event, initialState, initialDomainMapping, options, }) {
|
|
108
|
-
//
|
|
91
|
+
// Clone so the caller's initialState is never mutated.
|
|
109
92
|
const deepCloneInitialState = structuredClone(initialState);
|
|
110
93
|
const state = new ExtractionState({
|
|
111
94
|
event,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AirSyncEvent } from '../types/extraction';
|
|
2
1
|
import { InitialDomainMapping } from '../types/common';
|
|
2
|
+
import { AirSyncEvent } from '../types/extraction';
|
|
3
3
|
export declare function installInitialDomainMapping(event: AirSyncEvent, initialDomainMappingJson: InitialDomainMapping): Promise<void>;
|
|
4
4
|
//# sourceMappingURL=install-initial-domain-mapping.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"install-initial-domain-mapping.d.ts","sourceRoot":"","sources":["../../src/state/install-initial-domain-mapping.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,YAAY,EACnB,wBAAwB,EAAE,oBAAoB,GAC7C,OAAO,CAAC,IAAI,CAAC,CAwFf"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.installInitialDomainMapping = installInitialDomainMapping;
|
|
4
|
-
const
|
|
4
|
+
const client_1 = require("../http/client");
|
|
5
5
|
const logger_1 = require("../logger/logger");
|
|
6
6
|
async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
7
7
|
var _a, _b, _c, _d, _e, _f, _g;
|
|
@@ -12,8 +12,7 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
|
12
12
|
console.warn('No initial domain mapping found.');
|
|
13
13
|
return;
|
|
14
14
|
}
|
|
15
|
-
|
|
16
|
-
const snapInResponse = await axios_client_internal_1.axiosClient.get(devrevEndpoint + '/internal/snap-ins.get', {
|
|
15
|
+
const snapInResponse = await client_1.axiosClient.get(devrevEndpoint + '/internal/snap-ins.get', {
|
|
17
16
|
headers: {
|
|
18
17
|
Authorization: devrevToken,
|
|
19
18
|
},
|
|
@@ -28,12 +27,12 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
|
28
27
|
throw new Error(errorMessage);
|
|
29
28
|
}
|
|
30
29
|
const startingRecipeBlueprint = initialDomainMappingJson === null || initialDomainMappingJson === void 0 ? void 0 : initialDomainMappingJson.starting_recipe_blueprint;
|
|
31
|
-
//
|
|
30
|
+
// Recipe blueprint creation is best-effort; install proceeds without it.
|
|
32
31
|
let recipeBlueprintId;
|
|
33
32
|
if (startingRecipeBlueprint &&
|
|
34
33
|
Object.keys(startingRecipeBlueprint).length !== 0) {
|
|
35
34
|
try {
|
|
36
|
-
const recipeBlueprintResponse = await
|
|
35
|
+
const recipeBlueprintResponse = await client_1.axiosClient.post(`${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`, Object.assign({}, startingRecipeBlueprint), {
|
|
37
36
|
headers: {
|
|
38
37
|
Authorization: devrevToken,
|
|
39
38
|
},
|
|
@@ -45,9 +44,8 @@ async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
|
45
44
|
console.warn('Error while creating recipe blueprint. Continuing without it.', (0, logger_1.serializeError)(error));
|
|
46
45
|
}
|
|
47
46
|
}
|
|
48
|
-
// Install the initial domain mappings
|
|
49
47
|
const additionalMappings = initialDomainMappingJson.additional_mappings || {};
|
|
50
|
-
const initialDomainMappingInstallResponse = await
|
|
48
|
+
const initialDomainMappingInstallResponse = await client_1.axiosClient.post(`${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, Object.assign(Object.assign({ external_system_type: 'ADaaS', import_slug: importSlug, snap_in_slug: snapInSlug }, (recipeBlueprintId && {
|
|
51
49
|
starting_recipe_blueprint: recipeBlueprintId,
|
|
52
50
|
})), additionalMappings), {
|
|
53
51
|
headers: {
|
|
@@ -1,19 +1,8 @@
|
|
|
1
1
|
import { BaseState } from './base-state';
|
|
2
|
-
import {
|
|
3
|
-
/**
|
|
4
|
-
|
|
5
|
-
* to `LoadingSdkState` (files-to-load bookkeeping) on top of the shared
|
|
6
|
-
* lifecycle provided by `BaseState`.
|
|
7
|
-
*/
|
|
8
|
-
export declare class LoadingState<ConnectorState> extends BaseState<ConnectorState, LoadingSdkState> {
|
|
2
|
+
import { StateInterface } from './state.interfaces';
|
|
3
|
+
/** State for loading workers: seeds loading SDK state; no extraction-window resolution. */
|
|
4
|
+
export declare class LoadingState<ConnectorState> extends BaseState<ConnectorState> {
|
|
9
5
|
constructor(params: StateInterface<ConnectorState>);
|
|
10
6
|
}
|
|
11
|
-
/**
|
|
12
|
-
* Creates and initializes a `LoadingState` for a loading worker.
|
|
13
|
-
*
|
|
14
|
-
* For non-stateless events this fetches/migrates persisted state and installs
|
|
15
|
-
* the initial domain mapping if the snap-in version changed. Loading has no
|
|
16
|
-
* extraction-window resolution, so the factory is otherwise minimal.
|
|
17
|
-
*/
|
|
18
7
|
export declare function createLoadingState<ConnectorState>({ event, initialState, initialDomainMapping, options, }: StateInterface<ConnectorState>): Promise<LoadingState<ConnectorState>>;
|
|
19
8
|
//# sourceMappingURL=loading-state.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loading-state.d.ts","sourceRoot":"","sources":["../../src/state/loading-state.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,
|
|
1
|
+
{"version":3,"file":"loading-state.d.ts","sourceRoot":"","sources":["../../src/state/loading-state.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAmB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAErE,2FAA2F;AAC3F,qBAAa,YAAY,CAAC,cAAc,CAAE,SAAQ,SAAS,CAAC,cAAc,CAAC;gBAC7D,MAAM,EAAE,cAAc,CAAC,cAAc,CAAC;CAGnD;AAED,wBAAsB,kBAAkB,CAAC,cAAc,EAAE,EACvD,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,CAiBxE"}
|