@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -4
- package/dist/airsync-sdk.d.ts +1802 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +13 -2
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +201 -6
- package/dist/common/constants.d.ts +16 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +20 -6
- package/dist/common/control-protocol.d.ts +2 -2
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +1 -5
- package/dist/common/helpers.d.ts +8 -0
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +15 -0
- package/dist/common/install-initial-domain-mapping.d.ts +2 -2
- package/dist/common/install-initial-domain-mapping.test.js +5 -2
- package/dist/common/test-utils.d.ts +19 -0
- package/dist/common/test-utils.d.ts.map +1 -0
- package/dist/common/test-utils.js +101 -0
- package/dist/common/time-value-resolver.d.ts +47 -0
- package/dist/common/time-value-resolver.d.ts.map +1 -0
- package/dist/common/time-value-resolver.js +141 -0
- package/dist/common/time-value-resolver.test.d.ts +2 -0
- package/dist/common/time-value-resolver.test.d.ts.map +1 -0
- package/dist/common/time-value-resolver.test.js +268 -0
- package/dist/http/axios-client-internal.d.ts.map +1 -1
- package/dist/http/axios-client-internal.js +7 -0
- package/dist/http/axios-client-internal.test.js +21 -0
- package/dist/http/axios-client.d.ts.map +1 -1
- package/dist/http/axios-client.js +6 -3
- package/dist/http/index.d.ts +0 -1
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/index.js +0 -1
- package/dist/index.d.ts +9 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -10
- package/dist/logger/logger.d.ts +1 -17
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +2 -2
- package/dist/logger/logger.js +35 -27
- package/dist/logger/logger.test.js +66 -19
- package/dist/mappers/mappers.interface.d.ts +5 -5
- package/dist/mappers/mappers.interface.js +3 -3
- package/dist/mappers/mappers.test.js +8 -10
- package/dist/mock-server/mock-server.d.ts +47 -0
- package/dist/mock-server/mock-server.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
- package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.js +4 -0
- package/dist/mock-server/mock-server.js +278 -0
- package/dist/multithreading/create-worker.test.js +36 -17
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +15 -48
- package/dist/multithreading/process-task.test.d.ts +2 -0
- package/dist/multithreading/process-task.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.test.js +146 -0
- package/dist/multithreading/spawn/spawn.d.ts +3 -3
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +6 -14
- package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
- package/dist/multithreading/spawn/spawn.js +20 -25
- package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
- package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.test.js +232 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +232 -131
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +4 -2
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +4 -3
- package/dist/repo/repo.test.js +60 -7
- package/dist/state/state.d.ts +7 -1
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.extract-window.test.d.ts +2 -0
- package/dist/state/state.extract-window.test.d.ts.map +1 -0
- package/dist/state/state.extract-window.test.js +163 -0
- package/dist/state/state.interfaces.d.ts +17 -7
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +4 -2
- package/dist/state/state.js +111 -12
- package/dist/state/state.pending-boundaries.test.d.ts +2 -0
- package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
- package/dist/state/state.pending-boundaries.test.js +189 -0
- package/dist/state/state.post-state.test.d.ts +2 -0
- package/dist/state/state.post-state.test.d.ts.map +1 -0
- package/dist/state/state.post-state.test.js +77 -0
- package/dist/state/state.test.js +117 -73
- package/dist/state/state.time-value-resolution.test.d.ts +2 -0
- package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
- package/dist/state/state.time-value-resolution.test.js +175 -0
- package/dist/tsdoc-metadata.json +11 -0
- package/dist/types/common.d.ts +0 -25
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -11
- package/dist/types/external-domain-metadata.d.ts +20 -1
- package/dist/types/external-domain-metadata.d.ts.map +1 -1
- package/dist/types/extraction.d.ts +89 -218
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +56 -124
- package/dist/types/extraction.test.js +69 -43
- package/dist/types/index.d.ts +3 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +6 -3
- package/dist/types/loading.d.ts +4 -24
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +1 -22
- package/dist/types/workers.d.ts +31 -14
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +1 -0
- package/dist/uploader/uploader.d.ts +1 -0
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.test.js +0 -11
- package/dist/uploader/uploader.interfaces.d.ts +3 -3
- package/dist/uploader/uploader.js +53 -20
- package/dist/uploader/uploader.test.js +22 -10
- package/package.json +10 -13
- package/dist/common/event-type-translation.d.ts +0 -24
- package/dist/common/event-type-translation.d.ts.map +0 -1
- package/dist/common/event-type-translation.js +0 -117
- package/dist/deprecated/adapter/index.d.ts +0 -62
- package/dist/deprecated/adapter/index.d.ts.map +0 -1
- package/dist/deprecated/adapter/index.js +0 -151
- package/dist/deprecated/common/helpers.d.ts +0 -7
- package/dist/deprecated/common/helpers.d.ts.map +0 -1
- package/dist/deprecated/common/helpers.js +0 -47
- package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
- package/dist/deprecated/demo-extractor/index.d.ts +0 -18
- package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
- package/dist/deprecated/demo-extractor/index.js +0 -161
- package/dist/deprecated/http/client.d.ts +0 -22
- package/dist/deprecated/http/client.d.ts.map +0 -1
- package/dist/deprecated/http/client.js +0 -161
- package/dist/deprecated/uploader/index.d.ts +0 -35
- package/dist/deprecated/uploader/index.d.ts.map +0 -1
- package/dist/deprecated/uploader/index.js +0 -161
- package/dist/http/types.d.ts +0 -17
- package/dist/http/types.d.ts.map +0 -1
- package/dist/http/types.js +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Artifact } from '../uploader/uploader.interfaces';
|
|
2
|
-
import {
|
|
2
|
+
import { AirSyncEvent } from '../types/extraction';
|
|
3
3
|
import { WorkerAdapterOptions } from '../types/workers';
|
|
4
4
|
/**
|
|
5
5
|
* RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data.
|
|
@@ -7,12 +7,13 @@ import { WorkerAdapterOptions } from '../types/workers';
|
|
|
7
7
|
export interface RepoInterface {
|
|
8
8
|
itemType: string;
|
|
9
9
|
normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
|
|
10
|
+
overridenOptions?: WorkerAdapterOptions;
|
|
10
11
|
}
|
|
11
12
|
/**
|
|
12
13
|
* RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo.
|
|
13
14
|
*/
|
|
14
15
|
export interface RepoFactoryInterface {
|
|
15
|
-
event:
|
|
16
|
+
event: AirSyncEvent;
|
|
16
17
|
itemType: string;
|
|
17
18
|
normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
|
|
18
19
|
onUpload: (artifact: Artifact) => void;
|
|
@@ -37,6 +38,7 @@ export interface NormalizedAttachment {
|
|
|
37
38
|
parent_id: string;
|
|
38
39
|
author_id?: string;
|
|
39
40
|
inline?: boolean;
|
|
41
|
+
content_type?: string;
|
|
40
42
|
grand_parent_id?: number | string;
|
|
41
43
|
}
|
|
42
44
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IAItB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
|
package/dist/repo/repo.js
CHANGED
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Repo = void 0;
|
|
4
4
|
const constants_1 = require("../common/constants");
|
|
5
5
|
const uploader_1 = require("../uploader/uploader");
|
|
6
|
+
const logger_context_1 = require("../logger/logger.context");
|
|
6
7
|
class Repo {
|
|
7
8
|
constructor({ event, itemType, normalize, onUpload, options, }) {
|
|
8
9
|
this.items = [];
|
|
@@ -46,9 +47,9 @@ class Repo {
|
|
|
46
47
|
}
|
|
47
48
|
// Normalize items if needed
|
|
48
49
|
if (this.normalize &&
|
|
49
|
-
this.itemType != constants_1.
|
|
50
|
-
this.itemType != constants_1.
|
|
51
|
-
recordsToPush = items.map((item) => this.normalize(item));
|
|
50
|
+
this.itemType != constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
|
|
51
|
+
this.itemType != constants_1.SSOR_ATTACHMENT) {
|
|
52
|
+
recordsToPush = (0, logger_context_1.runWithUserLogContext)(() => items.map((item) => this.normalize(item)));
|
|
52
53
|
}
|
|
53
54
|
else {
|
|
54
55
|
recordsToPush = items;
|
package/dist/repo/repo.test.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const constants_1 = require("../common/constants");
|
|
4
4
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
|
+
const jest_setup_1 = require("../tests/jest.setup");
|
|
6
|
+
const test_utils_1 = require("../common/test-utils");
|
|
5
7
|
const types_1 = require("../types");
|
|
6
8
|
const repo_1 = require("./repo");
|
|
7
9
|
jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
|
|
@@ -11,7 +13,9 @@ describe(repo_1.Repo.name, () => {
|
|
|
11
13
|
beforeEach(() => {
|
|
12
14
|
normalize = jest.fn();
|
|
13
15
|
repo = new repo_1.Repo({
|
|
14
|
-
event: (0,
|
|
16
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
17
|
+
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
18
|
+
}),
|
|
15
19
|
itemType: 'test_item_type',
|
|
16
20
|
normalize,
|
|
17
21
|
onUpload: jest.fn(),
|
|
@@ -21,77 +25,111 @@ describe(repo_1.Repo.name, () => {
|
|
|
21
25
|
jest.clearAllMocks();
|
|
22
26
|
});
|
|
23
27
|
it('should normalize and push items when array contains items', async () => {
|
|
28
|
+
// Arrange
|
|
24
29
|
const items = (0, test_helpers_1.createItems)(10);
|
|
30
|
+
// Act
|
|
25
31
|
await repo.push(items);
|
|
32
|
+
// Assert
|
|
26
33
|
expect(normalize).toHaveBeenCalledTimes(10);
|
|
27
34
|
const normalizedItems = items.map((item) => (0, test_helpers_1.normalizeItem)(item));
|
|
28
35
|
expect(repo.getItems()).toEqual(normalizedItems);
|
|
29
36
|
});
|
|
30
37
|
it('should not normalize items when normalize function is not provided', async () => {
|
|
38
|
+
// Arrange
|
|
31
39
|
repo = new repo_1.Repo({
|
|
32
|
-
event: (0,
|
|
40
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
41
|
+
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
42
|
+
}),
|
|
33
43
|
itemType: 'test_item_type',
|
|
34
44
|
onUpload: jest.fn(),
|
|
35
45
|
options: {},
|
|
36
46
|
});
|
|
37
47
|
const items = (0, test_helpers_1.createItems)(10);
|
|
48
|
+
// Act
|
|
38
49
|
await repo.push(items);
|
|
50
|
+
// Assert
|
|
39
51
|
expect(normalize).not.toHaveBeenCalled();
|
|
40
52
|
});
|
|
41
53
|
it('[edge] should not push items when items array is empty', async () => {
|
|
54
|
+
// Act
|
|
42
55
|
await repo.push([]);
|
|
56
|
+
// Assert
|
|
43
57
|
expect(repo.getItems()).toEqual([]);
|
|
44
58
|
});
|
|
45
59
|
it('should not normalize items when item type is external_domain_metadata', async () => {
|
|
60
|
+
// Arrange
|
|
46
61
|
repo = new repo_1.Repo({
|
|
47
|
-
event: (0,
|
|
48
|
-
|
|
62
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
63
|
+
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
64
|
+
}),
|
|
65
|
+
itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA,
|
|
49
66
|
normalize,
|
|
50
67
|
onUpload: jest.fn(),
|
|
51
68
|
options: {},
|
|
52
69
|
});
|
|
53
70
|
const items = (0, test_helpers_1.createItems)(10);
|
|
71
|
+
// Act
|
|
54
72
|
await repo.push(items);
|
|
73
|
+
// Assert
|
|
55
74
|
expect(normalize).not.toHaveBeenCalled();
|
|
56
75
|
});
|
|
57
76
|
it('should not normalize items when item type is ssor_attachment', async () => {
|
|
77
|
+
// Arrange
|
|
58
78
|
repo = new repo_1.Repo({
|
|
59
|
-
event: (0,
|
|
60
|
-
|
|
79
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
80
|
+
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
81
|
+
}),
|
|
82
|
+
itemType: constants_1.SSOR_ATTACHMENT,
|
|
61
83
|
normalize,
|
|
62
84
|
onUpload: jest.fn(),
|
|
63
85
|
options: {},
|
|
64
86
|
});
|
|
65
87
|
const items = (0, test_helpers_1.createItems)(10);
|
|
88
|
+
// Act
|
|
66
89
|
await repo.push(items);
|
|
90
|
+
// Assert
|
|
67
91
|
expect(normalize).not.toHaveBeenCalled();
|
|
68
92
|
});
|
|
69
93
|
it('should leave 5 items in the items array after pushing 2005 items with batch size of 2000', async () => {
|
|
94
|
+
// Arrange
|
|
70
95
|
const items = (0, test_helpers_1.createItems)(2005);
|
|
96
|
+
// Act
|
|
71
97
|
await repo.push(items);
|
|
98
|
+
// Assert
|
|
72
99
|
expect(repo.getItems().length).toBe(5);
|
|
73
100
|
});
|
|
74
101
|
it('should normalize all items when pushing 4005 items with batch size of 2000', async () => {
|
|
102
|
+
// Arrange
|
|
75
103
|
const items = (0, test_helpers_1.createItems)(4005);
|
|
104
|
+
// Act
|
|
76
105
|
await repo.push(items);
|
|
106
|
+
// Assert
|
|
77
107
|
expect(normalize).toHaveBeenCalledTimes(4005);
|
|
78
108
|
});
|
|
79
109
|
it('should upload 2 batches when pushing 4005 items with batch size of 2000', async () => {
|
|
110
|
+
// Arrange
|
|
80
111
|
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
81
112
|
const items = (0, test_helpers_1.createItems)(4005);
|
|
113
|
+
// Act
|
|
82
114
|
await repo.push(items);
|
|
115
|
+
// Assert
|
|
83
116
|
expect(uploadSpy).toHaveBeenCalledTimes(2);
|
|
84
117
|
uploadSpy.mockRestore();
|
|
85
118
|
});
|
|
86
119
|
it('should leave 5 items in array after pushing 4005 items with batch size of 2000', async () => {
|
|
120
|
+
// Arrange
|
|
87
121
|
const items = (0, test_helpers_1.createItems)(4005);
|
|
122
|
+
// Act
|
|
88
123
|
await repo.push(items);
|
|
124
|
+
// Assert
|
|
89
125
|
expect(repo.getItems().length).toBe(5);
|
|
90
126
|
});
|
|
91
127
|
describe('should take batch size into account', () => {
|
|
92
128
|
beforeEach(() => {
|
|
93
129
|
repo = new repo_1.Repo({
|
|
94
|
-
event: (0,
|
|
130
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
131
|
+
payload: { event_type: types_1.EventType.StartExtractingData },
|
|
132
|
+
}),
|
|
95
133
|
itemType: 'test_item_type',
|
|
96
134
|
normalize,
|
|
97
135
|
onUpload: jest.fn(),
|
|
@@ -101,30 +139,45 @@ describe(repo_1.Repo.name, () => {
|
|
|
101
139
|
});
|
|
102
140
|
});
|
|
103
141
|
it('should empty the items array after pushing 50 items with batch size of 50', async () => {
|
|
142
|
+
// Arrange
|
|
104
143
|
const items = (0, test_helpers_1.createItems)(50);
|
|
144
|
+
// Act
|
|
105
145
|
await repo.push(items);
|
|
146
|
+
// Assert
|
|
106
147
|
expect(repo.getItems()).toEqual([]);
|
|
107
148
|
});
|
|
108
149
|
it('should leave 5 items in the items array after pushing 205 items with batch size of 50', async () => {
|
|
150
|
+
// Arrange
|
|
109
151
|
const items = (0, test_helpers_1.createItems)(205);
|
|
152
|
+
// Act
|
|
110
153
|
await repo.push(items);
|
|
154
|
+
// Assert
|
|
111
155
|
expect(repo.getItems().length).toBe(5);
|
|
112
156
|
});
|
|
113
157
|
it('should normalize all items when pushing 205 items with batch size of 50', async () => {
|
|
158
|
+
// Arrange
|
|
114
159
|
const items = (0, test_helpers_1.createItems)(205);
|
|
160
|
+
// Act
|
|
115
161
|
await repo.push(items);
|
|
162
|
+
// Assert
|
|
116
163
|
expect(normalize).toHaveBeenCalledTimes(205);
|
|
117
164
|
});
|
|
118
165
|
it('should upload 4 batches when pushing 205 items with batch size of 50', async () => {
|
|
166
|
+
// Arrange
|
|
119
167
|
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
120
168
|
const items = (0, test_helpers_1.createItems)(205);
|
|
169
|
+
// Act
|
|
121
170
|
await repo.push(items);
|
|
171
|
+
// Assert
|
|
122
172
|
expect(uploadSpy).toHaveBeenCalledTimes(4);
|
|
123
173
|
uploadSpy.mockRestore();
|
|
124
174
|
});
|
|
125
175
|
it('should leave 5 items in array after pushing 205 items with batch size of 50', async () => {
|
|
176
|
+
// Arrange
|
|
126
177
|
const items = (0, test_helpers_1.createItems)(205);
|
|
178
|
+
// Act
|
|
127
179
|
await repo.push(items);
|
|
180
|
+
// Assert
|
|
128
181
|
expect(repo.getItems().length).toBe(5);
|
|
129
182
|
});
|
|
130
183
|
});
|
package/dist/state/state.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { AdapterState, StateInterface } from './state.interfaces';
|
|
2
|
+
import { ExtractionScope } from '../types/workers';
|
|
2
3
|
export declare function createAdapterState<ConnectorState>({ event, initialState, initialDomainMapping, options, }: StateInterface<ConnectorState>): Promise<State<ConnectorState>>;
|
|
3
4
|
export declare class State<ConnectorState> {
|
|
4
5
|
private _state;
|
|
6
|
+
private _extractionScope;
|
|
5
7
|
private initialSdkState;
|
|
6
8
|
private workerUrl;
|
|
7
9
|
private devrevToken;
|
|
@@ -10,6 +12,7 @@ export declare class State<ConnectorState> {
|
|
|
10
12
|
constructor({ event, initialState }: StateInterface<ConnectorState>);
|
|
11
13
|
get state(): AdapterState<ConnectorState>;
|
|
12
14
|
set state(value: AdapterState<ConnectorState>);
|
|
15
|
+
get extractionScope(): ExtractionScope;
|
|
13
16
|
/**
|
|
14
17
|
* Initializes the state for this adapter instance by fetching from API
|
|
15
18
|
* or creating an initial state if none exists (404).
|
|
@@ -25,6 +28,9 @@ export declare class State<ConnectorState> {
|
|
|
25
28
|
* Fetches the state of the adapter from API.
|
|
26
29
|
* @return The raw state data from API
|
|
27
30
|
*/
|
|
28
|
-
fetchState(): Promise<
|
|
31
|
+
fetchState(): Promise<{
|
|
32
|
+
state: string;
|
|
33
|
+
objects?: string;
|
|
34
|
+
}>;
|
|
29
35
|
}
|
|
30
36
|
//# sourceMappingURL=state.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/state/state.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/state/state.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,YAAY,EAIZ,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,wBAAsB,kBAAkB,CAAC,cAAc,EAAE,EACvD,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAoIjE;AAED,qBAAa,KAAK,CAAC,cAAc;IAC/B,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAEd,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC;IAenE,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDvD;;;OAGG;IACG,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC;IAqDpD;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"state.extract-window.test.d.ts","sourceRoot":"","sources":["../../src/state/state.extract-window.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const jest_setup_1 = require("../tests/jest.setup");
|
|
4
|
+
const test_utils_1 = require("../common/test-utils");
|
|
5
|
+
const extraction_1 = require("../types/extraction");
|
|
6
|
+
const state_1 = require("./state");
|
|
7
|
+
describe('State — extraction window validation', () => {
|
|
8
|
+
let fetchStateSpy;
|
|
9
|
+
let processExitSpy;
|
|
10
|
+
beforeEach(() => {
|
|
11
|
+
jest.clearAllMocks();
|
|
12
|
+
jest.restoreAllMocks();
|
|
13
|
+
fetchStateSpy = jest.spyOn(state_1.State.prototype, 'fetchState');
|
|
14
|
+
processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => {
|
|
15
|
+
throw new Error('process.exit called');
|
|
16
|
+
});
|
|
17
|
+
});
|
|
18
|
+
describe('Enhanced Control Protocol - extraction window validation', () => {
|
|
19
|
+
it('should exit the process if extract_from >= extract_to', async () => {
|
|
20
|
+
// Arrange: start is after end (inverted window)
|
|
21
|
+
const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
22
|
+
payload: {
|
|
23
|
+
event_type: extraction_1.EventType.StartExtractingMetadata,
|
|
24
|
+
event_context: {
|
|
25
|
+
extraction_start_time: {
|
|
26
|
+
type: extraction_1.TimeValueType.ABSOLUTE_TIME,
|
|
27
|
+
value: '2025-06-01T00:00:00Z',
|
|
28
|
+
},
|
|
29
|
+
extraction_end_time: {
|
|
30
|
+
type: extraction_1.TimeValueType.ABSOLUTE_TIME,
|
|
31
|
+
value: '2024-01-01T00:00:00Z',
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
const stringifiedState = JSON.stringify({
|
|
37
|
+
snapInVersionId: 'test_snap_in_version_id',
|
|
38
|
+
});
|
|
39
|
+
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
40
|
+
// Act & Assert
|
|
41
|
+
await expect((0, state_1.createAdapterState)({
|
|
42
|
+
event,
|
|
43
|
+
initialState: {},
|
|
44
|
+
initialDomainMapping: {},
|
|
45
|
+
})).rejects.toThrow('process.exit called');
|
|
46
|
+
expect(processExitSpy).toHaveBeenCalledWith(1);
|
|
47
|
+
});
|
|
48
|
+
it('should exit the process if extract_from equals extract_to', async () => {
|
|
49
|
+
// Arrange: start equals end (zero-width window)
|
|
50
|
+
const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
51
|
+
payload: {
|
|
52
|
+
event_type: extraction_1.EventType.StartExtractingMetadata,
|
|
53
|
+
event_context: {
|
|
54
|
+
extraction_start_time: {
|
|
55
|
+
type: extraction_1.TimeValueType.ABSOLUTE_TIME,
|
|
56
|
+
value: '2024-06-01T00:00:00Z',
|
|
57
|
+
},
|
|
58
|
+
extraction_end_time: {
|
|
59
|
+
type: extraction_1.TimeValueType.ABSOLUTE_TIME,
|
|
60
|
+
value: '2024-06-01T00:00:00Z',
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
const stringifiedState = JSON.stringify({
|
|
66
|
+
snapInVersionId: 'test_snap_in_version_id',
|
|
67
|
+
});
|
|
68
|
+
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
69
|
+
// Act & Assert
|
|
70
|
+
await expect((0, state_1.createAdapterState)({
|
|
71
|
+
event,
|
|
72
|
+
initialState: {},
|
|
73
|
+
initialDomainMapping: {},
|
|
74
|
+
})).rejects.toThrow('process.exit called');
|
|
75
|
+
expect(processExitSpy).toHaveBeenCalledWith(1);
|
|
76
|
+
});
|
|
77
|
+
it('should not exit when extract_from < extract_to', async () => {
|
|
78
|
+
// Arrange: valid window
|
|
79
|
+
const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
80
|
+
payload: {
|
|
81
|
+
event_type: extraction_1.EventType.StartExtractingMetadata,
|
|
82
|
+
event_context: {
|
|
83
|
+
extraction_start_time: {
|
|
84
|
+
type: extraction_1.TimeValueType.ABSOLUTE_TIME,
|
|
85
|
+
value: '2024-01-01T00:00:00Z',
|
|
86
|
+
},
|
|
87
|
+
extraction_end_time: {
|
|
88
|
+
type: extraction_1.TimeValueType.ABSOLUTE_TIME,
|
|
89
|
+
value: '2025-06-01T00:00:00Z',
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
const stringifiedState = JSON.stringify({
|
|
95
|
+
snapInVersionId: 'test_snap_in_version_id',
|
|
96
|
+
});
|
|
97
|
+
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
98
|
+
// Act
|
|
99
|
+
await (0, state_1.createAdapterState)({
|
|
100
|
+
event,
|
|
101
|
+
initialState: {},
|
|
102
|
+
initialDomainMapping: {},
|
|
103
|
+
});
|
|
104
|
+
// Assert: process.exit should NOT have been called
|
|
105
|
+
expect(processExitSpy).not.toHaveBeenCalled();
|
|
106
|
+
});
|
|
107
|
+
it('should not validate when only extract_from is set', async () => {
|
|
108
|
+
// Arrange: only start, no end
|
|
109
|
+
const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
110
|
+
payload: {
|
|
111
|
+
event_type: extraction_1.EventType.StartExtractingMetadata,
|
|
112
|
+
event_context: {
|
|
113
|
+
extraction_start_time: {
|
|
114
|
+
type: extraction_1.TimeValueType.ABSOLUTE_TIME,
|
|
115
|
+
value: '2024-01-01T00:00:00Z',
|
|
116
|
+
},
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
const stringifiedState = JSON.stringify({
|
|
121
|
+
snapInVersionId: 'test_snap_in_version_id',
|
|
122
|
+
});
|
|
123
|
+
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
124
|
+
// Act
|
|
125
|
+
await (0, state_1.createAdapterState)({
|
|
126
|
+
event,
|
|
127
|
+
initialState: {},
|
|
128
|
+
initialDomainMapping: {},
|
|
129
|
+
});
|
|
130
|
+
// Assert: process.exit should NOT have been called
|
|
131
|
+
expect(processExitSpy).not.toHaveBeenCalled();
|
|
132
|
+
});
|
|
133
|
+
it('should not exit when extract_from is UNBOUNDED and extract_to is a real timestamp', async () => {
|
|
134
|
+
// Arrange: UNBOUNDED start (epoch) with a real ABSOLUTE end timestamp
|
|
135
|
+
const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
136
|
+
payload: {
|
|
137
|
+
event_type: extraction_1.EventType.StartExtractingMetadata,
|
|
138
|
+
event_context: {
|
|
139
|
+
extraction_start_time: {
|
|
140
|
+
type: extraction_1.TimeValueType.UNBOUNDED,
|
|
141
|
+
},
|
|
142
|
+
extraction_end_time: {
|
|
143
|
+
type: extraction_1.TimeValueType.ABSOLUTE_TIME,
|
|
144
|
+
value: '2025-06-01T00:00:00Z',
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
const stringifiedState = JSON.stringify({
|
|
150
|
+
snapInVersionId: 'test_snap_in_version_id',
|
|
151
|
+
});
|
|
152
|
+
fetchStateSpy.mockResolvedValue({ state: stringifiedState });
|
|
153
|
+
// Act
|
|
154
|
+
await (0, state_1.createAdapterState)({
|
|
155
|
+
event,
|
|
156
|
+
initialState: {},
|
|
157
|
+
initialDomainMapping: {},
|
|
158
|
+
});
|
|
159
|
+
// Assert: process.exit should NOT have been called
|
|
160
|
+
expect(processExitSpy).not.toHaveBeenCalled();
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
});
|
|
@@ -1,17 +1,25 @@
|
|
|
1
1
|
import { InitialDomainMapping } from '../types/common';
|
|
2
|
-
import {
|
|
2
|
+
import { AirSyncEvent } from '../types/extraction';
|
|
3
3
|
import { FileToLoad } from '../types/loading';
|
|
4
4
|
import { WorkerAdapterOptions } from '../types/workers';
|
|
5
5
|
export interface SdkState {
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
/** The pending (not yet committed) oldest extraction boundary (ISO 8601 timestamp).
|
|
7
|
+
* Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */
|
|
8
|
+
pendingWorkersOldest?: string;
|
|
9
|
+
/** The pending (not yet committed) newest extraction boundary (ISO 8601 timestamp).
|
|
10
|
+
* Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */
|
|
11
|
+
pendingWorkersNewest?: string;
|
|
12
|
+
/** The oldest point of extraction (ISO 8601 timestamp). */
|
|
13
|
+
workersOldest?: string;
|
|
14
|
+
/** The newest point of extraction (ISO 8601 timestamp). */
|
|
15
|
+
workersNewest?: string;
|
|
8
16
|
toDevRev?: ToDevRev;
|
|
9
17
|
fromDevRev?: FromDevRev;
|
|
10
18
|
snapInVersionId?: string;
|
|
11
19
|
}
|
|
12
20
|
/**
|
|
13
21
|
* AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor.
|
|
14
|
-
* It extends the connector state with additional fields
|
|
22
|
+
* It extends the connector state with additional fields including snapInVersionId and attachmentsMetadata.
|
|
15
23
|
*/
|
|
16
24
|
export type AdapterState<ConnectorState> = ConnectorState & SdkState;
|
|
17
25
|
export interface ToDevRev {
|
|
@@ -32,14 +40,16 @@ export interface FromDevRev {
|
|
|
32
40
|
filesToLoad: FileToLoad[];
|
|
33
41
|
}
|
|
34
42
|
export interface StateInterface<ConnectorState> {
|
|
35
|
-
event:
|
|
43
|
+
event: AirSyncEvent;
|
|
36
44
|
initialState: ConnectorState;
|
|
37
45
|
initialDomainMapping?: InitialDomainMapping;
|
|
38
46
|
options?: WorkerAdapterOptions;
|
|
39
47
|
}
|
|
40
48
|
export declare const extractionSdkState: {
|
|
41
|
-
|
|
42
|
-
|
|
49
|
+
pendingWorkersOldest: string;
|
|
50
|
+
pendingWorkersNewest: string;
|
|
51
|
+
workersOldest: string;
|
|
52
|
+
workersNewest: string;
|
|
43
53
|
snapInVersionId: string;
|
|
44
54
|
toDevRev: {
|
|
45
55
|
attachmentsMetadata: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state.interfaces.d.ts","sourceRoot":"","sources":["../../src/state/state.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,QAAQ;IACvB,
|
|
1
|
+
{"version":3,"file":"state.interfaces.d.ts","sourceRoot":"","sources":["../../src/state/state.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,QAAQ;IACvB;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;gHAC4G;IAC5G,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,CAAC,cAAc,IAAI,cAAc,GAAG,QAAQ,CAAC;AAErE,MAAM,WAAW,QAAQ;IACvB,mBAAmB,EAAE;QACnB,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,+BAA+B,CAAC,EAAE,mBAAmB,EAAE,CAAC;KACzD,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc,CAAC,cAAc;IAC5C,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;CAa9B,CAAC;AAEF,eAAO,MAAM,eAAe;;;;;CAK3B,CAAC"}
|
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.loadingSdkState = exports.extractionSdkState = void 0;
|
|
4
4
|
exports.extractionSdkState = {
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
pendingWorkersOldest: '',
|
|
6
|
+
pendingWorkersNewest: '',
|
|
7
|
+
workersOldest: '',
|
|
8
|
+
workersNewest: '',
|
|
7
9
|
snapInVersionId: '',
|
|
8
10
|
toDevRev: {
|
|
9
11
|
attachmentsMetadata: {
|