@devrev/ts-adaas 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -4
- package/dist/src/adapter/index.d.ts +35 -30
- package/dist/src/adapter/index.js +93 -56
- package/dist/src/common/constants.d.ts +2 -0
- package/dist/src/common/constants.js +10 -0
- package/dist/src/{adapter → common}/helpers.d.ts +4 -1
- package/dist/src/{adapter → common}/helpers.js +16 -4
- package/dist/src/common/install-initial-domain-mapping.d.ts +3 -0
- package/dist/src/common/install-initial-domain-mapping.js +60 -0
- package/dist/src/demo-extractor/external_domain_metadata.json +38 -0
- package/dist/src/demo-extractor/index.d.ts +8 -1
- package/dist/src/demo-extractor/index.js +79 -73
- package/dist/src/http/client.d.ts +1 -1
- package/dist/src/http/client.js +2 -2
- package/dist/src/http/index.d.ts +0 -1
- package/dist/src/http/index.js +0 -1
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.js +2 -0
- package/dist/src/logging/index.d.ts +5 -18
- package/dist/src/logging/index.js +20 -41
- package/dist/src/state/index.d.ts +23 -0
- package/dist/src/state/index.js +111 -0
- package/dist/src/types/common.d.ts +4 -0
- package/dist/src/types/extraction.d.ts +17 -5
- package/dist/src/uploader/index.d.ts +4 -2
- package/dist/src/uploader/index.js +61 -4
- package/dist/tests/adapter.helpers.test.js +60 -0
- package/dist/tests/adapter.test.js +107 -57
- package/dist/tests/demo-extractor.test.js +45 -81
- package/dist/tests/state.test.js +101 -0
- package/dist/tests/uploader.test.js +29 -0
- package/package.json +1 -1
- package/dist/src/adapter/index.test.js +0 -105
- package/dist/src/demo-extractor/recipe.json +0 -37
- package/dist/tests/helpers.test.js +0 -38
- package/dist/tests/test-helpers.d.ts +0 -2
- package/dist/tests/test-helpers.js +0 -33
- package/dist/tests/types.test.js +0 -71
- /package/dist/{src/adapter/index.test.d.ts → tests/adapter.helpers.test.d.ts} +0 -0
- /package/dist/tests/{helpers.test.d.ts → state.test.d.ts} +0 -0
- /package/dist/tests/{types.test.d.ts → uploader.test.d.ts} +0 -0
|
@@ -1,4 +1,27 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
2
25
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
27
|
};
|
|
@@ -6,7 +29,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
29
|
exports.Uploader = void 0;
|
|
7
30
|
const axios_1 = __importDefault(require("axios"));
|
|
8
31
|
const typescript_sdk_1 = require("@devrev/typescript-sdk");
|
|
9
|
-
const
|
|
32
|
+
const fs_1 = __importStar(require("fs"));
|
|
33
|
+
const helpers_1 = require("../common/helpers");
|
|
10
34
|
/**
|
|
11
35
|
* Uploader class is used to upload files to the DevRev platform.
|
|
12
36
|
* The class provides utilities to
|
|
@@ -18,14 +42,15 @@ const helpers_1 = require("../adapter/helpers");
|
|
|
18
42
|
* @constructor
|
|
19
43
|
* @param {string} endpoint - The endpoint of the DevRev platform
|
|
20
44
|
* @param {string} token - The token to authenticate with the DevRev platform
|
|
45
|
+
* @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
|
|
21
46
|
*/
|
|
22
47
|
class Uploader {
|
|
23
|
-
constructor(endpoint, token) {
|
|
48
|
+
constructor(endpoint, token, local = false) {
|
|
24
49
|
this.betaDevrevSdk = typescript_sdk_1.client.setupBeta({
|
|
25
50
|
endpoint,
|
|
26
51
|
token,
|
|
27
52
|
});
|
|
28
|
-
this.
|
|
53
|
+
this.local = local;
|
|
29
54
|
}
|
|
30
55
|
/**
|
|
31
56
|
*
|
|
@@ -39,6 +64,9 @@ class Uploader {
|
|
|
39
64
|
* @returns {Promise<UploadResponse>} - The response object containing the artifact information
|
|
40
65
|
*/
|
|
41
66
|
async upload(filename, entity, fetchedObjects, filetype = 'application/jsonl+json') {
|
|
67
|
+
if (this.local) {
|
|
68
|
+
this.downloadToLocal(filename, fetchedObjects);
|
|
69
|
+
}
|
|
42
70
|
const preparedArtifact = await this.prepareArtifact(filename, filetype);
|
|
43
71
|
if (!preparedArtifact) {
|
|
44
72
|
return {
|
|
@@ -60,6 +88,7 @@ class Uploader {
|
|
|
60
88
|
item_type: entity,
|
|
61
89
|
item_count: itemCount,
|
|
62
90
|
};
|
|
91
|
+
console.log(`Artifact uploaded successfully: ${artifact.id}`);
|
|
63
92
|
return { artifact, error: undefined };
|
|
64
93
|
}
|
|
65
94
|
async prepareArtifact(filename, filetype) {
|
|
@@ -71,7 +100,8 @@ class Uploader {
|
|
|
71
100
|
return response.data;
|
|
72
101
|
}
|
|
73
102
|
catch (error) {
|
|
74
|
-
|
|
103
|
+
console.error('Error while preparing artifact: ' + error);
|
|
104
|
+
return null;
|
|
75
105
|
}
|
|
76
106
|
}
|
|
77
107
|
async uploadToArtifact(
|
|
@@ -91,5 +121,32 @@ class Uploader {
|
|
|
91
121
|
return null;
|
|
92
122
|
}
|
|
93
123
|
}
|
|
124
|
+
async downloadToLocal(filePath, fetchedObjects) {
|
|
125
|
+
console.log(`Uploading ${filePath} to local file system`);
|
|
126
|
+
try {
|
|
127
|
+
if (!fs_1.default.existsSync('extracted_files')) {
|
|
128
|
+
fs_1.default.mkdirSync('extracted_files');
|
|
129
|
+
}
|
|
130
|
+
const timestamp = new Date().getTime();
|
|
131
|
+
const fileHandle = await fs_1.promises.open(`extracted_files/${timestamp}_${filePath}`, 'w');
|
|
132
|
+
let objArray = [];
|
|
133
|
+
if (!Array.isArray(fetchedObjects)) {
|
|
134
|
+
objArray.push(fetchedObjects);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
objArray = fetchedObjects;
|
|
138
|
+
}
|
|
139
|
+
for (const jsonObject of objArray) {
|
|
140
|
+
const jsonLine = JSON.stringify(jsonObject) + '\n';
|
|
141
|
+
await fileHandle.write(jsonLine);
|
|
142
|
+
}
|
|
143
|
+
await fileHandle.close();
|
|
144
|
+
console.log('Data successfully written to', filePath);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
console.error('Error writing data to file:', error);
|
|
148
|
+
return Promise.reject(error);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
94
151
|
}
|
|
95
152
|
exports.Uploader = Uploader;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const helpers_1 = require("../src/common/helpers");
|
|
4
|
+
const types_1 = require("../src/types");
|
|
5
|
+
describe('adapter.helpers.ts', () => {
|
|
6
|
+
it("should create a FormData object with the correct 'key' and 'value'", () => {
|
|
7
|
+
const preparedArtifact = {
|
|
8
|
+
form_data: [{ key: 'key', value: 'value' }],
|
|
9
|
+
};
|
|
10
|
+
const fetchedObjects = [{ key: 'value' }];
|
|
11
|
+
const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
|
|
12
|
+
expect(formData).toBeInstanceOf(FormData);
|
|
13
|
+
expect(formData.get('key')).toBe('value');
|
|
14
|
+
});
|
|
15
|
+
it("should create a FormData object with the correct 'file' key", () => {
|
|
16
|
+
const preparedArtifact = {
|
|
17
|
+
form_data: [{ key: 'key', value: 'value' }],
|
|
18
|
+
};
|
|
19
|
+
const fetchedObjects = [{ key: 'value' }];
|
|
20
|
+
const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
|
|
21
|
+
expect(formData.get('file')).toBeDefined();
|
|
22
|
+
});
|
|
23
|
+
it("should create an Artifact object with the correct 'item_count', 'id', and 'item_type'", () => {
|
|
24
|
+
const preparedArtifact = { id: 'id' };
|
|
25
|
+
const fetchedObjects = [{ key: 'value' }];
|
|
26
|
+
const entity = 'entity';
|
|
27
|
+
const artifact = (0, helpers_1.createArtifact)(preparedArtifact, fetchedObjects, entity);
|
|
28
|
+
expect(artifact).toEqual({
|
|
29
|
+
item_count: 1,
|
|
30
|
+
id: 'id',
|
|
31
|
+
item_type: 'entity',
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
it('should return the correct ExtractorEventType', () => {
|
|
35
|
+
expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionMetadataStart)).toStrictEqual({
|
|
36
|
+
eventType: types_1.ExtractorEventType.ExtractionMetadataError,
|
|
37
|
+
isError: true,
|
|
38
|
+
});
|
|
39
|
+
expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionDataStart)).toStrictEqual({
|
|
40
|
+
eventType: types_1.ExtractorEventType.ExtractionDataProgress,
|
|
41
|
+
isError: false,
|
|
42
|
+
});
|
|
43
|
+
expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionDataContinue)).toStrictEqual({
|
|
44
|
+
eventType: types_1.ExtractorEventType.ExtractionDataProgress,
|
|
45
|
+
isError: false,
|
|
46
|
+
});
|
|
47
|
+
expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionAttachmentsStart)).toStrictEqual({
|
|
48
|
+
eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
|
|
49
|
+
isError: false,
|
|
50
|
+
});
|
|
51
|
+
expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionAttachmentsContinue)).toStrictEqual({
|
|
52
|
+
eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
|
|
53
|
+
isError: false,
|
|
54
|
+
});
|
|
55
|
+
expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionExternalSyncUnitsStart)).toStrictEqual({
|
|
56
|
+
eventType: types_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
|
|
57
|
+
isError: true,
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
});
|
|
@@ -5,69 +5,119 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
const axios_1 = __importDefault(require("axios"));
|
|
7
7
|
const adapter_1 = require("../src/adapter");
|
|
8
|
+
const state_1 = require("../src/state");
|
|
8
9
|
const types_1 = require("../src/types");
|
|
9
|
-
const
|
|
10
|
+
const helpers_1 = require("../src/common/helpers");
|
|
11
|
+
jest.mock('axios');
|
|
12
|
+
jest.mock('../src/state');
|
|
13
|
+
jest.mock('../src/common/helpers');
|
|
14
|
+
jest.mock('../src/logging');
|
|
15
|
+
const mockedAxios = axios_1.default;
|
|
16
|
+
const mockedCreateAdapterState = state_1.createAdapterState;
|
|
17
|
+
const mockedGetTimeoutExtractorEventType = helpers_1.getTimeoutExtractorEventType;
|
|
10
18
|
describe('Adapter', () => {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
19
|
+
const event = {
|
|
20
|
+
execution_metadata: {
|
|
21
|
+
devrev_endpoint: 'devrev_endpoint',
|
|
22
|
+
},
|
|
23
|
+
context: {
|
|
24
|
+
secrets: {
|
|
25
|
+
service_account_token: 'service_account_token',
|
|
26
|
+
},
|
|
27
|
+
snap_in_version_id: 'snap_in_version_id',
|
|
28
|
+
},
|
|
29
|
+
payload: {
|
|
30
|
+
connection_data: {
|
|
31
|
+
org_id: 'org_id',
|
|
32
|
+
org_name: 'org_name',
|
|
33
|
+
key: 'key',
|
|
34
|
+
key_type: 'key_type',
|
|
35
|
+
},
|
|
36
|
+
event_context: {
|
|
37
|
+
mode: 'mode',
|
|
38
|
+
callback_url: 'callback_url',
|
|
39
|
+
dev_org_id: 'dev_org_id',
|
|
40
|
+
dev_user_id: 'dev_user_id',
|
|
41
|
+
external_system_id: 'external_system_id',
|
|
42
|
+
uuid: 'uuid',
|
|
43
|
+
sync_run_id: 'sync_run_id',
|
|
44
|
+
sync_unit_id: 'sync_unit_id',
|
|
45
|
+
worker_data_url: 'worker_data_url',
|
|
46
|
+
},
|
|
47
|
+
event_type: types_1.EventType.ExtractionDataStart,
|
|
48
|
+
},
|
|
49
|
+
input_data: {
|
|
50
|
+
global_values: {},
|
|
51
|
+
event_sources: {},
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
const initialState = {
|
|
55
|
+
customField: 'initial',
|
|
56
|
+
};
|
|
57
|
+
beforeEach(() => {
|
|
58
|
+
jest.clearAllMocks();
|
|
15
59
|
});
|
|
16
|
-
describe('
|
|
17
|
-
it('should
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
adapter.
|
|
22
|
-
expect(
|
|
23
|
-
});
|
|
24
|
-
it('should not add anything if artifact is not provided', () => {
|
|
25
|
-
const event = (0, test_helpers_1.createAirdropEvent)(types_1.EventType.ExtractionDataStart);
|
|
26
|
-
const adapter = new adapter_1.Adapter(event);
|
|
27
|
-
adapter.update({});
|
|
28
|
-
expect(adapter.getArtifacts()).toHaveLength(0);
|
|
60
|
+
describe('createAdapter', () => {
|
|
61
|
+
it('should create an Adapter instance', async () => {
|
|
62
|
+
const adapterState = new state_1.State(event, initialState);
|
|
63
|
+
mockedCreateAdapterState.mockResolvedValue(adapterState);
|
|
64
|
+
const adapter = await (0, adapter_1.createAdapter)(event, initialState, false);
|
|
65
|
+
expect(adapter).toBeInstanceOf(adapter_1.Adapter);
|
|
66
|
+
expect(mockedCreateAdapterState).toHaveBeenCalledWith(event, expect.anything());
|
|
29
67
|
});
|
|
30
68
|
});
|
|
31
|
-
describe('
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
69
|
+
describe('Adapter', () => {
|
|
70
|
+
let adapter;
|
|
71
|
+
let adapterState;
|
|
72
|
+
beforeEach(() => {
|
|
73
|
+
adapterState = new state_1.State(event, initialState);
|
|
74
|
+
adapter = new adapter_1.Adapter(event, adapterState, false);
|
|
75
|
+
});
|
|
76
|
+
it('should emit event and save state if event type is not stateless', async () => {
|
|
77
|
+
const data = {};
|
|
78
|
+
mockedAxios.post.mockResolvedValue({ data: {} });
|
|
79
|
+
await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone, data);
|
|
80
|
+
expect(mockedAxios.post).toHaveBeenCalledWith('callback_url', expect.objectContaining({
|
|
81
|
+
event_type: types_1.ExtractorEventType.ExtractionDataDone,
|
|
82
|
+
}), expect.any(Object));
|
|
83
|
+
expect(adapterState.postState).toHaveBeenCalledWith(adapter.state);
|
|
84
|
+
});
|
|
85
|
+
it('should not save state if event type is stateless', async () => {
|
|
86
|
+
const statelessEvent = Object.assign(Object.assign({}, event), { payload: Object.assign(Object.assign({}, event.payload), { event_type: types_1.EventType.ExtractionExternalSyncUnitsStart }) });
|
|
87
|
+
adapter = new adapter_1.Adapter(statelessEvent, adapterState, false);
|
|
88
|
+
const data = {};
|
|
89
|
+
mockedAxios.post.mockResolvedValue({ data: {} });
|
|
90
|
+
await adapter.emit(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, data);
|
|
91
|
+
expect(mockedAxios.post).toHaveBeenCalledWith('callback_url', expect.objectContaining({
|
|
92
|
+
event_type: types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone,
|
|
93
|
+
}), expect.any(Object));
|
|
94
|
+
expect(adapterState.postState).not.toHaveBeenCalled();
|
|
95
|
+
});
|
|
96
|
+
it('should exit adapter on heartbeat timeout', async () => {
|
|
97
|
+
mockedGetTimeoutExtractorEventType.mockReturnValue({
|
|
98
|
+
eventType: types_1.ExtractorEventType.ExtractionMetadataError,
|
|
99
|
+
isError: true,
|
|
100
|
+
});
|
|
101
|
+
jest
|
|
102
|
+
.spyOn(Date, 'now')
|
|
103
|
+
.mockImplementation(() => adapter['startTime'] + adapter['lambdaTimeout'] + 1);
|
|
104
|
+
const heartbeatResult = await adapter['heartbeat']();
|
|
105
|
+
expect(heartbeatResult).toBe(true);
|
|
106
|
+
expect(mockedGetTimeoutExtractorEventType).toHaveBeenCalledWith(event.payload.event_type);
|
|
107
|
+
expect(mockedAxios.post).toHaveBeenCalledWith('callback_url', expect.objectContaining({
|
|
108
|
+
event_type: types_1.ExtractorEventType.ExtractionMetadataError,
|
|
109
|
+
}), expect.any(Object));
|
|
110
|
+
});
|
|
111
|
+
it('should not emit event if adapter is in exit state', async () => {
|
|
112
|
+
adapter['exit'] = true;
|
|
113
|
+
await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone);
|
|
114
|
+
expect(mockedAxios.post).not.toHaveBeenCalled();
|
|
48
115
|
});
|
|
49
|
-
it('should
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
const data = {
|
|
55
|
-
error: { message: 'some error message' },
|
|
56
|
-
};
|
|
57
|
-
await adapter.emit(newEventType, data);
|
|
58
|
-
expect(axiosSpy).toHaveBeenCalledWith(event.payload.event_context.callback_url, expect.objectContaining({
|
|
59
|
-
event_type: newEventType,
|
|
60
|
-
event_data: expect.objectContaining({
|
|
61
|
-
error: data.error,
|
|
62
|
-
}),
|
|
63
|
-
}), expect.objectContaining({
|
|
64
|
-
headers: {
|
|
65
|
-
Accept: 'application/json, text/plain, */*',
|
|
66
|
-
Authorization: event.context.secrets["service_account_token"],
|
|
67
|
-
'Content-Type': 'application/json',
|
|
68
|
-
},
|
|
69
|
-
}));
|
|
70
|
-
axiosSpy.mockRestore();
|
|
116
|
+
it('should handle failed event emission gracefully', async () => {
|
|
117
|
+
mockedAxios.post.mockRejectedValue(new Error('Network error'));
|
|
118
|
+
await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone);
|
|
119
|
+
expect(mockedAxios.post).toHaveBeenCalled();
|
|
120
|
+
expect(adapter['exit']).toBe(true);
|
|
71
121
|
});
|
|
72
122
|
});
|
|
73
123
|
});
|
|
@@ -2,96 +2,60 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const types_1 = require("../src/types");
|
|
4
4
|
const adapter_1 = require("../src/adapter");
|
|
5
|
-
const uploader_1 = require("../src/uploader");
|
|
6
5
|
const demo_extractor_1 = require("../src/demo-extractor");
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
const mockEvent = {
|
|
7
|
+
execution_metadata: {
|
|
8
|
+
devrev_endpoint: 'devrev_endpoint',
|
|
9
|
+
},
|
|
10
|
+
context: {
|
|
11
|
+
secrets: {
|
|
12
|
+
service_account_token: 'service_account_token',
|
|
13
|
+
},
|
|
14
|
+
snap_in_version_id: 'snap_in_version_id',
|
|
15
|
+
},
|
|
16
|
+
payload: {
|
|
17
|
+
connection_data: {
|
|
18
|
+
org_id: 'org_id',
|
|
19
|
+
org_name: 'org_name',
|
|
20
|
+
key: 'key',
|
|
21
|
+
key_type: 'key_type',
|
|
22
|
+
},
|
|
23
|
+
event_context: {
|
|
24
|
+
mode: 'mode',
|
|
25
|
+
callback_url: 'callback_url',
|
|
26
|
+
dev_org_id: 'dev_org_id',
|
|
27
|
+
dev_user_id: 'dev_user_id',
|
|
28
|
+
external_system_id: 'external_system_id',
|
|
29
|
+
uuid: 'uuid',
|
|
30
|
+
sync_run_id: 'sync_run_id',
|
|
31
|
+
worker_data_url: 'worker_data_url',
|
|
32
|
+
},
|
|
33
|
+
event_type: types_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
34
|
+
},
|
|
35
|
+
input_data: {
|
|
36
|
+
global_values: {},
|
|
37
|
+
event_sources: {},
|
|
38
|
+
},
|
|
39
|
+
};
|
|
10
40
|
describe('DemoExtractor', () => {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
beforeEach(() => {
|
|
15
|
-
adapterMock = new adapter_1.Adapter({});
|
|
16
|
-
uploaderMock = new uploader_1.Uploader('', '');
|
|
17
|
-
adapter_1.Adapter.mockImplementation(() => adapterMock);
|
|
18
|
-
uploader_1.Uploader.mockImplementation(() => uploaderMock);
|
|
19
|
-
demoExtractor = new demo_extractor_1.DemoExtractor();
|
|
20
|
-
});
|
|
21
|
-
afterEach(() => {
|
|
22
|
-
jest.resetAllMocks();
|
|
23
|
-
});
|
|
24
|
-
it('should be able to create an instance of DemoExtractor', () => {
|
|
41
|
+
it('should create a new instance of the DemoExtractor', async () => {
|
|
42
|
+
const adapter = await (0, adapter_1.createAdapter)(mockEvent, {});
|
|
43
|
+
const demoExtractor = new demo_extractor_1.DemoExtractor(mockEvent, adapter);
|
|
25
44
|
expect(demoExtractor).toBeInstanceOf(demo_extractor_1.DemoExtractor);
|
|
26
45
|
});
|
|
27
|
-
it('should emit
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
46
|
+
it('should emit EXTRACTION_EXTERNAL_SYNC_UNITS_DONE with correct payload', async () => {
|
|
47
|
+
const adapter = await (0, adapter_1.createAdapter)(mockEvent, {});
|
|
48
|
+
const demoExtractor = new demo_extractor_1.DemoExtractor(mockEvent, adapter);
|
|
49
|
+
const spy = jest.spyOn(adapter, 'emit');
|
|
50
|
+
await demoExtractor.run();
|
|
51
|
+
expect(spy).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
|
|
31
52
|
external_sync_units: [
|
|
32
53
|
{
|
|
33
54
|
id: 'devrev',
|
|
34
55
|
name: 'devrev',
|
|
35
|
-
description: '
|
|
36
|
-
item_count: 0,
|
|
56
|
+
description: 'Demo external sync unit',
|
|
37
57
|
},
|
|
38
58
|
],
|
|
39
|
-
})
|
|
40
|
-
});
|
|
41
|
-
it('should emit ExtractionMetadataDone when metadata is uploaded successfully', async () => {
|
|
42
|
-
const event = (0, test_helpers_1.createAirdropEvent)(types_1.EventType.ExtractionMetadataStart);
|
|
43
|
-
const artifact = { id: 'id', item_type: 'item_type', item_count: 1 };
|
|
44
|
-
uploaderMock.upload.mockResolvedValueOnce({ artifact, error: undefined });
|
|
45
|
-
await demoExtractor.run(event);
|
|
46
|
-
expect(adapterMock.update).toHaveBeenCalledWith({ artifact });
|
|
47
|
-
expect(adapterMock.emit).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionMetadataDone);
|
|
48
|
-
});
|
|
49
|
-
it("should emit ExtractionMetadataError when there's an error uploading metadata", async () => {
|
|
50
|
-
const event = (0, test_helpers_1.createAirdropEvent)(types_1.EventType.ExtractionMetadataStart);
|
|
51
|
-
const error = new Error('Failed to upload metadata');
|
|
52
|
-
uploaderMock.upload.mockResolvedValueOnce({ artifact: undefined, error });
|
|
53
|
-
await demoExtractor.run(event);
|
|
54
|
-
expect(adapterMock.emit).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionMetadataError, { error });
|
|
55
|
-
});
|
|
56
|
-
it('should emit ExtractionDataProgress when data is uploaded successfully', async () => {
|
|
57
|
-
const event = (0, test_helpers_1.createAirdropEvent)(types_1.EventType.ExtractionDataStart);
|
|
58
|
-
const artifact = { id: 'id', item_type: 'item_type', item_count: 1 };
|
|
59
|
-
uploaderMock.upload.mockResolvedValueOnce({ artifact, error: undefined });
|
|
60
|
-
await demoExtractor.run(event);
|
|
61
|
-
expect(adapterMock.update).toHaveBeenCalledWith({ artifact });
|
|
62
|
-
expect(adapterMock.emit).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionDataProgress, { progress: 50 });
|
|
63
|
-
});
|
|
64
|
-
it('should emit ExtractionDataDone when data is uploaded successfully', async () => {
|
|
65
|
-
const event = (0, test_helpers_1.createAirdropEvent)(types_1.EventType.ExtractionDataContinue);
|
|
66
|
-
const artifact = { id: 'id', item_type: 'item_type', item_count: 1 };
|
|
67
|
-
uploaderMock.upload
|
|
68
|
-
.mockResolvedValueOnce({ artifact, error: undefined }) // First call to upload (users data)
|
|
69
|
-
.mockResolvedValueOnce({ artifact, error: undefined }); // Second call to upload (recipe.json)
|
|
70
|
-
await demoExtractor.run(event);
|
|
71
|
-
expect(adapterMock.update).toHaveBeenCalledTimes(2);
|
|
72
|
-
expect(adapterMock.update).toHaveBeenCalledWith({ artifact });
|
|
73
|
-
expect(adapterMock.emit).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionDataDone, { progress: 100 });
|
|
74
|
-
});
|
|
75
|
-
it("should emit ExtractionDataError when there's an error uploading data", async () => {
|
|
76
|
-
const event = (0, test_helpers_1.createAirdropEvent)(types_1.EventType.ExtractionDataStart);
|
|
77
|
-
const error = new Error('Failed to upload data');
|
|
78
|
-
uploaderMock.upload.mockResolvedValueOnce({ artifact: undefined, error });
|
|
79
|
-
await demoExtractor.run(event);
|
|
80
|
-
expect(adapterMock.emit).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionDataError, { error });
|
|
81
|
-
});
|
|
82
|
-
it('should emit ExtractionAttachmentsDone when attachments are uploaded successfully', async () => {
|
|
83
|
-
const event = (0, test_helpers_1.createAirdropEvent)(types_1.EventType.ExtractionAttachmentsStart);
|
|
84
|
-
const artifact = { id: 'id', item_type: 'item_type', item_count: 1 };
|
|
85
|
-
uploaderMock.upload.mockResolvedValueOnce({ artifact, error: undefined });
|
|
86
|
-
await demoExtractor.run(event);
|
|
87
|
-
expect(adapterMock.update).toHaveBeenCalledWith({ artifact });
|
|
88
|
-
expect(adapterMock.emit).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionAttachmentsProgress);
|
|
89
|
-
});
|
|
90
|
-
it("should emit ExtractionAttachmentsError when there's an error uploading attachments", async () => {
|
|
91
|
-
const event = (0, test_helpers_1.createAirdropEvent)(types_1.EventType.ExtractionAttachmentsStart);
|
|
92
|
-
const error = new Error('Failed to upload attachment');
|
|
93
|
-
uploaderMock.upload.mockResolvedValueOnce({ artifact: undefined, error });
|
|
94
|
-
await demoExtractor.run(event);
|
|
95
|
-
expect(adapterMock.emit).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionAttachmentsError, { error });
|
|
59
|
+
});
|
|
96
60
|
});
|
|
97
61
|
});
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const axios_1 = __importDefault(require("axios"));
|
|
7
|
+
const state_1 = require("../src/state");
|
|
8
|
+
const types_1 = require("../src/types");
|
|
9
|
+
const constants_1 = require("../src/common/constants");
|
|
10
|
+
jest.mock('axios');
|
|
11
|
+
const mockedAxios = axios_1.default;
|
|
12
|
+
describe('AdapterState', () => {
|
|
13
|
+
const event = {
|
|
14
|
+
execution_metadata: {
|
|
15
|
+
devrev_endpoint: 'devrev_endpoint',
|
|
16
|
+
},
|
|
17
|
+
context: {
|
|
18
|
+
secrets: {
|
|
19
|
+
service_account_token: 'service_account_token',
|
|
20
|
+
},
|
|
21
|
+
snap_in_version_id: 'snap_in_version_id',
|
|
22
|
+
},
|
|
23
|
+
payload: {
|
|
24
|
+
connection_data: {
|
|
25
|
+
org_id: 'org_id',
|
|
26
|
+
org_name: 'org_name',
|
|
27
|
+
key: 'key',
|
|
28
|
+
key_type: 'key_type',
|
|
29
|
+
},
|
|
30
|
+
event_context: {
|
|
31
|
+
mode: 'mode',
|
|
32
|
+
callback_url: 'callback_url',
|
|
33
|
+
dev_org_id: 'dev_org_id',
|
|
34
|
+
dev_user_id: 'dev_user_id',
|
|
35
|
+
external_system_id: 'external_system_id',
|
|
36
|
+
uuid: 'uuid',
|
|
37
|
+
sync_run_id: 'sync_run_id',
|
|
38
|
+
sync_unit_id: 'sync_unit_id',
|
|
39
|
+
worker_data_url: 'worker_data_url',
|
|
40
|
+
},
|
|
41
|
+
event_type: types_1.EventType.ExtractionDataStart,
|
|
42
|
+
},
|
|
43
|
+
input_data: {
|
|
44
|
+
global_values: {},
|
|
45
|
+
event_sources: {},
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
const initialState = {
|
|
49
|
+
customField: 'initial',
|
|
50
|
+
};
|
|
51
|
+
beforeEach(() => {
|
|
52
|
+
jest.clearAllMocks();
|
|
53
|
+
});
|
|
54
|
+
describe('createAdapterState', () => {
|
|
55
|
+
it('should create an AdapterState instance', async () => {
|
|
56
|
+
const adapterState = await (0, state_1.createAdapterState)(event, initialState);
|
|
57
|
+
expect(adapterState).toBeInstanceOf(state_1.State);
|
|
58
|
+
expect(adapterState.state).toEqual(Object.assign(Object.assign({}, initialState), { lastSyncStarted: '', lastSuccessfulSyncStarted: '' }));
|
|
59
|
+
});
|
|
60
|
+
it('should fetch state if event type is not stateless', async () => {
|
|
61
|
+
const spy = jest.spyOn(state_1.State.prototype, 'fetchState');
|
|
62
|
+
await (0, state_1.createAdapterState)(event, initialState);
|
|
63
|
+
expect(spy).toHaveBeenCalled();
|
|
64
|
+
});
|
|
65
|
+
it('should not fetch state if event type is stateless', async () => {
|
|
66
|
+
const statelessEvent = Object.assign(Object.assign({}, event), { payload: Object.assign(Object.assign({}, event.payload), { event_type: constants_1.STATELESS_EVENT_TYPES[0] }) });
|
|
67
|
+
const spy = jest.spyOn(state_1.State.prototype, 'fetchState');
|
|
68
|
+
await (0, state_1.createAdapterState)(statelessEvent, initialState);
|
|
69
|
+
expect(spy).not.toHaveBeenCalled();
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
describe('AdapterState', () => {
|
|
73
|
+
let adapterState;
|
|
74
|
+
beforeEach(() => {
|
|
75
|
+
adapterState = new state_1.State(event, initialState);
|
|
76
|
+
});
|
|
77
|
+
it('should initialize with given state', () => {
|
|
78
|
+
expect(adapterState.state).toEqual(Object.assign(Object.assign({}, initialState), { lastSyncStarted: '', lastSuccessfulSyncStarted: '' }));
|
|
79
|
+
});
|
|
80
|
+
it('should update state via postState', async () => {
|
|
81
|
+
const newState = Object.assign(Object.assign({}, initialState), { customField: 'updated' });
|
|
82
|
+
mockedAxios.post.mockResolvedValue({ data: {} });
|
|
83
|
+
await adapterState.postState(newState);
|
|
84
|
+
expect(adapterState.state).toEqual(newState);
|
|
85
|
+
expect(mockedAxios.post).toHaveBeenCalledWith('worker_data_url.update', { state: JSON.stringify(newState) }, {
|
|
86
|
+
headers: { Authorization: 'service_account_token' },
|
|
87
|
+
params: { sync_unit: 'sync_unit_id' },
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
it('should handle non-404 error in fetchState', async () => {
|
|
91
|
+
const error = new Error('Network error');
|
|
92
|
+
mockedAxios.post.mockRejectedValue(error);
|
|
93
|
+
const state = await adapterState.fetchState(initialState);
|
|
94
|
+
expect(state).toBe(error);
|
|
95
|
+
expect(mockedAxios.post).toHaveBeenCalledWith('worker_data_url.get', {}, {
|
|
96
|
+
headers: { Authorization: 'service_account_token' },
|
|
97
|
+
params: { sync_unit: 'sync_unit_id' },
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const uploader_1 = require("../src/uploader");
|
|
4
|
+
// mock uploader.upload method
|
|
5
|
+
jest.mock('../src/uploader', () => {
|
|
6
|
+
return {
|
|
7
|
+
Uploader: jest.fn().mockImplementation(() => {
|
|
8
|
+
return {
|
|
9
|
+
upload: jest.fn().mockResolvedValue({
|
|
10
|
+
artifact: { key: 'value' },
|
|
11
|
+
error: undefined,
|
|
12
|
+
}),
|
|
13
|
+
};
|
|
14
|
+
}),
|
|
15
|
+
};
|
|
16
|
+
});
|
|
17
|
+
describe('uploader.ts', () => {
|
|
18
|
+
const uploader = new uploader_1.Uploader('https://example.com', 'test-token', false);
|
|
19
|
+
it('should upload the file to the DevRev platform and return the artifact information', async () => {
|
|
20
|
+
const filename = 'filename';
|
|
21
|
+
const entity = 'entity';
|
|
22
|
+
const fetchedObjects = [{ key: 'value' }];
|
|
23
|
+
const uploadResponse = await uploader.upload(filename, entity, fetchedObjects);
|
|
24
|
+
expect(uploadResponse).toEqual({
|
|
25
|
+
artifact: { key: 'value' },
|
|
26
|
+
error: undefined,
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
});
|