@devrev/ts-adaas 0.0.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +182 -116
  2. package/dist/common/constants.d.ts +10 -0
  3. package/dist/common/constants.js +27 -0
  4. package/dist/common/control-protocol.d.ts +7 -0
  5. package/dist/common/control-protocol.js +41 -0
  6. package/dist/common/helpers.d.ts +4 -0
  7. package/dist/common/helpers.js +38 -0
  8. package/dist/common/install-initial-domain-mapping.d.ts +3 -0
  9. package/dist/common/install-initial-domain-mapping.js +87 -0
  10. package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
  11. package/dist/{src → deprecated}/adapter/index.js +13 -10
  12. package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
  13. package/dist/deprecated/common/helpers.js +47 -0
  14. package/dist/deprecated/demo-extractor/index.d.ts +17 -0
  15. package/dist/{src → deprecated}/demo-extractor/index.js +28 -22
  16. package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
  17. package/dist/{src → deprecated}/uploader/index.js +3 -1
  18. package/dist/{src/http → http}/client.d.ts +2 -1
  19. package/dist/index.d.ts +10 -0
  20. package/dist/{src/index.js → index.js} +11 -3
  21. package/dist/logger/logger.d.ts +14 -0
  22. package/dist/logger/logger.interfaces.d.ts +14 -0
  23. package/dist/logger/logger.interfaces.js +9 -0
  24. package/dist/logger/logger.js +91 -0
  25. package/dist/logger/logger.test.js +49 -0
  26. package/dist/repo/repo.d.ts +14 -0
  27. package/dist/repo/repo.interfaces.d.ts +43 -0
  28. package/dist/repo/repo.interfaces.js +2 -0
  29. package/dist/repo/repo.js +68 -0
  30. package/dist/repo/repo.test.js +74 -0
  31. package/dist/state/state.d.ts +24 -0
  32. package/dist/state/state.interfaces.d.ts +24 -0
  33. package/dist/state/state.interfaces.js +2 -0
  34. package/dist/state/state.js +115 -0
  35. package/dist/tests/test-helpers.d.ts +9 -0
  36. package/dist/tests/test-helpers.interfaces.d.ts +11 -0
  37. package/dist/tests/test-helpers.interfaces.js +2 -0
  38. package/dist/tests/test-helpers.js +88 -0
  39. package/dist/tests/test-worker.js +16 -0
  40. package/dist/types/common.d.ts +39 -0
  41. package/dist/{src/types → types}/common.js +4 -0
  42. package/dist/{src/types → types}/extraction.d.ts +56 -12
  43. package/dist/{src/types → types}/extraction.js +12 -29
  44. package/dist/types/index.d.ts +5 -0
  45. package/dist/types/index.js +11 -0
  46. package/dist/types/workers.d.ts +142 -0
  47. package/dist/types/workers.js +23 -0
  48. package/dist/uploader/uploader.d.ts +38 -0
  49. package/dist/uploader/uploader.interfaces.d.ts +63 -0
  50. package/dist/uploader/uploader.interfaces.js +2 -0
  51. package/dist/uploader/uploader.js +351 -0
  52. package/dist/{tests → uploader}/uploader.test.js +7 -5
  53. package/dist/workers/create-worker.d.ts +4 -0
  54. package/dist/workers/create-worker.js +30 -0
  55. package/dist/workers/create-worker.test.js +25 -0
  56. package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
  57. package/dist/workers/default-workers/attachments-deletion.js +13 -0
  58. package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
  59. package/dist/workers/default-workers/attachments-extraction.js +49 -0
  60. package/dist/workers/default-workers/data-deletion.d.ts +1 -0
  61. package/dist/workers/default-workers/data-deletion.js +15 -0
  62. package/dist/workers/default-workers/data-extraction.d.ts +1 -0
  63. package/dist/workers/default-workers/data-extraction.js +101 -0
  64. package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
  65. package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
  66. package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
  67. package/dist/workers/default-workers/metadata-extraction.js +26 -0
  68. package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
  69. package/dist/workers/dummy-extractor/data-normalization.js +41 -0
  70. package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
  71. package/dist/workers/process-task.d.ts +2 -0
  72. package/dist/workers/process-task.js +44 -0
  73. package/dist/workers/spawn.d.ts +25 -0
  74. package/dist/workers/spawn.js +163 -0
  75. package/dist/workers/worker-adapter.d.ts +48 -0
  76. package/dist/workers/worker-adapter.js +138 -0
  77. package/dist/workers/worker.d.ts +1 -0
  78. package/dist/workers/worker.js +6 -0
  79. package/package.json +9 -5
  80. package/dist/src/common/constants.d.ts +0 -2
  81. package/dist/src/common/constants.js +0 -10
  82. package/dist/src/common/helpers.js +0 -59
  83. package/dist/src/common/install-initial-domain-mapping.d.ts +0 -3
  84. package/dist/src/common/install-initial-domain-mapping.js +0 -60
  85. package/dist/src/demo-extractor/index.d.ts +0 -11
  86. package/dist/src/index.d.ts +0 -6
  87. package/dist/src/logging/index.d.ts +0 -18
  88. package/dist/src/logging/index.js +0 -39
  89. package/dist/src/state/index.d.ts +0 -23
  90. package/dist/src/state/index.js +0 -111
  91. package/dist/src/types/common.d.ts +0 -37
  92. package/dist/src/types/index.d.ts +0 -2
  93. package/dist/src/types/index.js +0 -18
  94. package/dist/tests/adapter.helpers.test.js +0 -60
  95. package/dist/tests/adapter.test.js +0 -123
  96. package/dist/tests/demo-extractor.test.js +0 -61
  97. package/dist/tests/state.test.js +0 -101
  98. /package/dist/{src → deprecated}/demo-extractor/external_domain_metadata.json +0 -0
  99. /package/dist/{src/http → http}/client.js +0 -0
  100. /package/dist/{src/http → http}/constants.d.ts +0 -0
  101. /package/dist/{src/http → http}/constants.js +0 -0
  102. /package/dist/{src/http → http}/index.d.ts +0 -0
  103. /package/dist/{src/http → http}/index.js +0 -0
  104. /package/dist/{src/http → http}/types.d.ts +0 -0
  105. /package/dist/{src/http → http}/types.js +0 -0
  106. /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
  107. /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
  108. /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
  109. /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
  110. /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
@@ -1,60 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const helpers_1 = require("../src/common/helpers");
4
- const types_1 = require("../src/types");
5
- describe('adapter.helpers.ts', () => {
6
- it("should create a FormData object with the correct 'key' and 'value'", () => {
7
- const preparedArtifact = {
8
- form_data: [{ key: 'key', value: 'value' }],
9
- };
10
- const fetchedObjects = [{ key: 'value' }];
11
- const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
12
- expect(formData).toBeInstanceOf(FormData);
13
- expect(formData.get('key')).toBe('value');
14
- });
15
- it("should create a FormData object with the correct 'file' key", () => {
16
- const preparedArtifact = {
17
- form_data: [{ key: 'key', value: 'value' }],
18
- };
19
- const fetchedObjects = [{ key: 'value' }];
20
- const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
21
- expect(formData.get('file')).toBeDefined();
22
- });
23
- it("should create an Artifact object with the correct 'item_count', 'id', and 'item_type'", () => {
24
- const preparedArtifact = { id: 'id' };
25
- const fetchedObjects = [{ key: 'value' }];
26
- const entity = 'entity';
27
- const artifact = (0, helpers_1.createArtifact)(preparedArtifact, fetchedObjects, entity);
28
- expect(artifact).toEqual({
29
- item_count: 1,
30
- id: 'id',
31
- item_type: 'entity',
32
- });
33
- });
34
- it('should return the correct ExtractorEventType', () => {
35
- expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionMetadataStart)).toStrictEqual({
36
- eventType: types_1.ExtractorEventType.ExtractionMetadataError,
37
- isError: true,
38
- });
39
- expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionDataStart)).toStrictEqual({
40
- eventType: types_1.ExtractorEventType.ExtractionDataProgress,
41
- isError: false,
42
- });
43
- expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionDataContinue)).toStrictEqual({
44
- eventType: types_1.ExtractorEventType.ExtractionDataProgress,
45
- isError: false,
46
- });
47
- expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionAttachmentsStart)).toStrictEqual({
48
- eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
49
- isError: false,
50
- });
51
- expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionAttachmentsContinue)).toStrictEqual({
52
- eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
53
- isError: false,
54
- });
55
- expect((0, helpers_1.getTimeoutExtractorEventType)(types_1.EventType.ExtractionExternalSyncUnitsStart)).toStrictEqual({
56
- eventType: types_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
57
- isError: true,
58
- });
59
- });
60
- });
@@ -1,123 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- const axios_1 = __importDefault(require("axios"));
7
- const adapter_1 = require("../src/adapter");
8
- const state_1 = require("../src/state");
9
- const types_1 = require("../src/types");
10
- const helpers_1 = require("../src/common/helpers");
11
- jest.mock('axios');
12
- jest.mock('../src/state');
13
- jest.mock('../src/common/helpers');
14
- jest.mock('../src/logging');
15
- const mockedAxios = axios_1.default;
16
- const mockedCreateAdapterState = state_1.createAdapterState;
17
- const mockedGetTimeoutExtractorEventType = helpers_1.getTimeoutExtractorEventType;
18
- describe('Adapter', () => {
19
- const event = {
20
- execution_metadata: {
21
- devrev_endpoint: 'devrev_endpoint',
22
- },
23
- context: {
24
- secrets: {
25
- service_account_token: 'service_account_token',
26
- },
27
- snap_in_version_id: 'snap_in_version_id',
28
- },
29
- payload: {
30
- connection_data: {
31
- org_id: 'org_id',
32
- org_name: 'org_name',
33
- key: 'key',
34
- key_type: 'key_type',
35
- },
36
- event_context: {
37
- mode: 'mode',
38
- callback_url: 'callback_url',
39
- dev_org_id: 'dev_org_id',
40
- dev_user_id: 'dev_user_id',
41
- external_system_id: 'external_system_id',
42
- uuid: 'uuid',
43
- sync_run_id: 'sync_run_id',
44
- sync_unit_id: 'sync_unit_id',
45
- worker_data_url: 'worker_data_url',
46
- },
47
- event_type: types_1.EventType.ExtractionDataStart,
48
- },
49
- input_data: {
50
- global_values: {},
51
- event_sources: {},
52
- },
53
- };
54
- const initialState = {
55
- customField: 'initial',
56
- };
57
- beforeEach(() => {
58
- jest.clearAllMocks();
59
- });
60
- describe('createAdapter', () => {
61
- it('should create an Adapter instance', async () => {
62
- const adapterState = new state_1.State(event, initialState);
63
- mockedCreateAdapterState.mockResolvedValue(adapterState);
64
- const adapter = await (0, adapter_1.createAdapter)(event, initialState, false);
65
- expect(adapter).toBeInstanceOf(adapter_1.Adapter);
66
- expect(mockedCreateAdapterState).toHaveBeenCalledWith(event, expect.anything());
67
- });
68
- });
69
- describe('Adapter', () => {
70
- let adapter;
71
- let adapterState;
72
- beforeEach(() => {
73
- adapterState = new state_1.State(event, initialState);
74
- adapter = new adapter_1.Adapter(event, adapterState, false);
75
- });
76
- it('should emit event and save state if event type is not stateless', async () => {
77
- const data = {};
78
- mockedAxios.post.mockResolvedValue({ data: {} });
79
- await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone, data);
80
- expect(mockedAxios.post).toHaveBeenCalledWith('callback_url', expect.objectContaining({
81
- event_type: types_1.ExtractorEventType.ExtractionDataDone,
82
- }), expect.any(Object));
83
- expect(adapterState.postState).toHaveBeenCalledWith(adapter.state);
84
- });
85
- it('should not save state if event type is stateless', async () => {
86
- const statelessEvent = Object.assign(Object.assign({}, event), { payload: Object.assign(Object.assign({}, event.payload), { event_type: types_1.EventType.ExtractionExternalSyncUnitsStart }) });
87
- adapter = new adapter_1.Adapter(statelessEvent, adapterState, false);
88
- const data = {};
89
- mockedAxios.post.mockResolvedValue({ data: {} });
90
- await adapter.emit(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, data);
91
- expect(mockedAxios.post).toHaveBeenCalledWith('callback_url', expect.objectContaining({
92
- event_type: types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone,
93
- }), expect.any(Object));
94
- expect(adapterState.postState).not.toHaveBeenCalled();
95
- });
96
- it('should exit adapter on heartbeat timeout', async () => {
97
- mockedGetTimeoutExtractorEventType.mockReturnValue({
98
- eventType: types_1.ExtractorEventType.ExtractionMetadataError,
99
- isError: true,
100
- });
101
- jest
102
- .spyOn(Date, 'now')
103
- .mockImplementation(() => adapter['startTime'] + adapter['lambdaTimeout'] + 1);
104
- const heartbeatResult = await adapter['heartbeat']();
105
- expect(heartbeatResult).toBe(true);
106
- expect(mockedGetTimeoutExtractorEventType).toHaveBeenCalledWith(event.payload.event_type);
107
- expect(mockedAxios.post).toHaveBeenCalledWith('callback_url', expect.objectContaining({
108
- event_type: types_1.ExtractorEventType.ExtractionMetadataError,
109
- }), expect.any(Object));
110
- });
111
- it('should not emit event if adapter is in exit state', async () => {
112
- adapter['exit'] = true;
113
- await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone);
114
- expect(mockedAxios.post).not.toHaveBeenCalled();
115
- });
116
- it('should handle failed event emission gracefully', async () => {
117
- mockedAxios.post.mockRejectedValue(new Error('Network error'));
118
- await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone);
119
- expect(mockedAxios.post).toHaveBeenCalled();
120
- expect(adapter['exit']).toBe(true);
121
- });
122
- });
123
- });
@@ -1,61 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const types_1 = require("../src/types");
4
- const adapter_1 = require("../src/adapter");
5
- const demo_extractor_1 = require("../src/demo-extractor");
6
- const mockEvent = {
7
- execution_metadata: {
8
- devrev_endpoint: 'devrev_endpoint',
9
- },
10
- context: {
11
- secrets: {
12
- service_account_token: 'service_account_token',
13
- },
14
- snap_in_version_id: 'snap_in_version_id',
15
- },
16
- payload: {
17
- connection_data: {
18
- org_id: 'org_id',
19
- org_name: 'org_name',
20
- key: 'key',
21
- key_type: 'key_type',
22
- },
23
- event_context: {
24
- mode: 'mode',
25
- callback_url: 'callback_url',
26
- dev_org_id: 'dev_org_id',
27
- dev_user_id: 'dev_user_id',
28
- external_system_id: 'external_system_id',
29
- uuid: 'uuid',
30
- sync_run_id: 'sync_run_id',
31
- worker_data_url: 'worker_data_url',
32
- },
33
- event_type: types_1.EventType.ExtractionExternalSyncUnitsStart,
34
- },
35
- input_data: {
36
- global_values: {},
37
- event_sources: {},
38
- },
39
- };
40
- describe('DemoExtractor', () => {
41
- it('should create a new instance of the DemoExtractor', async () => {
42
- const adapter = await (0, adapter_1.createAdapter)(mockEvent, {});
43
- const demoExtractor = new demo_extractor_1.DemoExtractor(mockEvent, adapter);
44
- expect(demoExtractor).toBeInstanceOf(demo_extractor_1.DemoExtractor);
45
- });
46
- it('should emit EXTRACTION_EXTERNAL_SYNC_UNITS_DONE with correct payload', async () => {
47
- const adapter = await (0, adapter_1.createAdapter)(mockEvent, {});
48
- const demoExtractor = new demo_extractor_1.DemoExtractor(mockEvent, adapter);
49
- const spy = jest.spyOn(adapter, 'emit');
50
- await demoExtractor.run();
51
- expect(spy).toHaveBeenCalledWith(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
52
- external_sync_units: [
53
- {
54
- id: 'devrev',
55
- name: 'devrev',
56
- description: 'Demo external sync unit',
57
- },
58
- ],
59
- });
60
- });
61
- });
@@ -1,101 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- const axios_1 = __importDefault(require("axios"));
7
- const state_1 = require("../src/state");
8
- const types_1 = require("../src/types");
9
- const constants_1 = require("../src/common/constants");
10
- jest.mock('axios');
11
- const mockedAxios = axios_1.default;
12
- describe('AdapterState', () => {
13
- const event = {
14
- execution_metadata: {
15
- devrev_endpoint: 'devrev_endpoint',
16
- },
17
- context: {
18
- secrets: {
19
- service_account_token: 'service_account_token',
20
- },
21
- snap_in_version_id: 'snap_in_version_id',
22
- },
23
- payload: {
24
- connection_data: {
25
- org_id: 'org_id',
26
- org_name: 'org_name',
27
- key: 'key',
28
- key_type: 'key_type',
29
- },
30
- event_context: {
31
- mode: 'mode',
32
- callback_url: 'callback_url',
33
- dev_org_id: 'dev_org_id',
34
- dev_user_id: 'dev_user_id',
35
- external_system_id: 'external_system_id',
36
- uuid: 'uuid',
37
- sync_run_id: 'sync_run_id',
38
- sync_unit_id: 'sync_unit_id',
39
- worker_data_url: 'worker_data_url',
40
- },
41
- event_type: types_1.EventType.ExtractionDataStart,
42
- },
43
- input_data: {
44
- global_values: {},
45
- event_sources: {},
46
- },
47
- };
48
- const initialState = {
49
- customField: 'initial',
50
- };
51
- beforeEach(() => {
52
- jest.clearAllMocks();
53
- });
54
- describe('createAdapterState', () => {
55
- it('should create an AdapterState instance', async () => {
56
- const adapterState = await (0, state_1.createAdapterState)(event, initialState);
57
- expect(adapterState).toBeInstanceOf(state_1.State);
58
- expect(adapterState.state).toEqual(Object.assign(Object.assign({}, initialState), { lastSyncStarted: '', lastSuccessfulSyncStarted: '' }));
59
- });
60
- it('should fetch state if event type is not stateless', async () => {
61
- const spy = jest.spyOn(state_1.State.prototype, 'fetchState');
62
- await (0, state_1.createAdapterState)(event, initialState);
63
- expect(spy).toHaveBeenCalled();
64
- });
65
- it('should not fetch state if event type is stateless', async () => {
66
- const statelessEvent = Object.assign(Object.assign({}, event), { payload: Object.assign(Object.assign({}, event.payload), { event_type: constants_1.STATELESS_EVENT_TYPES[0] }) });
67
- const spy = jest.spyOn(state_1.State.prototype, 'fetchState');
68
- await (0, state_1.createAdapterState)(statelessEvent, initialState);
69
- expect(spy).not.toHaveBeenCalled();
70
- });
71
- });
72
- describe('AdapterState', () => {
73
- let adapterState;
74
- beforeEach(() => {
75
- adapterState = new state_1.State(event, initialState);
76
- });
77
- it('should initialize with given state', () => {
78
- expect(adapterState.state).toEqual(Object.assign(Object.assign({}, initialState), { lastSyncStarted: '', lastSuccessfulSyncStarted: '' }));
79
- });
80
- it('should update state via postState', async () => {
81
- const newState = Object.assign(Object.assign({}, initialState), { customField: 'updated' });
82
- mockedAxios.post.mockResolvedValue({ data: {} });
83
- await adapterState.postState(newState);
84
- expect(adapterState.state).toEqual(newState);
85
- expect(mockedAxios.post).toHaveBeenCalledWith('worker_data_url.update', { state: JSON.stringify(newState) }, {
86
- headers: { Authorization: 'service_account_token' },
87
- params: { sync_unit: 'sync_unit_id' },
88
- });
89
- });
90
- it('should handle non-404 error in fetchState', async () => {
91
- const error = new Error('Network error');
92
- mockedAxios.post.mockRejectedValue(error);
93
- const state = await adapterState.fetchState(initialState);
94
- expect(state).toBe(error);
95
- expect(mockedAxios.post).toHaveBeenCalledWith('worker_data_url.get', {}, {
96
- headers: { Authorization: 'service_account_token' },
97
- params: { sync_unit: 'sync_unit_id' },
98
- });
99
- });
100
- });
101
- });
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes