@devrev/ts-adaas 0.0.3 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +186 -116
  2. package/dist/common/constants.d.ts +10 -0
  3. package/dist/common/constants.js +27 -0
  4. package/dist/common/control-protocol.d.ts +7 -0
  5. package/dist/common/control-protocol.js +41 -0
  6. package/dist/common/helpers.d.ts +4 -0
  7. package/dist/common/helpers.js +38 -0
  8. package/dist/common/install-initial-domain-mapping.d.ts +3 -0
  9. package/dist/common/install-initial-domain-mapping.js +87 -0
  10. package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
  11. package/dist/{src → deprecated}/adapter/index.js +13 -10
  12. package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
  13. package/dist/deprecated/common/helpers.js +47 -0
  14. package/dist/deprecated/demo-extractor/index.d.ts +17 -0
  15. package/dist/{src → deprecated}/demo-extractor/index.js +28 -22
  16. package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
  17. package/dist/{src → deprecated}/uploader/index.js +3 -1
  18. package/dist/{src/http → http}/client.d.ts +2 -1
  19. package/dist/index.d.ts +10 -0
  20. package/dist/{src/index.js → index.js} +11 -3
  21. package/dist/logger/logger.d.ts +14 -0
  22. package/dist/logger/logger.interfaces.d.ts +14 -0
  23. package/dist/logger/logger.interfaces.js +9 -0
  24. package/dist/logger/logger.js +91 -0
  25. package/dist/logger/logger.test.js +49 -0
  26. package/dist/repo/repo.d.ts +14 -0
  27. package/dist/repo/repo.interfaces.d.ts +43 -0
  28. package/dist/repo/repo.interfaces.js +2 -0
  29. package/dist/repo/repo.js +68 -0
  30. package/dist/repo/repo.test.js +74 -0
  31. package/dist/state/state.d.ts +24 -0
  32. package/dist/state/state.interfaces.d.ts +24 -0
  33. package/dist/state/state.interfaces.js +2 -0
  34. package/dist/state/state.js +115 -0
  35. package/dist/tests/test-helpers.d.ts +9 -0
  36. package/dist/tests/test-helpers.interfaces.d.ts +11 -0
  37. package/dist/tests/test-helpers.interfaces.js +2 -0
  38. package/dist/tests/test-helpers.js +86 -0
  39. package/dist/tests/test-worker.js +16 -0
  40. package/dist/types/common.d.ts +38 -0
  41. package/dist/{src/types → types}/common.js +4 -0
  42. package/dist/{src/types → types}/extraction.d.ts +56 -14
  43. package/dist/{src/types → types}/extraction.js +12 -29
  44. package/dist/types/index.d.ts +5 -0
  45. package/dist/types/index.js +11 -0
  46. package/dist/types/workers.d.ts +142 -0
  47. package/dist/types/workers.js +23 -0
  48. package/dist/uploader/uploader.d.ts +38 -0
  49. package/dist/uploader/uploader.interfaces.d.ts +63 -0
  50. package/dist/uploader/uploader.interfaces.js +2 -0
  51. package/dist/uploader/uploader.js +351 -0
  52. package/dist/{tests → uploader}/uploader.test.js +7 -5
  53. package/dist/workers/create-worker.d.ts +4 -0
  54. package/dist/workers/create-worker.js +30 -0
  55. package/dist/workers/create-worker.test.js +25 -0
  56. package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
  57. package/dist/workers/default-workers/attachments-deletion.js +13 -0
  58. package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
  59. package/dist/workers/default-workers/attachments-extraction.js +49 -0
  60. package/dist/workers/default-workers/data-deletion.d.ts +1 -0
  61. package/dist/workers/default-workers/data-deletion.js +15 -0
  62. package/dist/workers/default-workers/data-extraction.d.ts +1 -0
  63. package/dist/workers/default-workers/data-extraction.js +101 -0
  64. package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
  65. package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
  66. package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
  67. package/dist/workers/default-workers/metadata-extraction.js +26 -0
  68. package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
  69. package/dist/workers/dummy-extractor/data-normalization.js +41 -0
  70. package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
  71. package/dist/workers/process-task.d.ts +2 -0
  72. package/dist/workers/process-task.js +44 -0
  73. package/dist/workers/spawn.d.ts +25 -0
  74. package/dist/workers/spawn.js +164 -0
  75. package/dist/workers/worker-adapter.d.ts +48 -0
  76. package/dist/workers/worker-adapter.js +138 -0
  77. package/dist/workers/worker.d.ts +1 -0
  78. package/dist/workers/worker.js +6 -0
  79. package/package.json +9 -5
  80. package/dist/src/common/constants.d.ts +0 -2
  81. package/dist/src/common/constants.js +0 -10
  82. package/dist/src/common/helpers.js +0 -59
  83. package/dist/src/common/install-initial-domain-mapping.d.ts +0 -3
  84. package/dist/src/common/install-initial-domain-mapping.js +0 -60
  85. package/dist/src/demo-extractor/index.d.ts +0 -11
  86. package/dist/src/index.d.ts +0 -6
  87. package/dist/src/logging/index.d.ts +0 -18
  88. package/dist/src/logging/index.js +0 -39
  89. package/dist/src/state/index.d.ts +0 -23
  90. package/dist/src/state/index.js +0 -111
  91. package/dist/src/types/common.d.ts +0 -37
  92. package/dist/src/types/index.d.ts +0 -2
  93. package/dist/src/types/index.js +0 -18
  94. package/dist/tests/adapter.helpers.test.js +0 -60
  95. package/dist/tests/adapter.test.js +0 -123
  96. package/dist/tests/demo-extractor.test.js +0 -61
  97. package/dist/tests/state.test.js +0 -101
  98. /package/dist/{src → deprecated}/demo-extractor/external_domain_metadata.json +0 -0
  99. /package/dist/{src/http → http}/client.js +0 -0
  100. /package/dist/{src/http → http}/constants.d.ts +0 -0
  101. /package/dist/{src/http → http}/constants.js +0 -0
  102. /package/dist/{src/http → http}/index.d.ts +0 -0
  103. /package/dist/{src/http → http}/index.js +0 -0
  104. /package/dist/{src/http → http}/types.d.ts +0 -0
  105. /package/dist/{src/http → http}/types.js +0 -0
  106. /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
  107. /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
  108. /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
  109. /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
  110. /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
package/README.md CHANGED
@@ -1,12 +1,18 @@
1
1
  # ADaaS Library
2
2
 
3
- Typescript ADaaS Library (@devrev/ts-adaas) provides:
3
+ ## Release Notes
4
4
 
5
- - type definitions for ADaaS control protocol,
6
- - an adapter for ADaaS control protocol,
7
- - helpers for uploading artifacts and manage the state for ADaaS snap-in.
5
+ #### v1.0.1
8
6
 
9
- ## Release Notes
7
+ - Bug fixes and improvements in logging.
8
+
9
+ #### v1.0.0
10
+
11
+ - Allow extractions to use full lambda runtime and gracefully handle execution context timeout.
12
+ - Simplified metadata and data normalization and uploading with repo implementation.
13
+ - Default handling of attachment extraction phase in ADaaS SDK library.
14
+ - Reduced file size, streamlined process by gzip compression.
15
+ - Bug fixes and improvements in error handling.
10
16
 
11
17
  #### v0.0.3
12
18
 
@@ -25,155 +31,219 @@ Typescript ADaaS Library (@devrev/ts-adaas) provides:
25
31
  - Adapter for ADaaS control protocol with helper functions
26
32
  - Uploader for uploading artifacts
27
33
 
28
- ## Usage
29
-
30
- Create a new ADaaS adapter on each ADaaS snap-in invocation:
34
+ # Overview
31
35
 
32
- ```javascript
33
- const adapter = new Adapter(event: AirdropEvent);
34
- ```
36
+ The ADaaS (Airdrop-as-a-Service) Library for TypeScript helps developers build Snap-ins that integrate with DevRev’s ADaaS platform. This library simplifies the workflow for handling data extraction, event-driven actions, state management, and artifact handling.
35
37
 
36
- Adapter class provides:
38
+ ## Features
37
39
 
38
- - helper function to emit response,
39
- - automatic emit event if ADaaS snap-in invocation runs out of time,
40
- - setter for updating ADaaS snap-in state and adding artifacts to the return ADaaS message.
40
+ - Type Definitions: Structured types for ADaaS control protocol
41
+ - Event Management: Easily emit events for different extraction phases
42
+ - State Handling: Update and access state in real-time within tasks
43
+ - Artifact Management: Supports batched storage of artifacts (2000 items per batch)
44
+ - Error & Timeout Support: Error handling and timeout management for long-running tasks
41
45
 
42
- ### Phases of Airdrop Extraction
46
+ # Installation
43
47
 
44
- Each ADaaS snap-in must handle all the phases of ADaaS extraction.
48
+ ```bash
49
+ npm install @devrev/ts-adaas
50
+ ```
45
51
 
46
- ADaaS library provides type definitions to ensure ADaaS snap-ins are compatible with ADaaS control protocol.
52
+ # Usage
47
53
 
48
- ```javascript
49
- async run() {
50
- switch (this.event.payload.event_type) {
51
- case EventType.ExtractionExternalSyncUnitsStart: {
54
+ ADaaS Snap-ins are composed of several phases, each with unique requirements for initialization, data extraction, and error handling. The ADaaS library exports processTask to structure the work within each phase. The processTask function accepts task and onTimeout handlers, giving access to the adapter to streamline state updates, upload of extracted data, and event emission.
52
55
 
53
- // extract available External Sync Units (projects, organizations, ...)
56
+ ### ADaaS Snap-in Invocation
54
57
 
55
- await this.adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsDone, {
56
- external_sync_units: externalSyncUnits,
57
- });
58
- break;
59
- }
58
+ Each ADaaS snap-in must handle all the phases of ADaaS extraction. In a Snap-in, you typically define a `run` function that iterates over events and invokes workers per extraction phase.
60
59
 
61
- case EventType.ExtractionMetadataStart: {
60
+ ```typescript
61
+ import { AirdropEvent, EventType, spawn } from '@devrev/ts-adaas';
62
62
 
63
- // provide mappings of domain objects by provioding initial_domain_mapping.json file
64
- // update ADaaS snap-in state
63
+ interface DummyExtractorState {
64
+ issues: { completed: boolean };
65
+ users: { completed: boolean };
66
+ attachments: { completed: boolean };
67
+ }
65
68
 
66
- await this.adapter.emit(ExtractorEventType.ExtractionMetadataDone);
69
+ const initialState: DummyExtractorState = {
70
+ issues: { completed: false },
71
+ users: { completed: false },
72
+ attachments: { completed: false },
73
+ };
74
+
75
+ function getWorkerPerExtractionPhase(event: AirdropEvent) {
76
+ let path;
77
+ switch (event.payload.event_type) {
78
+ case EventType.ExtractionExternalSyncUnitsStart:
79
+ path = __dirname + '/workers/external-sync-units-extraction';
67
80
  break;
68
- }
69
-
70
- case EventType.ExtractionDataStart: {
71
-
72
- // extract Data
73
- // upload Data
74
- // update ADaaS snap-in state
75
- // approximate progress done
76
-
77
- await this.adapter.emit(ExtractorEventType.ExtractionDataContinue, {
78
- progress: 10,
79
- });
80
-
81
+ case EventType.ExtractionMetadataStart:
82
+ path = __dirname + '/workers/metadata-extraction';
81
83
  break;
82
- }
83
-
84
- case EventType.ExtractionDataContinue: {
85
- await this.processExtractionData();
86
-
87
- // extract Data
88
- // upload Data
89
- // update ADaaS snap-in state
90
- // approximate progress done
91
-
92
- await this.adapter.emit(ExtractorEventType.ExtractionDataDone, {
93
- progress: 100,
94
- });
84
+ case EventType.ExtractionDataStart:
85
+ case EventType.ExtractionDataContinue:
86
+ path = __dirname + '/workers/data-extraction';
95
87
  break;
96
- }
97
-
98
- case EventType.ExtractionDataDelete: {
88
+ }
89
+ return path;
90
+ }
99
91
 
100
- // if an extraction has any side-effects to 3rd party systems cleanup should be done here.
92
+ const run = async (events: AirdropEvent[]) => {
93
+ for (const event of events) {
94
+ const file = getWorkerPerExtractionPhase(event);
95
+ await spawn<DummyExtractorState>({
96
+ event,
97
+ initialState,
98
+ workerPath: file,
99
+ options: {
100
+ isLocalDevelopment: true,
101
+ },
102
+ });
103
+ }
104
+ };
101
105
 
102
- await this.adapter.emit(ExtractorEventType.ExtractionDataDeleteDone);
103
- break;
104
- }
106
+ export default run;
107
+ ```
105
108
 
106
- case EventType.ExtractionAttachmentsStart: {
109
+ ## Extraction Phases
107
110
 
108
- // extract Attachments
109
- // upload Attachments
110
- // update ADaaS snap-in state
111
+ The ADaaS snap-in extraction lifecycle consists of three main phases: External Sync Units Extraction, Metadata Extraction, and Data Extraction. Each phase is defined in a separate file and is responsible for fetching the respective data.
111
112
 
112
- await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsContinue);
113
- break;
114
- }
113
+ ### 1. External Sync Units Extraction
115
114
 
116
- case EventType.ExtractionAttachmentsContinue: {
115
+ This phase is defined in `external-sync-units-extraction.ts` and is responsible for fetching the external sync units.
117
116
 
117
+ ```typescript
118
+ import {
119
+ ExternalSyncUnit,
120
+ ExtractorEventType,
121
+ processTask,
122
+ } from '@devrev/ts-adaas';
123
+
124
+ const externalSyncUnits: ExternalSyncUnit[] = [
125
+ {
126
+ id: 'devrev',
127
+ name: 'devrev',
128
+ description: 'Demo external sync unit',
129
+ item_count: 2,
130
+ item_type: 'issues',
131
+ },
132
+ ];
133
+
134
+ processTask({
135
+ task: async ({ adapter }) => {
136
+ await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsDone, {
137
+ external_sync_units: externalSyncUnits,
138
+ });
139
+ },
140
+ onTimeout: async ({ adapter }) => {
141
+ await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsError, {
142
+ error: {
143
+ message: 'Failed to extract external sync units. Lambda timeout.',
144
+ },
145
+ });
146
+ },
147
+ });
148
+ ```
118
149
 
119
- // extract Attachments
120
- // upload Attachments
121
- // update ADaaS snap-in state
150
+ ### 2. Metadata Extraction
122
151
 
123
- await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDone);
124
- break;
125
- }
152
+ This phase is defined in `metadata-extraction.ts` and is responsible for fetching the metadata.
126
153
 
127
- case EventType.ExtractionAttachmentsDelete: {
154
+ ```typescript
155
+ import { ExtractorEventType, processTask } from '@devrev/ts-adaas';
156
+ import externalDomainMetadata from '../dummy-extractor/external_domain_metadata.json';
157
+
158
+ const repos = [{ itemType: 'external_domain_metadata' }];
159
+
160
+ processTask({
161
+ task: async ({ adapter }) => {
162
+ adapter.initializeRepos(repos);
163
+ await adapter
164
+ .getRepo('external_domain_metadata')
165
+ ?.push([externalDomainMetadata]);
166
+ await adapter.emit(ExtractorEventType.ExtractionMetadataDone);
167
+ },
168
+ onTimeout: async ({ adapter }) => {
169
+ await adapter.emit(ExtractorEventType.ExtractionMetadataError, {
170
+ error: { message: 'Failed to extract metadata. Lambda timeout.' },
171
+ });
172
+ },
173
+ });
174
+ ```
128
175
 
129
- // if an extraction has any side-effects to 3rd party systems cleanup should be done here.
176
+ ### 3. Data Extraction
130
177
 
131
- await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDeleteDone);
132
- break;
133
- }
178
+ This phase is defined in `data-extraction.ts` and is responsible for fetching the data. In this phase also attachments metadata is extracted.
134
179
 
135
- default: {
136
- console.log('Event not supported' + JSON.stringify(this.event));
180
+ ```typescript
181
+ import { EventType, ExtractorEventType, processTask } from '@devrev/ts-adaas';
182
+ import { normalizeAttachment, normalizeIssue, normalizeUser } from '../dummy-extractor/data-normalization';
183
+
184
+ const issues = [
185
+ { id: 'issue-1', created_date: '1999-12-25T01:00:03+01:00', ... },
186
+ { id: 'issue-2', created_date: '1999-12-27T15:31:34+01:00', ... },
187
+ ];
188
+
189
+ const users = [
190
+ { id: 'user-1', created_date: '1999-12-25T01:00:03+01:00', ... },
191
+ { id: 'user-2', created_date: '1999-12-27T15:31:34+01:00', ... },
192
+ ];
193
+
194
+ const attachments = [
195
+ { url: 'https://app.dev.devrev-eng.ai/favicon.ico', id: 'attachment-1', ... },
196
+ { url: 'https://app.dev.devrev-eng.ai/favicon.ico', id: 'attachment-2', ... },
197
+ ];
198
+
199
+ const repos = [
200
+ { itemType: 'issues', normalize: normalizeIssue },
201
+ { itemType: 'users', normalize: normalizeUser },
202
+ { itemType: 'attachments', normalize: normalizeAttachment },
203
+ ];
204
+
205
+ processTask({
206
+ task: async ({ adapter }) => {
207
+ adapter.initializeRepos(repos);
208
+
209
+ if (adapter.event.payload.event_type === EventType.ExtractionDataStart) {
210
+ await adapter.getRepo('issues')?.push(issues);
211
+ await adapter.emit(ExtractorEventType.ExtractionDataProgress, { progress: 50 });
212
+ } else {
213
+ await adapter.getRepo('users')?.push(users);
214
+ await adapter.getRepo('attachments')?.push(attachments);
215
+ await adapter.emit(ExtractorEventType.ExtractionDataDone, { progress: 100 });
137
216
  }
138
- }
139
- }
217
+ },
218
+ onTimeout: async ({ adapter }) => {
219
+ await adapter.postState();
220
+ await adapter.emit(ExtractorEventType.ExtractionDataProgress, { progress: 50 });
221
+ },
222
+ });
140
223
  ```
141
224
 
142
- ## Uploading artifacts
225
+ ## 4. Attachments Streaming
143
226
 
144
- Create a new Uploader class for uploading artifacts:
227
+ The ADaaS library handles attachments streaming to improve efficiency and reduce complexity for developers. During the extraction phase, developers need only to provide metadata in a specific format for each attachment, and the library manages the streaming process.
145
228
 
146
- ```javascript
147
- const upload = new Uploader(
148
- event.execution_metadata.devrev_endpoint,
149
- event.context.secrets.service_account_token
150
- );
151
- ```
229
+ The Snap-in should provide attachment metadata following the `NormalizedAttachment` interface:
152
230
 
153
- Files with extracted domain objects must be in JSONL (JSON Lines) format. Data files should contain 2000 - 5000 records each.
154
-
155
- ```javascript
156
- const entity = 'users';
157
- const { artifact, error } = await this.uploader.upload(
158
- `extractor_${entity}_${i}.jsonl`,
159
- entity,
160
- data
161
- );
162
- if (error) {
163
- return error;
164
- } else {
165
- await this.adapter.update({ artifact });
231
+ ```typescript
232
+ export interface NormalizedAttachment {
233
+ url: string;
234
+ id: string;
235
+ file_name: string;
236
+ author_id: string;
237
+ parent_id: string;
166
238
  }
167
239
  ```
168
240
 
169
- Each uploaded file must be attached to ADaaS adapter as soon as it is uploaded to ensure it is included in the ADaaS response message in case of a lambda timeout.
241
+ ## Artifact Uploading and State Management
170
242
 
171
- ## Updating ADaaS snap-in state
243
+ The ADaaS library provides a repository management system to handle artifacts in batches. The `initializeRepos` function initializes the repositories, and the `push` function uploads the artifacts to the repositories. The `postState` function is used to post the state of the extraction task.
172
244
 
173
- ADaaS snap-ins keep their own state between sync runs, between the states of a particular sync run and between invocations within a particular state.
245
+ State management is crucial for ADaaS Snap-ins to maintain the state of the extraction task. The `postState` function is used to post the state of the extraction task. The state is stored in the adapter and can be retrieved using the `adapter.state` property.
174
246
 
175
- By managing its own state, the ADaaS snap-in keeps track of the process of extraction (what items have already been extracted and where to continue), the times of the last successful sync run and keeps record of progress of the extraction.
247
+ ## Timeout Handling
176
248
 
177
- ```typescript
178
- async update({ artifacts, extractor_state}: AdapterUpdateParams)
179
- ```
249
+ The ADaaS library provides a timeout handler to handle timeouts in long-running tasks. The `onTimeout` handler is called when the task exceeds the timeout limit. The handler can be used to post the state of the extraction task and emit an event when a timeout occurs.
@@ -0,0 +1,10 @@
1
+ import { EventType } from '../types/extraction';
2
+ export declare const STATELESS_EVENT_TYPES: EventType[];
3
+ export declare const ALLOWED_EVENT_TYPES: EventType[];
4
+ export declare const ARTIFACT_BATCH_SIZE = 2000;
5
+ export declare const MAX_DEVREV_ARTIFACT_SIZE = 536870912;
6
+ export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
7
+ EXTERNAL_DOMAIN_METADATA: string;
8
+ ATTACHMENTS: string;
9
+ SSOR_ATTACHMENT: string;
10
+ };
@@ -0,0 +1,27 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.ALLOWED_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = void 0;
4
+ const extraction_1 = require("../types/extraction");
5
+ exports.STATELESS_EVENT_TYPES = [
6
+ extraction_1.EventType.ExtractionExternalSyncUnitsStart,
7
+ extraction_1.EventType.ExtractionMetadataStart,
8
+ extraction_1.EventType.ExtractionDataDelete,
9
+ extraction_1.EventType.ExtractionAttachmentsDelete,
10
+ ];
11
+ exports.ALLOWED_EVENT_TYPES = [
12
+ extraction_1.EventType.ExtractionExternalSyncUnitsStart,
13
+ extraction_1.EventType.ExtractionMetadataStart,
14
+ extraction_1.EventType.ExtractionDataStart,
15
+ extraction_1.EventType.ExtractionDataContinue,
16
+ extraction_1.EventType.ExtractionDataDelete,
17
+ extraction_1.EventType.ExtractionAttachmentsStart,
18
+ extraction_1.EventType.ExtractionAttachmentsContinue,
19
+ extraction_1.EventType.ExtractionAttachmentsDelete,
20
+ ];
21
+ exports.ARTIFACT_BATCH_SIZE = 2000;
22
+ exports.MAX_DEVREV_ARTIFACT_SIZE = 536870912; // 512MB
23
+ exports.AIRDROP_DEFAULT_ITEM_TYPES = {
24
+ EXTERNAL_DOMAIN_METADATA: 'external_domain_metadata',
25
+ ATTACHMENTS: 'attachments',
26
+ SSOR_ATTACHMENT: 'ssor_attachment',
27
+ };
@@ -0,0 +1,7 @@
1
+ import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
2
+ export interface EmitInterface {
3
+ event: AirdropEvent;
4
+ eventType: ExtractorEventType;
5
+ data?: EventData;
6
+ }
7
+ export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<void | Error>;
@@ -0,0 +1,41 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.emit = void 0;
7
+ const axios_1 = __importDefault(require("axios"));
8
+ const logger_1 = require("../logger/logger");
9
+ const emit = async ({ event, eventType, data, }) => {
10
+ const newEvent = {
11
+ event_type: eventType,
12
+ event_context: Object.assign({ uuid: event.payload.event_context.uuid, sync_run: event.payload.event_context.sync_run_id }, (event.payload.event_context.sync_unit_id && {
13
+ sync_unit: event.payload.event_context.sync_unit_id,
14
+ })),
15
+ event_data: Object.assign({}, data),
16
+ };
17
+ return new Promise(async (resolve, reject) => {
18
+ console.info('Emitting event', newEvent);
19
+ try {
20
+ await axios_1.default.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
21
+ headers: {
22
+ Accept: 'application/json, text/plain, */*',
23
+ Authorization: event.context.secrets.service_account_token,
24
+ 'Content-Type': 'application/json',
25
+ },
26
+ });
27
+ resolve();
28
+ }
29
+ catch (error) {
30
+ if (axios_1.default.isAxiosError(error)) {
31
+ console.error(`Failed to emit event with event type ${eventType}.`, (0, logger_1.formatAxiosError)(error));
32
+ }
33
+ else {
34
+ // TODO: Stop it through UI or think about retrying this request. Implement exponential retry mechanism.
35
+ console.error(`Failed to emit event with event type ${eventType}.`, error);
36
+ }
37
+ reject(error);
38
+ }
39
+ });
40
+ };
41
+ exports.emit = emit;
@@ -0,0 +1,4 @@
1
+ import { EventType, ExtractorEventType } from '../types/extraction';
2
+ export declare function getErrorExtractorEventType(eventType: EventType): {
3
+ eventType: ExtractorEventType;
4
+ } | null;
@@ -0,0 +1,38 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.getErrorExtractorEventType = getErrorExtractorEventType;
4
+ const extraction_1 = require("../types/extraction");
5
+ function getErrorExtractorEventType(eventType) {
6
+ switch (eventType) {
7
+ case extraction_1.EventType.ExtractionMetadataStart:
8
+ return {
9
+ eventType: extraction_1.ExtractorEventType.ExtractionMetadataError,
10
+ };
11
+ case extraction_1.EventType.ExtractionDataStart:
12
+ case extraction_1.EventType.ExtractionDataContinue:
13
+ return {
14
+ eventType: extraction_1.ExtractorEventType.ExtractionDataError,
15
+ };
16
+ case extraction_1.EventType.ExtractionDataDelete:
17
+ return {
18
+ eventType: extraction_1.ExtractorEventType.ExtractionDataDeleteError,
19
+ };
20
+ case extraction_1.EventType.ExtractionAttachmentsStart:
21
+ case extraction_1.EventType.ExtractionAttachmentsContinue:
22
+ return {
23
+ eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsError,
24
+ };
25
+ case extraction_1.EventType.ExtractionAttachmentsDelete:
26
+ return {
27
+ eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteError,
28
+ };
29
+ case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
30
+ return {
31
+ eventType: extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
32
+ };
33
+ default:
34
+ console.error('Event type not recognized in getTimeoutExtractorEventType function: ' +
35
+ eventType);
36
+ return null;
37
+ }
38
+ }
@@ -0,0 +1,3 @@
1
+ import { FunctionInput } from '@devrev/typescript-sdk/dist/snap-ins';
2
+ import { InitialDomainMapping } from '../types/common';
3
+ export declare function installInitialDomainMapping(event: FunctionInput, initialDomainMappingJson: InitialDomainMapping): Promise<void>;
@@ -0,0 +1,87 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.installInitialDomainMapping = installInitialDomainMapping;
7
+ const axios_1 = __importDefault(require("axios"));
8
+ const logger_1 = require("../logger/logger");
9
+ async function installInitialDomainMapping(event, initialDomainMappingJson) {
10
+ var _a, _b, _c, _d, _e, _f, _g;
11
+ const devrevEndpoint = event.execution_metadata.devrev_endpoint;
12
+ const devrevToken = event.context.secrets.service_account_token;
13
+ const snapInId = event.context.snap_in_id;
14
+ if (!initialDomainMappingJson) {
15
+ console.warn('No initial domain mapping found.');
16
+ return;
17
+ }
18
+ try {
19
+ const snapInResponse = await axios_1.default.get(devrevEndpoint + '/internal/snap-ins.get', {
20
+ headers: {
21
+ Authorization: devrevToken,
22
+ },
23
+ params: {
24
+ id: snapInId,
25
+ },
26
+ });
27
+ const importSlug = (_d = (_c = (_b = (_a = snapInResponse.data) === null || _a === void 0 ? void 0 : _a.snap_in) === null || _b === void 0 ? void 0 : _b.imports) === null || _c === void 0 ? void 0 : _c[0]) === null || _d === void 0 ? void 0 : _d.name;
28
+ const snapInSlug = (_g = (_f = (_e = snapInResponse.data) === null || _e === void 0 ? void 0 : _e.snap_in) === null || _f === void 0 ? void 0 : _f.snap_in_version) === null || _g === void 0 ? void 0 : _g.slug;
29
+ if (!importSlug || !snapInSlug) {
30
+ console.error('No import slug and snap in slug found in . Snap in response:', snapInResponse.data);
31
+ return;
32
+ }
33
+ const startingRecipeBlueprint = initialDomainMappingJson === null || initialDomainMappingJson === void 0 ? void 0 : initialDomainMappingJson.starting_recipe_blueprint;
34
+ let recipeBlueprintId;
35
+ if (startingRecipeBlueprint &&
36
+ Object.keys(startingRecipeBlueprint).length !== 0) {
37
+ try {
38
+ const recipeBlueprintResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`, Object.assign({}, startingRecipeBlueprint), {
39
+ headers: {
40
+ Authorization: devrevToken,
41
+ },
42
+ });
43
+ recipeBlueprintId = recipeBlueprintResponse.data.recipe_blueprint.id;
44
+ console.log('Successfully created recipe blueprint with id: ' + recipeBlueprintId);
45
+ }
46
+ catch (error) {
47
+ if (axios_1.default.isAxiosError(error)) {
48
+ console.error('Error while creating recipe blueprint', (0, logger_1.formatAxiosError)(error));
49
+ }
50
+ else {
51
+ console.error('Error while creating recipe blueprint', error);
52
+ }
53
+ }
54
+ }
55
+ try {
56
+ // 2. Install the initial domain mappings
57
+ const additionalMappings = initialDomainMappingJson.additional_mappings || {};
58
+ const initialDomainMappingInstallResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, Object.assign(Object.assign({ external_system_type: 'ADaaS', import_slug: importSlug, snap_in_slug: snapInSlug }, (recipeBlueprintId && {
59
+ starting_recipe_blueprint: recipeBlueprintId,
60
+ })), additionalMappings), {
61
+ headers: {
62
+ Authorization: devrevToken,
63
+ },
64
+ });
65
+ console.log('Successfully installed initial domain mapping: ' +
66
+ JSON.stringify(initialDomainMappingInstallResponse.data));
67
+ }
68
+ catch (error) {
69
+ if (axios_1.default.isAxiosError(error)) {
70
+ console.error('Error while installing initial domain mapping', (0, logger_1.formatAxiosError)(error));
71
+ }
72
+ else {
73
+ console.error('Error while installing initial domain mapping', error);
74
+ }
75
+ return;
76
+ }
77
+ }
78
+ catch (error) {
79
+ if (axios_1.default.isAxiosError(error)) {
80
+ console.error('Error while fetching snap in', (0, logger_1.formatAxiosError)(error));
81
+ }
82
+ else {
83
+ console.error('Error while fetching snap in', error);
84
+ }
85
+ return;
86
+ }
87
+ }
@@ -1,5 +1,7 @@
1
- import { AirdropEvent, ExtractorEventType, EventData, AdapterState, Artifact } from '../types';
2
- import { State } from '../state';
1
+ import { Artifact } from '../../uploader/uploader.interfaces';
2
+ import { AirdropEvent, ExtractorEventType, EventData } from '../../types/extraction';
3
+ import { AdapterState } from '../../state/state.interfaces';
4
+ import { State } from '../../state/state';
3
5
  /**
4
6
  * Adapter class is used to interact with Airdrop platform. The class provides
5
7
  * utilities to
@@ -9,6 +11,7 @@ import { State } from '../state';
9
11
  *
10
12
  * @class Adapter
11
13
  * @constructor
14
+ * @deprecated
12
15
  * @param {AirdropEvent} event - The event object received from the platform
13
16
  * @param {object=} initialState - The initial state of the adapter
14
17
  * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
@@ -21,8 +24,8 @@ import { State } from '../state';
21
24
  * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
22
25
  * @return The adapter instance
23
26
  */
24
- export declare function createAdapter<ExtractorState>(event: AirdropEvent, initialState: ExtractorState, isLocalDevelopment?: boolean): Promise<Adapter<ExtractorState>>;
25
- export declare class Adapter<ExtractorState> {
27
+ export declare function createAdapter<ConnectorState>(event: AirdropEvent, initialState: ConnectorState, isLocalDevelopment?: boolean): Promise<Adapter<ConnectorState>>;
28
+ export declare class Adapter<ConnectorState> {
26
29
  private adapterState;
27
30
  private _artifacts;
28
31
  private event;
@@ -33,9 +36,9 @@ export declare class Adapter<ExtractorState> {
33
36
  private exit;
34
37
  private lambdaTimeout;
35
38
  private heartBeatInterval;
36
- constructor(event: AirdropEvent, adapterState: State<ExtractorState>, isLocalDevelopment?: boolean);
37
- get state(): AdapterState<ExtractorState>;
38
- set state(value: AdapterState<ExtractorState>);
39
+ constructor(event: AirdropEvent, adapterState: State<ConnectorState>, isLocalDevelopment?: boolean);
40
+ get state(): AdapterState<ConnectorState>;
41
+ set state(value: AdapterState<ConnectorState>);
39
42
  get artifacts(): Artifact[];
40
43
  set artifacts(value: Artifact[]);
41
44
  /**