@devrev/ts-adaas 0.0.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +186 -116
  2. package/dist/common/constants.d.ts +10 -0
  3. package/dist/common/constants.js +27 -0
  4. package/dist/common/control-protocol.d.ts +7 -0
  5. package/dist/common/control-protocol.js +41 -0
  6. package/dist/common/helpers.d.ts +4 -0
  7. package/dist/common/helpers.js +38 -0
  8. package/dist/common/install-initial-domain-mapping.d.ts +3 -0
  9. package/dist/common/install-initial-domain-mapping.js +87 -0
  10. package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
  11. package/dist/{src → deprecated}/adapter/index.js +13 -10
  12. package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
  13. package/dist/deprecated/common/helpers.js +47 -0
  14. package/dist/deprecated/demo-extractor/external_domain_metadata.json +38 -0
  15. package/dist/deprecated/demo-extractor/index.d.ts +17 -0
  16. package/dist/deprecated/demo-extractor/index.js +161 -0
  17. package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
  18. package/dist/{src → deprecated}/uploader/index.js +3 -1
  19. package/dist/{src/http → http}/client.d.ts +2 -1
  20. package/dist/index.d.ts +10 -0
  21. package/dist/index.js +30 -0
  22. package/dist/logger/logger.d.ts +14 -0
  23. package/dist/logger/logger.interfaces.d.ts +14 -0
  24. package/dist/logger/logger.interfaces.js +9 -0
  25. package/dist/logger/logger.js +91 -0
  26. package/dist/logger/logger.test.js +49 -0
  27. package/dist/repo/repo.d.ts +14 -0
  28. package/dist/repo/repo.interfaces.d.ts +43 -0
  29. package/dist/repo/repo.interfaces.js +2 -0
  30. package/dist/repo/repo.js +68 -0
  31. package/dist/repo/repo.test.js +74 -0
  32. package/dist/state/state.d.ts +24 -0
  33. package/dist/state/state.interfaces.d.ts +24 -0
  34. package/dist/state/state.interfaces.js +2 -0
  35. package/dist/state/state.js +115 -0
  36. package/dist/tests/test-helpers.d.ts +9 -0
  37. package/dist/tests/test-helpers.interfaces.d.ts +11 -0
  38. package/dist/tests/test-helpers.interfaces.js +2 -0
  39. package/dist/tests/test-helpers.js +88 -0
  40. package/dist/tests/test-worker.js +16 -0
  41. package/dist/types/common.d.ts +39 -0
  42. package/dist/{src/types → types}/common.js +4 -0
  43. package/dist/{src/types → types}/extraction.d.ts +57 -12
  44. package/dist/{src/types → types}/extraction.js +12 -29
  45. package/dist/types/index.d.ts +5 -0
  46. package/dist/types/index.js +11 -0
  47. package/dist/types/workers.d.ts +142 -0
  48. package/dist/types/workers.js +23 -0
  49. package/dist/uploader/uploader.d.ts +38 -0
  50. package/dist/uploader/uploader.interfaces.d.ts +63 -0
  51. package/dist/uploader/uploader.interfaces.js +2 -0
  52. package/dist/uploader/uploader.js +351 -0
  53. package/dist/{tests → uploader}/uploader.test.js +7 -5
  54. package/dist/workers/create-worker.d.ts +4 -0
  55. package/dist/workers/create-worker.js +30 -0
  56. package/dist/workers/create-worker.test.js +25 -0
  57. package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
  58. package/dist/workers/default-workers/attachments-deletion.js +13 -0
  59. package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
  60. package/dist/workers/default-workers/attachments-extraction.js +49 -0
  61. package/dist/workers/default-workers/data-deletion.d.ts +1 -0
  62. package/dist/workers/default-workers/data-deletion.js +15 -0
  63. package/dist/workers/default-workers/data-extraction.d.ts +1 -0
  64. package/dist/workers/default-workers/data-extraction.js +101 -0
  65. package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
  66. package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
  67. package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
  68. package/dist/workers/default-workers/metadata-extraction.js +26 -0
  69. package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
  70. package/dist/workers/dummy-extractor/data-normalization.js +41 -0
  71. package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
  72. package/dist/workers/process-task.d.ts +2 -0
  73. package/dist/workers/process-task.js +44 -0
  74. package/dist/workers/spawn.d.ts +25 -0
  75. package/dist/workers/spawn.js +163 -0
  76. package/dist/workers/worker-adapter.d.ts +48 -0
  77. package/dist/workers/worker-adapter.js +138 -0
  78. package/dist/workers/worker.d.ts +1 -0
  79. package/dist/workers/worker.js +6 -0
  80. package/package.json +9 -5
  81. package/dist/src/common/constants.d.ts +0 -2
  82. package/dist/src/common/constants.js +0 -10
  83. package/dist/src/common/helpers.js +0 -59
  84. package/dist/src/demo-extractor/index.d.ts +0 -11
  85. package/dist/src/demo-extractor/index.js +0 -157
  86. package/dist/src/demo-extractor/initial_domain_mapping.json +0 -107
  87. package/dist/src/index.d.ts +0 -5
  88. package/dist/src/index.js +0 -21
  89. package/dist/src/logging/index.d.ts +0 -18
  90. package/dist/src/logging/index.js +0 -39
  91. package/dist/src/state/index.d.ts +0 -23
  92. package/dist/src/state/index.js +0 -111
  93. package/dist/src/types/common.d.ts +0 -33
  94. package/dist/src/types/index.d.ts +0 -2
  95. package/dist/src/types/index.js +0 -18
  96. package/dist/tests/adapter.helpers.test.js +0 -60
  97. package/dist/tests/adapter.test.js +0 -122
  98. package/dist/tests/demo-extractor.test.js +0 -60
  99. package/dist/tests/state.test.js +0 -100
  100. /package/dist/{src/http → http}/client.js +0 -0
  101. /package/dist/{src/http → http}/constants.d.ts +0 -0
  102. /package/dist/{src/http → http}/constants.js +0 -0
  103. /package/dist/{src/http → http}/index.d.ts +0 -0
  104. /package/dist/{src/http → http}/index.js +0 -0
  105. /package/dist/{src/http → http}/types.d.ts +0 -0
  106. /package/dist/{src/http → http}/types.js +0 -0
  107. /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
  108. /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
  109. /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
  110. /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
  111. /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
package/README.md CHANGED
@@ -1,12 +1,18 @@
1
1
  # ADaaS Library
2
2
 
3
- Typescript ADaaS Library (@devrev/ts-adaas) provides:
3
+ ## Release Notes
4
4
 
5
- - type definitions for ADaaS control protocol,
6
- - an adapter for ADaaS control protocol,
7
- - helpers for uploading artifacts and manage the state for ADaaS snap-in.
5
+ #### v1.0.0
8
6
 
9
- ## Release Notes
7
+ - Allow extractions to use full lambda runtime and gracefully handle execution context timeout.
8
+ - Simplified metadata and data normalization and uploading with repo implementation.
9
+ - Default handling of attachment extraction phase in ADaaS SDK library.
10
+ - Reduced file size, streamlined process by gzip compression.
11
+ - Bug fixes and improvements in error handling.
12
+
13
+ #### v0.0.3
14
+
15
+ - Support for new recipe management
10
16
 
11
17
  #### v0.0.2
12
18
 
@@ -21,155 +27,219 @@ Typescript ADaaS Library (@devrev/ts-adaas) provides:
21
27
  - Adapter for ADaaS control protocol with helper functions
22
28
  - Uploader for uploading artifacts
23
29
 
24
- ## Usage
25
-
26
- Create a new ADaaS adapter on each ADaaS snap-in invocation:
30
+ # Overview
27
31
 
28
- ```javascript
29
- const adapter = new Adapter(event: AirdropEvent);
30
- ```
32
+ The ADaaS (Airdrop-as-a-Service) Library for TypeScript helps developers build Snap-ins that integrate with DevRev’s ADaaS platform. This library simplifies the workflow for handling data extraction, event-driven actions, state management, and artifact handling.
31
33
 
32
- Adapter class provides:
34
+ ## Features
33
35
 
34
- - helper function to emit response,
35
- - automatic emit event if ADaaS snap-in invocation runs out of time,
36
- - setter for updating ADaaS snap-in state and adding artifacts to the return ADaaS message.
36
+ - Type Definitions: Structured types for ADaaS control protocol
37
+ - Event Management: Easily emit events for different extraction phases
38
+ - State Handling: Update and access state in real-time within tasks
39
+ - Artifact Management: Supports batched storage of artifacts (2000 items per batch)
40
+ - Error & Timeout Support: Error handling and timeout management for long-running tasks
37
41
 
38
- ### Phases of Airdrop Extraction
42
+ # Installation
39
43
 
40
- Each ADaaS snap-in must handle all the phases of ADaaS extraction.
44
+ ```bash
45
+ npm install @devrev/ts-adaas
46
+ ```
41
47
 
42
- ADaaS library provides type definitions to ensure ADaaS snap-ins are compatible with ADaaS control protocol.
48
+ # Usage
43
49
 
44
- ```javascript
45
- async run() {
46
- switch (this.event.payload.event_type) {
47
- case EventType.ExtractionExternalSyncUnitsStart: {
50
+ ADaaS Snap-ins are composed of several phases, each with unique requirements for initialization, data extraction, and error handling. The ADaaS library exports processTask to structure the work within each phase. The processTask function accepts task and onTimeout handlers, giving access to the adapter to streamline state updates, upload of extracted data, and event emission.
48
51
 
49
- // extract available External Sync Units (projects, organizations, ...)
52
+ ### ADaaS Snap-in Invocation
50
53
 
51
- await this.adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsDone, {
52
- external_sync_units: externalSyncUnits,
53
- });
54
- break;
55
- }
54
+ Each ADaaS snap-in must handle all the phases of ADaaS extraction. In a Snap-in, you typically define a `run` function that iterates over events and invokes workers per extraction phase.
56
55
 
57
- case EventType.ExtractionMetadataStart: {
56
+ ```typescript
57
+ import { AirdropEvent, EventType, spawn } from '@devrev/ts-adaas';
58
58
 
59
- // provide mappings of domain objects by provioding initial_domain_mapping.json file
60
- // update ADaaS snap-in state
59
+ interface DummyExtractorState {
60
+ issues: { completed: boolean };
61
+ users: { completed: boolean };
62
+ attachments: { completed: boolean };
63
+ }
61
64
 
62
- await this.adapter.emit(ExtractorEventType.ExtractionMetadataDone);
65
+ const initialState: DummyExtractorState = {
66
+ issues: { completed: false },
67
+ users: { completed: false },
68
+ attachments: { completed: false },
69
+ };
70
+
71
+ function getWorkerPerExtractionPhase(event: AirdropEvent) {
72
+ let path;
73
+ switch (event.payload.event_type) {
74
+ case EventType.ExtractionExternalSyncUnitsStart:
75
+ path = __dirname + '/workers/external-sync-units-extraction';
63
76
  break;
64
- }
65
-
66
- case EventType.ExtractionDataStart: {
67
-
68
- // extract Data
69
- // upload Data
70
- // update ADaaS snap-in state
71
- // approximate progress done
72
-
73
- await this.adapter.emit(ExtractorEventType.ExtractionDataContinue, {
74
- progress: 10,
75
- });
76
-
77
+ case EventType.ExtractionMetadataStart:
78
+ path = __dirname + '/workers/metadata-extraction';
77
79
  break;
78
- }
79
-
80
- case EventType.ExtractionDataContinue: {
81
- await this.processExtractionData();
82
-
83
- // extract Data
84
- // upload Data
85
- // update ADaaS snap-in state
86
- // approximate progress done
87
-
88
- await this.adapter.emit(ExtractorEventType.ExtractionDataDone, {
89
- progress: 100,
90
- });
80
+ case EventType.ExtractionDataStart:
81
+ case EventType.ExtractionDataContinue:
82
+ path = __dirname + '/workers/data-extraction';
91
83
  break;
92
- }
93
-
94
- case EventType.ExtractionDataDelete: {
84
+ }
85
+ return path;
86
+ }
95
87
 
96
- // if an extraction has any side-effects to 3rd party systems cleanup should be done here.
88
+ const run = async (events: AirdropEvent[]) => {
89
+ for (const event of events) {
90
+ const file = getWorkerPerExtractionPhase(event);
91
+ await spawn<DummyExtractorState>({
92
+ event,
93
+ initialState,
94
+ workerPath: file,
95
+ options: {
96
+ isLocalDevelopment: true,
97
+ },
98
+ });
99
+ }
100
+ };
97
101
 
98
- await this.adapter.emit(ExtractorEventType.ExtractionDataDeleteDone);
99
- break;
100
- }
102
+ export default run;
103
+ ```
101
104
 
102
- case EventType.ExtractionAttachmentsStart: {
105
+ ## Extraction Phases
103
106
 
104
- // extract Attachments
105
- // upload Attachments
106
- // update ADaaS snap-in state
107
+ The ADaaS snap-in extraction lifecycle consists of three main phases: External Sync Units Extraction, Metadata Extraction, and Data Extraction. Each phase is defined in a separate file and is responsible for fetching the respective data.
107
108
 
108
- await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsContinue);
109
- break;
110
- }
109
+ ### 1. External Sync Units Extraction
111
110
 
112
- case EventType.ExtractionAttachmentsContinue: {
111
+ This phase is defined in `external-sync-units-extraction.ts` and is responsible for fetching the external sync units.
113
112
 
113
+ ```typescript
114
+ import {
115
+ ExternalSyncUnit,
116
+ ExtractorEventType,
117
+ processTask,
118
+ } from '@devrev/ts-adaas';
119
+
120
+ const externalSyncUnits: ExternalSyncUnit[] = [
121
+ {
122
+ id: 'devrev',
123
+ name: 'devrev',
124
+ description: 'Demo external sync unit',
125
+ item_count: 2,
126
+ item_type: 'issues',
127
+ },
128
+ ];
129
+
130
+ processTask({
131
+ task: async ({ adapter }) => {
132
+ await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsDone, {
133
+ external_sync_units: externalSyncUnits,
134
+ });
135
+ },
136
+ onTimeout: async ({ adapter }) => {
137
+ await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsError, {
138
+ error: {
139
+ message: 'Failed to extract external sync units. Lambda timeout.',
140
+ },
141
+ });
142
+ },
143
+ });
144
+ ```
114
145
 
115
- // extract Attachments
116
- // upload Attachments
117
- // update ADaaS snap-in state
146
+ ### 2. Metadata Extraction
118
147
 
119
- await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDone);
120
- break;
121
- }
148
+ This phase is defined in `metadata-extraction.ts` and is responsible for fetching the metadata.
122
149
 
123
- case EventType.ExtractionAttachmentsDelete: {
150
+ ```typescript
151
+ import { ExtractorEventType, processTask } from '@devrev/ts-adaas';
152
+ import externalDomainMetadata from '../dummy-extractor/external_domain_metadata.json';
153
+
154
+ const repos = [{ itemType: 'external_domain_metadata' }];
155
+
156
+ processTask({
157
+ task: async ({ adapter }) => {
158
+ adapter.initializeRepos(repos);
159
+ await adapter
160
+ .getRepo('external_domain_metadata')
161
+ ?.push([externalDomainMetadata]);
162
+ await adapter.emit(ExtractorEventType.ExtractionMetadataDone);
163
+ },
164
+ onTimeout: async ({ adapter }) => {
165
+ await adapter.emit(ExtractorEventType.ExtractionMetadataError, {
166
+ error: { message: 'Failed to extract metadata. Lambda timeout.' },
167
+ });
168
+ },
169
+ });
170
+ ```
124
171
 
125
- // if an extraction has any side-effects to 3rd party systems cleanup should be done here.
172
+ ### 3. Data Extraction
126
173
 
127
- await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDeleteDone);
128
- break;
129
- }
174
+ This phase is defined in `data-extraction.ts` and is responsible for fetching the data. In this phase also attachments metadata is extracted.
130
175
 
131
- default: {
132
- console.log('Event not supported' + JSON.stringify(this.event));
176
+ ```typescript
177
+ import { EventType, ExtractorEventType, processTask } from '@devrev/ts-adaas';
178
+ import { normalizeAttachment, normalizeIssue, normalizeUser } from '../dummy-extractor/data-normalization';
179
+
180
+ const issues = [
181
+ { id: 'issue-1', created_date: '1999-12-25T01:00:03+01:00', ... },
182
+ { id: 'issue-2', created_date: '1999-12-27T15:31:34+01:00', ... },
183
+ ];
184
+
185
+ const users = [
186
+ { id: 'user-1', created_date: '1999-12-25T01:00:03+01:00', ... },
187
+ { id: 'user-2', created_date: '1999-12-27T15:31:34+01:00', ... },
188
+ ];
189
+
190
+ const attachments = [
191
+ { url: 'https://app.dev.devrev-eng.ai/favicon.ico', id: 'attachment-1', ... },
192
+ { url: 'https://app.dev.devrev-eng.ai/favicon.ico', id: 'attachment-2', ... },
193
+ ];
194
+
195
+ const repos = [
196
+ { itemType: 'issues', normalize: normalizeIssue },
197
+ { itemType: 'users', normalize: normalizeUser },
198
+ { itemType: 'attachments', normalize: normalizeAttachment },
199
+ ];
200
+
201
+ processTask({
202
+ task: async ({ adapter }) => {
203
+ adapter.initializeRepos(repos);
204
+
205
+ if (adapter.event.payload.event_type === EventType.ExtractionDataStart) {
206
+ await adapter.getRepo('issues')?.push(issues);
207
+ await adapter.emit(ExtractorEventType.ExtractionDataProgress, { progress: 50 });
208
+ } else {
209
+ await adapter.getRepo('users')?.push(users);
210
+ await adapter.getRepo('attachments')?.push(attachments);
211
+ await adapter.emit(ExtractorEventType.ExtractionDataDone, { progress: 100 });
133
212
  }
134
- }
135
- }
213
+ },
214
+ onTimeout: async ({ adapter }) => {
215
+ await adapter.postState();
216
+ await adapter.emit(ExtractorEventType.ExtractionDataProgress, { progress: 50 });
217
+ },
218
+ });
136
219
  ```
137
220
 
138
- ## Uploading artifacts
221
+ ## 4. Attachments Streaming
139
222
 
140
- Create a new Uploader class for uploading artifacts:
223
+ The ADaaS library handles attachments streaming to improve efficiency and reduce complexity for developers. During the extraction phase, developers need only to provide metadata in a specific format for each attachment, and the library manages the streaming process.
141
224
 
142
- ```javascript
143
- const upload = new Uploader(
144
- event.execution_metadata.devrev_endpoint,
145
- event.context.secrets.service_account_token
146
- );
147
- ```
225
+ The Snap-in should provide attachment metadata following the `NormalizedAttachment` interface:
148
226
 
149
- Files with extracted domain objects must be in JSONL (JSON Lines) format. Data files should contain 2000 - 5000 records each.
150
-
151
- ```javascript
152
- const entity = 'users';
153
- const { artifact, error } = await this.uploader.upload(
154
- `extractor_${entity}_${i}.jsonl`,
155
- entity,
156
- data
157
- );
158
- if (error) {
159
- return error;
160
- } else {
161
- await this.adapter.update({ artifact });
227
+ ```typescript
228
+ export interface NormalizedAttachment {
229
+ url: string;
230
+ id: string;
231
+ file_name: string;
232
+ author_id: string;
233
+ parent_id: string;
162
234
  }
163
235
  ```
164
236
 
165
- Each uploaded file must be attached to ADaaS adapter as soon as it is uploaded to ensure it is included in the ADaaS response message in case of a lambda timeout.
237
+ ## Artifact Uploading and State Management
166
238
 
167
- ## Updating ADaaS snap-in state
239
+ The ADaaS library provides a repository management system to handle artifacts in batches. The `initializeRepos` function initializes the repositories, and the `push` function uploads the artifacts to the repositories. The `postState` function is used to post the state of the extraction task.
168
240
 
169
- ADaaS snap-ins keep their own state between sync runs, between the states of a particular sync run and between invocations within a particular state.
241
+ State management is crucial for ADaaS Snap-ins to maintain the state of the extraction task. The `postState` function is used to post the state of the extraction task. The state is stored in the adapter and can be retrieved using the `adapter.state` property.
170
242
 
171
- By managing its own state, the ADaaS snap-in keeps track of the process of extraction (what items have already been extracted and where to continue), the times of the last successful sync run and keeps record of progress of the extraction.
243
+ ## Timeout Handling
172
244
 
173
- ```typescript
174
- async update({ artifacts, extractor_state}: AdapterUpdateParams)
175
- ```
245
+ The ADaaS library provides a timeout handler to handle timeouts in long-running tasks. The `onTimeout` handler is called when the task exceeds the timeout limit. The handler can be used to post the state of the extraction task and emit an event when a timeout occurs.
@@ -0,0 +1,10 @@
1
+ import { EventType } from '../types/extraction';
2
+ export declare const STATELESS_EVENT_TYPES: EventType[];
3
+ export declare const ALLOWED_EVENT_TYPES: EventType[];
4
+ export declare const ARTIFACT_BATCH_SIZE = 2000;
5
+ export declare const MAX_DEVREV_ARTIFACT_SIZE = 536870912;
6
+ export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
7
+ EXTERNAL_DOMAIN_METADATA: string;
8
+ ATTACHMENTS: string;
9
+ SSOR_ATTACHMENT: string;
10
+ };
@@ -0,0 +1,27 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.ALLOWED_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = void 0;
4
+ const extraction_1 = require("../types/extraction");
5
+ exports.STATELESS_EVENT_TYPES = [
6
+ extraction_1.EventType.ExtractionExternalSyncUnitsStart,
7
+ extraction_1.EventType.ExtractionMetadataStart,
8
+ extraction_1.EventType.ExtractionDataDelete,
9
+ extraction_1.EventType.ExtractionAttachmentsDelete,
10
+ ];
11
+ exports.ALLOWED_EVENT_TYPES = [
12
+ extraction_1.EventType.ExtractionExternalSyncUnitsStart,
13
+ extraction_1.EventType.ExtractionMetadataStart,
14
+ extraction_1.EventType.ExtractionDataStart,
15
+ extraction_1.EventType.ExtractionDataContinue,
16
+ extraction_1.EventType.ExtractionDataDelete,
17
+ extraction_1.EventType.ExtractionAttachmentsStart,
18
+ extraction_1.EventType.ExtractionAttachmentsContinue,
19
+ extraction_1.EventType.ExtractionAttachmentsDelete,
20
+ ];
21
+ exports.ARTIFACT_BATCH_SIZE = 2000;
22
+ exports.MAX_DEVREV_ARTIFACT_SIZE = 536870912; // 512MB
23
+ exports.AIRDROP_DEFAULT_ITEM_TYPES = {
24
+ EXTERNAL_DOMAIN_METADATA: 'external_domain_metadata',
25
+ ATTACHMENTS: 'attachments',
26
+ SSOR_ATTACHMENT: 'ssor_attachment',
27
+ };
@@ -0,0 +1,7 @@
1
+ import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
2
+ export interface EmitInterface {
3
+ event: AirdropEvent;
4
+ eventType: ExtractorEventType;
5
+ data?: EventData;
6
+ }
7
+ export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<void | Error>;
@@ -0,0 +1,41 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.emit = void 0;
7
+ const axios_1 = __importDefault(require("axios"));
8
+ const logger_1 = require("../logger/logger");
9
+ const emit = async ({ event, eventType, data, }) => {
10
+ const newEvent = {
11
+ event_type: eventType,
12
+ event_context: Object.assign({ uuid: event.payload.event_context.uuid, sync_run: event.payload.event_context.sync_run_id }, (event.payload.event_context.sync_unit_id && {
13
+ sync_unit: event.payload.event_context.sync_unit_id,
14
+ })),
15
+ event_data: Object.assign({}, data),
16
+ };
17
+ return new Promise(async (resolve, reject) => {
18
+ console.info('Emitting event', newEvent);
19
+ try {
20
+ await axios_1.default.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
21
+ headers: {
22
+ Accept: 'application/json, text/plain, */*',
23
+ Authorization: event.context.secrets.service_account_token,
24
+ 'Content-Type': 'application/json',
25
+ },
26
+ });
27
+ resolve();
28
+ }
29
+ catch (error) {
30
+ if (axios_1.default.isAxiosError(error)) {
31
+ console.error(`Failed to emit event with event type ${eventType}.`, (0, logger_1.formatAxiosError)(error));
32
+ }
33
+ else {
34
+ // TODO: Stop it through UI or think about retrying this request. Implement exponential retry mechanism.
35
+ console.error(`Failed to emit event with event type ${eventType}.`, error);
36
+ }
37
+ reject(error);
38
+ }
39
+ });
40
+ };
41
+ exports.emit = emit;
@@ -0,0 +1,4 @@
1
+ import { EventType, ExtractorEventType } from '../types/extraction';
2
+ export declare function getErrorExtractorEventType(eventType: EventType): {
3
+ eventType: ExtractorEventType;
4
+ } | null;
@@ -0,0 +1,38 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.getErrorExtractorEventType = getErrorExtractorEventType;
4
+ const extraction_1 = require("../types/extraction");
5
+ function getErrorExtractorEventType(eventType) {
6
+ switch (eventType) {
7
+ case extraction_1.EventType.ExtractionMetadataStart:
8
+ return {
9
+ eventType: extraction_1.ExtractorEventType.ExtractionMetadataError,
10
+ };
11
+ case extraction_1.EventType.ExtractionDataStart:
12
+ case extraction_1.EventType.ExtractionDataContinue:
13
+ return {
14
+ eventType: extraction_1.ExtractorEventType.ExtractionDataError,
15
+ };
16
+ case extraction_1.EventType.ExtractionDataDelete:
17
+ return {
18
+ eventType: extraction_1.ExtractorEventType.ExtractionDataDeleteError,
19
+ };
20
+ case extraction_1.EventType.ExtractionAttachmentsStart:
21
+ case extraction_1.EventType.ExtractionAttachmentsContinue:
22
+ return {
23
+ eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsError,
24
+ };
25
+ case extraction_1.EventType.ExtractionAttachmentsDelete:
26
+ return {
27
+ eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteError,
28
+ };
29
+ case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
30
+ return {
31
+ eventType: extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
32
+ };
33
+ default:
34
+ console.error('Event type not recognized in getTimeoutExtractorEventType function: ' +
35
+ eventType);
36
+ return null;
37
+ }
38
+ }
@@ -0,0 +1,3 @@
1
+ import { FunctionInput } from '@devrev/typescript-sdk/dist/snap-ins';
2
+ import { InitialDomainMapping } from '../types/common';
3
+ export declare function installInitialDomainMapping(event: FunctionInput, initialDomainMappingJson: InitialDomainMapping): Promise<void>;
@@ -0,0 +1,87 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.installInitialDomainMapping = installInitialDomainMapping;
7
+ const axios_1 = __importDefault(require("axios"));
8
+ const logger_1 = require("../logger/logger");
9
+ async function installInitialDomainMapping(event, initialDomainMappingJson) {
10
+ var _a, _b, _c, _d, _e, _f, _g;
11
+ const devrevEndpoint = event.execution_metadata.devrev_endpoint;
12
+ const devrevToken = event.context.secrets.service_account_token;
13
+ const snapInId = event.context.snap_in_id;
14
+ if (!initialDomainMappingJson) {
15
+ console.warn('No initial domain mapping found.');
16
+ return;
17
+ }
18
+ try {
19
+ const snapInResponse = await axios_1.default.get(devrevEndpoint + '/internal/snap-ins.get', {
20
+ headers: {
21
+ Authorization: devrevToken,
22
+ },
23
+ params: {
24
+ id: snapInId,
25
+ },
26
+ });
27
+ const importSlug = (_d = (_c = (_b = (_a = snapInResponse.data) === null || _a === void 0 ? void 0 : _a.snap_in) === null || _b === void 0 ? void 0 : _b.imports) === null || _c === void 0 ? void 0 : _c[0]) === null || _d === void 0 ? void 0 : _d.name;
28
+ const snapInSlug = (_g = (_f = (_e = snapInResponse.data) === null || _e === void 0 ? void 0 : _e.snap_in) === null || _f === void 0 ? void 0 : _f.snap_in_version) === null || _g === void 0 ? void 0 : _g.slug;
29
+ if (!importSlug || !snapInSlug) {
30
+ console.error('No import slug and snap in slug found in . Snap in response:', snapInResponse.data);
31
+ return;
32
+ }
33
+ const startingRecipeBlueprint = initialDomainMappingJson === null || initialDomainMappingJson === void 0 ? void 0 : initialDomainMappingJson.starting_recipe_blueprint;
34
+ let recipeBlueprintId;
35
+ if (startingRecipeBlueprint &&
36
+ Object.keys(startingRecipeBlueprint).length !== 0) {
37
+ try {
38
+ const recipeBlueprintResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`, Object.assign({}, startingRecipeBlueprint), {
39
+ headers: {
40
+ Authorization: devrevToken,
41
+ },
42
+ });
43
+ recipeBlueprintId = recipeBlueprintResponse.data.recipe_blueprint.id;
44
+ console.log('Successfully created recipe blueprint with id: ' + recipeBlueprintId);
45
+ }
46
+ catch (error) {
47
+ if (axios_1.default.isAxiosError(error)) {
48
+ console.error('Error while creating recipe blueprint', (0, logger_1.formatAxiosError)(error));
49
+ }
50
+ else {
51
+ console.error('Error while creating recipe blueprint', error);
52
+ }
53
+ }
54
+ }
55
+ try {
56
+ // 2. Install the initial domain mappings
57
+ const additionalMappings = initialDomainMappingJson.additional_mappings || {};
58
+ const initialDomainMappingInstallResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, Object.assign(Object.assign({ external_system_type: 'ADaaS', import_slug: importSlug, snap_in_slug: snapInSlug }, (recipeBlueprintId && {
59
+ starting_recipe_blueprint: recipeBlueprintId,
60
+ })), additionalMappings), {
61
+ headers: {
62
+ Authorization: devrevToken,
63
+ },
64
+ });
65
+ console.log('Successfully installed initial domain mapping: ' +
66
+ JSON.stringify(initialDomainMappingInstallResponse.data));
67
+ }
68
+ catch (error) {
69
+ if (axios_1.default.isAxiosError(error)) {
70
+ console.error('Error while installing initial domain mapping', (0, logger_1.formatAxiosError)(error));
71
+ }
72
+ else {
73
+ console.error('Error while installing initial domain mapping', error);
74
+ }
75
+ return;
76
+ }
77
+ }
78
+ catch (error) {
79
+ if (axios_1.default.isAxiosError(error)) {
80
+ console.error('Error while fetching snap in', (0, logger_1.formatAxiosError)(error));
81
+ }
82
+ else {
83
+ console.error('Error while fetching snap in', error);
84
+ }
85
+ return;
86
+ }
87
+ }
@@ -1,5 +1,7 @@
1
- import { AirdropEvent, ExtractorEventType, EventData, AdapterState, Artifact } from '../types';
2
- import { State } from '../state';
1
+ import { Artifact } from '../../uploader/uploader.interfaces';
2
+ import { AirdropEvent, ExtractorEventType, EventData } from '../../types/extraction';
3
+ import { AdapterState } from '../../state/state.interfaces';
4
+ import { State } from '../../state/state';
3
5
  /**
4
6
  * Adapter class is used to interact with Airdrop platform. The class provides
5
7
  * utilities to
@@ -9,6 +11,7 @@ import { State } from '../state';
9
11
  *
10
12
  * @class Adapter
11
13
  * @constructor
14
+ * @deprecated
12
15
  * @param {AirdropEvent} event - The event object received from the platform
13
16
  * @param {object=} initialState - The initial state of the adapter
14
17
  * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
@@ -21,8 +24,8 @@ import { State } from '../state';
21
24
  * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
22
25
  * @return The adapter instance
23
26
  */
24
- export declare function createAdapter<ExtractorState>(event: AirdropEvent, initialState: ExtractorState, isLocalDevelopment?: boolean): Promise<Adapter<ExtractorState>>;
25
- export declare class Adapter<ExtractorState> {
27
+ export declare function createAdapter<ConnectorState>(event: AirdropEvent, initialState: ConnectorState, isLocalDevelopment?: boolean): Promise<Adapter<ConnectorState>>;
28
+ export declare class Adapter<ConnectorState> {
26
29
  private adapterState;
27
30
  private _artifacts;
28
31
  private event;
@@ -33,9 +36,9 @@ export declare class Adapter<ExtractorState> {
33
36
  private exit;
34
37
  private lambdaTimeout;
35
38
  private heartBeatInterval;
36
- constructor(event: AirdropEvent, adapterState: State<ExtractorState>, isLocalDevelopment?: boolean);
37
- get state(): AdapterState<ExtractorState>;
38
- set state(value: AdapterState<ExtractorState>);
39
+ constructor(event: AirdropEvent, adapterState: State<ConnectorState>, isLocalDevelopment?: boolean);
40
+ get state(): AdapterState<ConnectorState>;
41
+ set state(value: AdapterState<ConnectorState>);
39
42
  get artifacts(): Artifact[];
40
43
  set artifacts(value: Artifact[]);
41
44
  /**