@devrev/ts-adaas 0.0.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +182 -116
- package/dist/common/constants.d.ts +10 -0
- package/dist/common/constants.js +27 -0
- package/dist/common/control-protocol.d.ts +7 -0
- package/dist/common/control-protocol.js +41 -0
- package/dist/common/helpers.d.ts +4 -0
- package/dist/common/helpers.js +38 -0
- package/dist/common/install-initial-domain-mapping.d.ts +3 -0
- package/dist/common/install-initial-domain-mapping.js +87 -0
- package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
- package/dist/{src → deprecated}/adapter/index.js +13 -10
- package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
- package/dist/deprecated/common/helpers.js +47 -0
- package/dist/deprecated/demo-extractor/index.d.ts +17 -0
- package/dist/{src → deprecated}/demo-extractor/index.js +28 -22
- package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
- package/dist/{src → deprecated}/uploader/index.js +3 -1
- package/dist/{src/http → http}/client.d.ts +2 -1
- package/dist/index.d.ts +10 -0
- package/dist/{src/index.js → index.js} +11 -3
- package/dist/logger/logger.d.ts +14 -0
- package/dist/logger/logger.interfaces.d.ts +14 -0
- package/dist/logger/logger.interfaces.js +9 -0
- package/dist/logger/logger.js +91 -0
- package/dist/logger/logger.test.js +49 -0
- package/dist/repo/repo.d.ts +14 -0
- package/dist/repo/repo.interfaces.d.ts +43 -0
- package/dist/repo/repo.interfaces.js +2 -0
- package/dist/repo/repo.js +68 -0
- package/dist/repo/repo.test.js +74 -0
- package/dist/state/state.d.ts +24 -0
- package/dist/state/state.interfaces.d.ts +24 -0
- package/dist/state/state.interfaces.js +2 -0
- package/dist/state/state.js +115 -0
- package/dist/tests/test-helpers.d.ts +9 -0
- package/dist/tests/test-helpers.interfaces.d.ts +11 -0
- package/dist/tests/test-helpers.interfaces.js +2 -0
- package/dist/tests/test-helpers.js +88 -0
- package/dist/tests/test-worker.js +16 -0
- package/dist/types/common.d.ts +39 -0
- package/dist/{src/types → types}/common.js +4 -0
- package/dist/{src/types → types}/extraction.d.ts +56 -12
- package/dist/{src/types → types}/extraction.js +12 -29
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.js +11 -0
- package/dist/types/workers.d.ts +142 -0
- package/dist/types/workers.js +23 -0
- package/dist/uploader/uploader.d.ts +38 -0
- package/dist/uploader/uploader.interfaces.d.ts +63 -0
- package/dist/uploader/uploader.interfaces.js +2 -0
- package/dist/uploader/uploader.js +351 -0
- package/dist/{tests → uploader}/uploader.test.js +7 -5
- package/dist/workers/create-worker.d.ts +4 -0
- package/dist/workers/create-worker.js +30 -0
- package/dist/workers/create-worker.test.js +25 -0
- package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
- package/dist/workers/default-workers/attachments-deletion.js +13 -0
- package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
- package/dist/workers/default-workers/attachments-extraction.js +49 -0
- package/dist/workers/default-workers/data-deletion.d.ts +1 -0
- package/dist/workers/default-workers/data-deletion.js +15 -0
- package/dist/workers/default-workers/data-extraction.d.ts +1 -0
- package/dist/workers/default-workers/data-extraction.js +101 -0
- package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
- package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
- package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
- package/dist/workers/default-workers/metadata-extraction.js +26 -0
- package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
- package/dist/workers/dummy-extractor/data-normalization.js +41 -0
- package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
- package/dist/workers/process-task.d.ts +2 -0
- package/dist/workers/process-task.js +44 -0
- package/dist/workers/spawn.d.ts +25 -0
- package/dist/workers/spawn.js +163 -0
- package/dist/workers/worker-adapter.d.ts +48 -0
- package/dist/workers/worker-adapter.js +138 -0
- package/dist/workers/worker.d.ts +1 -0
- package/dist/workers/worker.js +6 -0
- package/package.json +9 -5
- package/dist/src/common/constants.d.ts +0 -2
- package/dist/src/common/constants.js +0 -10
- package/dist/src/common/helpers.js +0 -59
- package/dist/src/common/install-initial-domain-mapping.d.ts +0 -3
- package/dist/src/common/install-initial-domain-mapping.js +0 -60
- package/dist/src/demo-extractor/index.d.ts +0 -11
- package/dist/src/index.d.ts +0 -6
- package/dist/src/logging/index.d.ts +0 -18
- package/dist/src/logging/index.js +0 -39
- package/dist/src/state/index.d.ts +0 -23
- package/dist/src/state/index.js +0 -111
- package/dist/src/types/common.d.ts +0 -37
- package/dist/src/types/index.d.ts +0 -2
- package/dist/src/types/index.js +0 -18
- package/dist/tests/adapter.helpers.test.js +0 -60
- package/dist/tests/adapter.test.js +0 -123
- package/dist/tests/demo-extractor.test.js +0 -61
- package/dist/tests/state.test.js +0 -101
- /package/dist/{src → deprecated}/demo-extractor/external_domain_metadata.json +0 -0
- /package/dist/{src/http → http}/client.js +0 -0
- /package/dist/{src/http → http}/constants.d.ts +0 -0
- /package/dist/{src/http → http}/constants.js +0 -0
- /package/dist/{src/http → http}/index.d.ts +0 -0
- /package/dist/{src/http → http}/index.js +0 -0
- /package/dist/{src/http → http}/types.d.ts +0 -0
- /package/dist/{src/http → http}/types.js +0 -0
- /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
- /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
- /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
- /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
- /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
package/README.md
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
# ADaaS Library
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## Release Notes
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
- an adapter for ADaaS control protocol,
|
|
7
|
-
- helpers for uploading artifacts and manage the state for ADaaS snap-in.
|
|
5
|
+
#### v1.0.0
|
|
8
6
|
|
|
9
|
-
|
|
7
|
+
- Allow extractions to use full lambda runtime and gracefully handle execution context timeout.
|
|
8
|
+
- Simplified metadata and data normalization and uploading with repo implementation.
|
|
9
|
+
- Default handling of attachment extraction phase in ADaaS SDK library.
|
|
10
|
+
- Reduced file size, streamlined process by gzip compression.
|
|
11
|
+
- Bug fixes and improvements in error handling.
|
|
10
12
|
|
|
11
13
|
#### v0.0.3
|
|
12
14
|
|
|
@@ -25,155 +27,219 @@ Typescript ADaaS Library (@devrev/ts-adaas) provides:
|
|
|
25
27
|
- Adapter for ADaaS control protocol with helper functions
|
|
26
28
|
- Uploader for uploading artifacts
|
|
27
29
|
|
|
28
|
-
|
|
30
|
+
# Overview
|
|
29
31
|
|
|
30
|
-
|
|
32
|
+
The ADaaS (Airdrop-as-a-Service) Library for TypeScript helps developers build Snap-ins that integrate with DevRev’s ADaaS platform. This library simplifies the workflow for handling data extraction, event-driven actions, state management, and artifact handling.
|
|
31
33
|
|
|
32
|
-
|
|
33
|
-
const adapter = new Adapter(event: AirdropEvent);
|
|
34
|
-
```
|
|
34
|
+
## Features
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
- Type Definitions: Structured types for ADaaS control protocol
|
|
37
|
+
- Event Management: Easily emit events for different extraction phases
|
|
38
|
+
- State Handling: Update and access state in real-time within tasks
|
|
39
|
+
- Artifact Management: Supports batched storage of artifacts (2000 items per batch)
|
|
40
|
+
- Error & Timeout Support: Error handling and timeout management for long-running tasks
|
|
37
41
|
|
|
38
|
-
|
|
39
|
-
- automatic emit event if ADaaS snap-in invocation runs out of time,
|
|
40
|
-
- setter for updating ADaaS snap-in state and adding artifacts to the return ADaaS message.
|
|
42
|
+
# Installation
|
|
41
43
|
|
|
42
|
-
|
|
44
|
+
```bash
|
|
45
|
+
npm install @devrev/ts-adaas
|
|
46
|
+
```
|
|
43
47
|
|
|
44
|
-
|
|
48
|
+
# Usage
|
|
45
49
|
|
|
46
|
-
ADaaS library
|
|
50
|
+
ADaaS Snap-ins are composed of several phases, each with unique requirements for initialization, data extraction, and error handling. The ADaaS library exports processTask to structure the work within each phase. The processTask function accepts task and onTimeout handlers, giving access to the adapter to streamline state updates, upload of extracted data, and event emission.
|
|
47
51
|
|
|
48
|
-
|
|
49
|
-
async run() {
|
|
50
|
-
switch (this.event.payload.event_type) {
|
|
51
|
-
case EventType.ExtractionExternalSyncUnitsStart: {
|
|
52
|
+
### ADaaS Snap-in Invocation
|
|
52
53
|
|
|
53
|
-
|
|
54
|
+
Each ADaaS snap-in must handle all the phases of ADaaS extraction. In a Snap-in, you typically define a `run` function that iterates over events and invokes workers per extraction phase.
|
|
54
55
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
});
|
|
58
|
-
break;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
case EventType.ExtractionMetadataStart: {
|
|
56
|
+
```typescript
|
|
57
|
+
import { AirdropEvent, EventType, spawn } from '@devrev/ts-adaas';
|
|
62
58
|
|
|
63
|
-
|
|
64
|
-
|
|
59
|
+
interface DummyExtractorState {
|
|
60
|
+
issues: { completed: boolean };
|
|
61
|
+
users: { completed: boolean };
|
|
62
|
+
attachments: { completed: boolean };
|
|
63
|
+
}
|
|
65
64
|
|
|
66
|
-
|
|
65
|
+
const initialState: DummyExtractorState = {
|
|
66
|
+
issues: { completed: false },
|
|
67
|
+
users: { completed: false },
|
|
68
|
+
attachments: { completed: false },
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
function getWorkerPerExtractionPhase(event: AirdropEvent) {
|
|
72
|
+
let path;
|
|
73
|
+
switch (event.payload.event_type) {
|
|
74
|
+
case EventType.ExtractionExternalSyncUnitsStart:
|
|
75
|
+
path = __dirname + '/workers/external-sync-units-extraction';
|
|
67
76
|
break;
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
case EventType.ExtractionDataStart: {
|
|
71
|
-
|
|
72
|
-
// extract Data
|
|
73
|
-
// upload Data
|
|
74
|
-
// update ADaaS snap-in state
|
|
75
|
-
// approximate progress done
|
|
76
|
-
|
|
77
|
-
await this.adapter.emit(ExtractorEventType.ExtractionDataContinue, {
|
|
78
|
-
progress: 10,
|
|
79
|
-
});
|
|
80
|
-
|
|
77
|
+
case EventType.ExtractionMetadataStart:
|
|
78
|
+
path = __dirname + '/workers/metadata-extraction';
|
|
81
79
|
break;
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
await this.processExtractionData();
|
|
86
|
-
|
|
87
|
-
// extract Data
|
|
88
|
-
// upload Data
|
|
89
|
-
// update ADaaS snap-in state
|
|
90
|
-
// approximate progress done
|
|
91
|
-
|
|
92
|
-
await this.adapter.emit(ExtractorEventType.ExtractionDataDone, {
|
|
93
|
-
progress: 100,
|
|
94
|
-
});
|
|
80
|
+
case EventType.ExtractionDataStart:
|
|
81
|
+
case EventType.ExtractionDataContinue:
|
|
82
|
+
path = __dirname + '/workers/data-extraction';
|
|
95
83
|
break;
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
84
|
+
}
|
|
85
|
+
return path;
|
|
86
|
+
}
|
|
99
87
|
|
|
100
|
-
|
|
88
|
+
const run = async (events: AirdropEvent[]) => {
|
|
89
|
+
for (const event of events) {
|
|
90
|
+
const file = getWorkerPerExtractionPhase(event);
|
|
91
|
+
await spawn<DummyExtractorState>({
|
|
92
|
+
event,
|
|
93
|
+
initialState,
|
|
94
|
+
workerPath: file,
|
|
95
|
+
options: {
|
|
96
|
+
isLocalDevelopment: true,
|
|
97
|
+
},
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
101
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
}
|
|
102
|
+
export default run;
|
|
103
|
+
```
|
|
105
104
|
|
|
106
|
-
|
|
105
|
+
## Extraction Phases
|
|
107
106
|
|
|
108
|
-
|
|
109
|
-
// upload Attachments
|
|
110
|
-
// update ADaaS snap-in state
|
|
107
|
+
The ADaaS snap-in extraction lifecycle consists of three main phases: External Sync Units Extraction, Metadata Extraction, and Data Extraction. Each phase is defined in a separate file and is responsible for fetching the respective data.
|
|
111
108
|
|
|
112
|
-
|
|
113
|
-
break;
|
|
114
|
-
}
|
|
109
|
+
### 1. External Sync Units Extraction
|
|
115
110
|
|
|
116
|
-
|
|
111
|
+
This phase is defined in `external-sync-units-extraction.ts` and is responsible for fetching the external sync units.
|
|
117
112
|
|
|
113
|
+
```typescript
|
|
114
|
+
import {
|
|
115
|
+
ExternalSyncUnit,
|
|
116
|
+
ExtractorEventType,
|
|
117
|
+
processTask,
|
|
118
|
+
} from '@devrev/ts-adaas';
|
|
119
|
+
|
|
120
|
+
const externalSyncUnits: ExternalSyncUnit[] = [
|
|
121
|
+
{
|
|
122
|
+
id: 'devrev',
|
|
123
|
+
name: 'devrev',
|
|
124
|
+
description: 'Demo external sync unit',
|
|
125
|
+
item_count: 2,
|
|
126
|
+
item_type: 'issues',
|
|
127
|
+
},
|
|
128
|
+
];
|
|
129
|
+
|
|
130
|
+
processTask({
|
|
131
|
+
task: async ({ adapter }) => {
|
|
132
|
+
await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsDone, {
|
|
133
|
+
external_sync_units: externalSyncUnits,
|
|
134
|
+
});
|
|
135
|
+
},
|
|
136
|
+
onTimeout: async ({ adapter }) => {
|
|
137
|
+
await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsError, {
|
|
138
|
+
error: {
|
|
139
|
+
message: 'Failed to extract external sync units. Lambda timeout.',
|
|
140
|
+
},
|
|
141
|
+
});
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
```
|
|
118
145
|
|
|
119
|
-
|
|
120
|
-
// upload Attachments
|
|
121
|
-
// update ADaaS snap-in state
|
|
146
|
+
### 2. Metadata Extraction
|
|
122
147
|
|
|
123
|
-
|
|
124
|
-
break;
|
|
125
|
-
}
|
|
148
|
+
This phase is defined in `metadata-extraction.ts` and is responsible for fetching the metadata.
|
|
126
149
|
|
|
127
|
-
|
|
150
|
+
```typescript
|
|
151
|
+
import { ExtractorEventType, processTask } from '@devrev/ts-adaas';
|
|
152
|
+
import externalDomainMetadata from '../dummy-extractor/external_domain_metadata.json';
|
|
153
|
+
|
|
154
|
+
const repos = [{ itemType: 'external_domain_metadata' }];
|
|
155
|
+
|
|
156
|
+
processTask({
|
|
157
|
+
task: async ({ adapter }) => {
|
|
158
|
+
adapter.initializeRepos(repos);
|
|
159
|
+
await adapter
|
|
160
|
+
.getRepo('external_domain_metadata')
|
|
161
|
+
?.push([externalDomainMetadata]);
|
|
162
|
+
await adapter.emit(ExtractorEventType.ExtractionMetadataDone);
|
|
163
|
+
},
|
|
164
|
+
onTimeout: async ({ adapter }) => {
|
|
165
|
+
await adapter.emit(ExtractorEventType.ExtractionMetadataError, {
|
|
166
|
+
error: { message: 'Failed to extract metadata. Lambda timeout.' },
|
|
167
|
+
});
|
|
168
|
+
},
|
|
169
|
+
});
|
|
170
|
+
```
|
|
128
171
|
|
|
129
|
-
|
|
172
|
+
### 3. Data Extraction
|
|
130
173
|
|
|
131
|
-
|
|
132
|
-
break;
|
|
133
|
-
}
|
|
174
|
+
This phase is defined in `data-extraction.ts` and is responsible for fetching the data. In this phase also attachments metadata is extracted.
|
|
134
175
|
|
|
135
|
-
|
|
136
|
-
|
|
176
|
+
```typescript
|
|
177
|
+
import { EventType, ExtractorEventType, processTask } from '@devrev/ts-adaas';
|
|
178
|
+
import { normalizeAttachment, normalizeIssue, normalizeUser } from '../dummy-extractor/data-normalization';
|
|
179
|
+
|
|
180
|
+
const issues = [
|
|
181
|
+
{ id: 'issue-1', created_date: '1999-12-25T01:00:03+01:00', ... },
|
|
182
|
+
{ id: 'issue-2', created_date: '1999-12-27T15:31:34+01:00', ... },
|
|
183
|
+
];
|
|
184
|
+
|
|
185
|
+
const users = [
|
|
186
|
+
{ id: 'user-1', created_date: '1999-12-25T01:00:03+01:00', ... },
|
|
187
|
+
{ id: 'user-2', created_date: '1999-12-27T15:31:34+01:00', ... },
|
|
188
|
+
];
|
|
189
|
+
|
|
190
|
+
const attachments = [
|
|
191
|
+
{ url: 'https://app.dev.devrev-eng.ai/favicon.ico', id: 'attachment-1', ... },
|
|
192
|
+
{ url: 'https://app.dev.devrev-eng.ai/favicon.ico', id: 'attachment-2', ... },
|
|
193
|
+
];
|
|
194
|
+
|
|
195
|
+
const repos = [
|
|
196
|
+
{ itemType: 'issues', normalize: normalizeIssue },
|
|
197
|
+
{ itemType: 'users', normalize: normalizeUser },
|
|
198
|
+
{ itemType: 'attachments', normalize: normalizeAttachment },
|
|
199
|
+
];
|
|
200
|
+
|
|
201
|
+
processTask({
|
|
202
|
+
task: async ({ adapter }) => {
|
|
203
|
+
adapter.initializeRepos(repos);
|
|
204
|
+
|
|
205
|
+
if (adapter.event.payload.event_type === EventType.ExtractionDataStart) {
|
|
206
|
+
await adapter.getRepo('issues')?.push(issues);
|
|
207
|
+
await adapter.emit(ExtractorEventType.ExtractionDataProgress, { progress: 50 });
|
|
208
|
+
} else {
|
|
209
|
+
await adapter.getRepo('users')?.push(users);
|
|
210
|
+
await adapter.getRepo('attachments')?.push(attachments);
|
|
211
|
+
await adapter.emit(ExtractorEventType.ExtractionDataDone, { progress: 100 });
|
|
137
212
|
}
|
|
138
|
-
}
|
|
139
|
-
}
|
|
213
|
+
},
|
|
214
|
+
onTimeout: async ({ adapter }) => {
|
|
215
|
+
await adapter.postState();
|
|
216
|
+
await adapter.emit(ExtractorEventType.ExtractionDataProgress, { progress: 50 });
|
|
217
|
+
},
|
|
218
|
+
});
|
|
140
219
|
```
|
|
141
220
|
|
|
142
|
-
##
|
|
221
|
+
## 4. Attachments Streaming
|
|
143
222
|
|
|
144
|
-
|
|
223
|
+
The ADaaS library handles attachments streaming to improve efficiency and reduce complexity for developers. During the extraction phase, developers need only to provide metadata in a specific format for each attachment, and the library manages the streaming process.
|
|
145
224
|
|
|
146
|
-
|
|
147
|
-
const upload = new Uploader(
|
|
148
|
-
event.execution_metadata.devrev_endpoint,
|
|
149
|
-
event.context.secrets.service_account_token
|
|
150
|
-
);
|
|
151
|
-
```
|
|
225
|
+
The Snap-in should provide attachment metadata following the `NormalizedAttachment` interface:
|
|
152
226
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
data
|
|
161
|
-
);
|
|
162
|
-
if (error) {
|
|
163
|
-
return error;
|
|
164
|
-
} else {
|
|
165
|
-
await this.adapter.update({ artifact });
|
|
227
|
+
```typescript
|
|
228
|
+
export interface NormalizedAttachment {
|
|
229
|
+
url: string;
|
|
230
|
+
id: string;
|
|
231
|
+
file_name: string;
|
|
232
|
+
author_id: string;
|
|
233
|
+
parent_id: string;
|
|
166
234
|
}
|
|
167
235
|
```
|
|
168
236
|
|
|
169
|
-
|
|
237
|
+
## Artifact Uploading and State Management
|
|
170
238
|
|
|
171
|
-
|
|
239
|
+
The ADaaS library provides a repository management system to handle artifacts in batches. The `initializeRepos` function initializes the repositories, and the `push` function uploads the artifacts to the repositories. The `postState` function is used to post the state of the extraction task.
|
|
172
240
|
|
|
173
|
-
ADaaS
|
|
241
|
+
State management is crucial for ADaaS Snap-ins to maintain the state of the extraction task. The `postState` function is used to post the state of the extraction task. The state is stored in the adapter and can be retrieved using the `adapter.state` property.
|
|
174
242
|
|
|
175
|
-
|
|
243
|
+
## Timeout Handling
|
|
176
244
|
|
|
177
|
-
|
|
178
|
-
async update({ artifacts, extractor_state}: AdapterUpdateParams)
|
|
179
|
-
```
|
|
245
|
+
The ADaaS library provides a timeout handler to handle timeouts in long-running tasks. The `onTimeout` handler is called when the task exceeds the timeout limit. The handler can be used to post the state of the extraction task and emit an event when a timeout occurs.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { EventType } from '../types/extraction';
|
|
2
|
+
export declare const STATELESS_EVENT_TYPES: EventType[];
|
|
3
|
+
export declare const ALLOWED_EVENT_TYPES: EventType[];
|
|
4
|
+
export declare const ARTIFACT_BATCH_SIZE = 2000;
|
|
5
|
+
export declare const MAX_DEVREV_ARTIFACT_SIZE = 536870912;
|
|
6
|
+
export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
|
|
7
|
+
EXTERNAL_DOMAIN_METADATA: string;
|
|
8
|
+
ATTACHMENTS: string;
|
|
9
|
+
SSOR_ATTACHMENT: string;
|
|
10
|
+
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.ALLOWED_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = void 0;
|
|
4
|
+
const extraction_1 = require("../types/extraction");
|
|
5
|
+
exports.STATELESS_EVENT_TYPES = [
|
|
6
|
+
extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
7
|
+
extraction_1.EventType.ExtractionMetadataStart,
|
|
8
|
+
extraction_1.EventType.ExtractionDataDelete,
|
|
9
|
+
extraction_1.EventType.ExtractionAttachmentsDelete,
|
|
10
|
+
];
|
|
11
|
+
exports.ALLOWED_EVENT_TYPES = [
|
|
12
|
+
extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
13
|
+
extraction_1.EventType.ExtractionMetadataStart,
|
|
14
|
+
extraction_1.EventType.ExtractionDataStart,
|
|
15
|
+
extraction_1.EventType.ExtractionDataContinue,
|
|
16
|
+
extraction_1.EventType.ExtractionDataDelete,
|
|
17
|
+
extraction_1.EventType.ExtractionAttachmentsStart,
|
|
18
|
+
extraction_1.EventType.ExtractionAttachmentsContinue,
|
|
19
|
+
extraction_1.EventType.ExtractionAttachmentsDelete,
|
|
20
|
+
];
|
|
21
|
+
exports.ARTIFACT_BATCH_SIZE = 2000;
|
|
22
|
+
exports.MAX_DEVREV_ARTIFACT_SIZE = 536870912; // 512MB
|
|
23
|
+
exports.AIRDROP_DEFAULT_ITEM_TYPES = {
|
|
24
|
+
EXTERNAL_DOMAIN_METADATA: 'external_domain_metadata',
|
|
25
|
+
ATTACHMENTS: 'attachments',
|
|
26
|
+
SSOR_ATTACHMENT: 'ssor_attachment',
|
|
27
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
|
|
2
|
+
export interface EmitInterface {
|
|
3
|
+
event: AirdropEvent;
|
|
4
|
+
eventType: ExtractorEventType;
|
|
5
|
+
data?: EventData;
|
|
6
|
+
}
|
|
7
|
+
export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<void | Error>;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.emit = void 0;
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const logger_1 = require("../logger/logger");
|
|
9
|
+
const emit = async ({ event, eventType, data, }) => {
|
|
10
|
+
const newEvent = {
|
|
11
|
+
event_type: eventType,
|
|
12
|
+
event_context: Object.assign({ uuid: event.payload.event_context.uuid, sync_run: event.payload.event_context.sync_run_id }, (event.payload.event_context.sync_unit_id && {
|
|
13
|
+
sync_unit: event.payload.event_context.sync_unit_id,
|
|
14
|
+
})),
|
|
15
|
+
event_data: Object.assign({}, data),
|
|
16
|
+
};
|
|
17
|
+
return new Promise(async (resolve, reject) => {
|
|
18
|
+
console.info('Emitting event', newEvent);
|
|
19
|
+
try {
|
|
20
|
+
await axios_1.default.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
|
|
21
|
+
headers: {
|
|
22
|
+
Accept: 'application/json, text/plain, */*',
|
|
23
|
+
Authorization: event.context.secrets.service_account_token,
|
|
24
|
+
'Content-Type': 'application/json',
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
resolve();
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
31
|
+
console.error(`Failed to emit event with event type ${eventType}.`, (0, logger_1.formatAxiosError)(error));
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
// TODO: Stop it through UI or think about retrying this request. Implement exponential retry mechanism.
|
|
35
|
+
console.error(`Failed to emit event with event type ${eventType}.`, error);
|
|
36
|
+
}
|
|
37
|
+
reject(error);
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
};
|
|
41
|
+
exports.emit = emit;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getErrorExtractorEventType = getErrorExtractorEventType;
|
|
4
|
+
const extraction_1 = require("../types/extraction");
|
|
5
|
+
function getErrorExtractorEventType(eventType) {
|
|
6
|
+
switch (eventType) {
|
|
7
|
+
case extraction_1.EventType.ExtractionMetadataStart:
|
|
8
|
+
return {
|
|
9
|
+
eventType: extraction_1.ExtractorEventType.ExtractionMetadataError,
|
|
10
|
+
};
|
|
11
|
+
case extraction_1.EventType.ExtractionDataStart:
|
|
12
|
+
case extraction_1.EventType.ExtractionDataContinue:
|
|
13
|
+
return {
|
|
14
|
+
eventType: extraction_1.ExtractorEventType.ExtractionDataError,
|
|
15
|
+
};
|
|
16
|
+
case extraction_1.EventType.ExtractionDataDelete:
|
|
17
|
+
return {
|
|
18
|
+
eventType: extraction_1.ExtractorEventType.ExtractionDataDeleteError,
|
|
19
|
+
};
|
|
20
|
+
case extraction_1.EventType.ExtractionAttachmentsStart:
|
|
21
|
+
case extraction_1.EventType.ExtractionAttachmentsContinue:
|
|
22
|
+
return {
|
|
23
|
+
eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsError,
|
|
24
|
+
};
|
|
25
|
+
case extraction_1.EventType.ExtractionAttachmentsDelete:
|
|
26
|
+
return {
|
|
27
|
+
eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteError,
|
|
28
|
+
};
|
|
29
|
+
case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
|
|
30
|
+
return {
|
|
31
|
+
eventType: extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
|
|
32
|
+
};
|
|
33
|
+
default:
|
|
34
|
+
console.error('Event type not recognized in getTimeoutExtractorEventType function: ' +
|
|
35
|
+
eventType);
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.installInitialDomainMapping = installInitialDomainMapping;
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const logger_1 = require("../logger/logger");
|
|
9
|
+
async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
10
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
11
|
+
const devrevEndpoint = event.execution_metadata.devrev_endpoint;
|
|
12
|
+
const devrevToken = event.context.secrets.service_account_token;
|
|
13
|
+
const snapInId = event.context.snap_in_id;
|
|
14
|
+
if (!initialDomainMappingJson) {
|
|
15
|
+
console.warn('No initial domain mapping found.');
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
try {
|
|
19
|
+
const snapInResponse = await axios_1.default.get(devrevEndpoint + '/internal/snap-ins.get', {
|
|
20
|
+
headers: {
|
|
21
|
+
Authorization: devrevToken,
|
|
22
|
+
},
|
|
23
|
+
params: {
|
|
24
|
+
id: snapInId,
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
const importSlug = (_d = (_c = (_b = (_a = snapInResponse.data) === null || _a === void 0 ? void 0 : _a.snap_in) === null || _b === void 0 ? void 0 : _b.imports) === null || _c === void 0 ? void 0 : _c[0]) === null || _d === void 0 ? void 0 : _d.name;
|
|
28
|
+
const snapInSlug = (_g = (_f = (_e = snapInResponse.data) === null || _e === void 0 ? void 0 : _e.snap_in) === null || _f === void 0 ? void 0 : _f.snap_in_version) === null || _g === void 0 ? void 0 : _g.slug;
|
|
29
|
+
if (!importSlug || !snapInSlug) {
|
|
30
|
+
console.error('No import slug and snap in slug found in . Snap in response:', snapInResponse.data);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
const startingRecipeBlueprint = initialDomainMappingJson === null || initialDomainMappingJson === void 0 ? void 0 : initialDomainMappingJson.starting_recipe_blueprint;
|
|
34
|
+
let recipeBlueprintId;
|
|
35
|
+
if (startingRecipeBlueprint &&
|
|
36
|
+
Object.keys(startingRecipeBlueprint).length !== 0) {
|
|
37
|
+
try {
|
|
38
|
+
const recipeBlueprintResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`, Object.assign({}, startingRecipeBlueprint), {
|
|
39
|
+
headers: {
|
|
40
|
+
Authorization: devrevToken,
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
recipeBlueprintId = recipeBlueprintResponse.data.recipe_blueprint.id;
|
|
44
|
+
console.log('Successfully created recipe blueprint with id: ' + recipeBlueprintId);
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
48
|
+
console.error('Error while creating recipe blueprint', (0, logger_1.formatAxiosError)(error));
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
console.error('Error while creating recipe blueprint', error);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
// 2. Install the initial domain mappings
|
|
57
|
+
const additionalMappings = initialDomainMappingJson.additional_mappings || {};
|
|
58
|
+
const initialDomainMappingInstallResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, Object.assign(Object.assign({ external_system_type: 'ADaaS', import_slug: importSlug, snap_in_slug: snapInSlug }, (recipeBlueprintId && {
|
|
59
|
+
starting_recipe_blueprint: recipeBlueprintId,
|
|
60
|
+
})), additionalMappings), {
|
|
61
|
+
headers: {
|
|
62
|
+
Authorization: devrevToken,
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
console.log('Successfully installed initial domain mapping: ' +
|
|
66
|
+
JSON.stringify(initialDomainMappingInstallResponse.data));
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
70
|
+
console.error('Error while installing initial domain mapping', (0, logger_1.formatAxiosError)(error));
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
console.error('Error while installing initial domain mapping', error);
|
|
74
|
+
}
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
80
|
+
console.error('Error while fetching snap in', (0, logger_1.formatAxiosError)(error));
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
console.error('Error while fetching snap in', error);
|
|
84
|
+
}
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Artifact } from '../../uploader/uploader.interfaces';
|
|
2
|
+
import { AirdropEvent, ExtractorEventType, EventData } from '../../types/extraction';
|
|
3
|
+
import { AdapterState } from '../../state/state.interfaces';
|
|
4
|
+
import { State } from '../../state/state';
|
|
3
5
|
/**
|
|
4
6
|
* Adapter class is used to interact with Airdrop platform. The class provides
|
|
5
7
|
* utilities to
|
|
@@ -9,6 +11,7 @@ import { State } from '../state';
|
|
|
9
11
|
*
|
|
10
12
|
* @class Adapter
|
|
11
13
|
* @constructor
|
|
14
|
+
* @deprecated
|
|
12
15
|
* @param {AirdropEvent} event - The event object received from the platform
|
|
13
16
|
* @param {object=} initialState - The initial state of the adapter
|
|
14
17
|
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
@@ -21,8 +24,8 @@ import { State } from '../state';
|
|
|
21
24
|
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
22
25
|
* @return The adapter instance
|
|
23
26
|
*/
|
|
24
|
-
export declare function createAdapter<
|
|
25
|
-
export declare class Adapter<
|
|
27
|
+
export declare function createAdapter<ConnectorState>(event: AirdropEvent, initialState: ConnectorState, isLocalDevelopment?: boolean): Promise<Adapter<ConnectorState>>;
|
|
28
|
+
export declare class Adapter<ConnectorState> {
|
|
26
29
|
private adapterState;
|
|
27
30
|
private _artifacts;
|
|
28
31
|
private event;
|
|
@@ -33,9 +36,9 @@ export declare class Adapter<ExtractorState> {
|
|
|
33
36
|
private exit;
|
|
34
37
|
private lambdaTimeout;
|
|
35
38
|
private heartBeatInterval;
|
|
36
|
-
constructor(event: AirdropEvent, adapterState: State<
|
|
37
|
-
get state(): AdapterState<
|
|
38
|
-
set state(value: AdapterState<
|
|
39
|
+
constructor(event: AirdropEvent, adapterState: State<ConnectorState>, isLocalDevelopment?: boolean);
|
|
40
|
+
get state(): AdapterState<ConnectorState>;
|
|
41
|
+
set state(value: AdapterState<ConnectorState>);
|
|
39
42
|
get artifacts(): Artifact[];
|
|
40
43
|
set artifacts(value: Artifact[]);
|
|
41
44
|
/**
|