@devrev/ts-adaas 0.0.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -116
- package/dist/common/constants.d.ts +10 -0
- package/dist/common/constants.js +27 -0
- package/dist/common/control-protocol.d.ts +7 -0
- package/dist/common/control-protocol.js +41 -0
- package/dist/common/helpers.d.ts +4 -0
- package/dist/common/helpers.js +38 -0
- package/dist/common/install-initial-domain-mapping.d.ts +3 -0
- package/dist/common/install-initial-domain-mapping.js +87 -0
- package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
- package/dist/{src → deprecated}/adapter/index.js +13 -10
- package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
- package/dist/deprecated/common/helpers.js +47 -0
- package/dist/deprecated/demo-extractor/external_domain_metadata.json +38 -0
- package/dist/deprecated/demo-extractor/index.d.ts +17 -0
- package/dist/deprecated/demo-extractor/index.js +161 -0
- package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
- package/dist/{src → deprecated}/uploader/index.js +3 -1
- package/dist/{src/http → http}/client.d.ts +2 -1
- package/dist/index.d.ts +10 -0
- package/dist/index.js +30 -0
- package/dist/logger/logger.d.ts +14 -0
- package/dist/logger/logger.interfaces.d.ts +14 -0
- package/dist/logger/logger.interfaces.js +9 -0
- package/dist/logger/logger.js +91 -0
- package/dist/logger/logger.test.js +49 -0
- package/dist/repo/repo.d.ts +14 -0
- package/dist/repo/repo.interfaces.d.ts +43 -0
- package/dist/repo/repo.interfaces.js +2 -0
- package/dist/repo/repo.js +68 -0
- package/dist/repo/repo.test.js +74 -0
- package/dist/state/state.d.ts +24 -0
- package/dist/state/state.interfaces.d.ts +24 -0
- package/dist/state/state.interfaces.js +2 -0
- package/dist/state/state.js +115 -0
- package/dist/tests/test-helpers.d.ts +9 -0
- package/dist/tests/test-helpers.interfaces.d.ts +11 -0
- package/dist/tests/test-helpers.interfaces.js +2 -0
- package/dist/tests/test-helpers.js +88 -0
- package/dist/tests/test-worker.js +16 -0
- package/dist/types/common.d.ts +39 -0
- package/dist/{src/types → types}/common.js +4 -0
- package/dist/{src/types → types}/extraction.d.ts +57 -12
- package/dist/{src/types → types}/extraction.js +12 -29
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.js +11 -0
- package/dist/types/workers.d.ts +142 -0
- package/dist/types/workers.js +23 -0
- package/dist/uploader/uploader.d.ts +38 -0
- package/dist/uploader/uploader.interfaces.d.ts +63 -0
- package/dist/uploader/uploader.interfaces.js +2 -0
- package/dist/uploader/uploader.js +351 -0
- package/dist/{tests → uploader}/uploader.test.js +7 -5
- package/dist/workers/create-worker.d.ts +4 -0
- package/dist/workers/create-worker.js +30 -0
- package/dist/workers/create-worker.test.js +25 -0
- package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
- package/dist/workers/default-workers/attachments-deletion.js +13 -0
- package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
- package/dist/workers/default-workers/attachments-extraction.js +49 -0
- package/dist/workers/default-workers/data-deletion.d.ts +1 -0
- package/dist/workers/default-workers/data-deletion.js +15 -0
- package/dist/workers/default-workers/data-extraction.d.ts +1 -0
- package/dist/workers/default-workers/data-extraction.js +101 -0
- package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
- package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
- package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
- package/dist/workers/default-workers/metadata-extraction.js +26 -0
- package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
- package/dist/workers/dummy-extractor/data-normalization.js +41 -0
- package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
- package/dist/workers/process-task.d.ts +2 -0
- package/dist/workers/process-task.js +44 -0
- package/dist/workers/spawn.d.ts +25 -0
- package/dist/workers/spawn.js +163 -0
- package/dist/workers/worker-adapter.d.ts +48 -0
- package/dist/workers/worker-adapter.js +138 -0
- package/dist/workers/worker.d.ts +1 -0
- package/dist/workers/worker.js +6 -0
- package/package.json +9 -5
- package/dist/src/common/constants.d.ts +0 -2
- package/dist/src/common/constants.js +0 -10
- package/dist/src/common/helpers.js +0 -59
- package/dist/src/demo-extractor/index.d.ts +0 -11
- package/dist/src/demo-extractor/index.js +0 -157
- package/dist/src/demo-extractor/initial_domain_mapping.json +0 -107
- package/dist/src/index.d.ts +0 -5
- package/dist/src/index.js +0 -21
- package/dist/src/logging/index.d.ts +0 -18
- package/dist/src/logging/index.js +0 -39
- package/dist/src/state/index.d.ts +0 -23
- package/dist/src/state/index.js +0 -111
- package/dist/src/types/common.d.ts +0 -33
- package/dist/src/types/index.d.ts +0 -2
- package/dist/src/types/index.js +0 -18
- package/dist/tests/adapter.helpers.test.js +0 -60
- package/dist/tests/adapter.test.js +0 -122
- package/dist/tests/demo-extractor.test.js +0 -60
- package/dist/tests/state.test.js +0 -100
- /package/dist/{src/http → http}/client.js +0 -0
- /package/dist/{src/http → http}/constants.d.ts +0 -0
- /package/dist/{src/http → http}/constants.js +0 -0
- /package/dist/{src/http → http}/index.d.ts +0 -0
- /package/dist/{src/http → http}/index.js +0 -0
- /package/dist/{src/http → http}/types.d.ts +0 -0
- /package/dist/{src/http → http}/types.js +0 -0
- /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
- /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
- /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
- /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
- /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
package/README.md
CHANGED
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
# ADaaS Library
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## Release Notes
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
- an adapter for ADaaS control protocol,
|
|
7
|
-
- helpers for uploading artifacts and manage the state for ADaaS snap-in.
|
|
5
|
+
#### v1.0.0
|
|
8
6
|
|
|
9
|
-
|
|
7
|
+
- Allow extractions to use full lambda runtime and gracefully handle execution context timeout.
|
|
8
|
+
- Simplified metadata and data normalization and uploading with repo implementation.
|
|
9
|
+
- Default handling of attachment extraction phase in ADaaS SDK library.
|
|
10
|
+
- Reduced file size, streamlined process by gzip compression.
|
|
11
|
+
- Bug fixes and improvements in error handling.
|
|
12
|
+
|
|
13
|
+
#### v0.0.3
|
|
14
|
+
|
|
15
|
+
- Support for new recipe management
|
|
10
16
|
|
|
11
17
|
#### v0.0.2
|
|
12
18
|
|
|
@@ -21,155 +27,219 @@ Typescript ADaaS Library (@devrev/ts-adaas) provides:
|
|
|
21
27
|
- Adapter for ADaaS control protocol with helper functions
|
|
22
28
|
- Uploader for uploading artifacts
|
|
23
29
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
Create a new ADaaS adapter on each ADaaS snap-in invocation:
|
|
30
|
+
# Overview
|
|
27
31
|
|
|
28
|
-
|
|
29
|
-
const adapter = new Adapter(event: AirdropEvent);
|
|
30
|
-
```
|
|
32
|
+
The ADaaS (Airdrop-as-a-Service) Library for TypeScript helps developers build Snap-ins that integrate with DevRev’s ADaaS platform. This library simplifies the workflow for handling data extraction, event-driven actions, state management, and artifact handling.
|
|
31
33
|
|
|
32
|
-
|
|
34
|
+
## Features
|
|
33
35
|
|
|
34
|
-
-
|
|
35
|
-
-
|
|
36
|
-
-
|
|
36
|
+
- Type Definitions: Structured types for ADaaS control protocol
|
|
37
|
+
- Event Management: Easily emit events for different extraction phases
|
|
38
|
+
- State Handling: Update and access state in real-time within tasks
|
|
39
|
+
- Artifact Management: Supports batched storage of artifacts (2000 items per batch)
|
|
40
|
+
- Error & Timeout Support: Error handling and timeout management for long-running tasks
|
|
37
41
|
|
|
38
|
-
|
|
42
|
+
# Installation
|
|
39
43
|
|
|
40
|
-
|
|
44
|
+
```bash
|
|
45
|
+
npm install @devrev/ts-adaas
|
|
46
|
+
```
|
|
41
47
|
|
|
42
|
-
|
|
48
|
+
# Usage
|
|
43
49
|
|
|
44
|
-
|
|
45
|
-
async run() {
|
|
46
|
-
switch (this.event.payload.event_type) {
|
|
47
|
-
case EventType.ExtractionExternalSyncUnitsStart: {
|
|
50
|
+
ADaaS Snap-ins are composed of several phases, each with unique requirements for initialization, data extraction, and error handling. The ADaaS library exports processTask to structure the work within each phase. The processTask function accepts task and onTimeout handlers, giving access to the adapter to streamline state updates, upload of extracted data, and event emission.
|
|
48
51
|
|
|
49
|
-
|
|
52
|
+
### ADaaS Snap-in Invocation
|
|
50
53
|
|
|
51
|
-
|
|
52
|
-
external_sync_units: externalSyncUnits,
|
|
53
|
-
});
|
|
54
|
-
break;
|
|
55
|
-
}
|
|
54
|
+
Each ADaaS snap-in must handle all the phases of ADaaS extraction. In a Snap-in, you typically define a `run` function that iterates over events and invokes workers per extraction phase.
|
|
56
55
|
|
|
57
|
-
|
|
56
|
+
```typescript
|
|
57
|
+
import { AirdropEvent, EventType, spawn } from '@devrev/ts-adaas';
|
|
58
58
|
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
interface DummyExtractorState {
|
|
60
|
+
issues: { completed: boolean };
|
|
61
|
+
users: { completed: boolean };
|
|
62
|
+
attachments: { completed: boolean };
|
|
63
|
+
}
|
|
61
64
|
|
|
62
|
-
|
|
65
|
+
const initialState: DummyExtractorState = {
|
|
66
|
+
issues: { completed: false },
|
|
67
|
+
users: { completed: false },
|
|
68
|
+
attachments: { completed: false },
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
function getWorkerPerExtractionPhase(event: AirdropEvent) {
|
|
72
|
+
let path;
|
|
73
|
+
switch (event.payload.event_type) {
|
|
74
|
+
case EventType.ExtractionExternalSyncUnitsStart:
|
|
75
|
+
path = __dirname + '/workers/external-sync-units-extraction';
|
|
63
76
|
break;
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
case EventType.ExtractionDataStart: {
|
|
67
|
-
|
|
68
|
-
// extract Data
|
|
69
|
-
// upload Data
|
|
70
|
-
// update ADaaS snap-in state
|
|
71
|
-
// approximate progress done
|
|
72
|
-
|
|
73
|
-
await this.adapter.emit(ExtractorEventType.ExtractionDataContinue, {
|
|
74
|
-
progress: 10,
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
+
case EventType.ExtractionMetadataStart:
|
|
78
|
+
path = __dirname + '/workers/metadata-extraction';
|
|
77
79
|
break;
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
await this.processExtractionData();
|
|
82
|
-
|
|
83
|
-
// extract Data
|
|
84
|
-
// upload Data
|
|
85
|
-
// update ADaaS snap-in state
|
|
86
|
-
// approximate progress done
|
|
87
|
-
|
|
88
|
-
await this.adapter.emit(ExtractorEventType.ExtractionDataDone, {
|
|
89
|
-
progress: 100,
|
|
90
|
-
});
|
|
80
|
+
case EventType.ExtractionDataStart:
|
|
81
|
+
case EventType.ExtractionDataContinue:
|
|
82
|
+
path = __dirname + '/workers/data-extraction';
|
|
91
83
|
break;
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
84
|
+
}
|
|
85
|
+
return path;
|
|
86
|
+
}
|
|
95
87
|
|
|
96
|
-
|
|
88
|
+
const run = async (events: AirdropEvent[]) => {
|
|
89
|
+
for (const event of events) {
|
|
90
|
+
const file = getWorkerPerExtractionPhase(event);
|
|
91
|
+
await spawn<DummyExtractorState>({
|
|
92
|
+
event,
|
|
93
|
+
initialState,
|
|
94
|
+
workerPath: file,
|
|
95
|
+
options: {
|
|
96
|
+
isLocalDevelopment: true,
|
|
97
|
+
},
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
};
|
|
97
101
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
}
|
|
102
|
+
export default run;
|
|
103
|
+
```
|
|
101
104
|
|
|
102
|
-
|
|
105
|
+
## Extraction Phases
|
|
103
106
|
|
|
104
|
-
|
|
105
|
-
// upload Attachments
|
|
106
|
-
// update ADaaS snap-in state
|
|
107
|
+
The ADaaS snap-in extraction lifecycle consists of three main phases: External Sync Units Extraction, Metadata Extraction, and Data Extraction. Each phase is defined in a separate file and is responsible for fetching the respective data.
|
|
107
108
|
|
|
108
|
-
|
|
109
|
-
break;
|
|
110
|
-
}
|
|
109
|
+
### 1. External Sync Units Extraction
|
|
111
110
|
|
|
112
|
-
|
|
111
|
+
This phase is defined in `external-sync-units-extraction.ts` and is responsible for fetching the external sync units.
|
|
113
112
|
|
|
113
|
+
```typescript
|
|
114
|
+
import {
|
|
115
|
+
ExternalSyncUnit,
|
|
116
|
+
ExtractorEventType,
|
|
117
|
+
processTask,
|
|
118
|
+
} from '@devrev/ts-adaas';
|
|
119
|
+
|
|
120
|
+
const externalSyncUnits: ExternalSyncUnit[] = [
|
|
121
|
+
{
|
|
122
|
+
id: 'devrev',
|
|
123
|
+
name: 'devrev',
|
|
124
|
+
description: 'Demo external sync unit',
|
|
125
|
+
item_count: 2,
|
|
126
|
+
item_type: 'issues',
|
|
127
|
+
},
|
|
128
|
+
];
|
|
129
|
+
|
|
130
|
+
processTask({
|
|
131
|
+
task: async ({ adapter }) => {
|
|
132
|
+
await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsDone, {
|
|
133
|
+
external_sync_units: externalSyncUnits,
|
|
134
|
+
});
|
|
135
|
+
},
|
|
136
|
+
onTimeout: async ({ adapter }) => {
|
|
137
|
+
await adapter.emit(ExtractorEventType.ExtractionExternalSyncUnitsError, {
|
|
138
|
+
error: {
|
|
139
|
+
message: 'Failed to extract external sync units. Lambda timeout.',
|
|
140
|
+
},
|
|
141
|
+
});
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
```
|
|
114
145
|
|
|
115
|
-
|
|
116
|
-
// upload Attachments
|
|
117
|
-
// update ADaaS snap-in state
|
|
146
|
+
### 2. Metadata Extraction
|
|
118
147
|
|
|
119
|
-
|
|
120
|
-
break;
|
|
121
|
-
}
|
|
148
|
+
This phase is defined in `metadata-extraction.ts` and is responsible for fetching the metadata.
|
|
122
149
|
|
|
123
|
-
|
|
150
|
+
```typescript
|
|
151
|
+
import { ExtractorEventType, processTask } from '@devrev/ts-adaas';
|
|
152
|
+
import externalDomainMetadata from '../dummy-extractor/external_domain_metadata.json';
|
|
153
|
+
|
|
154
|
+
const repos = [{ itemType: 'external_domain_metadata' }];
|
|
155
|
+
|
|
156
|
+
processTask({
|
|
157
|
+
task: async ({ adapter }) => {
|
|
158
|
+
adapter.initializeRepos(repos);
|
|
159
|
+
await adapter
|
|
160
|
+
.getRepo('external_domain_metadata')
|
|
161
|
+
?.push([externalDomainMetadata]);
|
|
162
|
+
await adapter.emit(ExtractorEventType.ExtractionMetadataDone);
|
|
163
|
+
},
|
|
164
|
+
onTimeout: async ({ adapter }) => {
|
|
165
|
+
await adapter.emit(ExtractorEventType.ExtractionMetadataError, {
|
|
166
|
+
error: { message: 'Failed to extract metadata. Lambda timeout.' },
|
|
167
|
+
});
|
|
168
|
+
},
|
|
169
|
+
});
|
|
170
|
+
```
|
|
124
171
|
|
|
125
|
-
|
|
172
|
+
### 3. Data Extraction
|
|
126
173
|
|
|
127
|
-
|
|
128
|
-
break;
|
|
129
|
-
}
|
|
174
|
+
This phase is defined in `data-extraction.ts` and is responsible for fetching the data. In this phase also attachments metadata is extracted.
|
|
130
175
|
|
|
131
|
-
|
|
132
|
-
|
|
176
|
+
```typescript
|
|
177
|
+
import { EventType, ExtractorEventType, processTask } from '@devrev/ts-adaas';
|
|
178
|
+
import { normalizeAttachment, normalizeIssue, normalizeUser } from '../dummy-extractor/data-normalization';
|
|
179
|
+
|
|
180
|
+
const issues = [
|
|
181
|
+
{ id: 'issue-1', created_date: '1999-12-25T01:00:03+01:00', ... },
|
|
182
|
+
{ id: 'issue-2', created_date: '1999-12-27T15:31:34+01:00', ... },
|
|
183
|
+
];
|
|
184
|
+
|
|
185
|
+
const users = [
|
|
186
|
+
{ id: 'user-1', created_date: '1999-12-25T01:00:03+01:00', ... },
|
|
187
|
+
{ id: 'user-2', created_date: '1999-12-27T15:31:34+01:00', ... },
|
|
188
|
+
];
|
|
189
|
+
|
|
190
|
+
const attachments = [
|
|
191
|
+
{ url: 'https://app.dev.devrev-eng.ai/favicon.ico', id: 'attachment-1', ... },
|
|
192
|
+
{ url: 'https://app.dev.devrev-eng.ai/favicon.ico', id: 'attachment-2', ... },
|
|
193
|
+
];
|
|
194
|
+
|
|
195
|
+
const repos = [
|
|
196
|
+
{ itemType: 'issues', normalize: normalizeIssue },
|
|
197
|
+
{ itemType: 'users', normalize: normalizeUser },
|
|
198
|
+
{ itemType: 'attachments', normalize: normalizeAttachment },
|
|
199
|
+
];
|
|
200
|
+
|
|
201
|
+
processTask({
|
|
202
|
+
task: async ({ adapter }) => {
|
|
203
|
+
adapter.initializeRepos(repos);
|
|
204
|
+
|
|
205
|
+
if (adapter.event.payload.event_type === EventType.ExtractionDataStart) {
|
|
206
|
+
await adapter.getRepo('issues')?.push(issues);
|
|
207
|
+
await adapter.emit(ExtractorEventType.ExtractionDataProgress, { progress: 50 });
|
|
208
|
+
} else {
|
|
209
|
+
await adapter.getRepo('users')?.push(users);
|
|
210
|
+
await adapter.getRepo('attachments')?.push(attachments);
|
|
211
|
+
await adapter.emit(ExtractorEventType.ExtractionDataDone, { progress: 100 });
|
|
133
212
|
}
|
|
134
|
-
}
|
|
135
|
-
}
|
|
213
|
+
},
|
|
214
|
+
onTimeout: async ({ adapter }) => {
|
|
215
|
+
await adapter.postState();
|
|
216
|
+
await adapter.emit(ExtractorEventType.ExtractionDataProgress, { progress: 50 });
|
|
217
|
+
},
|
|
218
|
+
});
|
|
136
219
|
```
|
|
137
220
|
|
|
138
|
-
##
|
|
221
|
+
## 4. Attachments Streaming
|
|
139
222
|
|
|
140
|
-
|
|
223
|
+
The ADaaS library handles attachments streaming to improve efficiency and reduce complexity for developers. During the extraction phase, developers need only to provide metadata in a specific format for each attachment, and the library manages the streaming process.
|
|
141
224
|
|
|
142
|
-
|
|
143
|
-
const upload = new Uploader(
|
|
144
|
-
event.execution_metadata.devrev_endpoint,
|
|
145
|
-
event.context.secrets.service_account_token
|
|
146
|
-
);
|
|
147
|
-
```
|
|
225
|
+
The Snap-in should provide attachment metadata following the `NormalizedAttachment` interface:
|
|
148
226
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
data
|
|
157
|
-
);
|
|
158
|
-
if (error) {
|
|
159
|
-
return error;
|
|
160
|
-
} else {
|
|
161
|
-
await this.adapter.update({ artifact });
|
|
227
|
+
```typescript
|
|
228
|
+
export interface NormalizedAttachment {
|
|
229
|
+
url: string;
|
|
230
|
+
id: string;
|
|
231
|
+
file_name: string;
|
|
232
|
+
author_id: string;
|
|
233
|
+
parent_id: string;
|
|
162
234
|
}
|
|
163
235
|
```
|
|
164
236
|
|
|
165
|
-
|
|
237
|
+
## Artifact Uploading and State Management
|
|
166
238
|
|
|
167
|
-
|
|
239
|
+
The ADaaS library provides a repository management system to handle artifacts in batches. The `initializeRepos` function initializes the repositories, and the `push` function uploads the artifacts to the repositories. The `postState` function is used to post the state of the extraction task.
|
|
168
240
|
|
|
169
|
-
ADaaS
|
|
241
|
+
State management is crucial for ADaaS Snap-ins to maintain the state of the extraction task. The `postState` function is used to post the state of the extraction task. The state is stored in the adapter and can be retrieved using the `adapter.state` property.
|
|
170
242
|
|
|
171
|
-
|
|
243
|
+
## Timeout Handling
|
|
172
244
|
|
|
173
|
-
|
|
174
|
-
async update({ artifacts, extractor_state}: AdapterUpdateParams)
|
|
175
|
-
```
|
|
245
|
+
The ADaaS library provides a timeout handler to handle timeouts in long-running tasks. The `onTimeout` handler is called when the task exceeds the timeout limit. The handler can be used to post the state of the extraction task and emit an event when a timeout occurs.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { EventType } from '../types/extraction';
|
|
2
|
+
export declare const STATELESS_EVENT_TYPES: EventType[];
|
|
3
|
+
export declare const ALLOWED_EVENT_TYPES: EventType[];
|
|
4
|
+
export declare const ARTIFACT_BATCH_SIZE = 2000;
|
|
5
|
+
export declare const MAX_DEVREV_ARTIFACT_SIZE = 536870912;
|
|
6
|
+
export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
|
|
7
|
+
EXTERNAL_DOMAIN_METADATA: string;
|
|
8
|
+
ATTACHMENTS: string;
|
|
9
|
+
SSOR_ATTACHMENT: string;
|
|
10
|
+
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.ALLOWED_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = void 0;
|
|
4
|
+
const extraction_1 = require("../types/extraction");
|
|
5
|
+
exports.STATELESS_EVENT_TYPES = [
|
|
6
|
+
extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
7
|
+
extraction_1.EventType.ExtractionMetadataStart,
|
|
8
|
+
extraction_1.EventType.ExtractionDataDelete,
|
|
9
|
+
extraction_1.EventType.ExtractionAttachmentsDelete,
|
|
10
|
+
];
|
|
11
|
+
exports.ALLOWED_EVENT_TYPES = [
|
|
12
|
+
extraction_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
13
|
+
extraction_1.EventType.ExtractionMetadataStart,
|
|
14
|
+
extraction_1.EventType.ExtractionDataStart,
|
|
15
|
+
extraction_1.EventType.ExtractionDataContinue,
|
|
16
|
+
extraction_1.EventType.ExtractionDataDelete,
|
|
17
|
+
extraction_1.EventType.ExtractionAttachmentsStart,
|
|
18
|
+
extraction_1.EventType.ExtractionAttachmentsContinue,
|
|
19
|
+
extraction_1.EventType.ExtractionAttachmentsDelete,
|
|
20
|
+
];
|
|
21
|
+
exports.ARTIFACT_BATCH_SIZE = 2000;
|
|
22
|
+
exports.MAX_DEVREV_ARTIFACT_SIZE = 536870912; // 512MB
|
|
23
|
+
exports.AIRDROP_DEFAULT_ITEM_TYPES = {
|
|
24
|
+
EXTERNAL_DOMAIN_METADATA: 'external_domain_metadata',
|
|
25
|
+
ATTACHMENTS: 'attachments',
|
|
26
|
+
SSOR_ATTACHMENT: 'ssor_attachment',
|
|
27
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
|
|
2
|
+
export interface EmitInterface {
|
|
3
|
+
event: AirdropEvent;
|
|
4
|
+
eventType: ExtractorEventType;
|
|
5
|
+
data?: EventData;
|
|
6
|
+
}
|
|
7
|
+
export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<void | Error>;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.emit = void 0;
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const logger_1 = require("../logger/logger");
|
|
9
|
+
const emit = async ({ event, eventType, data, }) => {
|
|
10
|
+
const newEvent = {
|
|
11
|
+
event_type: eventType,
|
|
12
|
+
event_context: Object.assign({ uuid: event.payload.event_context.uuid, sync_run: event.payload.event_context.sync_run_id }, (event.payload.event_context.sync_unit_id && {
|
|
13
|
+
sync_unit: event.payload.event_context.sync_unit_id,
|
|
14
|
+
})),
|
|
15
|
+
event_data: Object.assign({}, data),
|
|
16
|
+
};
|
|
17
|
+
return new Promise(async (resolve, reject) => {
|
|
18
|
+
console.info('Emitting event', newEvent);
|
|
19
|
+
try {
|
|
20
|
+
await axios_1.default.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
|
|
21
|
+
headers: {
|
|
22
|
+
Accept: 'application/json, text/plain, */*',
|
|
23
|
+
Authorization: event.context.secrets.service_account_token,
|
|
24
|
+
'Content-Type': 'application/json',
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
resolve();
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
31
|
+
console.error(`Failed to emit event with event type ${eventType}.`, (0, logger_1.formatAxiosError)(error));
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
// TODO: Stop it through UI or think about retrying this request. Implement exponential retry mechanism.
|
|
35
|
+
console.error(`Failed to emit event with event type ${eventType}.`, error);
|
|
36
|
+
}
|
|
37
|
+
reject(error);
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
};
|
|
41
|
+
exports.emit = emit;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getErrorExtractorEventType = getErrorExtractorEventType;
|
|
4
|
+
const extraction_1 = require("../types/extraction");
|
|
5
|
+
function getErrorExtractorEventType(eventType) {
|
|
6
|
+
switch (eventType) {
|
|
7
|
+
case extraction_1.EventType.ExtractionMetadataStart:
|
|
8
|
+
return {
|
|
9
|
+
eventType: extraction_1.ExtractorEventType.ExtractionMetadataError,
|
|
10
|
+
};
|
|
11
|
+
case extraction_1.EventType.ExtractionDataStart:
|
|
12
|
+
case extraction_1.EventType.ExtractionDataContinue:
|
|
13
|
+
return {
|
|
14
|
+
eventType: extraction_1.ExtractorEventType.ExtractionDataError,
|
|
15
|
+
};
|
|
16
|
+
case extraction_1.EventType.ExtractionDataDelete:
|
|
17
|
+
return {
|
|
18
|
+
eventType: extraction_1.ExtractorEventType.ExtractionDataDeleteError,
|
|
19
|
+
};
|
|
20
|
+
case extraction_1.EventType.ExtractionAttachmentsStart:
|
|
21
|
+
case extraction_1.EventType.ExtractionAttachmentsContinue:
|
|
22
|
+
return {
|
|
23
|
+
eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsError,
|
|
24
|
+
};
|
|
25
|
+
case extraction_1.EventType.ExtractionAttachmentsDelete:
|
|
26
|
+
return {
|
|
27
|
+
eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteError,
|
|
28
|
+
};
|
|
29
|
+
case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
|
|
30
|
+
return {
|
|
31
|
+
eventType: extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
|
|
32
|
+
};
|
|
33
|
+
default:
|
|
34
|
+
console.error('Event type not recognized in getTimeoutExtractorEventType function: ' +
|
|
35
|
+
eventType);
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.installInitialDomainMapping = installInitialDomainMapping;
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const logger_1 = require("../logger/logger");
|
|
9
|
+
async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
10
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
11
|
+
const devrevEndpoint = event.execution_metadata.devrev_endpoint;
|
|
12
|
+
const devrevToken = event.context.secrets.service_account_token;
|
|
13
|
+
const snapInId = event.context.snap_in_id;
|
|
14
|
+
if (!initialDomainMappingJson) {
|
|
15
|
+
console.warn('No initial domain mapping found.');
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
try {
|
|
19
|
+
const snapInResponse = await axios_1.default.get(devrevEndpoint + '/internal/snap-ins.get', {
|
|
20
|
+
headers: {
|
|
21
|
+
Authorization: devrevToken,
|
|
22
|
+
},
|
|
23
|
+
params: {
|
|
24
|
+
id: snapInId,
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
const importSlug = (_d = (_c = (_b = (_a = snapInResponse.data) === null || _a === void 0 ? void 0 : _a.snap_in) === null || _b === void 0 ? void 0 : _b.imports) === null || _c === void 0 ? void 0 : _c[0]) === null || _d === void 0 ? void 0 : _d.name;
|
|
28
|
+
const snapInSlug = (_g = (_f = (_e = snapInResponse.data) === null || _e === void 0 ? void 0 : _e.snap_in) === null || _f === void 0 ? void 0 : _f.snap_in_version) === null || _g === void 0 ? void 0 : _g.slug;
|
|
29
|
+
if (!importSlug || !snapInSlug) {
|
|
30
|
+
console.error('No import slug and snap in slug found in . Snap in response:', snapInResponse.data);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
const startingRecipeBlueprint = initialDomainMappingJson === null || initialDomainMappingJson === void 0 ? void 0 : initialDomainMappingJson.starting_recipe_blueprint;
|
|
34
|
+
let recipeBlueprintId;
|
|
35
|
+
if (startingRecipeBlueprint &&
|
|
36
|
+
Object.keys(startingRecipeBlueprint).length !== 0) {
|
|
37
|
+
try {
|
|
38
|
+
const recipeBlueprintResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`, Object.assign({}, startingRecipeBlueprint), {
|
|
39
|
+
headers: {
|
|
40
|
+
Authorization: devrevToken,
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
recipeBlueprintId = recipeBlueprintResponse.data.recipe_blueprint.id;
|
|
44
|
+
console.log('Successfully created recipe blueprint with id: ' + recipeBlueprintId);
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
48
|
+
console.error('Error while creating recipe blueprint', (0, logger_1.formatAxiosError)(error));
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
console.error('Error while creating recipe blueprint', error);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
// 2. Install the initial domain mappings
|
|
57
|
+
const additionalMappings = initialDomainMappingJson.additional_mappings || {};
|
|
58
|
+
const initialDomainMappingInstallResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, Object.assign(Object.assign({ external_system_type: 'ADaaS', import_slug: importSlug, snap_in_slug: snapInSlug }, (recipeBlueprintId && {
|
|
59
|
+
starting_recipe_blueprint: recipeBlueprintId,
|
|
60
|
+
})), additionalMappings), {
|
|
61
|
+
headers: {
|
|
62
|
+
Authorization: devrevToken,
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
console.log('Successfully installed initial domain mapping: ' +
|
|
66
|
+
JSON.stringify(initialDomainMappingInstallResponse.data));
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
70
|
+
console.error('Error while installing initial domain mapping', (0, logger_1.formatAxiosError)(error));
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
console.error('Error while installing initial domain mapping', error);
|
|
74
|
+
}
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
80
|
+
console.error('Error while fetching snap in', (0, logger_1.formatAxiosError)(error));
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
console.error('Error while fetching snap in', error);
|
|
84
|
+
}
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Artifact } from '../../uploader/uploader.interfaces';
|
|
2
|
+
import { AirdropEvent, ExtractorEventType, EventData } from '../../types/extraction';
|
|
3
|
+
import { AdapterState } from '../../state/state.interfaces';
|
|
4
|
+
import { State } from '../../state/state';
|
|
3
5
|
/**
|
|
4
6
|
* Adapter class is used to interact with Airdrop platform. The class provides
|
|
5
7
|
* utilities to
|
|
@@ -9,6 +11,7 @@ import { State } from '../state';
|
|
|
9
11
|
*
|
|
10
12
|
* @class Adapter
|
|
11
13
|
* @constructor
|
|
14
|
+
* @deprecated
|
|
12
15
|
* @param {AirdropEvent} event - The event object received from the platform
|
|
13
16
|
* @param {object=} initialState - The initial state of the adapter
|
|
14
17
|
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
@@ -21,8 +24,8 @@ import { State } from '../state';
|
|
|
21
24
|
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
22
25
|
* @return The adapter instance
|
|
23
26
|
*/
|
|
24
|
-
export declare function createAdapter<
|
|
25
|
-
export declare class Adapter<
|
|
27
|
+
export declare function createAdapter<ConnectorState>(event: AirdropEvent, initialState: ConnectorState, isLocalDevelopment?: boolean): Promise<Adapter<ConnectorState>>;
|
|
28
|
+
export declare class Adapter<ConnectorState> {
|
|
26
29
|
private adapterState;
|
|
27
30
|
private _artifacts;
|
|
28
31
|
private event;
|
|
@@ -33,9 +36,9 @@ export declare class Adapter<ExtractorState> {
|
|
|
33
36
|
private exit;
|
|
34
37
|
private lambdaTimeout;
|
|
35
38
|
private heartBeatInterval;
|
|
36
|
-
constructor(event: AirdropEvent, adapterState: State<
|
|
37
|
-
get state(): AdapterState<
|
|
38
|
-
set state(value: AdapterState<
|
|
39
|
+
constructor(event: AirdropEvent, adapterState: State<ConnectorState>, isLocalDevelopment?: boolean);
|
|
40
|
+
get state(): AdapterState<ConnectorState>;
|
|
41
|
+
set state(value: AdapterState<ConnectorState>);
|
|
39
42
|
get artifacts(): Artifact[];
|
|
40
43
|
set artifacts(value: Artifact[]);
|
|
41
44
|
/**
|