@devrev/ts-adaas 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/dist/src/adapter/index.d.ts +35 -30
- package/dist/src/adapter/index.js +93 -56
- package/dist/src/common/constants.d.ts +2 -0
- package/dist/src/common/constants.js +10 -0
- package/dist/src/{adapter → common}/helpers.d.ts +4 -1
- package/dist/src/{adapter → common}/helpers.js +16 -4
- package/dist/src/demo-extractor/index.d.ts +8 -1
- package/dist/src/demo-extractor/index.js +64 -56
- package/dist/src/demo-extractor/initial_domain_mapping.json +107 -0
- package/dist/src/http/client.d.ts +1 -1
- package/dist/src/http/client.js +2 -2
- package/dist/src/http/index.d.ts +0 -1
- package/dist/src/http/index.js +0 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/logging/index.d.ts +5 -18
- package/dist/src/logging/index.js +20 -41
- package/dist/src/state/index.d.ts +23 -0
- package/dist/src/state/index.js +111 -0
- package/dist/src/types/extraction.d.ts +16 -5
- package/dist/src/uploader/index.d.ts +4 -2
- package/dist/src/uploader/index.js +61 -4
- package/dist/tests/adapter.helpers.test.js +60 -0
- package/dist/tests/adapter.test.js +106 -57
- package/dist/tests/demo-extractor.test.js +44 -81
- package/dist/tests/state.test.js +100 -0
- package/dist/tests/uploader.test.js +29 -0
- package/package.json +1 -1
- package/dist/src/adapter/index.test.js +0 -105
- package/dist/src/demo-extractor/recipe.json +0 -37
- package/dist/tests/helpers.test.js +0 -38
- package/dist/tests/test-helpers.d.ts +0 -2
- package/dist/tests/test-helpers.js +0 -33
- package/dist/tests/types.test.js +0 -71
- /package/dist/{src/adapter/index.test.d.ts → tests/adapter.helpers.test.d.ts} +0 -0
- /package/dist/tests/{helpers.test.d.ts → state.test.d.ts} +0 -0
- /package/dist/tests/{types.test.d.ts → uploader.test.d.ts} +0 -0
package/README.md
CHANGED
|
@@ -1,14 +1,26 @@
|
|
|
1
1
|
# ADaaS Library
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
# ADaaS Library
|
|
5
|
-
|
|
6
3
|
Typescript ADaaS Library (@devrev/ts-adaas) provides:
|
|
7
4
|
|
|
8
5
|
- type definitions for ADaaS control protocol,
|
|
9
6
|
- an adapter for ADaaS control protocol,
|
|
10
7
|
- helpers for uploading artifacts and manage the state for ADaaS snap-in.
|
|
11
8
|
|
|
9
|
+
## Release Notes
|
|
10
|
+
|
|
11
|
+
#### v0.0.2
|
|
12
|
+
|
|
13
|
+
- Support for the State API
|
|
14
|
+
- HTTP client for API requests
|
|
15
|
+
- Local development environment creates local artifact files
|
|
16
|
+
- Improvements in logging
|
|
17
|
+
|
|
18
|
+
#### v0.0.1
|
|
19
|
+
|
|
20
|
+
- Demo implementation of ADaaS snap-in
|
|
21
|
+
- Adapter for ADaaS control protocol with helper functions
|
|
22
|
+
- Uploader for uploading artifacts
|
|
23
|
+
|
|
12
24
|
## Usage
|
|
13
25
|
|
|
14
26
|
Create a new ADaaS adapter on each ADaaS snap-in invocation:
|
|
@@ -160,4 +172,4 @@ By managing its own state, the ADaaS snap-in keeps track of the process of extra
|
|
|
160
172
|
|
|
161
173
|
```typescript
|
|
162
174
|
async update({ artifacts, extractor_state}: AdapterUpdateParams)
|
|
163
|
-
```
|
|
175
|
+
```
|
|
@@ -1,42 +1,43 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { AirdropEvent, ExtractorEventType, EventData, AdapterState, Artifact } from '../types';
|
|
2
|
+
import { State } from '../state';
|
|
3
3
|
/**
|
|
4
4
|
* Adapter class is used to interact with Airdrop platform. The class provides
|
|
5
5
|
* utilities to
|
|
6
6
|
* - emit control events to the platform
|
|
7
7
|
* - update the state of the extractor
|
|
8
|
-
* -
|
|
9
|
-
* - Return the last saved state and artifacts in case timeout
|
|
10
|
-
* - The event sent in case of timeout for each event type is as follows:
|
|
11
|
-
* - EXTRACTION_EXTERNAL_SYNC_UNITS_START => EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR
|
|
12
|
-
* - EXTRACTION_METADATA_START => EXTRACTION_METADATA_ERROR
|
|
13
|
-
* - EXTRACTION_DATA_START => EXTRACTION_DATA_PROGRESS
|
|
14
|
-
* - EXTRACTION_DATA_CONTINUE => EXTRACTION_DATA_PROGRESS
|
|
15
|
-
* - EXTRACTION_ATTACHMENTS_START => EXTRACTION_ATTACHMENTS_PROGRESS
|
|
16
|
-
* - EXTRACTION_ATTACHMENTS_CONTINUE => EXTRACTION_ATTACHMENTS_PROGRESS
|
|
8
|
+
* - set the last saved state in case of a timeout
|
|
17
9
|
*
|
|
18
10
|
* @class Adapter
|
|
19
11
|
* @constructor
|
|
20
12
|
* @param {AirdropEvent} event - The event object received from the platform
|
|
21
|
-
* @param {object=}
|
|
13
|
+
* @param {object=} initialState - The initial state of the adapter
|
|
14
|
+
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
22
15
|
*/
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
16
|
+
/**
|
|
17
|
+
* Creates an adapter instance.
|
|
18
|
+
*
|
|
19
|
+
* @param {AirdropEvent} event - The event object received from the platform
|
|
20
|
+
* @param initialState
|
|
21
|
+
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
22
|
+
* @return The adapter instance
|
|
23
|
+
*/
|
|
24
|
+
export declare function createAdapter<ExtractorState>(event: AirdropEvent, initialState: ExtractorState, isLocalDevelopment?: boolean): Promise<Adapter<ExtractorState>>;
|
|
25
|
+
export declare class Adapter<ExtractorState> {
|
|
26
|
+
private adapterState;
|
|
27
|
+
private _artifacts;
|
|
27
28
|
private event;
|
|
28
29
|
private callbackUrl;
|
|
29
30
|
private devrevToken;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
31
|
+
private startTime;
|
|
32
|
+
private heartBeatFn;
|
|
33
|
+
private exit;
|
|
34
|
+
private lambdaTimeout;
|
|
35
|
+
private heartBeatInterval;
|
|
36
|
+
constructor(event: AirdropEvent, adapterState: State<ExtractorState>, isLocalDevelopment?: boolean);
|
|
37
|
+
get state(): AdapterState<ExtractorState>;
|
|
38
|
+
set state(value: AdapterState<ExtractorState>);
|
|
39
|
+
get artifacts(): Artifact[];
|
|
40
|
+
set artifacts(value: Artifact[]);
|
|
40
41
|
/**
|
|
41
42
|
* Emits an event to the platform.
|
|
42
43
|
*
|
|
@@ -45,9 +46,13 @@ export declare class Adapter {
|
|
|
45
46
|
*/
|
|
46
47
|
emit(newEventType: ExtractorEventType, data?: EventData): Promise<void>;
|
|
47
48
|
/**
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
|
|
49
|
+
* Exit the adapter. This will stop the heartbeat and no
|
|
50
|
+
* further events will be emitted.
|
|
51
|
+
*/
|
|
52
|
+
private exitAdapter;
|
|
53
|
+
/**
|
|
54
|
+
* Heartbeat function to check if the lambda is about to timeout.
|
|
55
|
+
* @returns true if 10 minutes have passed since the start of the lambda.
|
|
51
56
|
*/
|
|
52
|
-
|
|
57
|
+
private heartbeat;
|
|
53
58
|
}
|
|
@@ -3,60 +3,73 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.Adapter = void 0;
|
|
6
|
+
exports.Adapter = exports.createAdapter = void 0;
|
|
7
7
|
const axios_1 = __importDefault(require("axios"));
|
|
8
|
-
const
|
|
8
|
+
const constants_1 = require("../common/constants");
|
|
9
|
+
const helpers_1 = require("../common/helpers");
|
|
10
|
+
const logging_1 = require("../logging");
|
|
11
|
+
const state_1 = require("../state");
|
|
9
12
|
/**
|
|
10
13
|
* Adapter class is used to interact with Airdrop platform. The class provides
|
|
11
14
|
* utilities to
|
|
12
15
|
* - emit control events to the platform
|
|
13
16
|
* - update the state of the extractor
|
|
14
|
-
* -
|
|
15
|
-
* - Return the last saved state and artifacts in case timeout
|
|
16
|
-
* - The event sent in case of timeout for each event type is as follows:
|
|
17
|
-
* - EXTRACTION_EXTERNAL_SYNC_UNITS_START => EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR
|
|
18
|
-
* - EXTRACTION_METADATA_START => EXTRACTION_METADATA_ERROR
|
|
19
|
-
* - EXTRACTION_DATA_START => EXTRACTION_DATA_PROGRESS
|
|
20
|
-
* - EXTRACTION_DATA_CONTINUE => EXTRACTION_DATA_PROGRESS
|
|
21
|
-
* - EXTRACTION_ATTACHMENTS_START => EXTRACTION_ATTACHMENTS_PROGRESS
|
|
22
|
-
* - EXTRACTION_ATTACHMENTS_CONTINUE => EXTRACTION_ATTACHMENTS_PROGRESS
|
|
17
|
+
* - set the last saved state in case of a timeout
|
|
23
18
|
*
|
|
24
19
|
* @class Adapter
|
|
25
20
|
* @constructor
|
|
26
21
|
* @param {AirdropEvent} event - The event object received from the platform
|
|
27
|
-
* @param {object=}
|
|
22
|
+
* @param {object=} initialState - The initial state of the adapter
|
|
23
|
+
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
28
24
|
*/
|
|
25
|
+
/**
|
|
26
|
+
* Creates an adapter instance.
|
|
27
|
+
*
|
|
28
|
+
* @param {AirdropEvent} event - The event object received from the platform
|
|
29
|
+
* @param initialState
|
|
30
|
+
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
31
|
+
* @return The adapter instance
|
|
32
|
+
*/
|
|
33
|
+
async function createAdapter(event, initialState, isLocalDevelopment = false) {
|
|
34
|
+
const newInitialState = structuredClone(initialState);
|
|
35
|
+
const adapterState = await (0, state_1.createAdapterState)(event, newInitialState);
|
|
36
|
+
const a = new Adapter(event, adapterState, isLocalDevelopment);
|
|
37
|
+
return a;
|
|
38
|
+
}
|
|
39
|
+
exports.createAdapter = createAdapter;
|
|
29
40
|
class Adapter {
|
|
30
|
-
constructor(event,
|
|
41
|
+
constructor(event, adapterState, isLocalDevelopment = false) {
|
|
42
|
+
this.exit = false;
|
|
43
|
+
this.lambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
|
|
44
|
+
this.heartBeatInterval = 30 * 1000; // 30 seconds in milliseconds
|
|
45
|
+
if (!isLocalDevelopment) {
|
|
46
|
+
logging_1.Logger.init(event);
|
|
47
|
+
}
|
|
48
|
+
this.adapterState = adapterState;
|
|
49
|
+
this._artifacts = [];
|
|
31
50
|
this.event = event;
|
|
32
|
-
this.artifacts = [];
|
|
33
|
-
this.extractorState = state || {};
|
|
34
51
|
this.callbackUrl = event.payload.event_context.callback_url;
|
|
35
|
-
this.devrevToken = event.context.secrets
|
|
36
|
-
|
|
37
|
-
//
|
|
38
|
-
|
|
39
|
-
const
|
|
40
|
-
if (
|
|
41
|
-
|
|
52
|
+
this.devrevToken = event.context.secrets.service_account_token;
|
|
53
|
+
this.startTime = Date.now();
|
|
54
|
+
// Run heartbeat every 30 seconds
|
|
55
|
+
this.heartBeatFn = setInterval(async () => {
|
|
56
|
+
const b = await this.heartbeat();
|
|
57
|
+
if (b) {
|
|
58
|
+
this.exitAdapter();
|
|
42
59
|
}
|
|
43
|
-
},
|
|
60
|
+
}, this.heartBeatInterval);
|
|
44
61
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
57
|
-
if (params.extractor_state) {
|
|
58
|
-
this.extractorState = params.extractor_state;
|
|
59
|
-
}
|
|
62
|
+
get state() {
|
|
63
|
+
return this.adapterState.state;
|
|
64
|
+
}
|
|
65
|
+
set state(value) {
|
|
66
|
+
this.adapterState.state = value;
|
|
67
|
+
}
|
|
68
|
+
get artifacts() {
|
|
69
|
+
return this._artifacts;
|
|
70
|
+
}
|
|
71
|
+
set artifacts(value) {
|
|
72
|
+
this._artifacts = value;
|
|
60
73
|
}
|
|
61
74
|
/**
|
|
62
75
|
* Emits an event to the platform.
|
|
@@ -65,21 +78,22 @@ class Adapter {
|
|
|
65
78
|
* @param {EventData=} data - The data to be sent with the event
|
|
66
79
|
*/
|
|
67
80
|
async emit(newEventType, data) {
|
|
81
|
+
if (this.exit) {
|
|
82
|
+
console.warn('Adapter is already in exit state. No more events can be emitted.');
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
// We want to save the state every time we emit an event, except for the start and delete events
|
|
86
|
+
if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
|
|
87
|
+
console.log(`Saving state before emitting event`);
|
|
88
|
+
await this.adapterState.postState(this.state);
|
|
89
|
+
}
|
|
68
90
|
const newEvent = {
|
|
69
|
-
extractor_state: JSON.stringify(this.extractorState),
|
|
70
91
|
event_type: newEventType,
|
|
71
|
-
event_context: {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
},
|
|
75
|
-
event_data: Object.assign(Object.assign({}, data), { artifacts: this.artifacts }),
|
|
92
|
+
event_context: Object.assign({ uuid: this.event.payload.event_context.uuid, sync_run: this.event.payload.event_context.sync_run_id }, (this.event.payload.event_context.sync_unit_id && {
|
|
93
|
+
sync_unit: this.event.payload.event_context.sync_unit_id,
|
|
94
|
+
})),
|
|
95
|
+
event_data: Object.assign({}, data),
|
|
76
96
|
};
|
|
77
|
-
// If sync_unit_id is present in the event, add it to the new event
|
|
78
|
-
if (this.event.payload.event_context.sync_unit_id) {
|
|
79
|
-
newEvent.event_context.sync_unit =
|
|
80
|
-
this.event.payload.event_context.sync_unit_id;
|
|
81
|
-
}
|
|
82
|
-
console.log('Event that will be emitted: ' + JSON.stringify(newEvent));
|
|
83
97
|
try {
|
|
84
98
|
await axios_1.default.post(this.callbackUrl, Object.assign({}, newEvent), {
|
|
85
99
|
headers: {
|
|
@@ -88,23 +102,46 @@ class Adapter {
|
|
|
88
102
|
'Content-Type': 'application/json',
|
|
89
103
|
},
|
|
90
104
|
});
|
|
105
|
+
console.log('Successfully emitted event: ' + JSON.stringify(newEvent));
|
|
91
106
|
}
|
|
92
107
|
catch (error) {
|
|
93
108
|
// If this request fails the extraction will be stuck in loop and
|
|
94
109
|
// we need to stop it through UI or think about retrying this request
|
|
95
|
-
console.log('
|
|
110
|
+
console.log('Failed to emit event: ' +
|
|
96
111
|
JSON.stringify(newEvent) +
|
|
97
112
|
', error: ' +
|
|
98
113
|
error);
|
|
99
114
|
}
|
|
115
|
+
finally {
|
|
116
|
+
this.exitAdapter();
|
|
117
|
+
}
|
|
100
118
|
}
|
|
101
119
|
/**
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
* @return The list of artifacts
|
|
120
|
+
* Exit the adapter. This will stop the heartbeat and no
|
|
121
|
+
* further events will be emitted.
|
|
105
122
|
*/
|
|
106
|
-
|
|
107
|
-
|
|
123
|
+
exitAdapter() {
|
|
124
|
+
this.exit = true;
|
|
125
|
+
clearInterval(this.heartBeatFn);
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Heartbeat function to check if the lambda is about to timeout.
|
|
129
|
+
* @returns true if 10 minutes have passed since the start of the lambda.
|
|
130
|
+
*/
|
|
131
|
+
async heartbeat() {
|
|
132
|
+
if (this.exit) {
|
|
133
|
+
return true;
|
|
134
|
+
}
|
|
135
|
+
if (Date.now() - this.startTime > this.lambdaTimeout) {
|
|
136
|
+
const timeoutEventType = (0, helpers_1.getTimeoutExtractorEventType)(this.event.payload.event_type);
|
|
137
|
+
if (timeoutEventType !== null) {
|
|
138
|
+
const { eventType, isError } = timeoutEventType;
|
|
139
|
+
const err = isError ? { message: 'Lambda Timeout' } : undefined;
|
|
140
|
+
await this.emit(eventType, { error: err, artifacts: this._artifacts });
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return false;
|
|
108
145
|
}
|
|
109
146
|
}
|
|
110
147
|
exports.Adapter = Adapter;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.STATELESS_EVENT_TYPES = void 0;
|
|
4
|
+
const types_1 = require("../types");
|
|
5
|
+
exports.STATELESS_EVENT_TYPES = [
|
|
6
|
+
types_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
7
|
+
types_1.EventType.ExtractionMetadataStart,
|
|
8
|
+
types_1.EventType.ExtractionDataDelete,
|
|
9
|
+
types_1.EventType.ExtractionAttachmentsDelete,
|
|
10
|
+
];
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { Artifact, EventType, ExtractorEventType } from '../types';
|
|
2
2
|
export declare function createFormData(preparedArtifact: any, fetchedObjects: object[] | object): FormData;
|
|
3
3
|
export declare function createArtifact(preparedArtifact: any, fetchedObjects: object[] | object, entity: string): Artifact;
|
|
4
|
-
export declare function getTimeoutExtractorEventType(eventType: EventType):
|
|
4
|
+
export declare function getTimeoutExtractorEventType(eventType: EventType): {
|
|
5
|
+
eventType: ExtractorEventType;
|
|
6
|
+
isError: boolean;
|
|
7
|
+
} | null;
|
|
@@ -29,15 +29,27 @@ exports.createArtifact = createArtifact;
|
|
|
29
29
|
function getTimeoutExtractorEventType(eventType) {
|
|
30
30
|
switch (eventType) {
|
|
31
31
|
case types_1.EventType.ExtractionMetadataStart:
|
|
32
|
-
return
|
|
32
|
+
return {
|
|
33
|
+
eventType: types_1.ExtractorEventType.ExtractionMetadataError,
|
|
34
|
+
isError: true,
|
|
35
|
+
};
|
|
33
36
|
case types_1.EventType.ExtractionDataStart:
|
|
34
37
|
case types_1.EventType.ExtractionDataContinue:
|
|
35
|
-
return
|
|
38
|
+
return {
|
|
39
|
+
eventType: types_1.ExtractorEventType.ExtractionDataProgress,
|
|
40
|
+
isError: false,
|
|
41
|
+
};
|
|
36
42
|
case types_1.EventType.ExtractionAttachmentsStart:
|
|
37
43
|
case types_1.EventType.ExtractionAttachmentsContinue:
|
|
38
|
-
return
|
|
44
|
+
return {
|
|
45
|
+
eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
|
|
46
|
+
isError: false,
|
|
47
|
+
};
|
|
39
48
|
case types_1.EventType.ExtractionExternalSyncUnitsStart:
|
|
40
|
-
return
|
|
49
|
+
return {
|
|
50
|
+
eventType: types_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
|
|
51
|
+
isError: true,
|
|
52
|
+
};
|
|
41
53
|
default:
|
|
42
54
|
console.log('Event type not recognized in getTimeoutExtractorEventType function: ' +
|
|
43
55
|
eventType);
|
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import { AirdropEvent } from '../types';
|
|
2
|
+
import { Adapter } from '../adapter';
|
|
3
|
+
type ExtractorState = object;
|
|
2
4
|
export declare class DemoExtractor {
|
|
3
|
-
|
|
5
|
+
private event;
|
|
6
|
+
private adapter;
|
|
7
|
+
private uploader;
|
|
8
|
+
constructor(event: AirdropEvent, adapter: Adapter<ExtractorState>);
|
|
9
|
+
run(): Promise<void>;
|
|
4
10
|
}
|
|
11
|
+
export {};
|
|
@@ -5,25 +5,25 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.DemoExtractor = void 0;
|
|
7
7
|
const types_1 = require("../types");
|
|
8
|
-
const adapter_1 = require("../adapter");
|
|
9
8
|
const uploader_1 = require("../uploader");
|
|
10
|
-
const
|
|
9
|
+
const initial_domain_mapping_json_1 = __importDefault(require("./initial_domain_mapping.json"));
|
|
11
10
|
class DemoExtractor {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
11
|
+
constructor(event, adapter) {
|
|
12
|
+
this.event = event;
|
|
13
|
+
this.adapter = adapter;
|
|
14
|
+
this.uploader = new uploader_1.Uploader(this.event.execution_metadata.devrev_endpoint, this.event.context.secrets.service_account_token);
|
|
15
|
+
}
|
|
16
|
+
async run() {
|
|
17
|
+
switch (this.event.payload.event_type) {
|
|
17
18
|
case types_1.EventType.ExtractionExternalSyncUnitsStart: {
|
|
18
19
|
const externalSyncUnits = [
|
|
19
20
|
{
|
|
20
21
|
id: 'devrev',
|
|
21
22
|
name: 'devrev',
|
|
22
|
-
description: '
|
|
23
|
-
item_count: 0,
|
|
23
|
+
description: 'Demo external sync unit',
|
|
24
24
|
},
|
|
25
25
|
];
|
|
26
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
|
|
26
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
|
|
27
27
|
external_sync_units: externalSyncUnits,
|
|
28
28
|
});
|
|
29
29
|
break;
|
|
@@ -32,116 +32,124 @@ class DemoExtractor {
|
|
|
32
32
|
const metadata = [
|
|
33
33
|
{
|
|
34
34
|
item: 'contacts',
|
|
35
|
-
fields: ['name', 'lastName'],
|
|
35
|
+
fields: ['id', 'name', 'lastName'],
|
|
36
36
|
},
|
|
37
37
|
{
|
|
38
38
|
item: 'users',
|
|
39
|
-
fields: ['name', 'lastName'],
|
|
39
|
+
fields: ['id', 'name', 'lastName'],
|
|
40
40
|
},
|
|
41
41
|
];
|
|
42
|
-
const { artifact, error } = await uploader.upload('loopback_metadata_1.jsonl', 'metadata', metadata);
|
|
43
|
-
if (error) {
|
|
44
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
|
|
42
|
+
const { artifact, error } = await this.uploader.upload('loopback_metadata_1.jsonl', 'metadata', metadata);
|
|
43
|
+
if (error || !artifact) {
|
|
44
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
|
|
45
45
|
error,
|
|
46
46
|
});
|
|
47
|
+
return;
|
|
47
48
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
await adapter.emit(types_1.ExtractorEventType.
|
|
49
|
+
const { artifact: recipe, error: recipeError } = await this.uploader.upload('recipe.json', 'initial_domain_mapping', initial_domain_mapping_json_1.default);
|
|
50
|
+
if (recipeError || !recipe) {
|
|
51
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
|
|
52
|
+
error: recipeError,
|
|
53
|
+
});
|
|
54
|
+
return;
|
|
51
55
|
}
|
|
56
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionMetadataDone, {
|
|
57
|
+
progress: 50,
|
|
58
|
+
artifacts: [artifact, recipe],
|
|
59
|
+
});
|
|
52
60
|
break;
|
|
53
61
|
}
|
|
54
62
|
case types_1.EventType.ExtractionDataStart: {
|
|
55
63
|
const contacts = [
|
|
56
64
|
{
|
|
65
|
+
id: 1,
|
|
57
66
|
name: 'John',
|
|
58
67
|
lastName: 'Doe',
|
|
59
68
|
},
|
|
60
69
|
{
|
|
70
|
+
id: 2,
|
|
61
71
|
name: 'Jane',
|
|
62
72
|
lastName: 'Doe',
|
|
63
73
|
},
|
|
64
74
|
];
|
|
65
|
-
const { artifact, error } = await uploader.upload('loopback_contacts_1.json', 'contacts', contacts);
|
|
66
|
-
if (error) {
|
|
67
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
|
|
75
|
+
const { artifact, error } = await this.uploader.upload('loopback_contacts_1.json', 'contacts', contacts);
|
|
76
|
+
if (error || !artifact) {
|
|
77
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
|
|
68
78
|
error,
|
|
69
79
|
});
|
|
80
|
+
return;
|
|
70
81
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
});
|
|
76
|
-
}
|
|
82
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataProgress, {
|
|
83
|
+
progress: 50,
|
|
84
|
+
artifacts: [artifact],
|
|
85
|
+
});
|
|
77
86
|
break;
|
|
78
87
|
}
|
|
79
88
|
case types_1.EventType.ExtractionDataContinue: {
|
|
80
89
|
const users = [
|
|
81
90
|
{
|
|
91
|
+
id: 1,
|
|
82
92
|
name: 'John',
|
|
83
93
|
lastName: 'Phd',
|
|
84
94
|
},
|
|
85
95
|
{
|
|
96
|
+
id: 2,
|
|
86
97
|
name: 'Jane',
|
|
87
98
|
lastName: 'Phd',
|
|
88
99
|
},
|
|
89
100
|
];
|
|
90
|
-
const { artifact, error } = await uploader.upload('loopback_users_1.json', 'users', users);
|
|
91
|
-
if (error) {
|
|
92
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
|
|
101
|
+
const { artifact, error } = await this.uploader.upload('loopback_users_1.json', 'users', users);
|
|
102
|
+
if (error || !artifact) {
|
|
103
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
|
|
93
104
|
error,
|
|
94
105
|
});
|
|
106
|
+
return;
|
|
95
107
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
await adapter.update({ artifact: recipe });
|
|
101
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionDataDone, {
|
|
102
|
-
progress: 100,
|
|
103
|
-
});
|
|
104
|
-
}
|
|
108
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataDone, {
|
|
109
|
+
progress: 100,
|
|
110
|
+
artifacts: [artifact],
|
|
111
|
+
});
|
|
105
112
|
break;
|
|
106
113
|
}
|
|
107
114
|
case types_1.EventType.ExtractionDataDelete: {
|
|
108
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionDataDeleteDone);
|
|
115
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataDeleteDone);
|
|
109
116
|
break;
|
|
110
117
|
}
|
|
111
118
|
case types_1.EventType.ExtractionAttachmentsStart: {
|
|
112
119
|
const attachment1 = ['This is attachment1.txt content'];
|
|
113
|
-
const { artifact, error } = await uploader.upload('attachment1.txt', 'attachment', attachment1);
|
|
114
|
-
if (error) {
|
|
115
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
|
|
120
|
+
const { artifact, error } = await this.uploader.upload('attachment1.txt', 'attachment', attachment1);
|
|
121
|
+
if (error || !artifact) {
|
|
122
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
|
|
116
123
|
error,
|
|
117
124
|
});
|
|
125
|
+
return;
|
|
118
126
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
}
|
|
127
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsProgress, {
|
|
128
|
+
artifacts: [artifact],
|
|
129
|
+
});
|
|
123
130
|
break;
|
|
124
131
|
}
|
|
125
132
|
case types_1.EventType.ExtractionAttachmentsContinue: {
|
|
126
133
|
const attachment2 = ['This is attachment2.txt content'];
|
|
127
|
-
const { artifact, error } = await uploader.upload('attachment2.txt', 'attachment', attachment2);
|
|
128
|
-
if (error) {
|
|
129
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
|
|
134
|
+
const { artifact, error } = await this.uploader.upload('attachment2.txt', 'attachment', attachment2);
|
|
135
|
+
if (error || !artifact) {
|
|
136
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
|
|
130
137
|
error,
|
|
131
138
|
});
|
|
139
|
+
return;
|
|
132
140
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
}
|
|
141
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDone, {
|
|
142
|
+
artifacts: [artifact],
|
|
143
|
+
});
|
|
137
144
|
break;
|
|
138
145
|
}
|
|
139
146
|
case types_1.EventType.ExtractionAttachmentsDelete: {
|
|
140
|
-
await adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
|
|
147
|
+
await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
|
|
141
148
|
break;
|
|
142
149
|
}
|
|
143
150
|
default: {
|
|
144
|
-
console.
|
|
151
|
+
console.error('Event in DemoExtractor run not recognized: ' +
|
|
152
|
+
JSON.stringify(this.event.payload.event_type));
|
|
145
153
|
}
|
|
146
154
|
}
|
|
147
155
|
}
|