@devrev/ts-adaas 0.0.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +182 -116
- package/dist/common/constants.d.ts +10 -0
- package/dist/common/constants.js +27 -0
- package/dist/common/control-protocol.d.ts +7 -0
- package/dist/common/control-protocol.js +41 -0
- package/dist/common/helpers.d.ts +4 -0
- package/dist/common/helpers.js +38 -0
- package/dist/common/install-initial-domain-mapping.d.ts +3 -0
- package/dist/common/install-initial-domain-mapping.js +87 -0
- package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
- package/dist/{src → deprecated}/adapter/index.js +13 -10
- package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
- package/dist/deprecated/common/helpers.js +47 -0
- package/dist/deprecated/demo-extractor/index.d.ts +17 -0
- package/dist/{src → deprecated}/demo-extractor/index.js +28 -22
- package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
- package/dist/{src → deprecated}/uploader/index.js +3 -1
- package/dist/{src/http → http}/client.d.ts +2 -1
- package/dist/index.d.ts +10 -0
- package/dist/{src/index.js → index.js} +11 -3
- package/dist/logger/logger.d.ts +14 -0
- package/dist/logger/logger.interfaces.d.ts +14 -0
- package/dist/logger/logger.interfaces.js +9 -0
- package/dist/logger/logger.js +91 -0
- package/dist/logger/logger.test.js +49 -0
- package/dist/repo/repo.d.ts +14 -0
- package/dist/repo/repo.interfaces.d.ts +43 -0
- package/dist/repo/repo.interfaces.js +2 -0
- package/dist/repo/repo.js +68 -0
- package/dist/repo/repo.test.js +74 -0
- package/dist/state/state.d.ts +24 -0
- package/dist/state/state.interfaces.d.ts +24 -0
- package/dist/state/state.interfaces.js +2 -0
- package/dist/state/state.js +115 -0
- package/dist/tests/test-helpers.d.ts +9 -0
- package/dist/tests/test-helpers.interfaces.d.ts +11 -0
- package/dist/tests/test-helpers.interfaces.js +2 -0
- package/dist/tests/test-helpers.js +88 -0
- package/dist/tests/test-worker.js +16 -0
- package/dist/types/common.d.ts +39 -0
- package/dist/{src/types → types}/common.js +4 -0
- package/dist/{src/types → types}/extraction.d.ts +56 -12
- package/dist/{src/types → types}/extraction.js +12 -29
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.js +11 -0
- package/dist/types/workers.d.ts +142 -0
- package/dist/types/workers.js +23 -0
- package/dist/uploader/uploader.d.ts +38 -0
- package/dist/uploader/uploader.interfaces.d.ts +63 -0
- package/dist/uploader/uploader.interfaces.js +2 -0
- package/dist/uploader/uploader.js +351 -0
- package/dist/{tests → uploader}/uploader.test.js +7 -5
- package/dist/workers/create-worker.d.ts +4 -0
- package/dist/workers/create-worker.js +30 -0
- package/dist/workers/create-worker.test.js +25 -0
- package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
- package/dist/workers/default-workers/attachments-deletion.js +13 -0
- package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
- package/dist/workers/default-workers/attachments-extraction.js +49 -0
- package/dist/workers/default-workers/data-deletion.d.ts +1 -0
- package/dist/workers/default-workers/data-deletion.js +15 -0
- package/dist/workers/default-workers/data-extraction.d.ts +1 -0
- package/dist/workers/default-workers/data-extraction.js +101 -0
- package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
- package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
- package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
- package/dist/workers/default-workers/metadata-extraction.js +26 -0
- package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
- package/dist/workers/dummy-extractor/data-normalization.js +41 -0
- package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
- package/dist/workers/process-task.d.ts +2 -0
- package/dist/workers/process-task.js +44 -0
- package/dist/workers/spawn.d.ts +25 -0
- package/dist/workers/spawn.js +163 -0
- package/dist/workers/worker-adapter.d.ts +48 -0
- package/dist/workers/worker-adapter.js +138 -0
- package/dist/workers/worker.d.ts +1 -0
- package/dist/workers/worker.js +6 -0
- package/package.json +9 -5
- package/dist/src/common/constants.d.ts +0 -2
- package/dist/src/common/constants.js +0 -10
- package/dist/src/common/helpers.js +0 -59
- package/dist/src/common/install-initial-domain-mapping.d.ts +0 -3
- package/dist/src/common/install-initial-domain-mapping.js +0 -60
- package/dist/src/demo-extractor/index.d.ts +0 -11
- package/dist/src/index.d.ts +0 -6
- package/dist/src/logging/index.d.ts +0 -18
- package/dist/src/logging/index.js +0 -39
- package/dist/src/state/index.d.ts +0 -23
- package/dist/src/state/index.js +0 -111
- package/dist/src/types/common.d.ts +0 -37
- package/dist/src/types/index.d.ts +0 -2
- package/dist/src/types/index.js +0 -18
- package/dist/tests/adapter.helpers.test.js +0 -60
- package/dist/tests/adapter.test.js +0 -123
- package/dist/tests/demo-extractor.test.js +0 -61
- package/dist/tests/state.test.js +0 -101
- /package/dist/{src → deprecated}/demo-extractor/external_domain_metadata.json +0 -0
- /package/dist/{src/http → http}/client.js +0 -0
- /package/dist/{src/http → http}/constants.d.ts +0 -0
- /package/dist/{src/http → http}/constants.js +0 -0
- /package/dist/{src/http → http}/index.d.ts +0 -0
- /package/dist/{src/http → http}/index.js +0 -0
- /package/dist/{src/http → http}/types.d.ts +0 -0
- /package/dist/{src/http → http}/types.js +0 -0
- /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
- /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
- /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
- /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
- /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { AirdropEvent, ExtractorEventType, EventData } from '../types/extraction';
|
|
2
|
+
import { AdapterState } from '../state/state.interfaces';
|
|
3
|
+
import { Artifact } from '../uploader/uploader.interfaces';
|
|
4
|
+
import { WorkerAdapterInterface, WorkerAdapterOptions } from '../types/workers';
|
|
5
|
+
import { Repo } from '../repo/repo';
|
|
6
|
+
import { RepoInterface } from '../repo/repo.interfaces';
|
|
7
|
+
export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
|
|
8
|
+
/**
|
|
9
|
+
* WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
|
|
10
|
+
* as parameter in processTask and onTimeout functions. The class provides
|
|
11
|
+
* utilities to emit control events to the platform, update the state of the connector,
|
|
12
|
+
* and upload artifacts to the platform.
|
|
13
|
+
* @class WorkerAdapter
|
|
14
|
+
* @constructor
|
|
15
|
+
* @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class
|
|
16
|
+
* @param {AirdropEvent} event - The event object received from the platform
|
|
17
|
+
* @param {object=} initialState - The initial state of the adapter
|
|
18
|
+
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
19
|
+
* @param {string} workerPath - The path to the worker file
|
|
20
|
+
*
|
|
21
|
+
*/
|
|
22
|
+
export declare class WorkerAdapter<ConnectorState> {
|
|
23
|
+
readonly event: AirdropEvent;
|
|
24
|
+
readonly options: WorkerAdapterOptions;
|
|
25
|
+
private adapterState;
|
|
26
|
+
private _artifacts;
|
|
27
|
+
private hasWorkerEmitted;
|
|
28
|
+
private parentPort;
|
|
29
|
+
private isTimeout;
|
|
30
|
+
private repos;
|
|
31
|
+
constructor({ event, adapterState, parentPort, options, }: WorkerAdapterInterface<ConnectorState>);
|
|
32
|
+
get state(): AdapterState<ConnectorState>;
|
|
33
|
+
set state(value: AdapterState<ConnectorState>);
|
|
34
|
+
initializeRepos(repos: RepoInterface[]): void;
|
|
35
|
+
getRepo(itemType: string): Repo | undefined;
|
|
36
|
+
postState(): Promise<void>;
|
|
37
|
+
get artifacts(): Artifact[];
|
|
38
|
+
set artifacts(artifacts: Artifact[]);
|
|
39
|
+
/**
|
|
40
|
+
* Emits an event to the platform.
|
|
41
|
+
*
|
|
42
|
+
* @param {ExtractorEventType} newEventType - The event type to be emitted
|
|
43
|
+
* @param {EventData=} data - The data to be sent with the event
|
|
44
|
+
*/
|
|
45
|
+
emit(newEventType: ExtractorEventType, data?: EventData): Promise<void>;
|
|
46
|
+
uploadAllRepos(): Promise<void>;
|
|
47
|
+
handleTimeout(): void;
|
|
48
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.WorkerAdapter = void 0;
|
|
4
|
+
exports.createWorkerAdapter = createWorkerAdapter;
|
|
5
|
+
const extraction_1 = require("../types/extraction");
|
|
6
|
+
const constants_1 = require("../common/constants");
|
|
7
|
+
const control_protocol_1 = require("../common/control-protocol");
|
|
8
|
+
const workers_1 = require("../types/workers");
|
|
9
|
+
const repo_1 = require("../repo/repo");
|
|
10
|
+
function createWorkerAdapter({ event, adapterState, parentPort, options, }) {
|
|
11
|
+
return new WorkerAdapter({
|
|
12
|
+
event,
|
|
13
|
+
adapterState,
|
|
14
|
+
parentPort,
|
|
15
|
+
options,
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
|
|
20
|
+
* as parameter in processTask and onTimeout functions. The class provides
|
|
21
|
+
* utilities to emit control events to the platform, update the state of the connector,
|
|
22
|
+
* and upload artifacts to the platform.
|
|
23
|
+
* @class WorkerAdapter
|
|
24
|
+
* @constructor
|
|
25
|
+
* @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class
|
|
26
|
+
* @param {AirdropEvent} event - The event object received from the platform
|
|
27
|
+
* @param {object=} initialState - The initial state of the adapter
|
|
28
|
+
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
29
|
+
* @param {string} workerPath - The path to the worker file
|
|
30
|
+
*
|
|
31
|
+
*/
|
|
32
|
+
class WorkerAdapter {
|
|
33
|
+
constructor({ event, adapterState, parentPort, options, }) {
|
|
34
|
+
this.repos = [];
|
|
35
|
+
this.event = event;
|
|
36
|
+
this.options = options;
|
|
37
|
+
this.adapterState = adapterState;
|
|
38
|
+
this._artifacts = [];
|
|
39
|
+
this.parentPort = parentPort;
|
|
40
|
+
this.hasWorkerEmitted = false;
|
|
41
|
+
this.isTimeout = false;
|
|
42
|
+
}
|
|
43
|
+
get state() {
|
|
44
|
+
return this.adapterState.state;
|
|
45
|
+
}
|
|
46
|
+
set state(value) {
|
|
47
|
+
if (!this.isTimeout) {
|
|
48
|
+
this.adapterState.state = value;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
initializeRepos(repos) {
|
|
52
|
+
this.repos = repos.map((repo) => {
|
|
53
|
+
const shouldNormalize = !Object.values(constants_1.AIRDROP_DEFAULT_ITEM_TYPES).includes(repo.itemType);
|
|
54
|
+
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
|
|
55
|
+
var _a;
|
|
56
|
+
this.artifacts.push(artifact);
|
|
57
|
+
// We need to store artifacts ids in state for later use when streaming attachments
|
|
58
|
+
if (repo.itemType === constants_1.AIRDROP_DEFAULT_ITEM_TYPES.ATTACHMENTS) {
|
|
59
|
+
(_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
60
|
+
}
|
|
61
|
+
}, options: this.options }));
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
getRepo(itemType) {
|
|
65
|
+
const repo = this.repos.find((repo) => repo.itemType === itemType);
|
|
66
|
+
if (!repo) {
|
|
67
|
+
console.error(`Repo not found for item type: ${itemType}.`);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
return repo;
|
|
71
|
+
}
|
|
72
|
+
async postState() {
|
|
73
|
+
await this.adapterState.postState();
|
|
74
|
+
}
|
|
75
|
+
get artifacts() {
|
|
76
|
+
return this._artifacts;
|
|
77
|
+
}
|
|
78
|
+
set artifacts(artifacts) {
|
|
79
|
+
this._artifacts = this._artifacts
|
|
80
|
+
.concat(artifacts)
|
|
81
|
+
.filter((value, index, self) => self.indexOf(value) === index);
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Emits an event to the platform.
|
|
85
|
+
*
|
|
86
|
+
* @param {ExtractorEventType} newEventType - The event type to be emitted
|
|
87
|
+
* @param {EventData=} data - The data to be sent with the event
|
|
88
|
+
*/
|
|
89
|
+
async emit(newEventType, data) {
|
|
90
|
+
if (this.hasWorkerEmitted) {
|
|
91
|
+
console.warn('Unable to emit with event type:' +
|
|
92
|
+
newEventType +
|
|
93
|
+
'. Lambda is going to be stopped.');
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
// We want to upload all the repos before emitting the event, except for the external sync units done event
|
|
97
|
+
if (newEventType !== extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone) {
|
|
98
|
+
await this.uploadAllRepos();
|
|
99
|
+
}
|
|
100
|
+
// If the extraction is done, we want to save the timestamp of the last successful sync
|
|
101
|
+
if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
|
|
102
|
+
this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
|
|
103
|
+
}
|
|
104
|
+
// We want to save the state every time we emit an event, except for the start and delete events
|
|
105
|
+
if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
|
|
106
|
+
console.log(`Saving state before emitting event with event type: ` +
|
|
107
|
+
newEventType +
|
|
108
|
+
'.');
|
|
109
|
+
await this.adapterState.postState(this.state);
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
await (0, control_protocol_1.emit)({
|
|
113
|
+
eventType: newEventType,
|
|
114
|
+
event: this.event,
|
|
115
|
+
data: Object.assign(Object.assign({}, data), { artifacts: this.artifacts }),
|
|
116
|
+
});
|
|
117
|
+
const message = {
|
|
118
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
|
|
119
|
+
payload: { eventType: newEventType },
|
|
120
|
+
};
|
|
121
|
+
this.artifacts = [];
|
|
122
|
+
this.parentPort.postMessage(message);
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
console.error('Error while emitting event with event type: ' + newEventType + '.', error);
|
|
126
|
+
this.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
async uploadAllRepos() {
|
|
130
|
+
for (const repo of this.repos) {
|
|
131
|
+
await repo.upload();
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
handleTimeout() {
|
|
135
|
+
this.isTimeout = true;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
exports.WorkerAdapter = WorkerAdapter;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devrev/ts-adaas",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "1.0.0",
|
|
4
4
|
"description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
|
|
5
5
|
"type": "commonjs",
|
|
6
|
-
"main": "./dist/
|
|
7
|
-
"typings": "./dist/
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"typings": "./dist/index.d.ts",
|
|
8
8
|
"scripts": {
|
|
9
|
-
"build": "tsc",
|
|
9
|
+
"build": "tsc -p ./tsconfig.json",
|
|
10
10
|
"start": "ts-node src/index.ts",
|
|
11
11
|
"lint": "eslint .",
|
|
12
12
|
"lint:fix": "eslint . --fix",
|
|
@@ -17,13 +17,15 @@
|
|
|
17
17
|
"url": "git+https://github.com/devrev/adaas-sdk.git"
|
|
18
18
|
},
|
|
19
19
|
"publishConfig": {
|
|
20
|
-
"registry": "https://
|
|
20
|
+
"registry": "https://npm.pkg.github.com"
|
|
21
21
|
},
|
|
22
22
|
"keywords": [],
|
|
23
23
|
"author": "devrev",
|
|
24
24
|
"license": "ISC",
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@types/jest": "^29.5.12",
|
|
27
|
+
"@types/lambda-log": "^3.0.3",
|
|
28
|
+
"@types/node": "20.16.11",
|
|
27
29
|
"@typescript-eslint/eslint-plugin": "^7.12.0",
|
|
28
30
|
"@typescript-eslint/parser": "^7.12.0",
|
|
29
31
|
"eslint": "^8.57.0",
|
|
@@ -36,7 +38,9 @@
|
|
|
36
38
|
"dependencies": {
|
|
37
39
|
"@devrev/typescript-sdk": "^1.1.27",
|
|
38
40
|
"axios": "^1.5.1",
|
|
41
|
+
"form-data": "^4.0.1",
|
|
39
42
|
"js-jsonl": "^1.1.1",
|
|
43
|
+
"lambda-log": "^3.1.0",
|
|
40
44
|
"ts-node": "^10.9.2"
|
|
41
45
|
},
|
|
42
46
|
"files": [
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.STATELESS_EVENT_TYPES = void 0;
|
|
4
|
-
const types_1 = require("../types");
|
|
5
|
-
exports.STATELESS_EVENT_TYPES = [
|
|
6
|
-
types_1.EventType.ExtractionExternalSyncUnitsStart,
|
|
7
|
-
types_1.EventType.ExtractionMetadataStart,
|
|
8
|
-
types_1.EventType.ExtractionDataDelete,
|
|
9
|
-
types_1.EventType.ExtractionAttachmentsDelete,
|
|
10
|
-
];
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getTimeoutExtractorEventType = exports.createArtifact = exports.createFormData = void 0;
|
|
4
|
-
const js_jsonl_1 = require("js-jsonl");
|
|
5
|
-
const types_1 = require("../types");
|
|
6
|
-
function createFormData(
|
|
7
|
-
//eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
8
|
-
preparedArtifact, fetchedObjects) {
|
|
9
|
-
const formData = new FormData();
|
|
10
|
-
for (const item of preparedArtifact.form_data) {
|
|
11
|
-
formData.append(item.key, item.value);
|
|
12
|
-
}
|
|
13
|
-
const output = js_jsonl_1.jsonl.stringify(fetchedObjects);
|
|
14
|
-
formData.append('file', output);
|
|
15
|
-
return formData;
|
|
16
|
-
}
|
|
17
|
-
exports.createFormData = createFormData;
|
|
18
|
-
function createArtifact(
|
|
19
|
-
//eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
20
|
-
preparedArtifact, fetchedObjects, entity) {
|
|
21
|
-
const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1;
|
|
22
|
-
return {
|
|
23
|
-
item_count: itemCount,
|
|
24
|
-
id: preparedArtifact.id,
|
|
25
|
-
item_type: entity,
|
|
26
|
-
};
|
|
27
|
-
}
|
|
28
|
-
exports.createArtifact = createArtifact;
|
|
29
|
-
function getTimeoutExtractorEventType(eventType) {
|
|
30
|
-
switch (eventType) {
|
|
31
|
-
case types_1.EventType.ExtractionMetadataStart:
|
|
32
|
-
return {
|
|
33
|
-
eventType: types_1.ExtractorEventType.ExtractionMetadataError,
|
|
34
|
-
isError: true,
|
|
35
|
-
};
|
|
36
|
-
case types_1.EventType.ExtractionDataStart:
|
|
37
|
-
case types_1.EventType.ExtractionDataContinue:
|
|
38
|
-
return {
|
|
39
|
-
eventType: types_1.ExtractorEventType.ExtractionDataProgress,
|
|
40
|
-
isError: false,
|
|
41
|
-
};
|
|
42
|
-
case types_1.EventType.ExtractionAttachmentsStart:
|
|
43
|
-
case types_1.EventType.ExtractionAttachmentsContinue:
|
|
44
|
-
return {
|
|
45
|
-
eventType: types_1.ExtractorEventType.ExtractionAttachmentsProgress,
|
|
46
|
-
isError: false,
|
|
47
|
-
};
|
|
48
|
-
case types_1.EventType.ExtractionExternalSyncUnitsStart:
|
|
49
|
-
return {
|
|
50
|
-
eventType: types_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
|
|
51
|
-
isError: true,
|
|
52
|
-
};
|
|
53
|
-
default:
|
|
54
|
-
console.log('Event type not recognized in getTimeoutExtractorEventType function: ' +
|
|
55
|
-
eventType);
|
|
56
|
-
return null;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
exports.getTimeoutExtractorEventType = getTimeoutExtractorEventType;
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.installInitialDomainMapping = void 0;
|
|
7
|
-
const axios_1 = __importDefault(require("axios"));
|
|
8
|
-
async function installInitialDomainMapping(event, initialDomainMappingJson) {
|
|
9
|
-
const devrevEndpoint = event.execution_metadata.devrev_endpoint;
|
|
10
|
-
const devrevToken = event.context.secrets.service_account_token;
|
|
11
|
-
const snapInVersionId = event.context.snap_in_version_id;
|
|
12
|
-
if (!initialDomainMappingJson) {
|
|
13
|
-
console.warn('No initial domain mapping found');
|
|
14
|
-
return;
|
|
15
|
-
}
|
|
16
|
-
const snapInVersionResponse = await axios_1.default.get(devrevEndpoint + '/internal/snap-in-versions.get', {
|
|
17
|
-
headers: {
|
|
18
|
-
Authorization: devrevToken,
|
|
19
|
-
},
|
|
20
|
-
params: {
|
|
21
|
-
id: snapInVersionId,
|
|
22
|
-
},
|
|
23
|
-
});
|
|
24
|
-
const importSlug = snapInVersionResponse.data.snap_in_version.imports[0].slug;
|
|
25
|
-
const snapInSlug = snapInVersionResponse.data.snap_in_version.slug;
|
|
26
|
-
const startingRecipeBlueprint = initialDomainMappingJson === null || initialDomainMappingJson === void 0 ? void 0 : initialDomainMappingJson.starting_recipe_blueprint;
|
|
27
|
-
let recipeBlueprintId;
|
|
28
|
-
if (startingRecipeBlueprint &&
|
|
29
|
-
Object.keys(startingRecipeBlueprint).length !== 0) {
|
|
30
|
-
try {
|
|
31
|
-
const recipeBlueprintResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`, Object.assign({}, startingRecipeBlueprint), {
|
|
32
|
-
headers: {
|
|
33
|
-
Authorization: devrevToken,
|
|
34
|
-
},
|
|
35
|
-
});
|
|
36
|
-
recipeBlueprintId = recipeBlueprintResponse.data.recipe_blueprint.id;
|
|
37
|
-
console.log('Successfully created recipe blueprint with id: ' + recipeBlueprintId);
|
|
38
|
-
}
|
|
39
|
-
catch (error) {
|
|
40
|
-
console.error('Error while creating recipe blueprint', error);
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
try {
|
|
44
|
-
// 2. Install the initial domain mappings
|
|
45
|
-
const additionalMappings = initialDomainMappingJson.additional_mappings || {};
|
|
46
|
-
const initialDomainMappingInstallResponse = await axios_1.default.post(`${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, Object.assign(Object.assign({ external_system_type: 'ADaaS', import_slug: importSlug, snap_in_slug: snapInSlug }, (recipeBlueprintId && {
|
|
47
|
-
starting_recipe_blueprint: recipeBlueprintId,
|
|
48
|
-
})), additionalMappings), {
|
|
49
|
-
headers: {
|
|
50
|
-
Authorization: devrevToken,
|
|
51
|
-
},
|
|
52
|
-
});
|
|
53
|
-
console.log('Successfully installed initial domain mapping', initialDomainMappingInstallResponse.data);
|
|
54
|
-
}
|
|
55
|
-
catch (error) {
|
|
56
|
-
console.error('Error while installing initial domain mapping', error);
|
|
57
|
-
return;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
exports.installInitialDomainMapping = installInitialDomainMapping;
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { AirdropEvent } from '../types';
|
|
2
|
-
import { Adapter } from '../adapter';
|
|
3
|
-
type ExtractorState = object;
|
|
4
|
-
export declare class DemoExtractor {
|
|
5
|
-
private event;
|
|
6
|
-
private adapter;
|
|
7
|
-
private uploader;
|
|
8
|
-
constructor(event: AirdropEvent, adapter: Adapter<ExtractorState>);
|
|
9
|
-
run(): Promise<void>;
|
|
10
|
-
}
|
|
11
|
-
export {};
|
package/dist/src/index.d.ts
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { AirdropEvent } from '../types';
|
|
2
|
-
export declare enum LogLevel {
|
|
3
|
-
INFO = "info",
|
|
4
|
-
WARN = "warn",
|
|
5
|
-
ERROR = "error"
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* Logger class to log messages based on the log level.
|
|
9
|
-
* The log level can be set to one of the following:
|
|
10
|
-
* - INFO
|
|
11
|
-
* - WARN
|
|
12
|
-
* - ERROR
|
|
13
|
-
*
|
|
14
|
-
* The log tags can be set to any key-value pair.
|
|
15
|
-
*/
|
|
16
|
-
export declare class Logger {
|
|
17
|
-
static init(event: AirdropEvent): void;
|
|
18
|
-
}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Logger = exports.LogLevel = void 0;
|
|
4
|
-
var LogLevel;
|
|
5
|
-
(function (LogLevel) {
|
|
6
|
-
LogLevel["INFO"] = "info";
|
|
7
|
-
LogLevel["WARN"] = "warn";
|
|
8
|
-
LogLevel["ERROR"] = "error";
|
|
9
|
-
})(LogLevel || (exports.LogLevel = LogLevel = {}));
|
|
10
|
-
/**
|
|
11
|
-
* Logger class to log messages based on the log level.
|
|
12
|
-
* The log level can be set to one of the following:
|
|
13
|
-
* - INFO
|
|
14
|
-
* - WARN
|
|
15
|
-
* - ERROR
|
|
16
|
-
*
|
|
17
|
-
* The log tags can be set to any key-value pair.
|
|
18
|
-
*/
|
|
19
|
-
class Logger {
|
|
20
|
-
static init(event) {
|
|
21
|
-
const origLog = console.log;
|
|
22
|
-
console.log = (message) => {
|
|
23
|
-
origLog(`[${LogLevel.INFO.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
|
|
24
|
-
};
|
|
25
|
-
const origInfo = console.info;
|
|
26
|
-
console.info = (message) => {
|
|
27
|
-
origInfo(`[${LogLevel.INFO.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
|
|
28
|
-
};
|
|
29
|
-
const origWarn = console.warn;
|
|
30
|
-
console.warn = (message) => {
|
|
31
|
-
origWarn(`[${LogLevel.WARN.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
|
|
32
|
-
};
|
|
33
|
-
const origError = console.error;
|
|
34
|
-
console.error = (message) => {
|
|
35
|
-
origError(`[${LogLevel.ERROR.toUpperCase()}]: ${message}`, Object.assign({}, event.payload.event_context));
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
exports.Logger = Logger;
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { AdapterState, AirdropEvent } from '../types';
|
|
2
|
-
export declare function createAdapterState<ExtractorState>(event: AirdropEvent, initialState: ExtractorState): Promise<State<ExtractorState>>;
|
|
3
|
-
export declare class State<ExtractorState> {
|
|
4
|
-
private _state;
|
|
5
|
-
private event;
|
|
6
|
-
private workerUrl;
|
|
7
|
-
private devrevToken;
|
|
8
|
-
constructor(event: AirdropEvent, initialState: ExtractorState);
|
|
9
|
-
get state(): AdapterState<ExtractorState>;
|
|
10
|
-
set state(value: AdapterState<ExtractorState>);
|
|
11
|
-
/**
|
|
12
|
-
* Updates the state of the adapter.
|
|
13
|
-
*
|
|
14
|
-
* @param {object} state - The state to be updated
|
|
15
|
-
*/
|
|
16
|
-
postState(state: AdapterState<ExtractorState>): Promise<void>;
|
|
17
|
-
/**
|
|
18
|
-
* Fetches the state of the adapter.
|
|
19
|
-
*
|
|
20
|
-
* @return The state of the adapter
|
|
21
|
-
*/
|
|
22
|
-
fetchState(initialState: ExtractorState): Promise<AdapterState<ExtractorState> | unknown>;
|
|
23
|
-
}
|
package/dist/src/state/index.js
DELETED
|
@@ -1,111 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.State = exports.createAdapterState = void 0;
|
|
27
|
-
const axios_1 = __importStar(require("axios"));
|
|
28
|
-
const constants_1 = require("../common/constants");
|
|
29
|
-
async function createAdapterState(event, initialState) {
|
|
30
|
-
const newInitialState = structuredClone(initialState);
|
|
31
|
-
const as = new State(event, newInitialState);
|
|
32
|
-
if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
|
|
33
|
-
console.log('Fetching state');
|
|
34
|
-
await as.fetchState(newInitialState);
|
|
35
|
-
}
|
|
36
|
-
return as;
|
|
37
|
-
}
|
|
38
|
-
exports.createAdapterState = createAdapterState;
|
|
39
|
-
class State {
|
|
40
|
-
constructor(event, initialState) {
|
|
41
|
-
this._state = Object.assign({ lastSyncStarted: '', lastSuccessfulSyncStarted: '' }, initialState);
|
|
42
|
-
this.event = event;
|
|
43
|
-
this.workerUrl = event.payload.event_context.worker_data_url;
|
|
44
|
-
this.devrevToken = event.context.secrets.service_account_token;
|
|
45
|
-
}
|
|
46
|
-
get state() {
|
|
47
|
-
return this._state;
|
|
48
|
-
}
|
|
49
|
-
set state(value) {
|
|
50
|
-
this._state = value;
|
|
51
|
-
}
|
|
52
|
-
/**
|
|
53
|
-
* Updates the state of the adapter.
|
|
54
|
-
*
|
|
55
|
-
* @param {object} state - The state to be updated
|
|
56
|
-
*/
|
|
57
|
-
async postState(state) {
|
|
58
|
-
try {
|
|
59
|
-
await axios_1.default.post(this.workerUrl + '.update', {
|
|
60
|
-
state: JSON.stringify(state),
|
|
61
|
-
}, {
|
|
62
|
-
headers: {
|
|
63
|
-
Authorization: this.devrevToken,
|
|
64
|
-
},
|
|
65
|
-
params: {
|
|
66
|
-
sync_unit: this.event.payload.event_context.sync_unit_id,
|
|
67
|
-
},
|
|
68
|
-
});
|
|
69
|
-
this.state = state;
|
|
70
|
-
console.log('State updated successfully');
|
|
71
|
-
}
|
|
72
|
-
catch (error) {
|
|
73
|
-
console.error('Failed to update state, error:' + error);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Fetches the state of the adapter.
|
|
78
|
-
*
|
|
79
|
-
* @return The state of the adapter
|
|
80
|
-
*/
|
|
81
|
-
async fetchState(initialState) {
|
|
82
|
-
var _a;
|
|
83
|
-
const state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: '', lastSuccessfulSyncStarted: '' });
|
|
84
|
-
console.log('Fetching state with sync unit id: ' +
|
|
85
|
-
this.event.payload.event_context.sync_unit_id);
|
|
86
|
-
try {
|
|
87
|
-
const response = await axios_1.default.post(this.workerUrl + '.get', {}, {
|
|
88
|
-
headers: {
|
|
89
|
-
Authorization: this.devrevToken,
|
|
90
|
-
},
|
|
91
|
-
params: {
|
|
92
|
-
sync_unit: this.event.payload.event_context.sync_unit_id,
|
|
93
|
-
},
|
|
94
|
-
});
|
|
95
|
-
this.state = JSON.parse(response.data.state);
|
|
96
|
-
console.log('State fetched successfully');
|
|
97
|
-
return this.state;
|
|
98
|
-
}
|
|
99
|
-
catch (error) {
|
|
100
|
-
if ((0, axios_1.isAxiosError)(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
101
|
-
console.log('State not found, returning initial state');
|
|
102
|
-
this.state = state;
|
|
103
|
-
this.postState(this.state);
|
|
104
|
-
return this.state;
|
|
105
|
-
}
|
|
106
|
-
console.error('Failed to fetch state, error:' + error);
|
|
107
|
-
return error;
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
exports.State = State;
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
export declare enum ErrorLevel {
|
|
2
|
-
Warning = "WARNING",
|
|
3
|
-
Error = "ERROR",
|
|
4
|
-
Info = "INFO"
|
|
5
|
-
}
|
|
6
|
-
export interface ErrorRecord {
|
|
7
|
-
message: string;
|
|
8
|
-
}
|
|
9
|
-
export interface LogRecord {
|
|
10
|
-
level: ErrorLevel;
|
|
11
|
-
message: string;
|
|
12
|
-
}
|
|
13
|
-
export interface Artifact {
|
|
14
|
-
id: string;
|
|
15
|
-
item_type: string;
|
|
16
|
-
item_count: number;
|
|
17
|
-
}
|
|
18
|
-
export interface ArtifactsPrepareResponse {
|
|
19
|
-
url: string;
|
|
20
|
-
id: string;
|
|
21
|
-
form_data: {
|
|
22
|
-
key: string;
|
|
23
|
-
value: string;
|
|
24
|
-
}[];
|
|
25
|
-
}
|
|
26
|
-
export interface AdapterUpdateParams {
|
|
27
|
-
artifact?: Artifact;
|
|
28
|
-
extractor_state?: object;
|
|
29
|
-
}
|
|
30
|
-
export interface UploadResponse {
|
|
31
|
-
artifact?: Artifact;
|
|
32
|
-
error?: ErrorRecord;
|
|
33
|
-
}
|
|
34
|
-
export interface InitialDomainMapping {
|
|
35
|
-
starting_recipe_blueprint?: object;
|
|
36
|
-
additional_mappings?: object;
|
|
37
|
-
}
|
package/dist/src/types/index.js
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./common"), exports);
|
|
18
|
-
__exportStar(require("./extraction"), exports);
|