@devrev/ts-adaas 0.0.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -116
- package/dist/common/constants.d.ts +10 -0
- package/dist/common/constants.js +27 -0
- package/dist/common/control-protocol.d.ts +7 -0
- package/dist/common/control-protocol.js +41 -0
- package/dist/common/helpers.d.ts +4 -0
- package/dist/common/helpers.js +38 -0
- package/dist/common/install-initial-domain-mapping.d.ts +3 -0
- package/dist/common/install-initial-domain-mapping.js +87 -0
- package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
- package/dist/{src → deprecated}/adapter/index.js +13 -10
- package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
- package/dist/deprecated/common/helpers.js +47 -0
- package/dist/deprecated/demo-extractor/external_domain_metadata.json +38 -0
- package/dist/deprecated/demo-extractor/index.d.ts +17 -0
- package/dist/deprecated/demo-extractor/index.js +161 -0
- package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
- package/dist/{src → deprecated}/uploader/index.js +3 -1
- package/dist/{src/http → http}/client.d.ts +2 -1
- package/dist/index.d.ts +10 -0
- package/dist/index.js +30 -0
- package/dist/logger/logger.d.ts +14 -0
- package/dist/logger/logger.interfaces.d.ts +14 -0
- package/dist/logger/logger.interfaces.js +9 -0
- package/dist/logger/logger.js +91 -0
- package/dist/logger/logger.test.js +49 -0
- package/dist/repo/repo.d.ts +14 -0
- package/dist/repo/repo.interfaces.d.ts +43 -0
- package/dist/repo/repo.interfaces.js +2 -0
- package/dist/repo/repo.js +68 -0
- package/dist/repo/repo.test.js +74 -0
- package/dist/state/state.d.ts +24 -0
- package/dist/state/state.interfaces.d.ts +24 -0
- package/dist/state/state.interfaces.js +2 -0
- package/dist/state/state.js +115 -0
- package/dist/tests/test-helpers.d.ts +9 -0
- package/dist/tests/test-helpers.interfaces.d.ts +11 -0
- package/dist/tests/test-helpers.interfaces.js +2 -0
- package/dist/tests/test-helpers.js +88 -0
- package/dist/tests/test-worker.js +16 -0
- package/dist/types/common.d.ts +39 -0
- package/dist/{src/types → types}/common.js +4 -0
- package/dist/{src/types → types}/extraction.d.ts +57 -12
- package/dist/{src/types → types}/extraction.js +12 -29
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.js +11 -0
- package/dist/types/workers.d.ts +142 -0
- package/dist/types/workers.js +23 -0
- package/dist/uploader/uploader.d.ts +38 -0
- package/dist/uploader/uploader.interfaces.d.ts +63 -0
- package/dist/uploader/uploader.interfaces.js +2 -0
- package/dist/uploader/uploader.js +351 -0
- package/dist/{tests → uploader}/uploader.test.js +7 -5
- package/dist/workers/create-worker.d.ts +4 -0
- package/dist/workers/create-worker.js +30 -0
- package/dist/workers/create-worker.test.js +25 -0
- package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
- package/dist/workers/default-workers/attachments-deletion.js +13 -0
- package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
- package/dist/workers/default-workers/attachments-extraction.js +49 -0
- package/dist/workers/default-workers/data-deletion.d.ts +1 -0
- package/dist/workers/default-workers/data-deletion.js +15 -0
- package/dist/workers/default-workers/data-extraction.d.ts +1 -0
- package/dist/workers/default-workers/data-extraction.js +101 -0
- package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
- package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
- package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
- package/dist/workers/default-workers/metadata-extraction.js +26 -0
- package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
- package/dist/workers/dummy-extractor/data-normalization.js +41 -0
- package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
- package/dist/workers/process-task.d.ts +2 -0
- package/dist/workers/process-task.js +44 -0
- package/dist/workers/spawn.d.ts +25 -0
- package/dist/workers/spawn.js +163 -0
- package/dist/workers/worker-adapter.d.ts +48 -0
- package/dist/workers/worker-adapter.js +138 -0
- package/dist/workers/worker.d.ts +1 -0
- package/dist/workers/worker.js +6 -0
- package/package.json +9 -5
- package/dist/src/common/constants.d.ts +0 -2
- package/dist/src/common/constants.js +0 -10
- package/dist/src/common/helpers.js +0 -59
- package/dist/src/demo-extractor/index.d.ts +0 -11
- package/dist/src/demo-extractor/index.js +0 -157
- package/dist/src/demo-extractor/initial_domain_mapping.json +0 -107
- package/dist/src/index.d.ts +0 -5
- package/dist/src/index.js +0 -21
- package/dist/src/logging/index.d.ts +0 -18
- package/dist/src/logging/index.js +0 -39
- package/dist/src/state/index.d.ts +0 -23
- package/dist/src/state/index.js +0 -111
- package/dist/src/types/common.d.ts +0 -33
- package/dist/src/types/index.d.ts +0 -2
- package/dist/src/types/index.js +0 -18
- package/dist/tests/adapter.helpers.test.js +0 -60
- package/dist/tests/adapter.test.js +0 -122
- package/dist/tests/demo-extractor.test.js +0 -60
- package/dist/tests/state.test.js +0 -100
- /package/dist/{src/http → http}/client.js +0 -0
- /package/dist/{src/http → http}/constants.d.ts +0 -0
- /package/dist/{src/http → http}/constants.js +0 -0
- /package/dist/{src/http → http}/index.d.ts +0 -0
- /package/dist/{src/http → http}/index.js +0 -0
- /package/dist/{src/http → http}/types.d.ts +0 -0
- /package/dist/{src/http → http}/types.js +0 -0
- /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
- /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
- /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
- /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
- /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
|
@@ -3,12 +3,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.Adapter =
|
|
6
|
+
exports.Adapter = void 0;
|
|
7
|
+
exports.createAdapter = createAdapter;
|
|
7
8
|
const axios_1 = __importDefault(require("axios"));
|
|
8
|
-
const constants_1 = require("
|
|
9
|
+
const constants_1 = require("../../common/constants");
|
|
9
10
|
const helpers_1 = require("../common/helpers");
|
|
10
|
-
|
|
11
|
-
const state_1 = require("
|
|
11
|
+
// import { Logger } from '../../logger/logger';
|
|
12
|
+
const state_1 = require("../../state/state");
|
|
12
13
|
/**
|
|
13
14
|
* Adapter class is used to interact with Airdrop platform. The class provides
|
|
14
15
|
* utilities to
|
|
@@ -18,6 +19,7 @@ const state_1 = require("../state");
|
|
|
18
19
|
*
|
|
19
20
|
* @class Adapter
|
|
20
21
|
* @constructor
|
|
22
|
+
* @deprecated
|
|
21
23
|
* @param {AirdropEvent} event - The event object received from the platform
|
|
22
24
|
* @param {object=} initialState - The initial state of the adapter
|
|
23
25
|
* @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
|
|
@@ -32,19 +34,21 @@ const state_1 = require("../state");
|
|
|
32
34
|
*/
|
|
33
35
|
async function createAdapter(event, initialState, isLocalDevelopment = false) {
|
|
34
36
|
const newInitialState = structuredClone(initialState);
|
|
35
|
-
const adapterState = await (0, state_1.createAdapterState)(
|
|
37
|
+
const adapterState = await (0, state_1.createAdapterState)({
|
|
38
|
+
event,
|
|
39
|
+
initialState: newInitialState,
|
|
40
|
+
});
|
|
36
41
|
const a = new Adapter(event, adapterState, isLocalDevelopment);
|
|
37
42
|
return a;
|
|
38
43
|
}
|
|
39
|
-
exports.createAdapter = createAdapter;
|
|
40
44
|
class Adapter {
|
|
41
45
|
constructor(event, adapterState, isLocalDevelopment = false) {
|
|
46
|
+
// if (!isLocalDevelopment) {
|
|
47
|
+
// Logger.init(event);
|
|
48
|
+
// }
|
|
42
49
|
this.exit = false;
|
|
43
50
|
this.lambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
|
|
44
51
|
this.heartBeatInterval = 30 * 1000; // 30 seconds in milliseconds
|
|
45
|
-
if (!isLocalDevelopment) {
|
|
46
|
-
logging_1.Logger.init(event);
|
|
47
|
-
}
|
|
48
52
|
this.adapterState = adapterState;
|
|
49
53
|
this._artifacts = [];
|
|
50
54
|
this.event = event;
|
|
@@ -122,7 +126,6 @@ class Adapter {
|
|
|
122
126
|
*/
|
|
123
127
|
exitAdapter() {
|
|
124
128
|
this.exit = true;
|
|
125
|
-
clearInterval(this.heartBeatFn);
|
|
126
129
|
}
|
|
127
130
|
/**
|
|
128
131
|
* Heartbeat function to check if the lambda is about to timeout.
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { EventType, ExtractorEventType } from '../../types/extraction';
|
|
2
2
|
export declare function createFormData(preparedArtifact: any, fetchedObjects: object[] | object): FormData;
|
|
3
|
-
export declare function createArtifact(preparedArtifact: any, fetchedObjects: object[] | object, entity: string): Artifact;
|
|
4
3
|
export declare function getTimeoutExtractorEventType(eventType: EventType): {
|
|
5
4
|
eventType: ExtractorEventType;
|
|
6
5
|
isError: boolean;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createFormData = createFormData;
|
|
4
|
+
exports.getTimeoutExtractorEventType = getTimeoutExtractorEventType;
|
|
5
|
+
const js_jsonl_1 = require("js-jsonl");
|
|
6
|
+
const extraction_1 = require("../../types/extraction");
|
|
7
|
+
function createFormData(
|
|
8
|
+
//eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
9
|
+
preparedArtifact, fetchedObjects) {
|
|
10
|
+
const formData = new FormData();
|
|
11
|
+
for (const item of preparedArtifact.form_data) {
|
|
12
|
+
formData.append(item.key, item.value);
|
|
13
|
+
}
|
|
14
|
+
const output = js_jsonl_1.jsonl.stringify(fetchedObjects);
|
|
15
|
+
formData.append('file', output);
|
|
16
|
+
return formData;
|
|
17
|
+
}
|
|
18
|
+
function getTimeoutExtractorEventType(eventType) {
|
|
19
|
+
switch (eventType) {
|
|
20
|
+
case extraction_1.EventType.ExtractionMetadataStart:
|
|
21
|
+
return {
|
|
22
|
+
eventType: extraction_1.ExtractorEventType.ExtractionMetadataError,
|
|
23
|
+
isError: true,
|
|
24
|
+
};
|
|
25
|
+
case extraction_1.EventType.ExtractionDataStart:
|
|
26
|
+
case extraction_1.EventType.ExtractionDataContinue:
|
|
27
|
+
return {
|
|
28
|
+
eventType: extraction_1.ExtractorEventType.ExtractionDataProgress,
|
|
29
|
+
isError: false,
|
|
30
|
+
};
|
|
31
|
+
case extraction_1.EventType.ExtractionAttachmentsStart:
|
|
32
|
+
case extraction_1.EventType.ExtractionAttachmentsContinue:
|
|
33
|
+
return {
|
|
34
|
+
eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsProgress,
|
|
35
|
+
isError: false,
|
|
36
|
+
};
|
|
37
|
+
case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
|
|
38
|
+
return {
|
|
39
|
+
eventType: extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
|
|
40
|
+
isError: true,
|
|
41
|
+
};
|
|
42
|
+
default:
|
|
43
|
+
console.log('Event type not recognized in getTimeoutExtractorEventType function: ' +
|
|
44
|
+
eventType);
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"record_types": {
|
|
3
|
+
"users": {
|
|
4
|
+
"fields": {
|
|
5
|
+
"name": {
|
|
6
|
+
"is_required": true,
|
|
7
|
+
"type": "text",
|
|
8
|
+
"name": "Name",
|
|
9
|
+
"text": {
|
|
10
|
+
"min_length": 1
|
|
11
|
+
}
|
|
12
|
+
},
|
|
13
|
+
"email": {
|
|
14
|
+
"type": "text",
|
|
15
|
+
"name": "Email",
|
|
16
|
+
"is_required": true
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"contacts": {
|
|
21
|
+
"fields": {
|
|
22
|
+
"name": {
|
|
23
|
+
"is_required": true,
|
|
24
|
+
"type": "text",
|
|
25
|
+
"name": "Name",
|
|
26
|
+
"text": {
|
|
27
|
+
"min_length": 1
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"email": {
|
|
31
|
+
"type": "text",
|
|
32
|
+
"name": "Email",
|
|
33
|
+
"is_required": true
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { AirdropEvent } from '../../types/extraction';
|
|
2
|
+
import { Adapter } from '../adapter';
|
|
3
|
+
type ConnectorState = object;
|
|
4
|
+
/**
|
|
5
|
+
* Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
|
|
6
|
+
*
|
|
7
|
+
* @class DemoExtractor
|
|
8
|
+
* @deprecated
|
|
9
|
+
**/
|
|
10
|
+
export declare class DemoExtractor {
|
|
11
|
+
private event;
|
|
12
|
+
private adapter;
|
|
13
|
+
private uploader;
|
|
14
|
+
constructor(event: AirdropEvent, adapter: Adapter<ConnectorState>);
|
|
15
|
+
run(): Promise<void>;
|
|
16
|
+
}
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.DemoExtractor = void 0;
|
|
7
|
+
const extraction_1 = require("../../types/extraction");
|
|
8
|
+
const uploader_1 = require("../uploader");
|
|
9
|
+
const external_domain_metadata_json_1 = __importDefault(require("./external_domain_metadata.json"));
|
|
10
|
+
/**
|
|
11
|
+
* Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
|
|
12
|
+
*
|
|
13
|
+
* @class DemoExtractor
|
|
14
|
+
* @deprecated
|
|
15
|
+
**/
|
|
16
|
+
class DemoExtractor {
|
|
17
|
+
constructor(event, adapter) {
|
|
18
|
+
this.event = event;
|
|
19
|
+
this.adapter = adapter;
|
|
20
|
+
this.uploader = new uploader_1.Uploader(this.event.execution_metadata.devrev_endpoint, this.event.context.secrets.service_account_token);
|
|
21
|
+
}
|
|
22
|
+
async run() {
|
|
23
|
+
switch (this.event.payload.event_type) {
|
|
24
|
+
case extraction_1.EventType.ExtractionExternalSyncUnitsStart: {
|
|
25
|
+
const externalSyncUnits = [
|
|
26
|
+
{
|
|
27
|
+
id: 'devrev',
|
|
28
|
+
name: 'devrev',
|
|
29
|
+
description: 'Demo external sync unit',
|
|
30
|
+
},
|
|
31
|
+
];
|
|
32
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
|
|
33
|
+
external_sync_units: externalSyncUnits,
|
|
34
|
+
});
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
case extraction_1.EventType.ExtractionMetadataStart: {
|
|
38
|
+
const { artifact, error } = await this.uploader.upload('metadata_1.jsonl', 'external_domain_metadata', external_domain_metadata_json_1.default);
|
|
39
|
+
if (error || !artifact) {
|
|
40
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataError, {
|
|
41
|
+
error,
|
|
42
|
+
});
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataDone, {
|
|
46
|
+
artifacts: [artifact],
|
|
47
|
+
});
|
|
48
|
+
break;
|
|
49
|
+
}
|
|
50
|
+
case extraction_1.EventType.ExtractionDataStart: {
|
|
51
|
+
const contacts = [
|
|
52
|
+
{
|
|
53
|
+
id: 'contact-1',
|
|
54
|
+
created_date: '1999-12-25T01:00:03+01:00',
|
|
55
|
+
modified_date: '1999-12-25T01:00:03+01:00',
|
|
56
|
+
data: {
|
|
57
|
+
email: 'johnsmith@test.com',
|
|
58
|
+
name: 'John Smith',
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
id: 'contact-2',
|
|
63
|
+
created_date: '1999-12-27T15:31:34+01:00',
|
|
64
|
+
modified_date: '2002-04-09T01:55:31+02:00',
|
|
65
|
+
data: {
|
|
66
|
+
email: 'janesmith@test.com',
|
|
67
|
+
name: 'Jane Smith',
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
];
|
|
71
|
+
const { artifact, error } = await this.uploader.upload('contacts_1.json', 'contacts', contacts);
|
|
72
|
+
if (error || !artifact) {
|
|
73
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
|
|
74
|
+
error,
|
|
75
|
+
});
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataProgress, {
|
|
79
|
+
progress: 50,
|
|
80
|
+
artifacts: [artifact],
|
|
81
|
+
});
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
case extraction_1.EventType.ExtractionDataContinue: {
|
|
85
|
+
const users = [
|
|
86
|
+
{
|
|
87
|
+
id: 'user-1',
|
|
88
|
+
created_date: '1999-12-25T01:00:03+01:00',
|
|
89
|
+
modified_date: '1999-12-25T01:00:03+01:00',
|
|
90
|
+
data: {
|
|
91
|
+
email: 'johndoe@test.com',
|
|
92
|
+
name: 'John Doe',
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
id: 'user-2',
|
|
97
|
+
created_date: '1999-12-27T15:31:34+01:00',
|
|
98
|
+
modified_date: '2002-04-09T01:55:31+02:00',
|
|
99
|
+
data: {
|
|
100
|
+
email: 'janedoe@test.com',
|
|
101
|
+
name: 'Jane Doe',
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
];
|
|
105
|
+
const { artifact, error } = await this.uploader.upload('users_1.json', 'users', users);
|
|
106
|
+
if (error || !artifact) {
|
|
107
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
|
|
108
|
+
error,
|
|
109
|
+
});
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDone, {
|
|
113
|
+
progress: 100,
|
|
114
|
+
artifacts: [artifact],
|
|
115
|
+
});
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
case extraction_1.EventType.ExtractionDataDelete: {
|
|
119
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDeleteDone);
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
case extraction_1.EventType.ExtractionAttachmentsStart: {
|
|
123
|
+
const attachment1 = ['This is attachment1.txt content'];
|
|
124
|
+
const { artifact, error } = await this.uploader.upload('attachment1.txt', 'attachment', attachment1);
|
|
125
|
+
if (error || !artifact) {
|
|
126
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
|
|
127
|
+
error,
|
|
128
|
+
});
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsProgress, {
|
|
132
|
+
artifacts: [artifact],
|
|
133
|
+
});
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
case extraction_1.EventType.ExtractionAttachmentsContinue: {
|
|
137
|
+
const attachment2 = ['This is attachment2.txt content'];
|
|
138
|
+
const { artifact, error } = await this.uploader.upload('attachment2.txt', 'attachment', attachment2);
|
|
139
|
+
if (error || !artifact) {
|
|
140
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
|
|
141
|
+
error,
|
|
142
|
+
});
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDone, {
|
|
146
|
+
artifacts: [artifact],
|
|
147
|
+
});
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
case extraction_1.EventType.ExtractionAttachmentsDelete: {
|
|
151
|
+
await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
default: {
|
|
155
|
+
console.error('Event in DemoExtractor run not recognized: ' +
|
|
156
|
+
JSON.stringify(this.event.payload.event_type));
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
exports.DemoExtractor = DemoExtractor;
|
|
@@ -106,7 +106,9 @@ class Uploader {
|
|
|
106
106
|
}
|
|
107
107
|
async uploadToArtifact(
|
|
108
108
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
109
|
-
preparedArtifact, fetchedObjects
|
|
109
|
+
preparedArtifact, fetchedObjects
|
|
110
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
111
|
+
) {
|
|
110
112
|
const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
|
|
111
113
|
try {
|
|
112
114
|
const response = await axios_1.default.post(preparedArtifact.url, formData, {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { RawAxiosRequestHeaders } from 'axios';
|
|
1
2
|
import { HTTPResponse } from './types';
|
|
2
3
|
export declare const defaultResponse: HTTPResponse;
|
|
3
4
|
export declare class HTTPClient {
|
|
@@ -12,5 +13,5 @@ export declare class HTTPClient {
|
|
|
12
13
|
* In case of rate limit exceeded, the function returns success as true and the delay time in seconds
|
|
13
14
|
* In case of any other error, the function returns success as false and the error message
|
|
14
15
|
*/
|
|
15
|
-
getCall(endpoint: string, headers:
|
|
16
|
+
getCall(endpoint: string, headers: RawAxiosRequestHeaders, params?: any): Promise<HTTPResponse>;
|
|
16
17
|
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export * from './deprecated/adapter';
|
|
2
|
+
export * from './deprecated/demo-extractor';
|
|
3
|
+
export * from './deprecated/uploader';
|
|
4
|
+
export * from './types';
|
|
5
|
+
export * from './http';
|
|
6
|
+
export * from './common/install-initial-domain-mapping';
|
|
7
|
+
export { WorkerAdapter } from './workers/worker-adapter';
|
|
8
|
+
export { processTask } from './workers/process-task';
|
|
9
|
+
export { spawn } from './workers/spawn';
|
|
10
|
+
export * from './types/workers';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.spawn = exports.processTask = exports.WorkerAdapter = void 0;
|
|
18
|
+
__exportStar(require("./deprecated/adapter"), exports);
|
|
19
|
+
__exportStar(require("./deprecated/demo-extractor"), exports);
|
|
20
|
+
__exportStar(require("./deprecated/uploader"), exports);
|
|
21
|
+
__exportStar(require("./types"), exports);
|
|
22
|
+
__exportStar(require("./http"), exports);
|
|
23
|
+
__exportStar(require("./common/install-initial-domain-mapping"), exports);
|
|
24
|
+
var worker_adapter_1 = require("./workers/worker-adapter");
|
|
25
|
+
Object.defineProperty(exports, "WorkerAdapter", { enumerable: true, get: function () { return worker_adapter_1.WorkerAdapter; } });
|
|
26
|
+
var process_task_1 = require("./workers/process-task");
|
|
27
|
+
Object.defineProperty(exports, "processTask", { enumerable: true, get: function () { return process_task_1.processTask; } });
|
|
28
|
+
var spawn_1 = require("./workers/spawn");
|
|
29
|
+
Object.defineProperty(exports, "spawn", { enumerable: true, get: function () { return spawn_1.spawn; } });
|
|
30
|
+
__exportStar(require("./types/workers"), exports);
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Console } from 'node:console';
|
|
2
|
+
import { LogLevel, PrintableState } from './logger.interfaces';
|
|
3
|
+
import { AirdropEvent } from '../types';
|
|
4
|
+
import { AxiosError } from 'axios';
|
|
5
|
+
export declare class Logger extends Console {
|
|
6
|
+
constructor(event: AirdropEvent);
|
|
7
|
+
logFn(args: unknown[], level: LogLevel): void;
|
|
8
|
+
log(...args: unknown[]): void;
|
|
9
|
+
info(...args: unknown[]): void;
|
|
10
|
+
warn(...args: unknown[]): void;
|
|
11
|
+
error(...args: unknown[]): void;
|
|
12
|
+
}
|
|
13
|
+
export declare function getPrintableState(state: Record<string, any>): PrintableState;
|
|
14
|
+
export declare function formatAxiosError(error: AxiosError): object;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare enum LogLevel {
|
|
2
|
+
INFO = "info",
|
|
3
|
+
WARN = "warn",
|
|
4
|
+
ERROR = "error"
|
|
5
|
+
}
|
|
6
|
+
export interface PrintableArray {
|
|
7
|
+
type: 'array';
|
|
8
|
+
length: number;
|
|
9
|
+
firstItem?: any;
|
|
10
|
+
lastItem?: any;
|
|
11
|
+
}
|
|
12
|
+
export interface PrintableState {
|
|
13
|
+
[key: string]: any | PrintableArray | PrintableState;
|
|
14
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LogLevel = void 0;
|
|
4
|
+
var LogLevel;
|
|
5
|
+
(function (LogLevel) {
|
|
6
|
+
LogLevel["INFO"] = "info";
|
|
7
|
+
LogLevel["WARN"] = "warn";
|
|
8
|
+
LogLevel["ERROR"] = "error";
|
|
9
|
+
})(LogLevel || (exports.LogLevel = LogLevel = {}));
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Logger = void 0;
|
|
7
|
+
exports.getPrintableState = getPrintableState;
|
|
8
|
+
exports.formatAxiosError = formatAxiosError;
|
|
9
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
10
|
+
const lambda_log_1 = __importDefault(require("lambda-log"));
|
|
11
|
+
const node_console_1 = require("node:console");
|
|
12
|
+
const logger_interfaces_1 = require("./logger.interfaces");
|
|
13
|
+
const node_worker_threads_1 = require("node:worker_threads");
|
|
14
|
+
const workers_1 = require("../types/workers");
|
|
15
|
+
class Logger extends node_console_1.Console {
|
|
16
|
+
constructor(event) {
|
|
17
|
+
super(process.stdout, process.stderr);
|
|
18
|
+
lambda_log_1.default.options.levelKey = null;
|
|
19
|
+
lambda_log_1.default.options.tagsKey = null;
|
|
20
|
+
lambda_log_1.default.options.messageKey = 'message';
|
|
21
|
+
lambda_log_1.default.options.meta = Object.assign({}, event.payload.event_context);
|
|
22
|
+
}
|
|
23
|
+
logFn(args, level) {
|
|
24
|
+
if (node_worker_threads_1.isMainThread) {
|
|
25
|
+
lambda_log_1.default.log(level, JSON.stringify(args));
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
|
|
29
|
+
subject: workers_1.WorkerMessageSubject.WorkerMessageLog,
|
|
30
|
+
payload: {
|
|
31
|
+
args,
|
|
32
|
+
level,
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
log(...args) {
|
|
38
|
+
this.logFn(args, logger_interfaces_1.LogLevel.INFO);
|
|
39
|
+
}
|
|
40
|
+
info(...args) {
|
|
41
|
+
this.logFn(args, logger_interfaces_1.LogLevel.INFO);
|
|
42
|
+
}
|
|
43
|
+
warn(...args) {
|
|
44
|
+
this.logFn(args, logger_interfaces_1.LogLevel.WARN);
|
|
45
|
+
}
|
|
46
|
+
error(...args) {
|
|
47
|
+
this.logFn(args, logger_interfaces_1.LogLevel.ERROR);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
exports.Logger = Logger;
|
|
51
|
+
// Helper function to process each value in the state
|
|
52
|
+
function getPrintableState(state) {
|
|
53
|
+
function processValue(value) {
|
|
54
|
+
if (Array.isArray(value)) {
|
|
55
|
+
// If the value is an array, summarize it
|
|
56
|
+
return {
|
|
57
|
+
type: 'array',
|
|
58
|
+
length: value.length,
|
|
59
|
+
firstItem: value.length > 0 ? value[0] : undefined,
|
|
60
|
+
lastItem: value.length > 1 ? value[value.length - 1] : undefined,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
else if (typeof value === 'object' && value !== null) {
|
|
64
|
+
// If the value is an object, recursively process its properties
|
|
65
|
+
const processedObject = {};
|
|
66
|
+
for (const key in value) {
|
|
67
|
+
if (value.hasOwnProperty(key)) {
|
|
68
|
+
processedObject[key] = processValue(value[key]);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return processedObject;
|
|
72
|
+
}
|
|
73
|
+
// For primitive types, return the value as is
|
|
74
|
+
return value;
|
|
75
|
+
}
|
|
76
|
+
// Process the state object directly since it's guaranteed to be an object
|
|
77
|
+
return processValue(state);
|
|
78
|
+
}
|
|
79
|
+
function formatAxiosError(error) {
|
|
80
|
+
var _a, _b, _c;
|
|
81
|
+
if (error.response) {
|
|
82
|
+
return {
|
|
83
|
+
status: error.response.status,
|
|
84
|
+
data: error.response.data,
|
|
85
|
+
method: (_a = error.config) === null || _a === void 0 ? void 0 : _a.method,
|
|
86
|
+
baseURL: (_b = error.config) === null || _b === void 0 ? void 0 : _b.baseURL,
|
|
87
|
+
url: (_c = error.config) === null || _c === void 0 ? void 0 : _c.url,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
return error;
|
|
91
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const logger_1 = require("./logger");
|
|
4
|
+
it('getPrintableState should return printable state', () => {
|
|
5
|
+
const state = {
|
|
6
|
+
test_key: 'test_value',
|
|
7
|
+
big_array: Array.from({ length: 1000 }, (_, index) => index),
|
|
8
|
+
nested_object: {
|
|
9
|
+
nested_key: 'nested_value',
|
|
10
|
+
nested_array: Array.from({ length: 1000 }, (_, index) => index),
|
|
11
|
+
},
|
|
12
|
+
};
|
|
13
|
+
const printableState = (0, logger_1.getPrintableState)(state);
|
|
14
|
+
expect(printableState).toEqual({
|
|
15
|
+
test_key: 'test_value',
|
|
16
|
+
big_array: {
|
|
17
|
+
type: 'array',
|
|
18
|
+
length: 1000,
|
|
19
|
+
firstItem: 0,
|
|
20
|
+
lastItem: 999,
|
|
21
|
+
},
|
|
22
|
+
nested_object: {
|
|
23
|
+
nested_key: 'nested_value',
|
|
24
|
+
nested_array: {
|
|
25
|
+
type: 'array',
|
|
26
|
+
length: 1000,
|
|
27
|
+
firstItem: 0,
|
|
28
|
+
lastItem: 999,
|
|
29
|
+
},
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
it('formatAxiosError should return formatted error', () => {
|
|
34
|
+
const error = {
|
|
35
|
+
response: {
|
|
36
|
+
status: 500,
|
|
37
|
+
data: 'Internal server error',
|
|
38
|
+
},
|
|
39
|
+
config: {
|
|
40
|
+
method: 'GET',
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
const formattedError = (0, logger_1.formatAxiosError)(error);
|
|
44
|
+
expect(formattedError).toEqual({
|
|
45
|
+
status: 500,
|
|
46
|
+
data: 'Internal server error',
|
|
47
|
+
method: 'GET',
|
|
48
|
+
});
|
|
49
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { ErrorRecord } from '../types/common';
|
|
2
|
+
import { Item } from '../repo/repo.interfaces';
|
|
3
|
+
import { RepoFactoryInterface, NormalizedItem, NormalizedAttachment } from './repo.interfaces';
|
|
4
|
+
export declare class Repo {
|
|
5
|
+
readonly itemType: string;
|
|
6
|
+
private items;
|
|
7
|
+
private normalize?;
|
|
8
|
+
private uploader;
|
|
9
|
+
private onUpload;
|
|
10
|
+
constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
|
|
11
|
+
getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
|
|
12
|
+
upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
|
|
13
|
+
push(items: Item[]): Promise<void | ErrorRecord>;
|
|
14
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { Artifact } from '../uploader/uploader.interfaces';
|
|
2
|
+
import { AirdropEvent } from '../types/extraction';
|
|
3
|
+
import { WorkerAdapterOptions } from '../types/workers';
|
|
4
|
+
/**
|
|
5
|
+
* RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data.
|
|
6
|
+
*/
|
|
7
|
+
export interface RepoInterface {
|
|
8
|
+
itemType: string;
|
|
9
|
+
normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo.
|
|
13
|
+
*/
|
|
14
|
+
export interface RepoFactoryInterface {
|
|
15
|
+
event: AirdropEvent;
|
|
16
|
+
itemType: string;
|
|
17
|
+
normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
|
|
18
|
+
onUpload: (artifact: Artifact) => void;
|
|
19
|
+
options?: WorkerAdapterOptions;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* NormalizedItem is an interface of item after normalization.
|
|
23
|
+
*/
|
|
24
|
+
export interface NormalizedItem {
|
|
25
|
+
id: string;
|
|
26
|
+
created_date: string;
|
|
27
|
+
modified_date: string;
|
|
28
|
+
data: object;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* NormalizedAttachment is an interface of attachment after normalization.
|
|
32
|
+
*/
|
|
33
|
+
export interface NormalizedAttachment {
|
|
34
|
+
url: string;
|
|
35
|
+
id: string;
|
|
36
|
+
file_name: string;
|
|
37
|
+
author_id: string;
|
|
38
|
+
parent_id: string;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Item is an interface that defines the structure of an item.
|
|
42
|
+
*/
|
|
43
|
+
export type Item = Record<string, any>;
|