@devrev/ts-adaas 1.2.0-beta.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/dist/common/control-protocol.d.ts +2 -1
- package/dist/common/control-protocol.js +11 -23
- package/dist/common/helpers.d.ts +1 -0
- package/dist/common/helpers.js +47 -0
- package/dist/{http → deprecated/http}/client.d.ts +1 -1
- package/dist/{http → deprecated/http}/client.js +1 -1
- package/dist/http/axios-client.js +13 -21
- package/dist/http/index.d.ts +0 -1
- package/dist/http/index.js +0 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/repo/repo.interfaces.d.ts +1 -1
- package/dist/repo/repo.js +4 -1
- package/dist/state/state.js +13 -7
- package/dist/types/extraction.d.ts +37 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/loading.d.ts +2 -0
- package/dist/types/workers.d.ts +1 -6
- package/dist/uploader/uploader.js +13 -13
- package/dist/workers/default-workers/attachments-extraction.js +22 -13
- package/dist/workers/default-workers/load-attachments.js +2 -2
- package/dist/workers/process-task.js +0 -1
- package/dist/workers/spawn.d.ts +3 -4
- package/dist/workers/spawn.js +71 -45
- package/dist/workers/worker-adapter.d.ts +10 -23
- package/dist/workers/worker-adapter.js +155 -125
- package/package.json +1 -1
- package/dist/workers/default-workers/data-loading.d.ts +0 -1
- package/dist/workers/default-workers/data-loading.js +0 -19
- package/dist/workers/default-workers/loader-state-deletion.d.ts +0 -1
- package/dist/workers/default-workers/loader-state-deletion.js +0 -15
package/README.md
CHANGED
|
@@ -2,6 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
## Release Notes
|
|
4
4
|
|
|
5
|
+
### v1.2.2
|
|
6
|
+
|
|
7
|
+
- Add library version as a part of control protocol.
|
|
8
|
+
- Improve axios client and adapter logging.
|
|
9
|
+
- Fix bugs related to state handling.
|
|
10
|
+
|
|
11
|
+
### v1.2.1
|
|
12
|
+
|
|
13
|
+
- Reduced the `delayFactor` to minimize unnecessary delays.
|
|
14
|
+
- Correct the setting of the `lastSyncStarted` timestamp.
|
|
15
|
+
- Improve logging for attachment extraction and loading.
|
|
16
|
+
- Fix several bugs related to the control protocol.
|
|
17
|
+
|
|
5
18
|
### v1.2.0
|
|
6
19
|
|
|
7
20
|
- Add support for loading attachments from DevRev to external system.
|
|
@@ -85,6 +98,8 @@
|
|
|
85
98
|
|
|
86
99
|
# Overview
|
|
87
100
|
|
|
101
|
+
[](https://coveralls.io/github/devrev/adaas-sdk?branch=main)
|
|
102
|
+
|
|
88
103
|
The ADaaS (Airdrop-as-a-Service) Library for TypeScript helps developers build Snap-ins that integrate with DevRev’s ADaaS platform. This library simplifies the workflow for handling data extraction and loading, event-driven actions, state management, and artifact handling.
|
|
89
104
|
|
|
90
105
|
It provides features such as:
|
|
@@ -359,7 +374,7 @@ This phase is defined in `load-attachments.ts` and is responsible for loading th
|
|
|
359
374
|
Loading is done by providing the create function to create attachments in the external system.
|
|
360
375
|
|
|
361
376
|
```typescript
|
|
362
|
-
|
|
377
|
+
processTask({
|
|
363
378
|
task: async ({ adapter }) => {
|
|
364
379
|
const { reports, processed_files } = await adapter.loadAttachments({
|
|
365
380
|
create,
|
|
@@ -378,7 +393,6 @@ Loading is done by providing the create function to create attachments in the ex
|
|
|
378
393
|
});
|
|
379
394
|
},
|
|
380
395
|
});
|
|
381
|
-
|
|
382
396
|
```
|
|
383
397
|
|
|
384
398
|
The loading function `create` provides loading to the external system, to make API calls to the external system to create the attachments and handle errors and external system's rate limiting.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AxiosResponse } from 'axios';
|
|
1
2
|
import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
|
|
2
3
|
import { LoaderEventType } from '../types/loading';
|
|
3
4
|
export interface EmitInterface {
|
|
@@ -5,4 +6,4 @@ export interface EmitInterface {
|
|
|
5
6
|
eventType: ExtractorEventType | LoaderEventType;
|
|
6
7
|
data?: EventData;
|
|
7
8
|
}
|
|
8
|
-
export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<
|
|
9
|
+
export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<AxiosResponse>;
|
|
@@ -2,35 +2,23 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.emit = void 0;
|
|
4
4
|
const axios_client_1 = require("../http/axios-client");
|
|
5
|
-
const
|
|
5
|
+
const helpers_1 = require("./helpers");
|
|
6
6
|
const emit = async ({ event, eventType, data, }) => {
|
|
7
7
|
const newEvent = {
|
|
8
8
|
event_type: eventType,
|
|
9
9
|
event_context: event.payload.event_context,
|
|
10
10
|
event_data: Object.assign({}, data),
|
|
11
|
+
worker_metadata: {
|
|
12
|
+
adaas_library_version: (0, helpers_1.getLibraryVersion)(),
|
|
13
|
+
},
|
|
11
14
|
};
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
'Content-Type': 'application/json',
|
|
20
|
-
},
|
|
21
|
-
});
|
|
22
|
-
resolve();
|
|
23
|
-
}
|
|
24
|
-
catch (error) {
|
|
25
|
-
if (axios_client_1.axios.isAxiosError(error)) {
|
|
26
|
-
console.error(`Failed to emit event with event type ${eventType}.`, (0, logger_1.serializeAxiosError)(error));
|
|
27
|
-
}
|
|
28
|
-
else {
|
|
29
|
-
// TODO: Stop it through UI or think about retrying this request. Implement exponential retry mechanism.
|
|
30
|
-
console.error(`Failed to emit event with event type ${eventType}.`, error);
|
|
31
|
-
}
|
|
32
|
-
reject();
|
|
33
|
-
}
|
|
15
|
+
console.info('Emitting event', JSON.stringify(newEvent));
|
|
16
|
+
return axios_client_1.axiosClient.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
|
|
17
|
+
headers: {
|
|
18
|
+
Accept: 'application/json, text/plain, */*',
|
|
19
|
+
Authorization: event.context.secrets.service_account_token,
|
|
20
|
+
'Content-Type': 'application/json',
|
|
21
|
+
},
|
|
34
22
|
});
|
|
35
23
|
};
|
|
36
24
|
exports.emit = emit;
|
package/dist/common/helpers.d.ts
CHANGED
package/dist/common/helpers.js
CHANGED
|
@@ -1,12 +1,48 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
36
|
exports.getTimeoutErrorEventType = getTimeoutErrorEventType;
|
|
4
37
|
exports.getSyncDirection = getSyncDirection;
|
|
5
38
|
exports.getFilesToLoad = getFilesToLoad;
|
|
6
39
|
exports.addReportToLoaderReport = addReportToLoaderReport;
|
|
7
40
|
exports.getCircularReplacer = getCircularReplacer;
|
|
41
|
+
exports.getLibraryVersion = getLibraryVersion;
|
|
8
42
|
const extraction_1 = require("../types/extraction");
|
|
9
43
|
const loading_1 = require("../types/loading");
|
|
44
|
+
const fs_1 = require("fs");
|
|
45
|
+
const path = __importStar(require("path"));
|
|
10
46
|
function getTimeoutErrorEventType(eventType) {
|
|
11
47
|
switch (eventType) {
|
|
12
48
|
case extraction_1.EventType.ExtractionMetadataStart:
|
|
@@ -120,3 +156,14 @@ function getCircularReplacer() {
|
|
|
120
156
|
return value;
|
|
121
157
|
};
|
|
122
158
|
}
|
|
159
|
+
// read adaas library version from package.json
|
|
160
|
+
function getLibraryVersion() {
|
|
161
|
+
var _a;
|
|
162
|
+
try {
|
|
163
|
+
return (_a = JSON.parse((0, fs_1.readFileSync)(path.resolve(__dirname, '../../package.json'), 'utf8'))) === null || _a === void 0 ? void 0 : _a.version;
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
console.error('Error reading adaas library version from package.json', error);
|
|
167
|
+
return '';
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -35,7 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.HTTPClient = exports.defaultResponse = void 0;
|
|
37
37
|
const axios_1 = __importStar(require("axios"));
|
|
38
|
-
const constants_1 = require("
|
|
38
|
+
const constants_1 = require("../../http/constants");
|
|
39
39
|
exports.defaultResponse = {
|
|
40
40
|
data: {
|
|
41
41
|
delay: 0,
|
|
@@ -12,30 +12,22 @@ exports.axiosClient = axiosClient;
|
|
|
12
12
|
(0, axios_retry_1.default)(axiosClient, {
|
|
13
13
|
retries: 5,
|
|
14
14
|
retryDelay: (retryCount, error) => {
|
|
15
|
-
var _a;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
var _a, _b, _c;
|
|
16
|
+
// exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms
|
|
17
|
+
const delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
|
|
18
|
+
console.warn(`Request to ${(_a = error.config) === null || _a === void 0 ? void 0 : _a.url} failed with response status code ${(_b = error.response) === null || _b === void 0 ? void 0 : _b.status}. Method ${(_c = error.config) === null || _c === void 0 ? void 0 : _c.method}. Retry count: ${retryCount}. Retrying in ${Math.round(delay / 1000)}s.`);
|
|
19
|
+
return delay;
|
|
19
20
|
},
|
|
20
21
|
retryCondition: (error) => {
|
|
21
|
-
var _a, _b, _c, _d;
|
|
22
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) &&
|
|
23
|
-
((_b = error.response) === null || _b === void 0 ? void 0 : _b.status) >= 500 &&
|
|
24
|
-
((_c = error.response) === null || _c === void 0 ? void 0 : _c.status) <= 599) {
|
|
25
|
-
return true;
|
|
26
|
-
}
|
|
27
|
-
else if (((_d = error.response) === null || _d === void 0 ? void 0 : _d.status) === 429) {
|
|
28
|
-
console.log('Rate limit exceeded. Delay: ' + error.response.headers['retry-after']);
|
|
29
|
-
return false;
|
|
30
|
-
}
|
|
31
|
-
else {
|
|
32
|
-
return false;
|
|
33
|
-
}
|
|
34
|
-
},
|
|
35
|
-
onMaxRetryTimesExceeded(error, retryCount) {
|
|
36
22
|
var _a;
|
|
37
|
-
|
|
38
|
-
|
|
23
|
+
return (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
|
|
24
|
+
((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) !== 429);
|
|
25
|
+
},
|
|
26
|
+
onMaxRetryTimesExceeded(error) {
|
|
27
|
+
var _a, _b, _c, _d;
|
|
28
|
+
(_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
|
|
29
|
+
(_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
|
|
39
30
|
delete error.request._header;
|
|
31
|
+
console.warn('Max retry times exceeded. Error', error);
|
|
40
32
|
},
|
|
41
33
|
});
|
package/dist/http/index.d.ts
CHANGED
package/dist/http/index.js
CHANGED
|
@@ -14,6 +14,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./client"), exports);
|
|
18
17
|
__exportStar(require("./types"), exports);
|
|
19
18
|
__exportStar(require("./axios-client"), exports);
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export * from './deprecated/adapter';
|
|
2
2
|
export * from './deprecated/demo-extractor';
|
|
3
3
|
export * from './deprecated/uploader';
|
|
4
|
+
export * from './deprecated/http/client';
|
|
4
5
|
export * from './types';
|
|
5
6
|
export * from './http';
|
|
6
7
|
export * from './common/install-initial-domain-mapping';
|
package/dist/index.js
CHANGED
|
@@ -18,6 +18,7 @@ exports.serializeAxiosError = exports.formatAxiosError = exports.spawn = exports
|
|
|
18
18
|
__exportStar(require("./deprecated/adapter"), exports);
|
|
19
19
|
__exportStar(require("./deprecated/demo-extractor"), exports);
|
|
20
20
|
__exportStar(require("./deprecated/uploader"), exports);
|
|
21
|
+
__exportStar(require("./deprecated/http/client"), exports);
|
|
21
22
|
__exportStar(require("./types"), exports);
|
|
22
23
|
__exportStar(require("./http"), exports);
|
|
23
24
|
__exportStar(require("./common/install-initial-domain-mapping"), exports);
|
package/dist/repo/repo.js
CHANGED
|
@@ -36,6 +36,10 @@ class Repo {
|
|
|
36
36
|
}
|
|
37
37
|
async push(items) {
|
|
38
38
|
let recordsToPush;
|
|
39
|
+
if (!items || items.length === 0) {
|
|
40
|
+
console.log(`No items to push for type ${this.itemType}. Skipping push.`);
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
39
43
|
// Normalize items if needed
|
|
40
44
|
if (this.normalize &&
|
|
41
45
|
this.itemType != constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA &&
|
|
@@ -47,7 +51,6 @@ class Repo {
|
|
|
47
51
|
}
|
|
48
52
|
// Add the new records to the items array
|
|
49
53
|
this.items.push(...recordsToPush);
|
|
50
|
-
console.info(`Extracted ${recordsToPush.length} new items of type ${this.itemType}. Total number of items in repo: ${this.items.length}.`);
|
|
51
54
|
// Upload in batches while the number of items exceeds the batch size
|
|
52
55
|
while (this.items.length >= constants_1.ARTIFACT_BATCH_SIZE) {
|
|
53
56
|
// Slice out a batch of ARTIFACT_BATCH_SIZE items to upload
|
package/dist/state/state.js
CHANGED
|
@@ -16,6 +16,11 @@ async function createAdapterState({ event, initialState, options, }) {
|
|
|
16
16
|
});
|
|
17
17
|
if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
|
|
18
18
|
await as.fetchState(newInitialState);
|
|
19
|
+
if (event.payload.event_type === extraction_1.EventType.ExtractionDataStart &&
|
|
20
|
+
!as.state.lastSyncStarted) {
|
|
21
|
+
as.state.lastSyncStarted = new Date().toISOString();
|
|
22
|
+
console.log(`Setting lastSyncStarted to ${as.state.lastSyncStarted}.`);
|
|
23
|
+
}
|
|
19
24
|
}
|
|
20
25
|
return as;
|
|
21
26
|
}
|
|
@@ -29,7 +34,7 @@ class State {
|
|
|
29
34
|
},
|
|
30
35
|
}
|
|
31
36
|
: {
|
|
32
|
-
lastSyncStarted:
|
|
37
|
+
lastSyncStarted: '',
|
|
33
38
|
lastSuccessfulSyncStarted: '',
|
|
34
39
|
toDevRev: {
|
|
35
40
|
attachmentsMetadata: {
|
|
@@ -68,15 +73,16 @@ class State {
|
|
|
68
73
|
},
|
|
69
74
|
});
|
|
70
75
|
this.state = state || this.state;
|
|
71
|
-
console.log('State updated successfully to
|
|
76
|
+
console.log('State updated successfully to', (0, logger_1.getPrintableState)(this.state));
|
|
72
77
|
}
|
|
73
78
|
catch (error) {
|
|
74
79
|
if (axios_client_1.axios.isAxiosError(error)) {
|
|
75
|
-
console.error('Failed to update state
|
|
80
|
+
console.error('Failed to update state', (0, logger_1.serializeAxiosError)(error));
|
|
76
81
|
}
|
|
77
82
|
else {
|
|
78
|
-
console.error('Failed to update state
|
|
83
|
+
console.error('Failed to update state', error);
|
|
79
84
|
}
|
|
85
|
+
process.exit(1);
|
|
80
86
|
}
|
|
81
87
|
}
|
|
82
88
|
/**
|
|
@@ -100,20 +106,20 @@ class State {
|
|
|
100
106
|
},
|
|
101
107
|
});
|
|
102
108
|
this.state = JSON.parse(response.data.state);
|
|
103
|
-
console.log('State fetched successfully. Current state
|
|
109
|
+
console.log('State fetched successfully. Current state', (0, logger_1.getPrintableState)(this.state));
|
|
104
110
|
return this.state;
|
|
105
111
|
}
|
|
106
112
|
catch (error) {
|
|
107
113
|
if (axios_client_1.axios.isAxiosError(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
108
114
|
const state = Object.assign(Object.assign({}, initialState), this.initialSdkState);
|
|
109
115
|
this.state = state;
|
|
110
|
-
console.log('State not found, returning initial state. Current state
|
|
116
|
+
console.log('State not found, returning initial state. Current state', (0, logger_1.getPrintableState)(this.state));
|
|
111
117
|
await this.postState(this.state);
|
|
112
118
|
return this.state;
|
|
113
119
|
}
|
|
114
120
|
else {
|
|
115
121
|
console.error('Failed to fetch state.', error);
|
|
116
|
-
|
|
122
|
+
process.exit(1);
|
|
117
123
|
}
|
|
118
124
|
}
|
|
119
125
|
}
|
|
@@ -2,8 +2,9 @@ import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
|
|
|
2
2
|
import { Artifact } from '../uploader/uploader.interfaces';
|
|
3
3
|
import { ErrorRecord } from './common';
|
|
4
4
|
import { DonV2, LoaderReport, RateLimited } from './loading';
|
|
5
|
-
import { NormalizedAttachment } from 'repo/repo.interfaces';
|
|
5
|
+
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
6
6
|
import { AxiosResponse } from 'axios';
|
|
7
|
+
import { WorkerAdapter } from '../workers/worker-adapter';
|
|
7
8
|
/**
|
|
8
9
|
* EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
|
|
9
10
|
* The external extractor can use these events to know what to do next in the extraction process.
|
|
@@ -164,6 +165,12 @@ export interface EventData {
|
|
|
164
165
|
processed_files?: string[];
|
|
165
166
|
stats_file?: string;
|
|
166
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS.
|
|
170
|
+
*/
|
|
171
|
+
export interface WorkerMetadata {
|
|
172
|
+
adaas_library_version: string;
|
|
173
|
+
}
|
|
167
174
|
/**
|
|
168
175
|
* DomainObject is an interface that defines the structure of a domain object that can be extracted.
|
|
169
176
|
* It must contain a name, a next chunk ID, the pages, the last modified date, whether it is done, and the count.
|
|
@@ -213,6 +220,7 @@ export interface ExtractorEvent {
|
|
|
213
220
|
event_type: string;
|
|
214
221
|
event_context: EventContext;
|
|
215
222
|
event_data?: EventData;
|
|
223
|
+
worker_metadata?: WorkerMetadata;
|
|
216
224
|
}
|
|
217
225
|
/**
|
|
218
226
|
* LoaderEvent
|
|
@@ -221,6 +229,7 @@ export interface LoaderEvent {
|
|
|
221
229
|
event_type: string;
|
|
222
230
|
event_context: EventContext;
|
|
223
231
|
event_data?: EventData;
|
|
232
|
+
worker_metadata?: WorkerMetadata;
|
|
224
233
|
}
|
|
225
234
|
export type ExternalSystemAttachmentStreamingFunction = ({ item, event, }: ExternalSystemAttachmentStreamingParams) => Promise<ExternalSystemAttachmentStreamingResponse>;
|
|
226
235
|
export interface ExternalSystemAttachmentStreamingParams {
|
|
@@ -237,3 +246,30 @@ export interface StreamAttachmentsResponse {
|
|
|
237
246
|
report?: LoaderReport;
|
|
238
247
|
rateLimit?: RateLimited;
|
|
239
248
|
}
|
|
249
|
+
export type ProcessAttachmentReturnType = {
|
|
250
|
+
delay?: number;
|
|
251
|
+
error?: {
|
|
252
|
+
message: string;
|
|
253
|
+
};
|
|
254
|
+
} | undefined;
|
|
255
|
+
export type StreamAttachmentsReturnType = {
|
|
256
|
+
delay?: number;
|
|
257
|
+
error?: ErrorRecord;
|
|
258
|
+
} | undefined;
|
|
259
|
+
export type ExternalSystemAttachmentReducerFunction<Batch, NewBatch, ConnectorState> = ({ attachments, adapter, }: {
|
|
260
|
+
attachments: Batch;
|
|
261
|
+
adapter: WorkerAdapter<ConnectorState>;
|
|
262
|
+
}) => NewBatch;
|
|
263
|
+
export type ExternalProcessAttachmentFunction = ({ attachment, stream, }: {
|
|
264
|
+
attachment: NormalizedAttachment;
|
|
265
|
+
stream: ExternalSystemAttachmentStreamingFunction;
|
|
266
|
+
}) => Promise<ProcessAttachmentReturnType>;
|
|
267
|
+
export type ExternalSystemAttachmentIteratorFunction<NewBatch, ConnectorState> = ({ reducedAttachments, adapter, stream, }: {
|
|
268
|
+
reducedAttachments: NewBatch;
|
|
269
|
+
adapter: WorkerAdapter<ConnectorState>;
|
|
270
|
+
stream: ExternalSystemAttachmentStreamingFunction;
|
|
271
|
+
}) => Promise<ProcessAttachmentReturnType>;
|
|
272
|
+
export interface ExternalSystemAttachmentProcessors<ConnectorState, Batch, NewBatch> {
|
|
273
|
+
reducer: ExternalSystemAttachmentReducerFunction<Batch, NewBatch, ConnectorState>;
|
|
274
|
+
iterator: ExternalSystemAttachmentIteratorFunction<NewBatch, ConnectorState>;
|
|
275
|
+
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
|
|
2
|
-
export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, } from './extraction';
|
|
2
|
+
export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, ExternalProcessAttachmentFunction, ExternalSystemAttachmentReducerFunction, ExternalSystemAttachmentIteratorFunction, } from './extraction';
|
|
3
3
|
export { LoaderEventType, ExternalSystemItem, ExternalSystemItemLoadingResponse, ExternalSystemItemLoadingParams, ExternalSystemAttachment, } from './loading';
|
|
4
4
|
export { NormalizedItem, NormalizedAttachment, RepoInterface, } from '../repo/repo.interfaces';
|
|
5
5
|
export { AdapterState } from '../state/state.interfaces';
|
package/dist/types/loading.d.ts
CHANGED
package/dist/types/workers.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Worker } from 'worker_threads';
|
|
2
|
-
import { MessagePort } from 'node:worker_threads';
|
|
3
2
|
import { State } from '../state/state';
|
|
4
3
|
import { WorkerAdapter } from '../workers/worker-adapter';
|
|
5
4
|
import { ExtractorEventType, AirdropEvent } from './extraction';
|
|
@@ -10,13 +9,11 @@ import { LoaderEventType } from './loading';
|
|
|
10
9
|
* @constructor
|
|
11
10
|
* @param {AirdropEvent} event - The event object received from the platform
|
|
12
11
|
* @param {object=} initialState - The initial state of the adapter
|
|
13
|
-
* @param {MessagePort} parentPort - The parent port of the worker thread
|
|
14
12
|
* @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class
|
|
15
13
|
*/
|
|
16
14
|
export interface WorkerAdapterInterface<ConnectorState> {
|
|
17
15
|
event: AirdropEvent;
|
|
18
16
|
adapterState: State<ConnectorState>;
|
|
19
|
-
parentPort: MessagePort;
|
|
20
17
|
options?: WorkerAdapterOptions;
|
|
21
18
|
}
|
|
22
19
|
/**
|
|
@@ -30,7 +27,6 @@ export interface WorkerAdapterOptions {
|
|
|
30
27
|
isLocalDevelopment?: boolean;
|
|
31
28
|
timeout?: number;
|
|
32
29
|
}
|
|
33
|
-
export type SpawnResolve = (value: boolean | PromiseLike<boolean>) => void;
|
|
34
30
|
/**
|
|
35
31
|
* SpawnInterface is an interface for Spawn class.
|
|
36
32
|
* @interface SpawnInterface
|
|
@@ -42,7 +38,7 @@ export interface SpawnInterface {
|
|
|
42
38
|
event: AirdropEvent;
|
|
43
39
|
worker: Worker;
|
|
44
40
|
options?: WorkerAdapterOptions;
|
|
45
|
-
resolve:
|
|
41
|
+
resolve: (value: void | PromiseLike<void>) => void;
|
|
46
42
|
}
|
|
47
43
|
/**
|
|
48
44
|
* SpawnFactoryInterface is an interface for Spawn class factory.
|
|
@@ -140,5 +136,4 @@ export interface WorkerData<ConnectorState> {
|
|
|
140
136
|
export interface GetWorkerPathInterface {
|
|
141
137
|
event: AirdropEvent;
|
|
142
138
|
connectorWorkerPath?: string | null;
|
|
143
|
-
options?: WorkerAdapterOptions;
|
|
144
139
|
}
|
|
@@ -38,7 +38,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
39
|
exports.Uploader = void 0;
|
|
40
40
|
const fs_1 = __importStar(require("fs"));
|
|
41
|
-
const
|
|
41
|
+
const axios_client_1 = require("../http/axios-client");
|
|
42
42
|
const zlib_1 = __importDefault(require("zlib"));
|
|
43
43
|
const js_jsonl_1 = require("js-jsonl");
|
|
44
44
|
const form_data_1 = __importDefault(require("form-data"));
|
|
@@ -67,7 +67,7 @@ class Uploader {
|
|
|
67
67
|
if (this.isLocalDevelopment) {
|
|
68
68
|
await this.downloadToLocal(itemType, fetchedObjects);
|
|
69
69
|
}
|
|
70
|
-
//
|
|
70
|
+
// compress the fetched objects to a gzipped jsonl object
|
|
71
71
|
const file = this.compressGzip(js_jsonl_1.jsonl.stringify(fetchedObjects));
|
|
72
72
|
if (!file) {
|
|
73
73
|
return {
|
|
@@ -76,27 +76,27 @@ class Uploader {
|
|
|
76
76
|
}
|
|
77
77
|
const filename = itemType + '.jsonl.gz';
|
|
78
78
|
const fileType = 'application/x-gzip';
|
|
79
|
-
//
|
|
79
|
+
// prepare the artifact for uploading
|
|
80
80
|
const preparedArtifact = await this.prepareArtifact(filename, fileType);
|
|
81
81
|
if (!preparedArtifact) {
|
|
82
82
|
return {
|
|
83
83
|
error: { message: 'Error while preparing artifact.' },
|
|
84
84
|
};
|
|
85
85
|
}
|
|
86
|
-
//
|
|
86
|
+
// upload the file to the prepared artifact
|
|
87
87
|
const uploadedArtifact = await this.uploadToArtifact(preparedArtifact, file);
|
|
88
88
|
if (!uploadedArtifact) {
|
|
89
89
|
return {
|
|
90
90
|
error: { message: 'Error while uploading artifact.' },
|
|
91
91
|
};
|
|
92
92
|
}
|
|
93
|
-
//
|
|
93
|
+
// return the artifact information to the platform
|
|
94
94
|
const artifact = {
|
|
95
95
|
id: preparedArtifact.id,
|
|
96
96
|
item_type: itemType,
|
|
97
97
|
item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1,
|
|
98
98
|
};
|
|
99
|
-
console.log('Successful upload of artifact
|
|
99
|
+
console.log('Successful upload of artifact', artifact);
|
|
100
100
|
return { artifact };
|
|
101
101
|
}
|
|
102
102
|
async prepareArtifact(filename, fileType) {
|
|
@@ -108,7 +108,7 @@ class Uploader {
|
|
|
108
108
|
return response.data;
|
|
109
109
|
}
|
|
110
110
|
catch (error) {
|
|
111
|
-
if (
|
|
111
|
+
if (axios_client_1.axios.isAxiosError(error)) {
|
|
112
112
|
console.error('Error while preparing artifact.', (0, logger_1.serializeAxiosError)(error));
|
|
113
113
|
}
|
|
114
114
|
else {
|
|
@@ -125,13 +125,13 @@ class Uploader {
|
|
|
125
125
|
}
|
|
126
126
|
formData.append('file', file);
|
|
127
127
|
try {
|
|
128
|
-
const response = await
|
|
128
|
+
const response = await axios_client_1.axiosClient.post(preparedArtifact.url, formData, {
|
|
129
129
|
headers: Object.assign({}, formData.getHeaders()),
|
|
130
130
|
});
|
|
131
131
|
return response;
|
|
132
132
|
}
|
|
133
133
|
catch (error) {
|
|
134
|
-
if (
|
|
134
|
+
if (axios_client_1.axios.isAxiosError(error)) {
|
|
135
135
|
console.error('Error while uploading artifact.', (0, logger_1.serializeAxiosError)(error));
|
|
136
136
|
}
|
|
137
137
|
else {
|
|
@@ -151,7 +151,7 @@ class Uploader {
|
|
|
151
151
|
return;
|
|
152
152
|
}
|
|
153
153
|
try {
|
|
154
|
-
const response = await
|
|
154
|
+
const response = await axios_client_1.axiosClient.post(preparedArtifact.url, formData, {
|
|
155
155
|
headers: Object.assign(Object.assign({}, formData.getHeaders()), (!fileStreamResponse.headers['content-length'] && {
|
|
156
156
|
'Content-Length': constants_1.MAX_DEVREV_ARTIFACT_SIZE,
|
|
157
157
|
})),
|
|
@@ -159,7 +159,7 @@ class Uploader {
|
|
|
159
159
|
return response;
|
|
160
160
|
}
|
|
161
161
|
catch (error) {
|
|
162
|
-
if (
|
|
162
|
+
if (axios_client_1.axios.isAxiosError(error)) {
|
|
163
163
|
console.error('Error while streaming artifact.', (0, logger_1.serializeAxiosError)(error));
|
|
164
164
|
}
|
|
165
165
|
else {
|
|
@@ -212,13 +212,13 @@ class Uploader {
|
|
|
212
212
|
}
|
|
213
213
|
async downloadArtifact(artifactUrl) {
|
|
214
214
|
try {
|
|
215
|
-
const response = await
|
|
215
|
+
const response = await axios_client_1.axiosClient.get(artifactUrl, {
|
|
216
216
|
responseType: 'arraybuffer',
|
|
217
217
|
});
|
|
218
218
|
return response.data;
|
|
219
219
|
}
|
|
220
220
|
catch (error) {
|
|
221
|
-
if (
|
|
221
|
+
if (axios_client_1.axios.isAxiosError(error)) {
|
|
222
222
|
console.error('Error while downloading artifact from URL.', (0, logger_1.serializeAxiosError)(error));
|
|
223
223
|
}
|
|
224
224
|
else {
|