@devrev/ts-adaas 0.0.3 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -116
- package/dist/common/constants.d.ts +10 -0
- package/dist/common/constants.js +27 -0
- package/dist/common/control-protocol.d.ts +7 -0
- package/dist/common/control-protocol.js +41 -0
- package/dist/common/helpers.d.ts +4 -0
- package/dist/common/helpers.js +38 -0
- package/dist/common/install-initial-domain-mapping.d.ts +3 -0
- package/dist/common/install-initial-domain-mapping.js +87 -0
- package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
- package/dist/{src → deprecated}/adapter/index.js +13 -10
- package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
- package/dist/deprecated/common/helpers.js +47 -0
- package/dist/deprecated/demo-extractor/index.d.ts +17 -0
- package/dist/{src → deprecated}/demo-extractor/index.js +28 -22
- package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
- package/dist/{src → deprecated}/uploader/index.js +3 -1
- package/dist/{src/http → http}/client.d.ts +2 -1
- package/dist/index.d.ts +10 -0
- package/dist/{src/index.js → index.js} +11 -3
- package/dist/logger/logger.d.ts +14 -0
- package/dist/logger/logger.interfaces.d.ts +14 -0
- package/dist/logger/logger.interfaces.js +9 -0
- package/dist/logger/logger.js +91 -0
- package/dist/logger/logger.test.js +49 -0
- package/dist/repo/repo.d.ts +14 -0
- package/dist/repo/repo.interfaces.d.ts +43 -0
- package/dist/repo/repo.interfaces.js +2 -0
- package/dist/repo/repo.js +68 -0
- package/dist/repo/repo.test.js +74 -0
- package/dist/state/state.d.ts +24 -0
- package/dist/state/state.interfaces.d.ts +24 -0
- package/dist/state/state.interfaces.js +2 -0
- package/dist/state/state.js +115 -0
- package/dist/tests/test-helpers.d.ts +9 -0
- package/dist/tests/test-helpers.interfaces.d.ts +11 -0
- package/dist/tests/test-helpers.interfaces.js +2 -0
- package/dist/tests/test-helpers.js +86 -0
- package/dist/tests/test-worker.js +16 -0
- package/dist/types/common.d.ts +38 -0
- package/dist/{src/types → types}/common.js +4 -0
- package/dist/{src/types → types}/extraction.d.ts +56 -14
- package/dist/{src/types → types}/extraction.js +12 -29
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.js +11 -0
- package/dist/types/workers.d.ts +142 -0
- package/dist/types/workers.js +23 -0
- package/dist/uploader/uploader.d.ts +38 -0
- package/dist/uploader/uploader.interfaces.d.ts +63 -0
- package/dist/uploader/uploader.interfaces.js +2 -0
- package/dist/uploader/uploader.js +351 -0
- package/dist/{tests → uploader}/uploader.test.js +7 -5
- package/dist/workers/create-worker.d.ts +4 -0
- package/dist/workers/create-worker.js +30 -0
- package/dist/workers/create-worker.test.js +25 -0
- package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
- package/dist/workers/default-workers/attachments-deletion.js +13 -0
- package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
- package/dist/workers/default-workers/attachments-extraction.js +49 -0
- package/dist/workers/default-workers/data-deletion.d.ts +1 -0
- package/dist/workers/default-workers/data-deletion.js +15 -0
- package/dist/workers/default-workers/data-extraction.d.ts +1 -0
- package/dist/workers/default-workers/data-extraction.js +101 -0
- package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
- package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
- package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
- package/dist/workers/default-workers/metadata-extraction.js +26 -0
- package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
- package/dist/workers/dummy-extractor/data-normalization.js +41 -0
- package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
- package/dist/workers/process-task.d.ts +2 -0
- package/dist/workers/process-task.js +44 -0
- package/dist/workers/spawn.d.ts +25 -0
- package/dist/workers/spawn.js +164 -0
- package/dist/workers/worker-adapter.d.ts +48 -0
- package/dist/workers/worker-adapter.js +138 -0
- package/dist/workers/worker.d.ts +1 -0
- package/dist/workers/worker.js +6 -0
- package/package.json +9 -5
- package/dist/src/common/constants.d.ts +0 -2
- package/dist/src/common/constants.js +0 -10
- package/dist/src/common/helpers.js +0 -59
- package/dist/src/common/install-initial-domain-mapping.d.ts +0 -3
- package/dist/src/common/install-initial-domain-mapping.js +0 -60
- package/dist/src/demo-extractor/index.d.ts +0 -11
- package/dist/src/index.d.ts +0 -6
- package/dist/src/logging/index.d.ts +0 -18
- package/dist/src/logging/index.js +0 -39
- package/dist/src/state/index.d.ts +0 -23
- package/dist/src/state/index.js +0 -111
- package/dist/src/types/common.d.ts +0 -37
- package/dist/src/types/index.d.ts +0 -2
- package/dist/src/types/index.js +0 -18
- package/dist/tests/adapter.helpers.test.js +0 -60
- package/dist/tests/adapter.test.js +0 -123
- package/dist/tests/demo-extractor.test.js +0 -61
- package/dist/tests/state.test.js +0 -101
- /package/dist/{src → deprecated}/demo-extractor/external_domain_metadata.json +0 -0
- /package/dist/{src/http → http}/client.js +0 -0
- /package/dist/{src/http → http}/constants.d.ts +0 -0
- /package/dist/{src/http → http}/constants.js +0 -0
- /package/dist/{src/http → http}/index.d.ts +0 -0
- /package/dist/{src/http → http}/index.js +0 -0
- /package/dist/{src/http → http}/types.d.ts +0 -0
- /package/dist/{src/http → http}/types.js +0 -0
- /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
- /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
- /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
- /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
- /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const constants_1 = require("../common/constants");
|
|
4
|
+
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
|
+
const types_1 = require("../types");
|
|
6
|
+
const repo_1 = require("./repo");
|
|
7
|
+
jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
|
|
8
|
+
describe('Repo class push method', () => {
|
|
9
|
+
let repo;
|
|
10
|
+
let normalize;
|
|
11
|
+
beforeEach(() => {
|
|
12
|
+
normalize = jest.fn();
|
|
13
|
+
repo = new repo_1.Repo({
|
|
14
|
+
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
15
|
+
itemType: 'test_item_type',
|
|
16
|
+
normalize,
|
|
17
|
+
onUpload: jest.fn(),
|
|
18
|
+
options: {},
|
|
19
|
+
});
|
|
20
|
+
});
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
jest.clearAllMocks();
|
|
23
|
+
});
|
|
24
|
+
it('should not push items if items array is empty', async () => {
|
|
25
|
+
await repo.push([]);
|
|
26
|
+
expect(repo.getItems()).toEqual([]);
|
|
27
|
+
});
|
|
28
|
+
it('should normalize and push 10 items if array is not empty', async () => {
|
|
29
|
+
const items = (0, test_helpers_1.createItems)(10);
|
|
30
|
+
await repo.push(items);
|
|
31
|
+
expect(normalize).toHaveBeenCalledTimes(10);
|
|
32
|
+
const normalizedItems = items.map((item) => (0, test_helpers_1.normalizeItem)(item));
|
|
33
|
+
expect(repo.getItems()).toEqual(normalizedItems);
|
|
34
|
+
});
|
|
35
|
+
it('should not normalize items if normalize function is not provided', async () => {
|
|
36
|
+
repo = new repo_1.Repo({
|
|
37
|
+
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
38
|
+
itemType: 'test_item_type',
|
|
39
|
+
onUpload: jest.fn(),
|
|
40
|
+
options: {},
|
|
41
|
+
});
|
|
42
|
+
const items = (0, test_helpers_1.createItems)(10);
|
|
43
|
+
await repo.push(items);
|
|
44
|
+
expect(normalize).not.toHaveBeenCalled();
|
|
45
|
+
});
|
|
46
|
+
describe('should not normalize items if they are airdrop default item types', () => {
|
|
47
|
+
it.each(Object.values(constants_1.AIRDROP_DEFAULT_ITEM_TYPES))('item type: %s', async (itemType) => {
|
|
48
|
+
repo = new repo_1.Repo({
|
|
49
|
+
event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
|
|
50
|
+
itemType,
|
|
51
|
+
normalize,
|
|
52
|
+
onUpload: jest.fn(),
|
|
53
|
+
options: {},
|
|
54
|
+
});
|
|
55
|
+
const items = (0, test_helpers_1.createItems)(10);
|
|
56
|
+
await repo.push(items);
|
|
57
|
+
expect(normalize).not.toHaveBeenCalled();
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
it('should leave 5 items in the items array after pushing 2005 items with batch size of 2000', async () => {
|
|
61
|
+
const items = (0, test_helpers_1.createItems)(2005);
|
|
62
|
+
await repo.push(items);
|
|
63
|
+
expect(repo.getItems().length).toBe(5);
|
|
64
|
+
});
|
|
65
|
+
it('should upload 2 batches of 2000 and leave 5 items in the items array after pushing 4005 items with batch size of 2000', async () => {
|
|
66
|
+
const uploadSpy = jest.spyOn(repo, 'upload');
|
|
67
|
+
const items = (0, test_helpers_1.createItems)(4005);
|
|
68
|
+
await repo.push(items);
|
|
69
|
+
expect(normalize).toHaveBeenCalledTimes(4005);
|
|
70
|
+
expect(repo.getItems().length).toBe(5);
|
|
71
|
+
expect(uploadSpy).toHaveBeenCalledTimes(2); // Check that upload was called twice
|
|
72
|
+
uploadSpy.mockRestore();
|
|
73
|
+
});
|
|
74
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { ErrorRecord } from '../types/common';
|
|
2
|
+
import { AdapterState, StateInterface } from './state.interfaces';
|
|
3
|
+
export declare function createAdapterState<ConnectorState>({ event, initialState, options, }: StateInterface<ConnectorState>): Promise<State<ConnectorState>>;
|
|
4
|
+
export declare class State<ConnectorState> {
|
|
5
|
+
private _state;
|
|
6
|
+
private event;
|
|
7
|
+
private workerUrl;
|
|
8
|
+
private devrevToken;
|
|
9
|
+
constructor({ event, initialState }: StateInterface<ConnectorState>);
|
|
10
|
+
get state(): AdapterState<ConnectorState>;
|
|
11
|
+
set state(value: AdapterState<ConnectorState>);
|
|
12
|
+
/**
|
|
13
|
+
* Updates the state of the adapter.
|
|
14
|
+
*
|
|
15
|
+
* @param {object} state - The state to be updated
|
|
16
|
+
*/
|
|
17
|
+
postState(state?: AdapterState<ConnectorState>): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Fetches the state of the adapter.
|
|
20
|
+
*
|
|
21
|
+
* @return The state of the adapter
|
|
22
|
+
*/
|
|
23
|
+
fetchState(initialState: ConnectorState): Promise<AdapterState<ConnectorState> | ErrorRecord>;
|
|
24
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { AirdropEvent } from '../types/extraction';
|
|
2
|
+
import { WorkerAdapterOptions } from '../types/workers';
|
|
3
|
+
/**
|
|
4
|
+
* AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, and attachmentsMetadata.
|
|
5
|
+
*/
|
|
6
|
+
export type AdapterState<ConnectorState> = ConnectorState & {
|
|
7
|
+
lastSyncStarted?: string;
|
|
8
|
+
lastSuccessfulSyncStarted?: string;
|
|
9
|
+
toDevRev?: ToDevRev;
|
|
10
|
+
fromDevRev?: FromDevRev;
|
|
11
|
+
};
|
|
12
|
+
export interface ToDevRev {
|
|
13
|
+
attachmentsMetadata: {
|
|
14
|
+
artifactIds: string[];
|
|
15
|
+
lastProcessed: number;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
export interface FromDevRev {
|
|
19
|
+
}
|
|
20
|
+
export interface StateInterface<ConnectorState> {
|
|
21
|
+
event: AirdropEvent;
|
|
22
|
+
initialState: ConnectorState;
|
|
23
|
+
options?: WorkerAdapterOptions;
|
|
24
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.State = void 0;
|
|
7
|
+
exports.createAdapterState = createAdapterState;
|
|
8
|
+
const axios_1 = __importDefault(require("axios"));
|
|
9
|
+
const constants_1 = require("../common/constants");
|
|
10
|
+
const logger_1 = require("../logger/logger");
|
|
11
|
+
async function createAdapterState({ event, initialState, options, }) {
|
|
12
|
+
const newInitialState = structuredClone(initialState);
|
|
13
|
+
const as = new State({
|
|
14
|
+
event,
|
|
15
|
+
initialState: newInitialState,
|
|
16
|
+
options,
|
|
17
|
+
});
|
|
18
|
+
if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
|
|
19
|
+
await as.fetchState(newInitialState);
|
|
20
|
+
}
|
|
21
|
+
return as;
|
|
22
|
+
}
|
|
23
|
+
class State {
|
|
24
|
+
constructor({ event, initialState }) {
|
|
25
|
+
this._state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: new Date().toISOString(), lastSuccessfulSyncStarted: '', fromDevRev: {
|
|
26
|
+
attachmentsMetadata: {
|
|
27
|
+
artifactIds: [],
|
|
28
|
+
lastProcessed: 0,
|
|
29
|
+
},
|
|
30
|
+
}, toDevRev: {} });
|
|
31
|
+
this.event = event;
|
|
32
|
+
this.workerUrl = event.payload.event_context.worker_data_url;
|
|
33
|
+
this.devrevToken = event.context.secrets.service_account_token;
|
|
34
|
+
}
|
|
35
|
+
get state() {
|
|
36
|
+
return this._state;
|
|
37
|
+
}
|
|
38
|
+
set state(value) {
|
|
39
|
+
this._state = value;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Updates the state of the adapter.
|
|
43
|
+
*
|
|
44
|
+
* @param {object} state - The state to be updated
|
|
45
|
+
*/
|
|
46
|
+
async postState(state) {
|
|
47
|
+
try {
|
|
48
|
+
await axios_1.default.post(this.workerUrl + '.update', {
|
|
49
|
+
state: JSON.stringify(state || this.state),
|
|
50
|
+
}, {
|
|
51
|
+
headers: {
|
|
52
|
+
Authorization: this.devrevToken,
|
|
53
|
+
},
|
|
54
|
+
params: {
|
|
55
|
+
sync_unit: this.event.payload.event_context.sync_unit_id,
|
|
56
|
+
request_id: this.event.payload.event_context.uuid,
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
this.state = state || this.state;
|
|
60
|
+
console.log('State updated successfully to:', (0, logger_1.getPrintableState)(this.state));
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
64
|
+
console.error('Failed to update state.', (0, logger_1.formatAxiosError)(error));
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
console.error('Failed to update state.', error);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Fetches the state of the adapter.
|
|
73
|
+
*
|
|
74
|
+
* @return The state of the adapter
|
|
75
|
+
*/
|
|
76
|
+
async fetchState(initialState) {
|
|
77
|
+
var _a;
|
|
78
|
+
console.log('Fetching state with sync unit id: ' +
|
|
79
|
+
this.event.payload.event_context.sync_unit_id +
|
|
80
|
+
'.');
|
|
81
|
+
try {
|
|
82
|
+
const response = await axios_1.default.post(this.workerUrl + '.get', {}, {
|
|
83
|
+
headers: {
|
|
84
|
+
Authorization: this.devrevToken,
|
|
85
|
+
},
|
|
86
|
+
params: {
|
|
87
|
+
sync_unit: this.event.payload.event_context.sync_unit_id,
|
|
88
|
+
request_id: this.event.payload.event_context.uuid,
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
this.state = JSON.parse(response.data.state);
|
|
92
|
+
console.log('State fetched successfully. Current state:', (0, logger_1.getPrintableState)(this.state));
|
|
93
|
+
return this.state;
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
if (axios_1.default.isAxiosError(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
|
|
97
|
+
const state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: new Date().toISOString(), lastSuccessfulSyncStarted: '', toDevRev: {
|
|
98
|
+
attachmentsMetadata: {
|
|
99
|
+
artifactIds: [],
|
|
100
|
+
lastProcessed: 0,
|
|
101
|
+
},
|
|
102
|
+
}, fromDevRev: {} });
|
|
103
|
+
this.state = state;
|
|
104
|
+
console.log('State not found, returning initial state. Current state:', (0, logger_1.getPrintableState)(this.state));
|
|
105
|
+
await this.postState(this.state);
|
|
106
|
+
return this.state;
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
console.error('Failed to fetch state.', error);
|
|
110
|
+
return { message: 'Failed to fetch state.' };
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
exports.State = State;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Item, NormalizedAttachment, NormalizedItem } from '../repo/repo.interfaces';
|
|
2
|
+
import { AirdropEvent } from '../types/extraction';
|
|
3
|
+
import { CreateEventInterface } from './test-helpers.interfaces';
|
|
4
|
+
export declare function createEvent({ eventType, externalSyncUnits, progress, error, delay, contextOverrides, payloadOverrides, }: CreateEventInterface): AirdropEvent;
|
|
5
|
+
export declare function createItem(id: number): Item;
|
|
6
|
+
export declare function createItems(count: number): Item[];
|
|
7
|
+
export declare function normalizeItem(item: Item): NormalizedItem;
|
|
8
|
+
export declare function createAttachment(id: number): NormalizedAttachment;
|
|
9
|
+
export declare function createAttachments(count: number): NormalizedAttachment[];
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { ErrorRecord } from '../types/common';
|
|
2
|
+
import { AirdropEvent, EventType, ExternalSyncUnit } from '../types/extraction';
|
|
3
|
+
export interface CreateEventInterface {
|
|
4
|
+
eventType: EventType;
|
|
5
|
+
externalSyncUnits?: ExternalSyncUnit[];
|
|
6
|
+
progress?: number;
|
|
7
|
+
error?: ErrorRecord;
|
|
8
|
+
delay?: number;
|
|
9
|
+
contextOverrides?: Partial<AirdropEvent['context']>;
|
|
10
|
+
payloadOverrides?: Partial<AirdropEvent['payload']>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createEvent = createEvent;
|
|
4
|
+
exports.createItem = createItem;
|
|
5
|
+
exports.createItems = createItems;
|
|
6
|
+
exports.normalizeItem = normalizeItem;
|
|
7
|
+
exports.createAttachment = createAttachment;
|
|
8
|
+
exports.createAttachments = createAttachments;
|
|
9
|
+
function createEvent({ eventType, externalSyncUnits = [], progress, error, delay, contextOverrides = {}, payloadOverrides = {}, }) {
|
|
10
|
+
return {
|
|
11
|
+
context: Object.assign({ secrets: {
|
|
12
|
+
service_account_token: 'test_token',
|
|
13
|
+
}, snap_in_version_id: 'test_snap_in_version_id' }, contextOverrides),
|
|
14
|
+
payload: Object.assign({ connection_data: {
|
|
15
|
+
org_id: 'test_org_id',
|
|
16
|
+
org_name: 'test_org_name',
|
|
17
|
+
key: 'test_key',
|
|
18
|
+
key_type: 'test_key_type',
|
|
19
|
+
}, event_context: {
|
|
20
|
+
mode: 'test_mode',
|
|
21
|
+
callback_url: 'test_callback_url',
|
|
22
|
+
dev_org_id: 'test_dev_org_id',
|
|
23
|
+
dev_user_id: 'test_dev_user_id',
|
|
24
|
+
external_sync_unit_id: 'test_external_sync_unit_id',
|
|
25
|
+
sync_unit_id: 'test_sync_unit_id',
|
|
26
|
+
sync_run_id: 'test_sync_run_id',
|
|
27
|
+
external_system_id: 'test_external_system_id',
|
|
28
|
+
uuid: 'test_uuid',
|
|
29
|
+
worker_data_url: 'test_worker_data_url',
|
|
30
|
+
external_system: 'test_external_system',
|
|
31
|
+
external_system_type: 'test_external_system_type',
|
|
32
|
+
import_slug: 'test_import_slug',
|
|
33
|
+
snap_in_slug: 'test_snap_in_slug',
|
|
34
|
+
sync_tier: 'test_sync_tier',
|
|
35
|
+
}, event_type: eventType, event_data: {
|
|
36
|
+
external_sync_units: externalSyncUnits,
|
|
37
|
+
progress,
|
|
38
|
+
error,
|
|
39
|
+
delay,
|
|
40
|
+
} }, payloadOverrides),
|
|
41
|
+
execution_metadata: {
|
|
42
|
+
devrev_endpoint: 'test_devrev_endpoint',
|
|
43
|
+
},
|
|
44
|
+
input_data: {
|
|
45
|
+
global_values: {
|
|
46
|
+
test_global_key: 'test_global_value',
|
|
47
|
+
},
|
|
48
|
+
event_sources: {
|
|
49
|
+
test_event_source_key: 'test_event_source_id',
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function createItem(id) {
|
|
55
|
+
return {
|
|
56
|
+
id,
|
|
57
|
+
created_at: '2021-01-01',
|
|
58
|
+
updated_at: '2021-01-01',
|
|
59
|
+
name: 'item' + id,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
function createItems(count) {
|
|
63
|
+
return Array.from({ length: count }, (_, index) => createItem(index));
|
|
64
|
+
}
|
|
65
|
+
function normalizeItem(item) {
|
|
66
|
+
return {
|
|
67
|
+
id: item.id,
|
|
68
|
+
created_date: item.created_at,
|
|
69
|
+
modified_date: item.updated_at,
|
|
70
|
+
data: {
|
|
71
|
+
name: item.name,
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
function createAttachment(id) {
|
|
76
|
+
return {
|
|
77
|
+
id: id.toString(),
|
|
78
|
+
url: 'https://test.com/' + id,
|
|
79
|
+
author_id: 'author' + id,
|
|
80
|
+
file_name: 'file' + id,
|
|
81
|
+
parent_id: 'parent' + id,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
function createAttachments(count) {
|
|
85
|
+
return Array.from({ length: count }, (_, index) => createAttachment(index));
|
|
86
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const process_task_1 = require("../workers/process-task");
|
|
4
|
+
const extraction_1 = require("../types/extraction");
|
|
5
|
+
(0, process_task_1.processTask)({
|
|
6
|
+
task: async ({ adapter }) => {
|
|
7
|
+
await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {});
|
|
8
|
+
return 0;
|
|
9
|
+
},
|
|
10
|
+
onTimeout: async ({ adapter }) => {
|
|
11
|
+
await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError, {
|
|
12
|
+
error: { message: 'External sync unit failed.' },
|
|
13
|
+
});
|
|
14
|
+
return 0;
|
|
15
|
+
},
|
|
16
|
+
});
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { Artifact } from '../uploader/uploader.interfaces';
|
|
2
|
+
/**
|
|
3
|
+
* ErrorLevel is an enum that represents the level of an error.
|
|
4
|
+
* @deprecated
|
|
5
|
+
*/
|
|
6
|
+
export declare enum ErrorLevel {
|
|
7
|
+
Warning = "WARNING",
|
|
8
|
+
Error = "ERROR",
|
|
9
|
+
Info = "INFO"
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* ErrorRecord is an interface that defines the structure of an error record.
|
|
13
|
+
*/
|
|
14
|
+
export interface ErrorRecord {
|
|
15
|
+
message: string;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* LogRecord is an interface that defines the structure of a log record.
|
|
19
|
+
* @deprecated
|
|
20
|
+
*/
|
|
21
|
+
export interface LogRecord {
|
|
22
|
+
level: ErrorLevel;
|
|
23
|
+
message: string;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* AdapterUpdateParams is an interface that defines the structure of the parameters that can be passed to the update adapter.
|
|
27
|
+
* @deprecated
|
|
28
|
+
*/
|
|
29
|
+
export interface AdapterUpdateParams {
|
|
30
|
+
artifact?: Artifact;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* InitialDomainMapping is an interface that defines the structure of the initial domain mapping.
|
|
34
|
+
*/
|
|
35
|
+
export interface InitialDomainMapping {
|
|
36
|
+
starting_recipe_blueprint?: object;
|
|
37
|
+
additional_mappings?: object;
|
|
38
|
+
}
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ErrorLevel = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* ErrorLevel is an enum that represents the level of an error.
|
|
6
|
+
* @deprecated
|
|
7
|
+
*/
|
|
4
8
|
var ErrorLevel;
|
|
5
9
|
(function (ErrorLevel) {
|
|
6
10
|
ErrorLevel["Warning"] = "WARNING";
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
|
|
2
|
-
import { Artifact
|
|
2
|
+
import { Artifact } from '../uploader/uploader.interfaces';
|
|
3
|
+
import { ErrorRecord } from './common';
|
|
4
|
+
/**
|
|
5
|
+
* EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
|
|
6
|
+
* The external extractor can use these events to know what to do next in the extraction process.
|
|
7
|
+
*/
|
|
3
8
|
export declare enum EventType {
|
|
4
9
|
ExtractionExternalSyncUnitsStart = "EXTRACTION_EXTERNAL_SYNC_UNITS_START",
|
|
5
10
|
ExtractionMetadataStart = "EXTRACTION_METADATA_START",
|
|
@@ -10,6 +15,10 @@ export declare enum EventType {
|
|
|
10
15
|
ExtractionAttachmentsContinue = "EXTRACTION_ATTACHMENTS_CONTINUE",
|
|
11
16
|
ExtractionAttachmentsDelete = "EXTRACTION_ATTACHMENTS_DELETE"
|
|
12
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
|
|
20
|
+
* The external extractor can use these events to inform ADaaS about the progress of the extraction process.
|
|
21
|
+
*/
|
|
13
22
|
export declare enum ExtractorEventType {
|
|
14
23
|
ExtractionExternalSyncUnitsDone = "EXTRACTION_EXTERNAL_SYNC_UNITS_DONE",
|
|
15
24
|
ExtractionExternalSyncUnitsError = "EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR",
|
|
@@ -28,23 +37,28 @@ export declare enum ExtractorEventType {
|
|
|
28
37
|
ExtractionAttachmentsDeleteDone = "EXTRACTION_ATTACHMENTS_DELETE_DONE",
|
|
29
38
|
ExtractionAttachmentsDeleteError = "EXTRACTION_ATTACHMENTS_DELETE_ERROR"
|
|
30
39
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
delay?: number;
|
|
36
|
-
artifacts?: Artifact[];
|
|
37
|
-
}
|
|
40
|
+
/**
|
|
41
|
+
* ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor.
|
|
42
|
+
* It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs.
|
|
43
|
+
*/
|
|
38
44
|
export declare enum ExtractionMode {
|
|
39
45
|
INITIAL = "INITIAL",
|
|
40
46
|
INCREMENTAL = "INCREMENTAL"
|
|
41
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* ExternalSyncUnit is an interface that defines the structure of an external sync unit (repos, projects, ...) that can be extracted.
|
|
50
|
+
* It must contain an ID, a name, and a description. It can also contain the number of items in the external sync unit.
|
|
51
|
+
*/
|
|
42
52
|
export interface ExternalSyncUnit {
|
|
43
53
|
id: string;
|
|
44
54
|
name: string;
|
|
45
55
|
description: string;
|
|
46
56
|
item_count?: number;
|
|
57
|
+
item_type?: string;
|
|
47
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* EventContextIn is an interface that defines the structure of the input event context that is sent to the external extractor from ADaaS.
|
|
61
|
+
*/
|
|
48
62
|
export interface EventContextIn {
|
|
49
63
|
mode: string;
|
|
50
64
|
callback_url: string;
|
|
@@ -56,25 +70,48 @@ export interface EventContextIn {
|
|
|
56
70
|
external_system_id: string;
|
|
57
71
|
uuid: string;
|
|
58
72
|
worker_data_url: string;
|
|
73
|
+
external_system: string;
|
|
74
|
+
external_system_type: string;
|
|
75
|
+
import_slug: string;
|
|
76
|
+
snap_in_slug: string;
|
|
77
|
+
sync_tier: string;
|
|
59
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* EventContextOut is an interface that defines the structure of the output event context that is sent from the external extractor to ADaaS.
|
|
81
|
+
*/
|
|
60
82
|
export interface EventContextOut {
|
|
61
83
|
uuid: string;
|
|
62
84
|
sync_run: string;
|
|
63
85
|
sync_unit?: string;
|
|
64
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* ConnectionData is an interface that defines the structure of the connection data that is sent to the external extractor from ADaaS.
|
|
89
|
+
* It contains the organization ID, organization name, key, and key type.
|
|
90
|
+
*/
|
|
65
91
|
export interface ConnectionData {
|
|
66
92
|
org_id: string;
|
|
67
93
|
org_name: string;
|
|
68
94
|
key: string;
|
|
69
95
|
key_type: string;
|
|
70
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* EventData is an interface that defines the structure of the event data that is sent from the external extractor to ADaaS.
|
|
99
|
+
*/
|
|
71
100
|
export interface EventData {
|
|
72
101
|
external_sync_units?: ExternalSyncUnit[];
|
|
73
102
|
progress?: number;
|
|
74
103
|
error?: ErrorRecord;
|
|
75
104
|
delay?: number;
|
|
105
|
+
/**
|
|
106
|
+
* @deprecated This field is deprecated and should not be used.
|
|
107
|
+
*/
|
|
76
108
|
artifacts?: Artifact[];
|
|
77
109
|
}
|
|
110
|
+
/**
|
|
111
|
+
* DomainObject is an interface that defines the structure of a domain object that can be extracted.
|
|
112
|
+
* It must contain a name, a next chunk ID, the pages, the last modified date, whether it is done, and the count.
|
|
113
|
+
* @deprecated
|
|
114
|
+
*/
|
|
78
115
|
export interface DomainObjectState {
|
|
79
116
|
name: string;
|
|
80
117
|
nextChunkId: number;
|
|
@@ -85,6 +122,10 @@ export interface DomainObjectState {
|
|
|
85
122
|
isDone: boolean;
|
|
86
123
|
count: number;
|
|
87
124
|
}
|
|
125
|
+
/**
|
|
126
|
+
* AirdropEvent is an interface that defines the structure of the event that is sent to the external extractor from ADaaS.
|
|
127
|
+
* It contains the context, payload, execution metadata, and input data as common snap-ins.
|
|
128
|
+
*/
|
|
88
129
|
export interface AirdropEvent {
|
|
89
130
|
context: {
|
|
90
131
|
secrets: {
|
|
@@ -98,20 +139,21 @@ export interface AirdropEvent {
|
|
|
98
139
|
};
|
|
99
140
|
input_data: InputData;
|
|
100
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* AirdropMessage is an interface that defines the structure of the payload/message that is sent to the external extractor from ADaaS.
|
|
144
|
+
*/
|
|
101
145
|
export interface AirdropMessage {
|
|
102
146
|
connection_data: ConnectionData;
|
|
103
147
|
event_context: EventContextIn;
|
|
104
148
|
event_type: EventType;
|
|
105
149
|
event_data?: EventData;
|
|
106
|
-
extractor_state?: any;
|
|
107
150
|
}
|
|
151
|
+
/**
|
|
152
|
+
* ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to ADaaS.
|
|
153
|
+
* It contains the event type, event context, extractor state, and event data.
|
|
154
|
+
*/
|
|
108
155
|
export interface ExtractorEvent {
|
|
109
156
|
event_type: string;
|
|
110
157
|
event_context: EventContextOut;
|
|
111
|
-
extractor_state?: string;
|
|
112
158
|
event_data?: EventData;
|
|
113
159
|
}
|
|
114
|
-
export type AdapterState<ExtractorState> = ExtractorState & {
|
|
115
|
-
lastSyncStarted?: string;
|
|
116
|
-
lastSuccessfulSyncStarted?: string;
|
|
117
|
-
};
|
|
@@ -1,65 +1,48 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ExtractionMode = exports.ExtractorEventType = exports.EventType = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
|
|
6
|
+
* The external extractor can use these events to know what to do next in the extraction process.
|
|
7
|
+
*/
|
|
4
8
|
var EventType;
|
|
5
9
|
(function (EventType) {
|
|
6
|
-
// Get the list of sync units (repos, projects, ...) that can be extracted
|
|
7
10
|
EventType["ExtractionExternalSyncUnitsStart"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_START";
|
|
8
|
-
// Start the extraction of external sync unit's metadata (repos, projects, ...)
|
|
9
11
|
EventType["ExtractionMetadataStart"] = "EXTRACTION_METADATA_START";
|
|
10
|
-
// Start the extraction of a specific external sync unit
|
|
11
12
|
EventType["ExtractionDataStart"] = "EXTRACTION_DATA_START";
|
|
12
|
-
// Continue the extraction of a specific external sync unit
|
|
13
13
|
EventType["ExtractionDataContinue"] = "EXTRACTION_DATA_CONTINUE";
|
|
14
|
-
// Give the external extractor an opportunity to clean up after itself
|
|
15
14
|
EventType["ExtractionDataDelete"] = "EXTRACTION_DATA_DELETE";
|
|
16
|
-
// Extract all the attachments for a specific external sync unit
|
|
17
15
|
EventType["ExtractionAttachmentsStart"] = "EXTRACTION_ATTACHMENTS_START";
|
|
18
|
-
// Continue the extraction of attachments for a specific external sync unit
|
|
19
16
|
EventType["ExtractionAttachmentsContinue"] = "EXTRACTION_ATTACHMENTS_CONTINUE";
|
|
20
|
-
// Give the external extractor an opportunity to clean up after extractions of external sync units
|
|
21
17
|
EventType["ExtractionAttachmentsDelete"] = "EXTRACTION_ATTACHMENTS_DELETE";
|
|
22
18
|
})(EventType || (exports.EventType = EventType = {}));
|
|
19
|
+
/**
|
|
20
|
+
* ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
|
|
21
|
+
* The external extractor can use these events to inform ADaaS about the progress of the extraction process.
|
|
22
|
+
*/
|
|
23
23
|
var ExtractorEventType;
|
|
24
24
|
(function (ExtractorEventType) {
|
|
25
|
-
/* Sync Units */
|
|
26
|
-
// Sent when the extraction of external sync units finished
|
|
27
25
|
ExtractorEventType["ExtractionExternalSyncUnitsDone"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_DONE";
|
|
28
|
-
// Sent when there was an unrecoverable error for extraction of external sync units.
|
|
29
|
-
// Must contain a list of error records.
|
|
30
26
|
ExtractorEventType["ExtractionExternalSyncUnitsError"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR";
|
|
31
|
-
/* Metadata */
|
|
32
|
-
// Sent when the extraction of metadata finished
|
|
33
27
|
ExtractorEventType["ExtractionMetadataDone"] = "EXTRACTION_METADATA_DONE";
|
|
34
|
-
// Sent when there was an unrecoverable error for extraction of metadata.
|
|
35
28
|
ExtractorEventType["ExtractionMetadataError"] = "EXTRACTION_METADATA_ERROR";
|
|
36
|
-
/* Data */
|
|
37
|
-
// Sent after a batch was extracted, contains artifact IDs of the uploaded files
|
|
38
29
|
ExtractorEventType["ExtractionDataProgress"] = "EXTRACTION_DATA_PROGRESS";
|
|
39
|
-
// Sent when there is a rate limit of more than ~1m, adapter will restart the extraction after the delay
|
|
40
30
|
ExtractorEventType["ExtractionDataDelay"] = "EXTRACTION_DATA_DELAY";
|
|
41
|
-
// Sent when the extraction of data finished
|
|
42
31
|
ExtractorEventType["ExtractionDataDone"] = "EXTRACTION_DATA_DONE";
|
|
43
|
-
// Sent when there was an unrecoverable error for extraction of data
|
|
44
32
|
ExtractorEventType["ExtractionDataError"] = "EXTRACTION_DATA_ERROR";
|
|
45
|
-
// Sent when the external extractor has finished cleaning up after itself
|
|
46
33
|
ExtractorEventType["ExtractionDataDeleteDone"] = "EXTRACTION_DATA_DELETE_DONE";
|
|
47
|
-
// Sent when there was an unrecoverable error for extraction of data
|
|
48
34
|
ExtractorEventType["ExtractionDataDeleteError"] = "EXTRACTION_DATA_DELETE_ERROR";
|
|
49
|
-
/* Attachments */
|
|
50
|
-
// Sent after a batch was extracted, contains artifact IDs of the uploaded files
|
|
51
35
|
ExtractorEventType["ExtractionAttachmentsProgress"] = "EXTRACTION_ATTACHMENTS_PROGRESS";
|
|
52
|
-
// Sent when there is a rate limit of more than ~30s, adapter will restart the extraction after the delay
|
|
53
36
|
ExtractorEventType["ExtractionAttachmentsDelay"] = "EXTRACTION_ATTACHMENTS_DELAY";
|
|
54
|
-
// Sent when the extraction of attachements is finished
|
|
55
37
|
ExtractorEventType["ExtractionAttachmentsDone"] = "EXTRACTION_ATTACHMENTS_DONE";
|
|
56
|
-
// Sent when there was an unrecoverable error for extraction of attachments
|
|
57
38
|
ExtractorEventType["ExtractionAttachmentsError"] = "EXTRACTION_ATTACHMENTS_ERROR";
|
|
58
|
-
// Sent when the external extractor has finished cleaning up after itself
|
|
59
39
|
ExtractorEventType["ExtractionAttachmentsDeleteDone"] = "EXTRACTION_ATTACHMENTS_DELETE_DONE";
|
|
60
|
-
// Sent when there was an unrecoverable error for extraction of attachments
|
|
61
40
|
ExtractorEventType["ExtractionAttachmentsDeleteError"] = "EXTRACTION_ATTACHMENTS_DELETE_ERROR";
|
|
62
41
|
})(ExtractorEventType || (exports.ExtractorEventType = ExtractorEventType = {}));
|
|
42
|
+
/**
|
|
43
|
+
* ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor.
|
|
44
|
+
* It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs.
|
|
45
|
+
*/
|
|
63
46
|
var ExtractionMode;
|
|
64
47
|
(function (ExtractionMode) {
|
|
65
48
|
ExtractionMode["INITIAL"] = "INITIAL";
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
|
|
2
|
+
export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, } from './extraction';
|
|
3
|
+
export { NormalizedItem, NormalizedAttachment } from '../repo/repo.interfaces';
|
|
4
|
+
export { AdapterState } from '../state/state.interfaces';
|
|
5
|
+
export { Artifact, ArtifactsPrepareResponse, UploadResponse, StreamResponse, StreamAttachmentsResponse, SsorAttachment, } from '../uploader/uploader.interfaces';
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ExtractionMode = exports.ExtractorEventType = exports.EventType = exports.ErrorLevel = void 0;
|
|
4
|
+
// Common
|
|
5
|
+
var common_1 = require("./common");
|
|
6
|
+
Object.defineProperty(exports, "ErrorLevel", { enumerable: true, get: function () { return common_1.ErrorLevel; } });
|
|
7
|
+
// Extraction
|
|
8
|
+
var extraction_1 = require("./extraction");
|
|
9
|
+
Object.defineProperty(exports, "EventType", { enumerable: true, get: function () { return extraction_1.EventType; } });
|
|
10
|
+
Object.defineProperty(exports, "ExtractorEventType", { enumerable: true, get: function () { return extraction_1.ExtractorEventType; } });
|
|
11
|
+
Object.defineProperty(exports, "ExtractionMode", { enumerable: true, get: function () { return extraction_1.ExtractionMode; } });
|