@devrev/ts-adaas 0.0.3 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +186 -116
  2. package/dist/common/constants.d.ts +10 -0
  3. package/dist/common/constants.js +27 -0
  4. package/dist/common/control-protocol.d.ts +7 -0
  5. package/dist/common/control-protocol.js +41 -0
  6. package/dist/common/helpers.d.ts +4 -0
  7. package/dist/common/helpers.js +38 -0
  8. package/dist/common/install-initial-domain-mapping.d.ts +3 -0
  9. package/dist/common/install-initial-domain-mapping.js +87 -0
  10. package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
  11. package/dist/{src → deprecated}/adapter/index.js +13 -10
  12. package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
  13. package/dist/deprecated/common/helpers.js +47 -0
  14. package/dist/deprecated/demo-extractor/index.d.ts +17 -0
  15. package/dist/{src → deprecated}/demo-extractor/index.js +28 -22
  16. package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
  17. package/dist/{src → deprecated}/uploader/index.js +3 -1
  18. package/dist/{src/http → http}/client.d.ts +2 -1
  19. package/dist/index.d.ts +10 -0
  20. package/dist/{src/index.js → index.js} +11 -3
  21. package/dist/logger/logger.d.ts +14 -0
  22. package/dist/logger/logger.interfaces.d.ts +14 -0
  23. package/dist/logger/logger.interfaces.js +9 -0
  24. package/dist/logger/logger.js +91 -0
  25. package/dist/logger/logger.test.js +49 -0
  26. package/dist/repo/repo.d.ts +14 -0
  27. package/dist/repo/repo.interfaces.d.ts +43 -0
  28. package/dist/repo/repo.interfaces.js +2 -0
  29. package/dist/repo/repo.js +68 -0
  30. package/dist/repo/repo.test.js +74 -0
  31. package/dist/state/state.d.ts +24 -0
  32. package/dist/state/state.interfaces.d.ts +24 -0
  33. package/dist/state/state.interfaces.js +2 -0
  34. package/dist/state/state.js +115 -0
  35. package/dist/tests/test-helpers.d.ts +9 -0
  36. package/dist/tests/test-helpers.interfaces.d.ts +11 -0
  37. package/dist/tests/test-helpers.interfaces.js +2 -0
  38. package/dist/tests/test-helpers.js +86 -0
  39. package/dist/tests/test-worker.js +16 -0
  40. package/dist/types/common.d.ts +38 -0
  41. package/dist/{src/types → types}/common.js +4 -0
  42. package/dist/{src/types → types}/extraction.d.ts +56 -14
  43. package/dist/{src/types → types}/extraction.js +12 -29
  44. package/dist/types/index.d.ts +5 -0
  45. package/dist/types/index.js +11 -0
  46. package/dist/types/workers.d.ts +142 -0
  47. package/dist/types/workers.js +23 -0
  48. package/dist/uploader/uploader.d.ts +38 -0
  49. package/dist/uploader/uploader.interfaces.d.ts +63 -0
  50. package/dist/uploader/uploader.interfaces.js +2 -0
  51. package/dist/uploader/uploader.js +351 -0
  52. package/dist/{tests → uploader}/uploader.test.js +7 -5
  53. package/dist/workers/create-worker.d.ts +4 -0
  54. package/dist/workers/create-worker.js +30 -0
  55. package/dist/workers/create-worker.test.js +25 -0
  56. package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
  57. package/dist/workers/default-workers/attachments-deletion.js +13 -0
  58. package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
  59. package/dist/workers/default-workers/attachments-extraction.js +49 -0
  60. package/dist/workers/default-workers/data-deletion.d.ts +1 -0
  61. package/dist/workers/default-workers/data-deletion.js +15 -0
  62. package/dist/workers/default-workers/data-extraction.d.ts +1 -0
  63. package/dist/workers/default-workers/data-extraction.js +101 -0
  64. package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
  65. package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
  66. package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
  67. package/dist/workers/default-workers/metadata-extraction.js +26 -0
  68. package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
  69. package/dist/workers/dummy-extractor/data-normalization.js +41 -0
  70. package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
  71. package/dist/workers/process-task.d.ts +2 -0
  72. package/dist/workers/process-task.js +44 -0
  73. package/dist/workers/spawn.d.ts +25 -0
  74. package/dist/workers/spawn.js +164 -0
  75. package/dist/workers/worker-adapter.d.ts +48 -0
  76. package/dist/workers/worker-adapter.js +138 -0
  77. package/dist/workers/worker.d.ts +1 -0
  78. package/dist/workers/worker.js +6 -0
  79. package/package.json +9 -5
  80. package/dist/src/common/constants.d.ts +0 -2
  81. package/dist/src/common/constants.js +0 -10
  82. package/dist/src/common/helpers.js +0 -59
  83. package/dist/src/common/install-initial-domain-mapping.d.ts +0 -3
  84. package/dist/src/common/install-initial-domain-mapping.js +0 -60
  85. package/dist/src/demo-extractor/index.d.ts +0 -11
  86. package/dist/src/index.d.ts +0 -6
  87. package/dist/src/logging/index.d.ts +0 -18
  88. package/dist/src/logging/index.js +0 -39
  89. package/dist/src/state/index.d.ts +0 -23
  90. package/dist/src/state/index.js +0 -111
  91. package/dist/src/types/common.d.ts +0 -37
  92. package/dist/src/types/index.d.ts +0 -2
  93. package/dist/src/types/index.js +0 -18
  94. package/dist/tests/adapter.helpers.test.js +0 -60
  95. package/dist/tests/adapter.test.js +0 -123
  96. package/dist/tests/demo-extractor.test.js +0 -61
  97. package/dist/tests/state.test.js +0 -101
  98. /package/dist/{src → deprecated}/demo-extractor/external_domain_metadata.json +0 -0
  99. /package/dist/{src/http → http}/client.js +0 -0
  100. /package/dist/{src/http → http}/constants.d.ts +0 -0
  101. /package/dist/{src/http → http}/constants.js +0 -0
  102. /package/dist/{src/http → http}/index.d.ts +0 -0
  103. /package/dist/{src/http → http}/index.js +0 -0
  104. /package/dist/{src/http → http}/types.d.ts +0 -0
  105. /package/dist/{src/http → http}/types.js +0 -0
  106. /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
  107. /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
  108. /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
  109. /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
  110. /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
@@ -0,0 +1,74 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const constants_1 = require("../common/constants");
4
+ const test_helpers_1 = require("../tests/test-helpers");
5
+ const types_1 = require("../types");
6
+ const repo_1 = require("./repo");
7
+ jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
8
+ describe('Repo class push method', () => {
9
+ let repo;
10
+ let normalize;
11
+ beforeEach(() => {
12
+ normalize = jest.fn();
13
+ repo = new repo_1.Repo({
14
+ event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
15
+ itemType: 'test_item_type',
16
+ normalize,
17
+ onUpload: jest.fn(),
18
+ options: {},
19
+ });
20
+ });
21
+ afterEach(() => {
22
+ jest.clearAllMocks();
23
+ });
24
+ it('should not push items if items array is empty', async () => {
25
+ await repo.push([]);
26
+ expect(repo.getItems()).toEqual([]);
27
+ });
28
+ it('should normalize and push 10 items if array is not empty', async () => {
29
+ const items = (0, test_helpers_1.createItems)(10);
30
+ await repo.push(items);
31
+ expect(normalize).toHaveBeenCalledTimes(10);
32
+ const normalizedItems = items.map((item) => (0, test_helpers_1.normalizeItem)(item));
33
+ expect(repo.getItems()).toEqual(normalizedItems);
34
+ });
35
+ it('should not normalize items if normalize function is not provided', async () => {
36
+ repo = new repo_1.Repo({
37
+ event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
38
+ itemType: 'test_item_type',
39
+ onUpload: jest.fn(),
40
+ options: {},
41
+ });
42
+ const items = (0, test_helpers_1.createItems)(10);
43
+ await repo.push(items);
44
+ expect(normalize).not.toHaveBeenCalled();
45
+ });
46
+ describe('should not normalize items if they are airdrop default item types', () => {
47
+ it.each(Object.values(constants_1.AIRDROP_DEFAULT_ITEM_TYPES))('item type: %s', async (itemType) => {
48
+ repo = new repo_1.Repo({
49
+ event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
50
+ itemType,
51
+ normalize,
52
+ onUpload: jest.fn(),
53
+ options: {},
54
+ });
55
+ const items = (0, test_helpers_1.createItems)(10);
56
+ await repo.push(items);
57
+ expect(normalize).not.toHaveBeenCalled();
58
+ });
59
+ });
60
+ it('should leave 5 items in the items array after pushing 2005 items with batch size of 2000', async () => {
61
+ const items = (0, test_helpers_1.createItems)(2005);
62
+ await repo.push(items);
63
+ expect(repo.getItems().length).toBe(5);
64
+ });
65
+ it('should upload 2 batches of 2000 and leave 5 items in the items array after pushing 4005 items with batch size of 2000', async () => {
66
+ const uploadSpy = jest.spyOn(repo, 'upload');
67
+ const items = (0, test_helpers_1.createItems)(4005);
68
+ await repo.push(items);
69
+ expect(normalize).toHaveBeenCalledTimes(4005);
70
+ expect(repo.getItems().length).toBe(5);
71
+ expect(uploadSpy).toHaveBeenCalledTimes(2); // Check that upload was called twice
72
+ uploadSpy.mockRestore();
73
+ });
74
+ });
@@ -0,0 +1,24 @@
1
+ import { ErrorRecord } from '../types/common';
2
+ import { AdapterState, StateInterface } from './state.interfaces';
3
+ export declare function createAdapterState<ConnectorState>({ event, initialState, options, }: StateInterface<ConnectorState>): Promise<State<ConnectorState>>;
4
+ export declare class State<ConnectorState> {
5
+ private _state;
6
+ private event;
7
+ private workerUrl;
8
+ private devrevToken;
9
+ constructor({ event, initialState }: StateInterface<ConnectorState>);
10
+ get state(): AdapterState<ConnectorState>;
11
+ set state(value: AdapterState<ConnectorState>);
12
+ /**
13
+ * Updates the state of the adapter.
14
+ *
15
+ * @param {object} state - The state to be updated
16
+ */
17
+ postState(state?: AdapterState<ConnectorState>): Promise<void>;
18
+ /**
19
+ * Fetches the state of the adapter.
20
+ *
21
+ * @return The state of the adapter
22
+ */
23
+ fetchState(initialState: ConnectorState): Promise<AdapterState<ConnectorState> | ErrorRecord>;
24
+ }
@@ -0,0 +1,24 @@
1
+ import { AirdropEvent } from '../types/extraction';
2
+ import { WorkerAdapterOptions } from '../types/workers';
3
+ /**
4
+ * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, and attachmentsMetadata.
5
+ */
6
+ export type AdapterState<ConnectorState> = ConnectorState & {
7
+ lastSyncStarted?: string;
8
+ lastSuccessfulSyncStarted?: string;
9
+ toDevRev?: ToDevRev;
10
+ fromDevRev?: FromDevRev;
11
+ };
12
+ export interface ToDevRev {
13
+ attachmentsMetadata: {
14
+ artifactIds: string[];
15
+ lastProcessed: number;
16
+ };
17
+ }
18
+ export interface FromDevRev {
19
+ }
20
+ export interface StateInterface<ConnectorState> {
21
+ event: AirdropEvent;
22
+ initialState: ConnectorState;
23
+ options?: WorkerAdapterOptions;
24
+ }
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -0,0 +1,115 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.State = void 0;
7
+ exports.createAdapterState = createAdapterState;
8
+ const axios_1 = __importDefault(require("axios"));
9
+ const constants_1 = require("../common/constants");
10
+ const logger_1 = require("../logger/logger");
11
+ async function createAdapterState({ event, initialState, options, }) {
12
+ const newInitialState = structuredClone(initialState);
13
+ const as = new State({
14
+ event,
15
+ initialState: newInitialState,
16
+ options,
17
+ });
18
+ if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
19
+ await as.fetchState(newInitialState);
20
+ }
21
+ return as;
22
+ }
23
+ class State {
24
+ constructor({ event, initialState }) {
25
+ this._state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: new Date().toISOString(), lastSuccessfulSyncStarted: '', fromDevRev: {
26
+ attachmentsMetadata: {
27
+ artifactIds: [],
28
+ lastProcessed: 0,
29
+ },
30
+ }, toDevRev: {} });
31
+ this.event = event;
32
+ this.workerUrl = event.payload.event_context.worker_data_url;
33
+ this.devrevToken = event.context.secrets.service_account_token;
34
+ }
35
+ get state() {
36
+ return this._state;
37
+ }
38
+ set state(value) {
39
+ this._state = value;
40
+ }
41
+ /**
42
+ * Updates the state of the adapter.
43
+ *
44
+ * @param {object} state - The state to be updated
45
+ */
46
+ async postState(state) {
47
+ try {
48
+ await axios_1.default.post(this.workerUrl + '.update', {
49
+ state: JSON.stringify(state || this.state),
50
+ }, {
51
+ headers: {
52
+ Authorization: this.devrevToken,
53
+ },
54
+ params: {
55
+ sync_unit: this.event.payload.event_context.sync_unit_id,
56
+ request_id: this.event.payload.event_context.uuid,
57
+ },
58
+ });
59
+ this.state = state || this.state;
60
+ console.log('State updated successfully to:', (0, logger_1.getPrintableState)(this.state));
61
+ }
62
+ catch (error) {
63
+ if (axios_1.default.isAxiosError(error)) {
64
+ console.error('Failed to update state.', (0, logger_1.formatAxiosError)(error));
65
+ }
66
+ else {
67
+ console.error('Failed to update state.', error);
68
+ }
69
+ }
70
+ }
71
+ /**
72
+ * Fetches the state of the adapter.
73
+ *
74
+ * @return The state of the adapter
75
+ */
76
+ async fetchState(initialState) {
77
+ var _a;
78
+ console.log('Fetching state with sync unit id: ' +
79
+ this.event.payload.event_context.sync_unit_id +
80
+ '.');
81
+ try {
82
+ const response = await axios_1.default.post(this.workerUrl + '.get', {}, {
83
+ headers: {
84
+ Authorization: this.devrevToken,
85
+ },
86
+ params: {
87
+ sync_unit: this.event.payload.event_context.sync_unit_id,
88
+ request_id: this.event.payload.event_context.uuid,
89
+ },
90
+ });
91
+ this.state = JSON.parse(response.data.state);
92
+ console.log('State fetched successfully. Current state:', (0, logger_1.getPrintableState)(this.state));
93
+ return this.state;
94
+ }
95
+ catch (error) {
96
+ if (axios_1.default.isAxiosError(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
97
+ const state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: new Date().toISOString(), lastSuccessfulSyncStarted: '', toDevRev: {
98
+ attachmentsMetadata: {
99
+ artifactIds: [],
100
+ lastProcessed: 0,
101
+ },
102
+ }, fromDevRev: {} });
103
+ this.state = state;
104
+ console.log('State not found, returning initial state. Current state:', (0, logger_1.getPrintableState)(this.state));
105
+ await this.postState(this.state);
106
+ return this.state;
107
+ }
108
+ else {
109
+ console.error('Failed to fetch state.', error);
110
+ return { message: 'Failed to fetch state.' };
111
+ }
112
+ }
113
+ }
114
+ }
115
+ exports.State = State;
@@ -0,0 +1,9 @@
1
+ import { Item, NormalizedAttachment, NormalizedItem } from '../repo/repo.interfaces';
2
+ import { AirdropEvent } from '../types/extraction';
3
+ import { CreateEventInterface } from './test-helpers.interfaces';
4
+ export declare function createEvent({ eventType, externalSyncUnits, progress, error, delay, contextOverrides, payloadOverrides, }: CreateEventInterface): AirdropEvent;
5
+ export declare function createItem(id: number): Item;
6
+ export declare function createItems(count: number): Item[];
7
+ export declare function normalizeItem(item: Item): NormalizedItem;
8
+ export declare function createAttachment(id: number): NormalizedAttachment;
9
+ export declare function createAttachments(count: number): NormalizedAttachment[];
@@ -0,0 +1,11 @@
1
+ import { ErrorRecord } from '../types/common';
2
+ import { AirdropEvent, EventType, ExternalSyncUnit } from '../types/extraction';
3
+ export interface CreateEventInterface {
4
+ eventType: EventType;
5
+ externalSyncUnits?: ExternalSyncUnit[];
6
+ progress?: number;
7
+ error?: ErrorRecord;
8
+ delay?: number;
9
+ contextOverrides?: Partial<AirdropEvent['context']>;
10
+ payloadOverrides?: Partial<AirdropEvent['payload']>;
11
+ }
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -0,0 +1,86 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createEvent = createEvent;
4
+ exports.createItem = createItem;
5
+ exports.createItems = createItems;
6
+ exports.normalizeItem = normalizeItem;
7
+ exports.createAttachment = createAttachment;
8
+ exports.createAttachments = createAttachments;
9
+ function createEvent({ eventType, externalSyncUnits = [], progress, error, delay, contextOverrides = {}, payloadOverrides = {}, }) {
10
+ return {
11
+ context: Object.assign({ secrets: {
12
+ service_account_token: 'test_token',
13
+ }, snap_in_version_id: 'test_snap_in_version_id' }, contextOverrides),
14
+ payload: Object.assign({ connection_data: {
15
+ org_id: 'test_org_id',
16
+ org_name: 'test_org_name',
17
+ key: 'test_key',
18
+ key_type: 'test_key_type',
19
+ }, event_context: {
20
+ mode: 'test_mode',
21
+ callback_url: 'test_callback_url',
22
+ dev_org_id: 'test_dev_org_id',
23
+ dev_user_id: 'test_dev_user_id',
24
+ external_sync_unit_id: 'test_external_sync_unit_id',
25
+ sync_unit_id: 'test_sync_unit_id',
26
+ sync_run_id: 'test_sync_run_id',
27
+ external_system_id: 'test_external_system_id',
28
+ uuid: 'test_uuid',
29
+ worker_data_url: 'test_worker_data_url',
30
+ external_system: 'test_external_system',
31
+ external_system_type: 'test_external_system_type',
32
+ import_slug: 'test_import_slug',
33
+ snap_in_slug: 'test_snap_in_slug',
34
+ sync_tier: 'test_sync_tier',
35
+ }, event_type: eventType, event_data: {
36
+ external_sync_units: externalSyncUnits,
37
+ progress,
38
+ error,
39
+ delay,
40
+ } }, payloadOverrides),
41
+ execution_metadata: {
42
+ devrev_endpoint: 'test_devrev_endpoint',
43
+ },
44
+ input_data: {
45
+ global_values: {
46
+ test_global_key: 'test_global_value',
47
+ },
48
+ event_sources: {
49
+ test_event_source_key: 'test_event_source_id',
50
+ },
51
+ },
52
+ };
53
+ }
54
+ function createItem(id) {
55
+ return {
56
+ id,
57
+ created_at: '2021-01-01',
58
+ updated_at: '2021-01-01',
59
+ name: 'item' + id,
60
+ };
61
+ }
62
+ function createItems(count) {
63
+ return Array.from({ length: count }, (_, index) => createItem(index));
64
+ }
65
+ function normalizeItem(item) {
66
+ return {
67
+ id: item.id,
68
+ created_date: item.created_at,
69
+ modified_date: item.updated_at,
70
+ data: {
71
+ name: item.name,
72
+ },
73
+ };
74
+ }
75
+ function createAttachment(id) {
76
+ return {
77
+ id: id.toString(),
78
+ url: 'https://test.com/' + id,
79
+ author_id: 'author' + id,
80
+ file_name: 'file' + id,
81
+ parent_id: 'parent' + id,
82
+ };
83
+ }
84
+ function createAttachments(count) {
85
+ return Array.from({ length: count }, (_, index) => createAttachment(index));
86
+ }
@@ -0,0 +1,16 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const process_task_1 = require("../workers/process-task");
4
+ const extraction_1 = require("../types/extraction");
5
+ (0, process_task_1.processTask)({
6
+ task: async ({ adapter }) => {
7
+ await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {});
8
+ return 0;
9
+ },
10
+ onTimeout: async ({ adapter }) => {
11
+ await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError, {
12
+ error: { message: 'External sync unit failed.' },
13
+ });
14
+ return 0;
15
+ },
16
+ });
@@ -0,0 +1,38 @@
1
+ import { Artifact } from '../uploader/uploader.interfaces';
2
+ /**
3
+ * ErrorLevel is an enum that represents the level of an error.
4
+ * @deprecated
5
+ */
6
+ export declare enum ErrorLevel {
7
+ Warning = "WARNING",
8
+ Error = "ERROR",
9
+ Info = "INFO"
10
+ }
11
+ /**
12
+ * ErrorRecord is an interface that defines the structure of an error record.
13
+ */
14
+ export interface ErrorRecord {
15
+ message: string;
16
+ }
17
+ /**
18
+ * LogRecord is an interface that defines the structure of a log record.
19
+ * @deprecated
20
+ */
21
+ export interface LogRecord {
22
+ level: ErrorLevel;
23
+ message: string;
24
+ }
25
+ /**
26
+ * AdapterUpdateParams is an interface that defines the structure of the parameters that can be passed to the update adapter.
27
+ * @deprecated
28
+ */
29
+ export interface AdapterUpdateParams {
30
+ artifact?: Artifact;
31
+ }
32
+ /**
33
+ * InitialDomainMapping is an interface that defines the structure of the initial domain mapping.
34
+ */
35
+ export interface InitialDomainMapping {
36
+ starting_recipe_blueprint?: object;
37
+ additional_mappings?: object;
38
+ }
@@ -1,6 +1,10 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.ErrorLevel = void 0;
4
+ /**
5
+ * ErrorLevel is an enum that represents the level of an error.
6
+ * @deprecated
7
+ */
4
8
  var ErrorLevel;
5
9
  (function (ErrorLevel) {
6
10
  ErrorLevel["Warning"] = "WARNING";
@@ -1,5 +1,10 @@
1
1
  import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
2
- import { Artifact, ErrorRecord } from './common';
2
+ import { Artifact } from '../uploader/uploader.interfaces';
3
+ import { ErrorRecord } from './common';
4
+ /**
5
+ * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
6
+ * The external extractor can use these events to know what to do next in the extraction process.
7
+ */
3
8
  export declare enum EventType {
4
9
  ExtractionExternalSyncUnitsStart = "EXTRACTION_EXTERNAL_SYNC_UNITS_START",
5
10
  ExtractionMetadataStart = "EXTRACTION_METADATA_START",
@@ -10,6 +15,10 @@ export declare enum EventType {
10
15
  ExtractionAttachmentsContinue = "EXTRACTION_ATTACHMENTS_CONTINUE",
11
16
  ExtractionAttachmentsDelete = "EXTRACTION_ATTACHMENTS_DELETE"
12
17
  }
18
+ /**
19
+ * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
20
+ * The external extractor can use these events to inform ADaaS about the progress of the extraction process.
21
+ */
13
22
  export declare enum ExtractorEventType {
14
23
  ExtractionExternalSyncUnitsDone = "EXTRACTION_EXTERNAL_SYNC_UNITS_DONE",
15
24
  ExtractionExternalSyncUnitsError = "EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR",
@@ -28,23 +37,28 @@ export declare enum ExtractorEventType {
28
37
  ExtractionAttachmentsDeleteDone = "EXTRACTION_ATTACHMENTS_DELETE_DONE",
29
38
  ExtractionAttachmentsDeleteError = "EXTRACTION_ATTACHMENTS_DELETE_ERROR"
30
39
  }
31
- export interface EventData {
32
- external_sync_units?: ExternalSyncUnit[];
33
- progress?: number;
34
- error?: ErrorRecord;
35
- delay?: number;
36
- artifacts?: Artifact[];
37
- }
40
+ /**
41
+ * ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor.
42
+ * It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs.
43
+ */
38
44
  export declare enum ExtractionMode {
39
45
  INITIAL = "INITIAL",
40
46
  INCREMENTAL = "INCREMENTAL"
41
47
  }
48
+ /**
49
+ * ExternalSyncUnit is an interface that defines the structure of an external sync unit (repos, projects, ...) that can be extracted.
50
+ * It must contain an ID, a name, and a description. It can also contain the number of items in the external sync unit.
51
+ */
42
52
  export interface ExternalSyncUnit {
43
53
  id: string;
44
54
  name: string;
45
55
  description: string;
46
56
  item_count?: number;
57
+ item_type?: string;
47
58
  }
59
+ /**
60
+ * EventContextIn is an interface that defines the structure of the input event context that is sent to the external extractor from ADaaS.
61
+ */
48
62
  export interface EventContextIn {
49
63
  mode: string;
50
64
  callback_url: string;
@@ -56,25 +70,48 @@ export interface EventContextIn {
56
70
  external_system_id: string;
57
71
  uuid: string;
58
72
  worker_data_url: string;
73
+ external_system: string;
74
+ external_system_type: string;
75
+ import_slug: string;
76
+ snap_in_slug: string;
77
+ sync_tier: string;
59
78
  }
79
+ /**
80
+ * EventContextOut is an interface that defines the structure of the output event context that is sent from the external extractor to ADaaS.
81
+ */
60
82
  export interface EventContextOut {
61
83
  uuid: string;
62
84
  sync_run: string;
63
85
  sync_unit?: string;
64
86
  }
87
+ /**
88
+ * ConnectionData is an interface that defines the structure of the connection data that is sent to the external extractor from ADaaS.
89
+ * It contains the organization ID, organization name, key, and key type.
90
+ */
65
91
  export interface ConnectionData {
66
92
  org_id: string;
67
93
  org_name: string;
68
94
  key: string;
69
95
  key_type: string;
70
96
  }
97
+ /**
98
+ * EventData is an interface that defines the structure of the event data that is sent from the external extractor to ADaaS.
99
+ */
71
100
  export interface EventData {
72
101
  external_sync_units?: ExternalSyncUnit[];
73
102
  progress?: number;
74
103
  error?: ErrorRecord;
75
104
  delay?: number;
105
+ /**
106
+ * @deprecated This field is deprecated and should not be used.
107
+ */
76
108
  artifacts?: Artifact[];
77
109
  }
110
+ /**
111
+ * DomainObject is an interface that defines the structure of a domain object that can be extracted.
112
+ * It must contain a name, a next chunk ID, the pages, the last modified date, whether it is done, and the count.
113
+ * @deprecated
114
+ */
78
115
  export interface DomainObjectState {
79
116
  name: string;
80
117
  nextChunkId: number;
@@ -85,6 +122,10 @@ export interface DomainObjectState {
85
122
  isDone: boolean;
86
123
  count: number;
87
124
  }
125
+ /**
126
+ * AirdropEvent is an interface that defines the structure of the event that is sent to the external extractor from ADaaS.
127
+ * It contains the context, payload, execution metadata, and input data as common snap-ins.
128
+ */
88
129
  export interface AirdropEvent {
89
130
  context: {
90
131
  secrets: {
@@ -98,20 +139,21 @@ export interface AirdropEvent {
98
139
  };
99
140
  input_data: InputData;
100
141
  }
142
+ /**
143
+ * AirdropMessage is an interface that defines the structure of the payload/message that is sent to the external extractor from ADaaS.
144
+ */
101
145
  export interface AirdropMessage {
102
146
  connection_data: ConnectionData;
103
147
  event_context: EventContextIn;
104
148
  event_type: EventType;
105
149
  event_data?: EventData;
106
- extractor_state?: any;
107
150
  }
151
+ /**
152
+ * ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to ADaaS.
153
+ * It contains the event type, event context, extractor state, and event data.
154
+ */
108
155
  export interface ExtractorEvent {
109
156
  event_type: string;
110
157
  event_context: EventContextOut;
111
- extractor_state?: string;
112
158
  event_data?: EventData;
113
159
  }
114
- export type AdapterState<ExtractorState> = ExtractorState & {
115
- lastSyncStarted?: string;
116
- lastSuccessfulSyncStarted?: string;
117
- };
@@ -1,65 +1,48 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.ExtractionMode = exports.ExtractorEventType = exports.EventType = void 0;
4
+ /**
5
+ * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
6
+ * The external extractor can use these events to know what to do next in the extraction process.
7
+ */
4
8
  var EventType;
5
9
  (function (EventType) {
6
- // Get the list of sync units (repos, projects, ...) that can be extracted
7
10
  EventType["ExtractionExternalSyncUnitsStart"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_START";
8
- // Start the extraction of external sync unit's metadata (repos, projects, ...)
9
11
  EventType["ExtractionMetadataStart"] = "EXTRACTION_METADATA_START";
10
- // Start the extraction of a specific external sync unit
11
12
  EventType["ExtractionDataStart"] = "EXTRACTION_DATA_START";
12
- // Continue the extraction of a specific external sync unit
13
13
  EventType["ExtractionDataContinue"] = "EXTRACTION_DATA_CONTINUE";
14
- // Give the external extractor an opportunity to clean up after itself
15
14
  EventType["ExtractionDataDelete"] = "EXTRACTION_DATA_DELETE";
16
- // Extract all the attachments for a specific external sync unit
17
15
  EventType["ExtractionAttachmentsStart"] = "EXTRACTION_ATTACHMENTS_START";
18
- // Continue the extraction of attachments for a specific external sync unit
19
16
  EventType["ExtractionAttachmentsContinue"] = "EXTRACTION_ATTACHMENTS_CONTINUE";
20
- // Give the external extractor an opportunity to clean up after extractions of external sync units
21
17
  EventType["ExtractionAttachmentsDelete"] = "EXTRACTION_ATTACHMENTS_DELETE";
22
18
  })(EventType || (exports.EventType = EventType = {}));
19
+ /**
20
+ * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
21
+ * The external extractor can use these events to inform ADaaS about the progress of the extraction process.
22
+ */
23
23
  var ExtractorEventType;
24
24
  (function (ExtractorEventType) {
25
- /* Sync Units */
26
- // Sent when the extraction of external sync units finished
27
25
  ExtractorEventType["ExtractionExternalSyncUnitsDone"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_DONE";
28
- // Sent when there was an unrecoverable error for extraction of external sync units.
29
- // Must contain a list of error records.
30
26
  ExtractorEventType["ExtractionExternalSyncUnitsError"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR";
31
- /* Metadata */
32
- // Sent when the extraction of metadata finished
33
27
  ExtractorEventType["ExtractionMetadataDone"] = "EXTRACTION_METADATA_DONE";
34
- // Sent when there was an unrecoverable error for extraction of metadata.
35
28
  ExtractorEventType["ExtractionMetadataError"] = "EXTRACTION_METADATA_ERROR";
36
- /* Data */
37
- // Sent after a batch was extracted, contains artifact IDs of the uploaded files
38
29
  ExtractorEventType["ExtractionDataProgress"] = "EXTRACTION_DATA_PROGRESS";
39
- // Sent when there is a rate limit of more than ~1m, adapter will restart the extraction after the delay
40
30
  ExtractorEventType["ExtractionDataDelay"] = "EXTRACTION_DATA_DELAY";
41
- // Sent when the extraction of data finished
42
31
  ExtractorEventType["ExtractionDataDone"] = "EXTRACTION_DATA_DONE";
43
- // Sent when there was an unrecoverable error for extraction of data
44
32
  ExtractorEventType["ExtractionDataError"] = "EXTRACTION_DATA_ERROR";
45
- // Sent when the external extractor has finished cleaning up after itself
46
33
  ExtractorEventType["ExtractionDataDeleteDone"] = "EXTRACTION_DATA_DELETE_DONE";
47
- // Sent when there was an unrecoverable error for extraction of data
48
34
  ExtractorEventType["ExtractionDataDeleteError"] = "EXTRACTION_DATA_DELETE_ERROR";
49
- /* Attachments */
50
- // Sent after a batch was extracted, contains artifact IDs of the uploaded files
51
35
  ExtractorEventType["ExtractionAttachmentsProgress"] = "EXTRACTION_ATTACHMENTS_PROGRESS";
52
- // Sent when there is a rate limit of more than ~30s, adapter will restart the extraction after the delay
53
36
  ExtractorEventType["ExtractionAttachmentsDelay"] = "EXTRACTION_ATTACHMENTS_DELAY";
54
- // Sent when the extraction of attachements is finished
55
37
  ExtractorEventType["ExtractionAttachmentsDone"] = "EXTRACTION_ATTACHMENTS_DONE";
56
- // Sent when there was an unrecoverable error for extraction of attachments
57
38
  ExtractorEventType["ExtractionAttachmentsError"] = "EXTRACTION_ATTACHMENTS_ERROR";
58
- // Sent when the external extractor has finished cleaning up after itself
59
39
  ExtractorEventType["ExtractionAttachmentsDeleteDone"] = "EXTRACTION_ATTACHMENTS_DELETE_DONE";
60
- // Sent when there was an unrecoverable error for extraction of attachments
61
40
  ExtractorEventType["ExtractionAttachmentsDeleteError"] = "EXTRACTION_ATTACHMENTS_DELETE_ERROR";
62
41
  })(ExtractorEventType || (exports.ExtractorEventType = ExtractorEventType = {}));
42
+ /**
43
+ * ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor.
44
+ * It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs.
45
+ */
63
46
  var ExtractionMode;
64
47
  (function (ExtractionMode) {
65
48
  ExtractionMode["INITIAL"] = "INITIAL";
@@ -0,0 +1,5 @@
1
+ export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
2
+ export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, } from './extraction';
3
+ export { NormalizedItem, NormalizedAttachment } from '../repo/repo.interfaces';
4
+ export { AdapterState } from '../state/state.interfaces';
5
+ export { Artifact, ArtifactsPrepareResponse, UploadResponse, StreamResponse, StreamAttachmentsResponse, SsorAttachment, } from '../uploader/uploader.interfaces';
@@ -0,0 +1,11 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ExtractionMode = exports.ExtractorEventType = exports.EventType = exports.ErrorLevel = void 0;
4
+ // Common
5
+ var common_1 = require("./common");
6
+ Object.defineProperty(exports, "ErrorLevel", { enumerable: true, get: function () { return common_1.ErrorLevel; } });
7
+ // Extraction
8
+ var extraction_1 = require("./extraction");
9
+ Object.defineProperty(exports, "EventType", { enumerable: true, get: function () { return extraction_1.EventType; } });
10
+ Object.defineProperty(exports, "ExtractorEventType", { enumerable: true, get: function () { return extraction_1.ExtractorEventType; } });
11
+ Object.defineProperty(exports, "ExtractionMode", { enumerable: true, get: function () { return extraction_1.ExtractionMode; } });