@devrev/airsync-sdk 2.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +31 -0
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +16 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -0
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +9 -0
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -0
  6. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.js +2 -0
  7. package/dist/attachments-streaming/attachments-streaming-pool.js +97 -0
  8. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +2 -0
  9. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +1 -0
  10. package/dist/attachments-streaming/attachments-streaming-pool.test.js +267 -0
  11. package/dist/common/constants.d.ts +25 -0
  12. package/dist/common/constants.d.ts.map +1 -0
  13. package/dist/common/constants.js +58 -0
  14. package/dist/common/control-protocol.d.ts +10 -0
  15. package/dist/common/control-protocol.d.ts.map +1 -0
  16. package/dist/common/control-protocol.js +31 -0
  17. package/dist/common/errors.d.ts +6 -0
  18. package/dist/common/errors.d.ts.map +1 -0
  19. package/dist/common/errors.js +4 -0
  20. package/dist/common/event-type-translation.d.ts +24 -0
  21. package/dist/common/event-type-translation.d.ts.map +1 -0
  22. package/dist/common/event-type-translation.js +117 -0
  23. package/dist/common/helpers.d.ts +41 -0
  24. package/dist/common/helpers.d.ts.map +1 -0
  25. package/dist/common/helpers.js +124 -0
  26. package/dist/common/install-initial-domain-mapping.d.ts +4 -0
  27. package/dist/common/install-initial-domain-mapping.d.ts.map +1 -0
  28. package/dist/common/install-initial-domain-mapping.js +58 -0
  29. package/dist/common/install-initial-domain-mapping.test.d.ts +2 -0
  30. package/dist/common/install-initial-domain-mapping.test.d.ts.map +1 -0
  31. package/dist/common/install-initial-domain-mapping.test.js +207 -0
  32. package/dist/deprecated/adapter/index.d.ts +62 -0
  33. package/dist/deprecated/adapter/index.d.ts.map +1 -0
  34. package/dist/deprecated/adapter/index.js +151 -0
  35. package/dist/deprecated/common/helpers.d.ts +7 -0
  36. package/dist/deprecated/common/helpers.d.ts.map +1 -0
  37. package/dist/deprecated/common/helpers.js +47 -0
  38. package/dist/deprecated/demo-extractor/external_domain_metadata.json +38 -0
  39. package/dist/deprecated/demo-extractor/index.d.ts +18 -0
  40. package/dist/deprecated/demo-extractor/index.d.ts.map +1 -0
  41. package/dist/deprecated/demo-extractor/index.js +161 -0
  42. package/dist/deprecated/http/client.d.ts +22 -0
  43. package/dist/deprecated/http/client.d.ts.map +1 -0
  44. package/dist/deprecated/http/client.js +161 -0
  45. package/dist/deprecated/uploader/index.d.ts +35 -0
  46. package/dist/deprecated/uploader/index.d.ts.map +1 -0
  47. package/dist/deprecated/uploader/index.js +161 -0
  48. package/dist/http/axios-client-internal.d.ts +3 -0
  49. package/dist/http/axios-client-internal.d.ts.map +1 -0
  50. package/dist/http/axios-client-internal.js +66 -0
  51. package/dist/http/axios-client-internal.test.d.ts +2 -0
  52. package/dist/http/axios-client-internal.test.d.ts.map +1 -0
  53. package/dist/http/axios-client-internal.test.js +154 -0
  54. package/dist/http/axios-client.d.ts +27 -0
  55. package/dist/http/axios-client.d.ts.map +1 -0
  56. package/dist/http/axios-client.js +57 -0
  57. package/dist/http/constants.d.ts +4 -0
  58. package/dist/http/constants.d.ts.map +1 -0
  59. package/dist/http/constants.js +6 -0
  60. package/dist/http/index.d.ts +3 -0
  61. package/dist/http/index.d.ts.map +1 -0
  62. package/dist/http/index.js +18 -0
  63. package/dist/http/types.d.ts +17 -0
  64. package/dist/http/types.d.ts.map +1 -0
  65. package/dist/http/types.js +2 -0
  66. package/dist/index.d.ts +14 -0
  67. package/dist/index.d.ts.map +1 -0
  68. package/dist/index.js +34 -0
  69. package/dist/logger/logger.constants.d.ts +6 -0
  70. package/dist/logger/logger.constants.d.ts.map +1 -0
  71. package/dist/logger/logger.constants.js +13 -0
  72. package/dist/logger/logger.context.d.ts +58 -0
  73. package/dist/logger/logger.context.d.ts.map +1 -0
  74. package/dist/logger/logger.context.js +86 -0
  75. package/dist/logger/logger.d.ts +89 -0
  76. package/dist/logger/logger.d.ts.map +1 -0
  77. package/dist/logger/logger.interfaces.d.ts +43 -0
  78. package/dist/logger/logger.interfaces.d.ts.map +1 -0
  79. package/dist/logger/logger.interfaces.js +9 -0
  80. package/dist/logger/logger.js +196 -0
  81. package/dist/logger/logger.test.d.ts +2 -0
  82. package/dist/logger/logger.test.d.ts.map +1 -0
  83. package/dist/logger/logger.test.js +490 -0
  84. package/dist/mappers/mappers.d.ts +52 -0
  85. package/dist/mappers/mappers.d.ts.map +1 -0
  86. package/dist/mappers/mappers.interface.d.ts +294 -0
  87. package/dist/mappers/mappers.interface.d.ts.map +1 -0
  88. package/dist/mappers/mappers.interface.js +48 -0
  89. package/dist/mappers/mappers.js +83 -0
  90. package/dist/mappers/mappers.test.d.ts +2 -0
  91. package/dist/mappers/mappers.test.d.ts.map +1 -0
  92. package/dist/mappers/mappers.test.js +107 -0
  93. package/dist/multithreading/create-worker.d.ts +5 -0
  94. package/dist/multithreading/create-worker.d.ts.map +1 -0
  95. package/dist/multithreading/create-worker.js +28 -0
  96. package/dist/multithreading/create-worker.test.d.ts +2 -0
  97. package/dist/multithreading/create-worker.test.d.ts.map +1 -0
  98. package/dist/multithreading/create-worker.test.js +89 -0
  99. package/dist/multithreading/process-task.d.ts +3 -0
  100. package/dist/multithreading/process-task.d.ts.map +1 -0
  101. package/dist/multithreading/process-task.js +58 -0
  102. package/dist/multithreading/spawn/spawn.d.ts +30 -0
  103. package/dist/multithreading/spawn/spawn.d.ts.map +1 -0
  104. package/dist/multithreading/spawn/spawn.helpers.d.ts +21 -0
  105. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -0
  106. package/dist/multithreading/spawn/spawn.helpers.js +114 -0
  107. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +2 -0
  108. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +1 -0
  109. package/dist/multithreading/spawn/spawn.helpers.test.js +293 -0
  110. package/dist/multithreading/spawn/spawn.js +249 -0
  111. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +2 -0
  112. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +1 -0
  113. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +127 -0
  114. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +91 -0
  115. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -0
  116. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +22 -0
  117. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -0
  118. package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +64 -0
  119. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +2 -0
  120. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +1 -0
  121. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +514 -0
  122. package/dist/multithreading/worker-adapter/worker-adapter.js +747 -0
  123. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +2 -0
  124. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +1 -0
  125. package/dist/multithreading/worker-adapter/worker-adapter.test.js +483 -0
  126. package/dist/multithreading/worker.d.ts +2 -0
  127. package/dist/multithreading/worker.d.ts.map +1 -0
  128. package/dist/multithreading/worker.js +9 -0
  129. package/dist/repo/repo.d.ts +18 -0
  130. package/dist/repo/repo.d.ts.map +1 -0
  131. package/dist/repo/repo.interfaces.d.ts +46 -0
  132. package/dist/repo/repo.interfaces.d.ts.map +1 -0
  133. package/dist/repo/repo.interfaces.js +2 -0
  134. package/dist/repo/repo.js +75 -0
  135. package/dist/repo/repo.test.d.ts +2 -0
  136. package/dist/repo/repo.test.d.ts.map +1 -0
  137. package/dist/repo/repo.test.js +131 -0
  138. package/dist/state/state.d.ts +30 -0
  139. package/dist/state/state.d.ts.map +1 -0
  140. package/dist/state/state.interfaces.d.ts +51 -0
  141. package/dist/state/state.interfaces.d.ts.map +1 -0
  142. package/dist/state/state.interfaces.js +21 -0
  143. package/dist/state/state.js +166 -0
  144. package/dist/state/state.test.d.ts +2 -0
  145. package/dist/state/state.test.d.ts.map +1 -0
  146. package/dist/state/state.test.js +224 -0
  147. package/dist/types/common.d.ts +50 -0
  148. package/dist/types/common.d.ts.map +1 -0
  149. package/dist/types/common.js +25 -0
  150. package/dist/types/extraction.d.ts +417 -0
  151. package/dist/types/extraction.d.ts.map +1 -0
  152. package/dist/types/extraction.js +170 -0
  153. package/dist/types/extraction.test.d.ts +2 -0
  154. package/dist/types/extraction.test.d.ts.map +1 -0
  155. package/dist/types/extraction.test.js +70 -0
  156. package/dist/types/index.d.ts +9 -0
  157. package/dist/types/index.d.ts.map +1 -0
  158. package/dist/types/index.js +18 -0
  159. package/dist/types/loading.d.ts +147 -0
  160. package/dist/types/loading.d.ts.map +1 -0
  161. package/dist/types/loading.js +48 -0
  162. package/dist/types/workers.d.ts +161 -0
  163. package/dist/types/workers.d.ts.map +1 -0
  164. package/dist/types/workers.js +22 -0
  165. package/dist/uploader/uploader.d.ts +92 -0
  166. package/dist/uploader/uploader.d.ts.map +1 -0
  167. package/dist/uploader/uploader.helpers.d.ts +33 -0
  168. package/dist/uploader/uploader.helpers.d.ts.map +1 -0
  169. package/dist/uploader/uploader.helpers.js +139 -0
  170. package/dist/uploader/uploader.helpers.test.d.ts +2 -0
  171. package/dist/uploader/uploader.helpers.test.d.ts.map +1 -0
  172. package/dist/uploader/uploader.helpers.test.js +267 -0
  173. package/dist/uploader/uploader.interfaces.d.ts +95 -0
  174. package/dist/uploader/uploader.interfaces.d.ts.map +1 -0
  175. package/dist/uploader/uploader.interfaces.js +2 -0
  176. package/dist/uploader/uploader.js +305 -0
  177. package/dist/uploader/uploader.test.d.ts +2 -0
  178. package/dist/uploader/uploader.test.d.ts.map +1 -0
  179. package/dist/uploader/uploader.test.js +589 -0
  180. package/package.json +60 -0
@@ -0,0 +1,151 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.Adapter = void 0;
7
+ exports.createAdapter = createAdapter;
8
+ const axios_1 = __importDefault(require("axios"));
9
+ const constants_1 = require("../../common/constants");
10
+ const helpers_1 = require("../common/helpers");
11
+ // import { Logger } from '../../logger/logger';
12
+ const state_1 = require("../../state/state");
13
+ const event_type_translation_1 = require("../../common/event-type-translation");
14
+ const logger_context_1 = require("../../logger/logger.context");
15
+ /**
16
+ * Adapter class is used to interact with Airdrop platform. The class provides
17
+ * utilities to
18
+ * - emit control events to the platform
19
+ * - update the state of the extractor
20
+ * - set the last saved state in case of a timeout
21
+ *
22
+ * @class Adapter
23
+ * @constructor
24
+ * @deprecated
25
+ * @param {AirdropEvent} event - The event object received from the platform
26
+ * @param {object=} initialState - The initial state of the adapter
27
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
28
+ */
29
+ /**
30
+ * Creates an adapter instance.
31
+ *
32
+ * @param {AirdropEvent} event - The event object received from the platform
33
+ * @param initialState
34
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
35
+ * @return The adapter instance
36
+ */
37
+ async function createAdapter(event, initialState, isLocalDevelopment = false) {
38
+ event.payload.event_type = (0, event_type_translation_1.translateIncomingEventType)(event.payload.event_type);
39
+ const newInitialState = structuredClone(initialState);
40
+ const adapterState = await (0, state_1.createAdapterState)({
41
+ event,
42
+ initialState: newInitialState,
43
+ });
44
+ const a = new Adapter(event, adapterState, isLocalDevelopment);
45
+ return a;
46
+ }
47
+ class Adapter {
48
+ constructor(event, adapterState, isLocalDevelopment = false) {
49
+ this.exit = false;
50
+ this.lambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
51
+ this.heartBeatInterval = 30 * 1000; // 30 seconds in milliseconds
52
+ if (!isLocalDevelopment) {
53
+ // Logger.init(event);
54
+ }
55
+ this.adapterState = adapterState;
56
+ this._artifacts = [];
57
+ this.event = event;
58
+ this.callbackUrl = event.payload.event_context.callback_url;
59
+ this.devrevToken = event.context.secrets.service_account_token;
60
+ this.startTime = Date.now();
61
+ // Run heartbeat every 30 seconds
62
+ this.heartBeatFn = setInterval(async () => {
63
+ const b = await this.heartbeat();
64
+ if (b) {
65
+ this.exitAdapter();
66
+ }
67
+ }, this.heartBeatInterval);
68
+ }
69
+ get state() {
70
+ return this.adapterState.state;
71
+ }
72
+ set state(value) {
73
+ this.adapterState.state = value;
74
+ }
75
+ get artifacts() {
76
+ return this._artifacts;
77
+ }
78
+ set artifacts(value) {
79
+ this._artifacts = value;
80
+ }
81
+ /**
82
+ * Emits an event to the platform.
83
+ *
84
+ * @param {ExtractorEventType} newEventType - The event type to be emitted
85
+ * @param {EventData=} data - The data to be sent with the event
86
+ */
87
+ async emit(newEventType, data) {
88
+ if (this.exit) {
89
+ console.warn('Adapter is already in exit state. No more events can be emitted.');
90
+ return;
91
+ }
92
+ // We want to save the state every time we emit an event, except for the start and delete events
93
+ if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
94
+ (0, logger_context_1.runWithSdkLogContext)(() => console.log(`Saving state before emitting event`));
95
+ await this.adapterState.postState(this.state);
96
+ }
97
+ const newEvent = {
98
+ event_type: newEventType,
99
+ event_context: this.event.payload.event_context,
100
+ event_data: Object.assign({}, data),
101
+ };
102
+ try {
103
+ await axios_1.default.post(this.callbackUrl, Object.assign({}, newEvent), {
104
+ headers: {
105
+ Accept: 'application/json, text/plain, */*',
106
+ Authorization: this.devrevToken,
107
+ 'Content-Type': 'application/json',
108
+ },
109
+ });
110
+ console.log('Successfully emitted event: ' + JSON.stringify(newEvent));
111
+ }
112
+ catch (error) {
113
+ // If this request fails the extraction will be stuck in loop and
114
+ // we need to stop it through UI or think about retrying this request
115
+ console.log('Failed to emit event: ' +
116
+ JSON.stringify(newEvent) +
117
+ ', error: ' +
118
+ error);
119
+ }
120
+ finally {
121
+ this.exitAdapter();
122
+ }
123
+ }
124
+ /**
125
+ * Exit the adapter. This will stop the heartbeat and no
126
+ * further events will be emitted.
127
+ */
128
+ exitAdapter() {
129
+ this.exit = true;
130
+ }
131
+ /**
132
+ * Heartbeat function to check if the lambda is about to timeout.
133
+ * @returns true if 10 minutes have passed since the start of the lambda.
134
+ */
135
+ async heartbeat() {
136
+ if (this.exit) {
137
+ return true;
138
+ }
139
+ if (Date.now() - this.startTime > this.lambdaTimeout) {
140
+ const timeoutEventType = (0, helpers_1.getTimeoutExtractorEventType)(this.event.payload.event_type);
141
+ if (timeoutEventType !== null) {
142
+ const { eventType, isError } = timeoutEventType;
143
+ const err = isError ? { message: 'Lambda Timeout' } : undefined;
144
+ await this.emit(eventType, { error: err, artifacts: this._artifacts });
145
+ return true;
146
+ }
147
+ }
148
+ return false;
149
+ }
150
+ }
151
+ exports.Adapter = Adapter;
@@ -0,0 +1,7 @@
1
+ import { EventType, ExtractorEventType } from '../../types/extraction';
2
+ export declare function createFormData(preparedArtifact: any, fetchedObjects: object[] | object): FormData;
3
+ export declare function getTimeoutExtractorEventType(eventType: EventType): {
4
+ eventType: ExtractorEventType;
5
+ isError: boolean;
6
+ } | null;
7
+ //# sourceMappingURL=helpers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../src/deprecated/common/helpers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAEvE,wBAAgB,cAAc,CAE5B,gBAAgB,EAAE,GAAG,EACrB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,QAAQ,CAUV;AAED,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,SAAS,GAAG;IAClE,SAAS,EAAE,kBAAkB,CAAC;IAC9B,OAAO,EAAE,OAAO,CAAC;CAClB,GAAG,IAAI,CA+BP"}
@@ -0,0 +1,47 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createFormData = createFormData;
4
+ exports.getTimeoutExtractorEventType = getTimeoutExtractorEventType;
5
+ const js_jsonl_1 = require("js-jsonl");
6
+ const extraction_1 = require("../../types/extraction");
7
+ function createFormData(
8
+ //eslint-disable-next-line @typescript-eslint/no-explicit-any
9
+ preparedArtifact, fetchedObjects) {
10
+ const formData = new FormData();
11
+ for (const item of preparedArtifact.form_data) {
12
+ formData.append(item.key, item.value);
13
+ }
14
+ const output = js_jsonl_1.jsonl.stringify(fetchedObjects);
15
+ formData.append('file', output);
16
+ return formData;
17
+ }
18
+ function getTimeoutExtractorEventType(eventType) {
19
+ switch (eventType) {
20
+ case extraction_1.EventType.ExtractionMetadataStart:
21
+ return {
22
+ eventType: extraction_1.ExtractorEventType.ExtractionMetadataError,
23
+ isError: true,
24
+ };
25
+ case extraction_1.EventType.ExtractionDataStart:
26
+ case extraction_1.EventType.ExtractionDataContinue:
27
+ return {
28
+ eventType: extraction_1.ExtractorEventType.ExtractionDataProgress,
29
+ isError: false,
30
+ };
31
+ case extraction_1.EventType.ExtractionAttachmentsStart:
32
+ case extraction_1.EventType.ExtractionAttachmentsContinue:
33
+ return {
34
+ eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsProgress,
35
+ isError: false,
36
+ };
37
+ case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
38
+ return {
39
+ eventType: extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
40
+ isError: true,
41
+ };
42
+ default:
43
+ console.log('Event type not recognized in getTimeoutExtractorEventType function: ' +
44
+ eventType);
45
+ return null;
46
+ }
47
+ }
@@ -0,0 +1,38 @@
1
+ {
2
+ "record_types": {
3
+ "users": {
4
+ "fields": {
5
+ "name": {
6
+ "is_required": true,
7
+ "type": "text",
8
+ "name": "Name",
9
+ "text": {
10
+ "min_length": 1
11
+ }
12
+ },
13
+ "email": {
14
+ "type": "text",
15
+ "name": "Email",
16
+ "is_required": true
17
+ }
18
+ }
19
+ },
20
+ "contacts": {
21
+ "fields": {
22
+ "name": {
23
+ "is_required": true,
24
+ "type": "text",
25
+ "name": "Name",
26
+ "text": {
27
+ "min_length": 1
28
+ }
29
+ },
30
+ "email": {
31
+ "type": "text",
32
+ "name": "Email",
33
+ "is_required": true
34
+ }
35
+ }
36
+ }
37
+ }
38
+ }
@@ -0,0 +1,18 @@
1
+ import { AirdropEvent } from '../../types/extraction';
2
+ import { Adapter } from '../adapter';
3
+ type ConnectorState = object;
4
+ /**
5
+ * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
6
+ *
7
+ * @class DemoExtractor
8
+ * @deprecated
9
+ **/
10
+ export declare class DemoExtractor {
11
+ private event;
12
+ private adapter;
13
+ private uploader;
14
+ constructor(event: AirdropEvent, adapter: Adapter<ConnectorState>);
15
+ run(): Promise<void>;
16
+ }
17
+ export {};
18
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/deprecated/demo-extractor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EAIb,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAIrC,KAAK,cAAc,GAAG,MAAM,CAAC;AAE7B;;;;;IAKI;AACJ,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,QAAQ,CAAW;gBAEf,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC;IAS3D,GAAG;CA0MV"}
@@ -0,0 +1,161 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.DemoExtractor = void 0;
7
+ const extraction_1 = require("../../types/extraction");
8
+ const uploader_1 = require("../uploader");
9
+ const external_domain_metadata_json_1 = __importDefault(require("./external_domain_metadata.json"));
10
+ /**
11
+ * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
12
+ *
13
+ * @class DemoExtractor
14
+ * @deprecated
15
+ **/
16
+ class DemoExtractor {
17
+ constructor(event, adapter) {
18
+ this.event = event;
19
+ this.adapter = adapter;
20
+ this.uploader = new uploader_1.Uploader(this.event.execution_metadata.devrev_endpoint, this.event.context.secrets.service_account_token);
21
+ }
22
+ async run() {
23
+ switch (this.event.payload.event_type) {
24
+ case extraction_1.EventType.ExtractionExternalSyncUnitsStart: {
25
+ const externalSyncUnits = [
26
+ {
27
+ id: 'devrev',
28
+ name: 'devrev',
29
+ description: 'Demo external sync unit',
30
+ },
31
+ ];
32
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
33
+ external_sync_units: externalSyncUnits,
34
+ });
35
+ break;
36
+ }
37
+ case extraction_1.EventType.ExtractionMetadataStart: {
38
+ const { artifact, error } = await this.uploader.upload('metadata_1.jsonl', 'external_domain_metadata', external_domain_metadata_json_1.default);
39
+ if (error || !artifact) {
40
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataError, {
41
+ error,
42
+ });
43
+ return;
44
+ }
45
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataDone, {
46
+ artifacts: [artifact],
47
+ });
48
+ break;
49
+ }
50
+ case extraction_1.EventType.ExtractionDataStart: {
51
+ const contacts = [
52
+ {
53
+ id: 'contact-1',
54
+ created_date: '1999-12-25T01:00:03+01:00',
55
+ modified_date: '1999-12-25T01:00:03+01:00',
56
+ data: {
57
+ email: 'johnsmith@test.com',
58
+ name: 'John Smith',
59
+ },
60
+ },
61
+ {
62
+ id: 'contact-2',
63
+ created_date: '1999-12-27T15:31:34+01:00',
64
+ modified_date: '2002-04-09T01:55:31+02:00',
65
+ data: {
66
+ email: 'janesmith@test.com',
67
+ name: 'Jane Smith',
68
+ },
69
+ },
70
+ ];
71
+ const { artifact, error } = await this.uploader.upload('contacts_1.json', 'contacts', contacts);
72
+ if (error || !artifact) {
73
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
74
+ error,
75
+ });
76
+ return;
77
+ }
78
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataProgress, {
79
+ progress: 50,
80
+ artifacts: [artifact],
81
+ });
82
+ break;
83
+ }
84
+ case extraction_1.EventType.ExtractionDataContinue: {
85
+ const users = [
86
+ {
87
+ id: 'user-1',
88
+ created_date: '1999-12-25T01:00:03+01:00',
89
+ modified_date: '1999-12-25T01:00:03+01:00',
90
+ data: {
91
+ email: 'johndoe@test.com',
92
+ name: 'John Doe',
93
+ },
94
+ },
95
+ {
96
+ id: 'user-2',
97
+ created_date: '1999-12-27T15:31:34+01:00',
98
+ modified_date: '2002-04-09T01:55:31+02:00',
99
+ data: {
100
+ email: 'janedoe@test.com',
101
+ name: 'Jane Doe',
102
+ },
103
+ },
104
+ ];
105
+ const { artifact, error } = await this.uploader.upload('users_1.json', 'users', users);
106
+ if (error || !artifact) {
107
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
108
+ error,
109
+ });
110
+ return;
111
+ }
112
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDone, {
113
+ progress: 100,
114
+ artifacts: [artifact],
115
+ });
116
+ break;
117
+ }
118
+ case extraction_1.EventType.ExtractionDataDelete: {
119
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDeleteDone);
120
+ break;
121
+ }
122
+ case extraction_1.EventType.ExtractionAttachmentsStart: {
123
+ const attachment1 = ['This is attachment1.txt content'];
124
+ const { artifact, error } = await this.uploader.upload('attachment1.txt', 'attachment', attachment1);
125
+ if (error || !artifact) {
126
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
127
+ error,
128
+ });
129
+ return;
130
+ }
131
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsProgress, {
132
+ artifacts: [artifact],
133
+ });
134
+ break;
135
+ }
136
+ case extraction_1.EventType.ExtractionAttachmentsContinue: {
137
+ const attachment2 = ['This is attachment2.txt content'];
138
+ const { artifact, error } = await this.uploader.upload('attachment2.txt', 'attachment', attachment2);
139
+ if (error || !artifact) {
140
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
141
+ error,
142
+ });
143
+ return;
144
+ }
145
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDone, {
146
+ artifacts: [artifact],
147
+ });
148
+ break;
149
+ }
150
+ case extraction_1.EventType.ExtractionAttachmentsDelete: {
151
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
152
+ break;
153
+ }
154
+ default: {
155
+ console.error('Event in DemoExtractor run not recognized: ' +
156
+ JSON.stringify(this.event.payload.event_type));
157
+ }
158
+ }
159
+ }
160
+ }
161
+ exports.DemoExtractor = DemoExtractor;
@@ -0,0 +1,22 @@
1
+ import { RawAxiosRequestHeaders } from 'axios';
2
+ import { HTTPResponse } from '../../http/types';
3
+ export declare const defaultResponse: HTTPResponse;
4
+ /**
5
+ * HTTPClient class to make HTTP requests
6
+ * @deprecated
7
+ */
8
+ export declare class HTTPClient {
9
+ private retryAfter;
10
+ private retryAt;
11
+ private axiosInstance;
12
+ constructor();
13
+ /**
14
+ *
15
+ * Function to make a GET call to the endpoint.
16
+ * There is special handling for rate limit exceeded error.
17
+ * In case of rate limit exceeded, the function returns success as true and the delay time in seconds
18
+ * In case of any other error, the function returns success as false and the error message
19
+ */
20
+ getCall(endpoint: string, headers: RawAxiosRequestHeaders, params?: any): Promise<HTTPResponse>;
21
+ }
22
+ //# sourceMappingURL=client.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../src/deprecated/http/client.ts"],"names":[],"mappings":"AAAA,OAAc,EAGZ,sBAAsB,EACvB,MAAM,OAAO,CAAC;AAKf,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhD,eAAO,MAAM,eAAe,EAAE,YAQ7B,CAAC;AAEF;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,aAAa,CAAkB;;IA8BvC;;;;;;OAMG;IACG,OAAO,CACX,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,sBAAsB,EAE/B,MAAM,CAAC,EAAE,GAAG,GACX,OAAO,CAAC,YAAY,CAAC;CA8EzB"}
@@ -0,0 +1,161 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.HTTPClient = exports.defaultResponse = void 0;
37
+ const axios_1 = __importStar(require("axios"));
38
+ const constants_1 = require("../../http/constants");
39
+ exports.defaultResponse = {
40
+ data: {
41
+ delay: 0,
42
+ nextPage: 1,
43
+ records: [],
44
+ },
45
+ message: '',
46
+ success: false,
47
+ };
48
+ /**
49
+ * HTTPClient class to make HTTP requests
50
+ * @deprecated
51
+ */
52
+ class HTTPClient {
53
+ constructor() {
54
+ this.retryAfter = 0;
55
+ this.retryAt = 0;
56
+ this.axiosInstance = axios_1.default.create();
57
+ // Add request interceptor to check for retryAfter before making a request
58
+ this.axiosInstance.interceptors.request.use((config) => {
59
+ // Check if retryAfter is not 0 and return a LIMIT_EXCEEDED error
60
+ if (this.retryAfter !== 0) {
61
+ // check if the current time is greater than the retryAt time
62
+ const currentTime = new Date().getTime();
63
+ if (currentTime < this.retryAt) {
64
+ console.error('Rate limit exceeded. Interceptor has retryAfter: ' +
65
+ this.retryAfter);
66
+ // Rate limit exceeded.
67
+ return Promise.reject(constants_1.RATE_LIMIT_EXCEEDED);
68
+ }
69
+ else {
70
+ // Reset the retryAfter
71
+ this.retryAfter = 0;
72
+ }
73
+ }
74
+ return config;
75
+ }, (error) => {
76
+ return Promise.reject(error);
77
+ });
78
+ }
79
+ /**
80
+ *
81
+ * Function to make a GET call to the endpoint.
82
+ * There is special handling for rate limit exceeded error.
83
+ * In case of rate limit exceeded, the function returns success as true and the delay time in seconds
84
+ * In case of any other error, the function returns success as false and the error message
85
+ */
86
+ async getCall(endpoint, headers,
87
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
88
+ params) {
89
+ var _a;
90
+ // Return the LIMIT_EXCEEDED error if the retryAfter is not 0
91
+ try {
92
+ const res = await this.axiosInstance.get(endpoint, {
93
+ headers: headers,
94
+ params: params,
95
+ });
96
+ return Object.assign(Object.assign({}, exports.defaultResponse), { data: {
97
+ delay: 0,
98
+ records: res.data,
99
+ }, success: true });
100
+ }
101
+ catch (error) {
102
+ console.log('Error in getCall: ' + JSON.stringify(error));
103
+ // send error to adapter
104
+ if ((0, axios_1.isAxiosError)(error)) {
105
+ if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === constants_1.RATE_LIMIT_EXCEEDED_STATUS_CODE) {
106
+ this.retryAfter = error.response.headers['retry-after']
107
+ ? error.response.headers['retry-after']
108
+ : 0;
109
+ this.retryAt = new Date().getTime() + this.retryAfter * 1000;
110
+ console.warn('Rate limit exceeded. Error code: ' +
111
+ error.response.status +
112
+ ' RetryAfter: ' +
113
+ this.retryAfter +
114
+ ' RetryAt: ' +
115
+ this.retryAt);
116
+ return {
117
+ data: {
118
+ delay: this.retryAfter,
119
+ records: [],
120
+ },
121
+ message: constants_1.RATE_LIMIT_EXCEEDED,
122
+ success: true,
123
+ };
124
+ }
125
+ if (error.response) {
126
+ return Object.assign(Object.assign({}, exports.defaultResponse), { message: error.response.data });
127
+ }
128
+ else {
129
+ return Object.assign(Object.assign({}, exports.defaultResponse), { message: error.message });
130
+ }
131
+ }
132
+ else {
133
+ if (this.retryAfter !== 0) {
134
+ console.warn('Rate limit exceeded. Going to return the following response: ' +
135
+ JSON.stringify(error));
136
+ return {
137
+ data: {
138
+ delay: this.retryAfter,
139
+ records: [],
140
+ },
141
+ message: typeof error === 'string'
142
+ ? error
143
+ : JSON.stringify(error, Object.getOwnPropertyNames(error)),
144
+ success: true,
145
+ };
146
+ }
147
+ return {
148
+ data: {
149
+ delay: this.retryAfter,
150
+ records: [],
151
+ },
152
+ message: typeof error === 'string'
153
+ ? error
154
+ : JSON.stringify(error, Object.getOwnPropertyNames(error)),
155
+ success: false,
156
+ };
157
+ }
158
+ }
159
+ }
160
+ }
161
+ exports.HTTPClient = HTTPClient;
@@ -0,0 +1,35 @@
1
+ import { UploadResponse } from '../../uploader/uploader.interfaces';
2
+ /**
3
+ * Uploader class is used to upload files to the DevRev platform.
4
+ * The class provides utilities to
5
+ * - prepare artifact
6
+ * - upload artifact
7
+ * - return the artifact information to the platform
8
+ *
9
+ * @class Uploader
10
+ * @constructor
11
+ * @param {string} endpoint - The endpoint of the DevRev platform
12
+ * @param {string} token - The token to authenticate with the DevRev platform
13
+ * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
14
+ */
15
+ export declare class Uploader {
16
+ private betaDevrevSdk;
17
+ private local;
18
+ constructor(endpoint: string, token: string, local?: boolean);
19
+ /**
20
+ *
21
+ * Uploads the file to the DevRev platform. The file is uploaded to the platform
22
+ * and the artifact information is returned.
23
+ *
24
+ * @param {string} filename - The name of the file to be uploaded
25
+ * @param {string} entity - The entity type of the file to be uploaded
26
+ * @param {object[] | object} fetchedObjects - The objects to be uploaded
27
+ * @param filetype - The type of the file to be uploaded
28
+ * @returns {Promise<UploadResponse>} - The response object containing the artifact information
29
+ */
30
+ upload(filename: string, entity: string, fetchedObjects: object[] | object, filetype?: string): Promise<UploadResponse>;
31
+ private prepareArtifact;
32
+ private uploadToArtifact;
33
+ private downloadToLocal;
34
+ }
35
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/deprecated/uploader/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAY,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAG9E;;;;;;;;;;;;GAYG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,KAAK,CAAU;gBACX,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,UAAQ;IAQ1D;;;;;;;;;;OAUG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,EACjC,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAAC,cAAc,CAAC;YAuCZ,eAAe;YAiBf,gBAAgB;YAqBhB,eAAe;CAgC9B"}