@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. package/README.md +7 -21
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +6 -7
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +4 -3
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  6. package/dist/attachments-streaming/attachments-streaming-pool.js +69 -61
  7. package/dist/common/constants.d.ts +15 -12
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +20 -34
  10. package/dist/common/helpers.d.ts +3 -27
  11. package/dist/common/helpers.d.ts.map +1 -1
  12. package/dist/common/helpers.js +6 -36
  13. package/dist/http/client.d.ts +4 -0
  14. package/dist/http/client.d.ts.map +1 -0
  15. package/dist/http/client.js +80 -0
  16. package/dist/http/errors.d.ts +8 -0
  17. package/dist/http/errors.d.ts.map +1 -0
  18. package/dist/http/errors.js +31 -0
  19. package/dist/index.d.ts +30 -12
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +47 -28
  22. package/dist/logger/logger.d.ts +13 -70
  23. package/dist/logger/logger.d.ts.map +1 -1
  24. package/dist/logger/logger.interfaces.d.ts +4 -7
  25. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  26. package/dist/logger/logger.js +67 -92
  27. package/dist/mappers/mappers.d.ts +10 -43
  28. package/dist/mappers/mappers.d.ts.map +1 -1
  29. package/dist/mappers/mappers.interfaces.d.ts +173 -0
  30. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  31. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +1 -10
  32. package/dist/mappers/mappers.js +32 -46
  33. package/dist/multithreading/adapters/base-adapter.d.ts +39 -0
  34. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
  35. package/dist/multithreading/adapters/base-adapter.js +212 -0
  36. package/dist/multithreading/adapters/extraction-adapter.d.ts +49 -0
  37. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
  38. package/dist/multithreading/adapters/extraction-adapter.js +375 -0
  39. package/dist/multithreading/adapters/loading-adapter.d.ts +26 -0
  40. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
  41. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +12 -0
  42. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
  43. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +2 -12
  44. package/dist/multithreading/adapters/loading-adapter.js +279 -0
  45. package/dist/multithreading/emit.d.ts +11 -0
  46. package/dist/multithreading/emit.d.ts.map +1 -0
  47. package/dist/multithreading/emit.js +25 -0
  48. package/dist/multithreading/process-task.d.ts +6 -1
  49. package/dist/multithreading/process-task.d.ts.map +1 -1
  50. package/dist/multithreading/process-task.js +120 -82
  51. package/dist/multithreading/spawn/spawn.d.ts +10 -11
  52. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  53. package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
  54. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  55. package/dist/multithreading/spawn/spawn.helpers.js +142 -77
  56. package/dist/multithreading/spawn/spawn.js +59 -75
  57. package/dist/multithreading/worker.js +16 -5
  58. package/dist/repo/repo.d.ts +14 -5
  59. package/dist/repo/repo.d.ts.map +1 -1
  60. package/dist/repo/repo.helpers.d.ts +6 -0
  61. package/dist/repo/repo.helpers.d.ts.map +1 -0
  62. package/dist/repo/repo.helpers.js +16 -0
  63. package/dist/repo/repo.interfaces.d.ts +11 -20
  64. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  65. package/dist/repo/repo.js +36 -22
  66. package/dist/state/base-state.d.ts +58 -0
  67. package/dist/state/base-state.d.ts.map +1 -0
  68. package/dist/state/base-state.js +242 -0
  69. package/dist/state/extraction-state.d.ts +14 -0
  70. package/dist/state/extraction-state.d.ts.map +1 -0
  71. package/dist/state/extraction-state.js +146 -0
  72. package/dist/state/install-initial-domain-mapping.d.ts +4 -0
  73. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  74. package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
  75. package/dist/state/loading-state.d.ts +8 -0
  76. package/dist/state/loading-state.d.ts.map +1 -0
  77. package/dist/state/loading-state.js +29 -0
  78. package/dist/state/state.d.ts +3 -29
  79. package/dist/state/state.d.ts.map +1 -1
  80. package/dist/state/state.interfaces.d.ts +31 -12
  81. package/dist/state/state.interfaces.d.ts.map +1 -1
  82. package/dist/state/state.interfaces.js +17 -1
  83. package/dist/state/state.js +9 -164
  84. package/dist/state/time-value-resolver.d.ts +17 -0
  85. package/dist/state/time-value-resolver.d.ts.map +1 -0
  86. package/dist/state/time-value-resolver.js +110 -0
  87. package/dist/testing/mock-event.d.ts +10 -0
  88. package/dist/testing/mock-event.d.ts.map +1 -0
  89. package/dist/testing/mock-event.js +96 -0
  90. package/dist/testing/mock-server.d.ts +31 -0
  91. package/dist/testing/mock-server.d.ts.map +1 -0
  92. package/dist/testing/mock-server.interfaces.d.ts +54 -0
  93. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  94. package/dist/testing/mock-server.interfaces.js +4 -0
  95. package/dist/testing/mock-server.js +297 -0
  96. package/dist/testing/test-adapters.d.ts +22 -0
  97. package/dist/testing/test-adapters.d.ts.map +1 -0
  98. package/dist/testing/test-adapters.js +32 -0
  99. package/dist/types/common.d.ts +4 -37
  100. package/dist/types/common.d.ts.map +1 -1
  101. package/dist/types/common.js +2 -17
  102. package/dist/types/errors.d.ts +15 -0
  103. package/dist/types/errors.d.ts.map +1 -0
  104. package/dist/types/errors.js +28 -0
  105. package/dist/types/external-domain-metadata.d.ts +40 -162
  106. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  107. package/dist/types/extraction.d.ts +135 -278
  108. package/dist/types/extraction.d.ts.map +1 -1
  109. package/dist/types/extraction.js +46 -133
  110. package/dist/types/loading.d.ts +15 -55
  111. package/dist/types/loading.d.ts.map +1 -1
  112. package/dist/types/loading.js +1 -23
  113. package/dist/types/workers.d.ts +70 -103
  114. package/dist/types/workers.d.ts.map +1 -1
  115. package/dist/types/workers.js +1 -6
  116. package/dist/uploader/uploader.d.ts +10 -61
  117. package/dist/uploader/uploader.d.ts.map +1 -1
  118. package/dist/uploader/uploader.helpers.d.ts +7 -27
  119. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  120. package/dist/uploader/uploader.helpers.js +80 -30
  121. package/dist/uploader/uploader.interfaces.d.ts +19 -54
  122. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  123. package/dist/uploader/uploader.interfaces.js +8 -0
  124. package/dist/uploader/uploader.js +72 -124
  125. package/docs/MIGRATION.md +751 -0
  126. package/package.json +35 -22
  127. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  128. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  129. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -321
  130. package/dist/common/control-protocol.d.ts +0 -10
  131. package/dist/common/control-protocol.d.ts.map +0 -1
  132. package/dist/common/control-protocol.js +0 -31
  133. package/dist/common/errors.d.ts +0 -6
  134. package/dist/common/errors.d.ts.map +0 -1
  135. package/dist/common/errors.js +0 -4
  136. package/dist/common/event-type-translation.d.ts +0 -24
  137. package/dist/common/event-type-translation.d.ts.map +0 -1
  138. package/dist/common/event-type-translation.js +0 -117
  139. package/dist/common/install-initial-domain-mapping.d.ts +0 -4
  140. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  141. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  142. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  143. package/dist/common/install-initial-domain-mapping.test.js +0 -208
  144. package/dist/deprecated/adapter/index.d.ts +0 -62
  145. package/dist/deprecated/adapter/index.d.ts.map +0 -1
  146. package/dist/deprecated/adapter/index.js +0 -151
  147. package/dist/deprecated/common/helpers.d.ts +0 -7
  148. package/dist/deprecated/common/helpers.d.ts.map +0 -1
  149. package/dist/deprecated/common/helpers.js +0 -47
  150. package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
  151. package/dist/deprecated/demo-extractor/index.d.ts +0 -18
  152. package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
  153. package/dist/deprecated/demo-extractor/index.js +0 -161
  154. package/dist/deprecated/http/client.d.ts +0 -22
  155. package/dist/deprecated/http/client.d.ts.map +0 -1
  156. package/dist/deprecated/http/client.js +0 -161
  157. package/dist/deprecated/uploader/index.d.ts +0 -35
  158. package/dist/deprecated/uploader/index.d.ts.map +0 -1
  159. package/dist/deprecated/uploader/index.js +0 -161
  160. package/dist/http/axios-client-internal.d.ts +0 -3
  161. package/dist/http/axios-client-internal.d.ts.map +0 -1
  162. package/dist/http/axios-client-internal.js +0 -66
  163. package/dist/http/axios-client-internal.test.d.ts +0 -2
  164. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  165. package/dist/http/axios-client-internal.test.js +0 -154
  166. package/dist/http/axios-client.d.ts +0 -27
  167. package/dist/http/axios-client.d.ts.map +0 -1
  168. package/dist/http/axios-client.js +0 -57
  169. package/dist/http/constants.d.ts +0 -4
  170. package/dist/http/constants.d.ts.map +0 -1
  171. package/dist/http/constants.js +0 -6
  172. package/dist/http/index.d.ts +0 -3
  173. package/dist/http/index.d.ts.map +0 -1
  174. package/dist/http/index.js +0 -18
  175. package/dist/http/types.d.ts +0 -17
  176. package/dist/http/types.d.ts.map +0 -1
  177. package/dist/http/types.js +0 -2
  178. package/dist/logger/logger.constants.d.ts +0 -6
  179. package/dist/logger/logger.constants.d.ts.map +0 -1
  180. package/dist/logger/logger.constants.js +0 -13
  181. package/dist/logger/logger.context.d.ts +0 -58
  182. package/dist/logger/logger.context.d.ts.map +0 -1
  183. package/dist/logger/logger.context.js +0 -86
  184. package/dist/logger/logger.test.d.ts +0 -2
  185. package/dist/logger/logger.test.d.ts.map +0 -1
  186. package/dist/logger/logger.test.js +0 -490
  187. package/dist/mappers/mappers.interface.d.ts +0 -294
  188. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  189. package/dist/mappers/mappers.test.d.ts +0 -2
  190. package/dist/mappers/mappers.test.d.ts.map +0 -1
  191. package/dist/mappers/mappers.test.js +0 -107
  192. package/dist/multithreading/create-worker.test.d.ts +0 -2
  193. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  194. package/dist/multithreading/create-worker.test.js +0 -89
  195. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  196. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  197. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  198. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
  199. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
  200. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +0 -127
  201. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -91
  202. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
  203. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +0 -22
  204. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
  205. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
  206. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
  207. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +0 -514
  208. package/dist/multithreading/worker-adapter/worker-adapter.js +0 -745
  209. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
  210. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
  211. package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
  212. package/dist/repo/repo.test.d.ts +0 -2
  213. package/dist/repo/repo.test.d.ts.map +0 -1
  214. package/dist/repo/repo.test.js +0 -131
  215. package/dist/state/state.test.d.ts +0 -2
  216. package/dist/state/state.test.d.ts.map +0 -1
  217. package/dist/state/state.test.js +0 -224
  218. package/dist/types/extraction.test.d.ts +0 -2
  219. package/dist/types/extraction.test.d.ts.map +0 -1
  220. package/dist/types/extraction.test.js +0 -70
  221. package/dist/types/index.d.ts +0 -10
  222. package/dist/types/index.d.ts.map +0 -1
  223. package/dist/types/index.js +0 -18
  224. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  225. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  226. package/dist/uploader/uploader.helpers.test.js +0 -267
  227. package/dist/uploader/uploader.test.d.ts +0 -2
  228. package/dist/uploader/uploader.test.d.ts.map +0 -1
  229. package/dist/uploader/uploader.test.js +0 -589
@@ -1,161 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.DemoExtractor = void 0;
7
- const extraction_1 = require("../../types/extraction");
8
- const uploader_1 = require("../uploader");
9
- const external_domain_metadata_json_1 = __importDefault(require("./external_domain_metadata.json"));
10
- /**
11
- * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
12
- *
13
- * @class DemoExtractor
14
- * @deprecated
15
- **/
16
- class DemoExtractor {
17
- constructor(event, adapter) {
18
- this.event = event;
19
- this.adapter = adapter;
20
- this.uploader = new uploader_1.Uploader(this.event.execution_metadata.devrev_endpoint, this.event.context.secrets.service_account_token);
21
- }
22
- async run() {
23
- switch (this.event.payload.event_type) {
24
- case extraction_1.EventType.ExtractionExternalSyncUnitsStart: {
25
- const externalSyncUnits = [
26
- {
27
- id: 'devrev',
28
- name: 'devrev',
29
- description: 'Demo external sync unit',
30
- },
31
- ];
32
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
33
- external_sync_units: externalSyncUnits,
34
- });
35
- break;
36
- }
37
- case extraction_1.EventType.ExtractionMetadataStart: {
38
- const { artifact, error } = await this.uploader.upload('metadata_1.jsonl', 'external_domain_metadata', external_domain_metadata_json_1.default);
39
- if (error || !artifact) {
40
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataError, {
41
- error,
42
- });
43
- return;
44
- }
45
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataDone, {
46
- artifacts: [artifact],
47
- });
48
- break;
49
- }
50
- case extraction_1.EventType.ExtractionDataStart: {
51
- const contacts = [
52
- {
53
- id: 'contact-1',
54
- created_date: '1999-12-25T01:00:03+01:00',
55
- modified_date: '1999-12-25T01:00:03+01:00',
56
- data: {
57
- email: 'johnsmith@test.com',
58
- name: 'John Smith',
59
- },
60
- },
61
- {
62
- id: 'contact-2',
63
- created_date: '1999-12-27T15:31:34+01:00',
64
- modified_date: '2002-04-09T01:55:31+02:00',
65
- data: {
66
- email: 'janesmith@test.com',
67
- name: 'Jane Smith',
68
- },
69
- },
70
- ];
71
- const { artifact, error } = await this.uploader.upload('contacts_1.json', 'contacts', contacts);
72
- if (error || !artifact) {
73
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
74
- error,
75
- });
76
- return;
77
- }
78
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataProgress, {
79
- progress: 50,
80
- artifacts: [artifact],
81
- });
82
- break;
83
- }
84
- case extraction_1.EventType.ExtractionDataContinue: {
85
- const users = [
86
- {
87
- id: 'user-1',
88
- created_date: '1999-12-25T01:00:03+01:00',
89
- modified_date: '1999-12-25T01:00:03+01:00',
90
- data: {
91
- email: 'johndoe@test.com',
92
- name: 'John Doe',
93
- },
94
- },
95
- {
96
- id: 'user-2',
97
- created_date: '1999-12-27T15:31:34+01:00',
98
- modified_date: '2002-04-09T01:55:31+02:00',
99
- data: {
100
- email: 'janedoe@test.com',
101
- name: 'Jane Doe',
102
- },
103
- },
104
- ];
105
- const { artifact, error } = await this.uploader.upload('users_1.json', 'users', users);
106
- if (error || !artifact) {
107
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
108
- error,
109
- });
110
- return;
111
- }
112
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDone, {
113
- progress: 100,
114
- artifacts: [artifact],
115
- });
116
- break;
117
- }
118
- case extraction_1.EventType.ExtractionDataDelete: {
119
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDeleteDone);
120
- break;
121
- }
122
- case extraction_1.EventType.ExtractionAttachmentsStart: {
123
- const attachment1 = ['This is attachment1.txt content'];
124
- const { artifact, error } = await this.uploader.upload('attachment1.txt', 'attachment', attachment1);
125
- if (error || !artifact) {
126
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
127
- error,
128
- });
129
- return;
130
- }
131
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsProgress, {
132
- artifacts: [artifact],
133
- });
134
- break;
135
- }
136
- case extraction_1.EventType.ExtractionAttachmentsContinue: {
137
- const attachment2 = ['This is attachment2.txt content'];
138
- const { artifact, error } = await this.uploader.upload('attachment2.txt', 'attachment', attachment2);
139
- if (error || !artifact) {
140
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
141
- error,
142
- });
143
- return;
144
- }
145
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDone, {
146
- artifacts: [artifact],
147
- });
148
- break;
149
- }
150
- case extraction_1.EventType.ExtractionAttachmentsDelete: {
151
- await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
152
- break;
153
- }
154
- default: {
155
- console.error('Event in DemoExtractor run not recognized: ' +
156
- JSON.stringify(this.event.payload.event_type));
157
- }
158
- }
159
- }
160
- }
161
- exports.DemoExtractor = DemoExtractor;
@@ -1,22 +0,0 @@
1
- import { RawAxiosRequestHeaders } from 'axios';
2
- import { HTTPResponse } from '../../http/types';
3
- export declare const defaultResponse: HTTPResponse;
4
- /**
5
- * HTTPClient class to make HTTP requests
6
- * @deprecated
7
- */
8
- export declare class HTTPClient {
9
- private retryAfter;
10
- private retryAt;
11
- private axiosInstance;
12
- constructor();
13
- /**
14
- *
15
- * Function to make a GET call to the endpoint.
16
- * There is special handling for rate limit exceeded error.
17
- * In case of rate limit exceeded, the function returns success as true and the delay time in seconds
18
- * In case of any other error, the function returns success as false and the error message
19
- */
20
- getCall(endpoint: string, headers: RawAxiosRequestHeaders, params?: any): Promise<HTTPResponse>;
21
- }
22
- //# sourceMappingURL=client.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../src/deprecated/http/client.ts"],"names":[],"mappings":"AAAA,OAAc,EAGZ,sBAAsB,EACvB,MAAM,OAAO,CAAC;AAKf,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhD,eAAO,MAAM,eAAe,EAAE,YAQ7B,CAAC;AAEF;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,aAAa,CAAkB;;IA8BvC;;;;;;OAMG;IACG,OAAO,CACX,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,sBAAsB,EAE/B,MAAM,CAAC,EAAE,GAAG,GACX,OAAO,CAAC,YAAY,CAAC;CA8EzB"}
@@ -1,161 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.HTTPClient = exports.defaultResponse = void 0;
37
- const axios_1 = __importStar(require("axios"));
38
- const constants_1 = require("../../http/constants");
39
- exports.defaultResponse = {
40
- data: {
41
- delay: 0,
42
- nextPage: 1,
43
- records: [],
44
- },
45
- message: '',
46
- success: false,
47
- };
48
- /**
49
- * HTTPClient class to make HTTP requests
50
- * @deprecated
51
- */
52
- class HTTPClient {
53
- constructor() {
54
- this.retryAfter = 0;
55
- this.retryAt = 0;
56
- this.axiosInstance = axios_1.default.create();
57
- // Add request interceptor to check for retryAfter before making a request
58
- this.axiosInstance.interceptors.request.use((config) => {
59
- // Check if retryAfter is not 0 and return a LIMIT_EXCEEDED error
60
- if (this.retryAfter !== 0) {
61
- // check if the current time is greater than the retryAt time
62
- const currentTime = new Date().getTime();
63
- if (currentTime < this.retryAt) {
64
- console.error('Rate limit exceeded. Interceptor has retryAfter: ' +
65
- this.retryAfter);
66
- // Rate limit exceeded.
67
- return Promise.reject(constants_1.RATE_LIMIT_EXCEEDED);
68
- }
69
- else {
70
- // Reset the retryAfter
71
- this.retryAfter = 0;
72
- }
73
- }
74
- return config;
75
- }, (error) => {
76
- return Promise.reject(error);
77
- });
78
- }
79
- /**
80
- *
81
- * Function to make a GET call to the endpoint.
82
- * There is special handling for rate limit exceeded error.
83
- * In case of rate limit exceeded, the function returns success as true and the delay time in seconds
84
- * In case of any other error, the function returns success as false and the error message
85
- */
86
- async getCall(endpoint, headers,
87
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
88
- params) {
89
- var _a;
90
- // Return the LIMIT_EXCEEDED error if the retryAfter is not 0
91
- try {
92
- const res = await this.axiosInstance.get(endpoint, {
93
- headers: headers,
94
- params: params,
95
- });
96
- return Object.assign(Object.assign({}, exports.defaultResponse), { data: {
97
- delay: 0,
98
- records: res.data,
99
- }, success: true });
100
- }
101
- catch (error) {
102
- console.log('Error in getCall: ' + JSON.stringify(error));
103
- // send error to adapter
104
- if ((0, axios_1.isAxiosError)(error)) {
105
- if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === constants_1.RATE_LIMIT_EXCEEDED_STATUS_CODE) {
106
- this.retryAfter = error.response.headers['retry-after']
107
- ? error.response.headers['retry-after']
108
- : 0;
109
- this.retryAt = new Date().getTime() + this.retryAfter * 1000;
110
- console.warn('Rate limit exceeded. Error code: ' +
111
- error.response.status +
112
- ' RetryAfter: ' +
113
- this.retryAfter +
114
- ' RetryAt: ' +
115
- this.retryAt);
116
- return {
117
- data: {
118
- delay: this.retryAfter,
119
- records: [],
120
- },
121
- message: constants_1.RATE_LIMIT_EXCEEDED,
122
- success: true,
123
- };
124
- }
125
- if (error.response) {
126
- return Object.assign(Object.assign({}, exports.defaultResponse), { message: error.response.data });
127
- }
128
- else {
129
- return Object.assign(Object.assign({}, exports.defaultResponse), { message: error.message });
130
- }
131
- }
132
- else {
133
- if (this.retryAfter !== 0) {
134
- console.warn('Rate limit exceeded. Going to return the following response: ' +
135
- JSON.stringify(error));
136
- return {
137
- data: {
138
- delay: this.retryAfter,
139
- records: [],
140
- },
141
- message: typeof error === 'string'
142
- ? error
143
- : JSON.stringify(error, Object.getOwnPropertyNames(error)),
144
- success: true,
145
- };
146
- }
147
- return {
148
- data: {
149
- delay: this.retryAfter,
150
- records: [],
151
- },
152
- message: typeof error === 'string'
153
- ? error
154
- : JSON.stringify(error, Object.getOwnPropertyNames(error)),
155
- success: false,
156
- };
157
- }
158
- }
159
- }
160
- }
161
- exports.HTTPClient = HTTPClient;
@@ -1,35 +0,0 @@
1
- import { UploadResponse } from '../../uploader/uploader.interfaces';
2
- /**
3
- * Uploader class is used to upload files to the DevRev platform.
4
- * The class provides utilities to
5
- * - prepare artifact
6
- * - upload artifact
7
- * - return the artifact information to the platform
8
- *
9
- * @class Uploader
10
- * @constructor
11
- * @param {string} endpoint - The endpoint of the DevRev platform
12
- * @param {string} token - The token to authenticate with the DevRev platform
13
- * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
14
- */
15
- export declare class Uploader {
16
- private betaDevrevSdk;
17
- private local;
18
- constructor(endpoint: string, token: string, local?: boolean);
19
- /**
20
- *
21
- * Uploads the file to the DevRev platform. The file is uploaded to the platform
22
- * and the artifact information is returned.
23
- *
24
- * @param {string} filename - The name of the file to be uploaded
25
- * @param {string} entity - The entity type of the file to be uploaded
26
- * @param {object[] | object} fetchedObjects - The objects to be uploaded
27
- * @param filetype - The type of the file to be uploaded
28
- * @returns {Promise<UploadResponse>} - The response object containing the artifact information
29
- */
30
- upload(filename: string, entity: string, fetchedObjects: object[] | object, filetype?: string): Promise<UploadResponse>;
31
- private prepareArtifact;
32
- private uploadToArtifact;
33
- private downloadToLocal;
34
- }
35
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/deprecated/uploader/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAY,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAG9E;;;;;;;;;;;;GAYG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,KAAK,CAAU;gBACX,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,UAAQ;IAQ1D;;;;;;;;;;OAUG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,EACjC,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAAC,cAAc,CAAC;YAuCZ,eAAe;YAiBf,gBAAgB;YAqBhB,eAAe;CAgC9B"}
@@ -1,161 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.Uploader = void 0;
37
- const typescript_sdk_1 = require("@devrev/typescript-sdk");
38
- const fs_1 = __importStar(require("fs"));
39
- const axios_client_internal_1 = require("../../http/axios-client-internal");
40
- const helpers_1 = require("../common/helpers");
41
- /**
42
- * Uploader class is used to upload files to the DevRev platform.
43
- * The class provides utilities to
44
- * - prepare artifact
45
- * - upload artifact
46
- * - return the artifact information to the platform
47
- *
48
- * @class Uploader
49
- * @constructor
50
- * @param {string} endpoint - The endpoint of the DevRev platform
51
- * @param {string} token - The token to authenticate with the DevRev platform
52
- * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
53
- */
54
- class Uploader {
55
- constructor(endpoint, token, local = false) {
56
- this.betaDevrevSdk = typescript_sdk_1.client.setupBeta({
57
- endpoint,
58
- token,
59
- });
60
- this.local = local;
61
- }
62
- /**
63
- *
64
- * Uploads the file to the DevRev platform. The file is uploaded to the platform
65
- * and the artifact information is returned.
66
- *
67
- * @param {string} filename - The name of the file to be uploaded
68
- * @param {string} entity - The entity type of the file to be uploaded
69
- * @param {object[] | object} fetchedObjects - The objects to be uploaded
70
- * @param filetype - The type of the file to be uploaded
71
- * @returns {Promise<UploadResponse>} - The response object containing the artifact information
72
- */
73
- async upload(filename, entity, fetchedObjects, filetype = 'application/jsonl+json') {
74
- if (this.local) {
75
- await this.downloadToLocal(filename, fetchedObjects);
76
- }
77
- const preparedArtifact = await this.prepareArtifact(filename, filetype);
78
- if (!preparedArtifact) {
79
- return {
80
- artifact: undefined,
81
- error: { message: 'Error while preparing artifact' },
82
- };
83
- }
84
- const uploadedArtifact = await this.uploadToArtifact(preparedArtifact, fetchedObjects);
85
- if (!uploadedArtifact) {
86
- return {
87
- artifact: undefined,
88
- error: { message: 'Error while uploading artifact' },
89
- };
90
- }
91
- // If file was successfully uploaded we want to post data about that file when emitting
92
- const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1;
93
- const artifact = {
94
- id: preparedArtifact.id,
95
- item_type: entity,
96
- item_count: itemCount,
97
- };
98
- console.log(`Artifact uploaded successfully: ${artifact.id}`);
99
- return { artifact, error: undefined };
100
- }
101
- async prepareArtifact(filename, filetype) {
102
- try {
103
- const response = await this.betaDevrevSdk.artifactsPrepare({
104
- file_name: filename,
105
- file_type: filetype,
106
- });
107
- return response.data;
108
- }
109
- catch (error) {
110
- console.error('Error while preparing artifact: ' + error);
111
- return null;
112
- }
113
- }
114
- async uploadToArtifact(
115
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
116
- preparedArtifact, fetchedObjects
117
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
118
- ) {
119
- const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
120
- try {
121
- const response = await axios_client_internal_1.axiosClient.post(preparedArtifact.url, formData, {
122
- headers: {
123
- 'Content-Type': 'multipart/form-data',
124
- },
125
- });
126
- return response;
127
- }
128
- catch (error) {
129
- console.error('Error while uploading artifact: ' + error);
130
- return null;
131
- }
132
- }
133
- async downloadToLocal(filePath, fetchedObjects) {
134
- console.log(`Uploading ${filePath} to local file system`);
135
- try {
136
- if (!fs_1.default.existsSync('extracted_files')) {
137
- fs_1.default.mkdirSync('extracted_files');
138
- }
139
- const timestamp = new Date().getTime();
140
- const fileHandle = await fs_1.promises.open(`extracted_files/${timestamp}_${filePath}`, 'w');
141
- let objArray = [];
142
- if (!Array.isArray(fetchedObjects)) {
143
- objArray.push(fetchedObjects);
144
- }
145
- else {
146
- objArray = fetchedObjects;
147
- }
148
- for (const jsonObject of objArray) {
149
- const jsonLine = JSON.stringify(jsonObject) + '\n';
150
- await fileHandle.write(jsonLine);
151
- }
152
- await fileHandle.close();
153
- console.log('Data successfully written to', filePath);
154
- }
155
- catch (error) {
156
- console.error('Error writing data to file:', error);
157
- return Promise.reject(error);
158
- }
159
- }
160
- }
161
- exports.Uploader = Uploader;
@@ -1,3 +0,0 @@
1
- declare const axiosClient: import("axios").AxiosInstance;
2
- export { axiosClient };
3
- //# sourceMappingURL=axios-client-internal.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"axios-client-internal.d.ts","sourceRoot":"","sources":["../../src/http/axios-client-internal.ts"],"names":[],"mappings":"AAGA,QAAA,MAAM,WAAW,+BAEf,CAAC;AAiFH,OAAO,EAAE,WAAW,EAAE,CAAC"}
@@ -1,66 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.axiosClient = void 0;
7
- const axios_1 = __importDefault(require("axios"));
8
- const axios_retry_1 = __importDefault(require("axios-retry"));
9
- const axiosClient = axios_1.default.create({
10
- timeout: 30 * 1000,
11
- });
12
- exports.axiosClient = axiosClient;
13
- (0, axios_retry_1.default)(axiosClient, {
14
- retries: 5,
15
- shouldResetTimeout: true,
16
- retryDelay: (retryCount, error) => {
17
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
18
- let delay;
19
- if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
20
- const retryAfter = ((_c = (_b = error.response) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['retry-after']) ||
21
- ((_e = (_d = error.response) === null || _d === void 0 ? void 0 : _d.headers) === null || _e === void 0 ? void 0 : _e['Retry-After']);
22
- delay = parseInt(retryAfter, 10) * 1000;
23
- }
24
- else {
25
- delay = axios_retry_1.default.exponentialDelay(retryCount, error, 1000);
26
- }
27
- const requestId = ((_g = (_f = error.config) === null || _f === void 0 ? void 0 : _f.headers) === null || _g === void 0 ? void 0 : _g['x-request-id']) ||
28
- ((_j = (_h = error.config) === null || _h === void 0 ? void 0 : _h.headers) === null || _j === void 0 ? void 0 : _j['X-Request-ID']);
29
- const delayInSeconds = Math.round(delay / 1000);
30
- console.warn(`Retrying request to ${(_k = error.config) === null || _k === void 0 ? void 0 : _k.url} in ${delayInSeconds}s due to ${(_m = (_l = error.response) === null || _l === void 0 ? void 0 : _l.status) !== null && _m !== void 0 ? _m : 'unknown'} error.`, Object.assign(Object.assign({ method: (_o = error.config) === null || _o === void 0 ? void 0 : _o.method }, (retryCount && { retryCount })), (requestId && { requestId })));
31
- return delay;
32
- },
33
- retryCondition: (error) => {
34
- var _a, _b, _c, _d, _e, _f, _g;
35
- const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
36
- ((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
37
- // 5xx errors
38
- if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
39
- return true;
40
- }
41
- // 429 errors when retry-after header is present
42
- else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
43
- error.response.status === 429 &&
44
- retryAfter &&
45
- !isNaN(Number(retryAfter)) &&
46
- Number(retryAfter) >= 0) {
47
- return true;
48
- }
49
- // Network errors for idempotent requests if not 429, because 429 is handled above
50
- else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
51
- ((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
52
- return true;
53
- }
54
- // all other errors
55
- else {
56
- return false;
57
- }
58
- },
59
- onMaxRetryTimesExceeded(error) {
60
- var _a, _b, _c, _d, _e, _f;
61
- (_b = (_a = error.config) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? true : delete _b.authorization;
62
- (_d = (_c = error.config) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? true : delete _d.Authorization;
63
- (_e = error.request) === null || _e === void 0 ? true : delete _e._header;
64
- console.error(`Request to ${(_f = error.config) === null || _f === void 0 ? void 0 : _f.url} failed after max retries. Error`, error);
65
- },
66
- });
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=axios-client-internal.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"axios-client-internal.test.d.ts","sourceRoot":"","sources":["../../src/http/axios-client-internal.test.ts"],"names":[],"mappings":""}