@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. package/README.md +0 -4
  2. package/dist/airsync-sdk.d.ts +2059 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +2 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +8 -2
  6. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  7. package/dist/attachments-streaming/attachments-streaming-pool.js +27 -16
  8. package/dist/attachments-streaming/attachments-streaming-pool.test.js +246 -27
  9. package/dist/common/constants.d.ts +16 -5
  10. package/dist/common/constants.d.ts.map +1 -1
  11. package/dist/common/constants.js +20 -6
  12. package/dist/common/control-protocol.d.ts +2 -2
  13. package/dist/common/control-protocol.d.ts.map +1 -1
  14. package/dist/common/control-protocol.js +1 -5
  15. package/dist/common/helpers.d.ts +8 -0
  16. package/dist/common/helpers.d.ts.map +1 -1
  17. package/dist/common/helpers.js +15 -0
  18. package/dist/common/install-initial-domain-mapping.d.ts +2 -2
  19. package/dist/common/install-initial-domain-mapping.test.js +5 -2
  20. package/dist/common/test-utils.d.ts +19 -0
  21. package/dist/common/test-utils.d.ts.map +1 -0
  22. package/dist/common/test-utils.js +101 -0
  23. package/dist/common/time-value-resolver.d.ts +47 -0
  24. package/dist/common/time-value-resolver.d.ts.map +1 -0
  25. package/dist/common/time-value-resolver.js +141 -0
  26. package/dist/common/time-value-resolver.test.d.ts +2 -0
  27. package/dist/common/time-value-resolver.test.d.ts.map +1 -0
  28. package/dist/common/time-value-resolver.test.js +268 -0
  29. package/dist/http/axios-client-internal.d.ts.map +1 -1
  30. package/dist/http/axios-client-internal.js +7 -0
  31. package/dist/http/axios-client-internal.test.js +21 -0
  32. package/dist/http/axios-client.d.ts.map +1 -1
  33. package/dist/http/axios-client.js +6 -3
  34. package/dist/http/index.d.ts +0 -1
  35. package/dist/http/index.d.ts.map +1 -1
  36. package/dist/http/index.js +0 -1
  37. package/dist/index.d.ts +12 -10
  38. package/dist/index.d.ts.map +1 -1
  39. package/dist/index.js +18 -13
  40. package/dist/logger/logger.d.ts +1 -17
  41. package/dist/logger/logger.d.ts.map +1 -1
  42. package/dist/logger/logger.interfaces.d.ts +2 -2
  43. package/dist/logger/logger.js +35 -27
  44. package/dist/logger/logger.test.js +66 -19
  45. package/dist/mappers/mappers.interface.d.ts +5 -5
  46. package/dist/mappers/mappers.interface.js +3 -3
  47. package/dist/mappers/mappers.test.js +8 -10
  48. package/dist/mock-server/mock-server.d.ts +47 -0
  49. package/dist/mock-server/mock-server.d.ts.map +1 -0
  50. package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
  51. package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
  52. package/dist/mock-server/mock-server.interfaces.js +4 -0
  53. package/dist/mock-server/mock-server.js +278 -0
  54. package/dist/multithreading/adapters/base-adapter.d.ts +56 -0
  55. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -0
  56. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +2 -0
  57. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +1 -0
  58. package/dist/multithreading/adapters/base-adapter.emit.test.js +403 -0
  59. package/dist/multithreading/adapters/base-adapter.js +105 -0
  60. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +2 -0
  61. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +1 -0
  62. package/dist/multithreading/{worker-adapter/worker-adapter.artifacts.test.js → adapters/extraction-adapter.artifacts.test.js} +53 -5
  63. package/dist/multithreading/adapters/extraction-adapter.d.ts +58 -0
  64. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -0
  65. package/dist/multithreading/adapters/extraction-adapter.js +325 -0
  66. package/dist/multithreading/adapters/extraction-adapter.test.d.ts +2 -0
  67. package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +1 -0
  68. package/dist/multithreading/adapters/extraction-adapter.test.js +789 -0
  69. package/dist/multithreading/adapters/loading-adapter.d.ts +44 -0
  70. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -0
  71. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts} +1 -1
  72. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -0
  73. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +2 -0
  74. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +1 -0
  75. package/dist/multithreading/{worker-adapter/worker-adapter.helpers.test.js → adapters/loading-adapter.helpers.test.js} +26 -26
  76. package/dist/multithreading/adapters/loading-adapter.js +461 -0
  77. package/dist/multithreading/adapters/loading-adapter.test.d.ts +2 -0
  78. package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +1 -0
  79. package/dist/multithreading/adapters/loading-adapter.test.js +589 -0
  80. package/dist/multithreading/adapters/serialization.test.d.ts +2 -0
  81. package/dist/multithreading/adapters/serialization.test.d.ts.map +1 -0
  82. package/dist/multithreading/adapters/serialization.test.js +71 -0
  83. package/dist/multithreading/create-worker.test.js +36 -17
  84. package/dist/multithreading/process-task.d.ts +16 -1
  85. package/dist/multithreading/process-task.d.ts.map +1 -1
  86. package/dist/multithreading/process-task.js +97 -82
  87. package/dist/multithreading/process-task.test.d.ts +2 -0
  88. package/dist/multithreading/process-task.test.d.ts.map +1 -0
  89. package/dist/multithreading/process-task.test.js +146 -0
  90. package/dist/multithreading/spawn/spawn.d.ts +3 -3
  91. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  92. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  93. package/dist/multithreading/spawn/spawn.helpers.js +6 -14
  94. package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
  95. package/dist/multithreading/spawn/spawn.js +20 -25
  96. package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
  97. package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
  98. package/dist/multithreading/spawn/spawn.test.js +232 -0
  99. package/dist/repo/repo.d.ts.map +1 -1
  100. package/dist/repo/repo.interfaces.d.ts +4 -2
  101. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  102. package/dist/repo/repo.js +4 -3
  103. package/dist/repo/repo.test.js +60 -7
  104. package/dist/state/base-state.d.ts +75 -0
  105. package/dist/state/base-state.d.ts.map +1 -0
  106. package/dist/state/base-state.js +261 -0
  107. package/dist/state/extraction-state.d.ts +29 -0
  108. package/dist/state/extraction-state.d.ts.map +1 -0
  109. package/dist/state/extraction-state.js +122 -0
  110. package/dist/state/loading-state.d.ts +19 -0
  111. package/dist/state/loading-state.d.ts.map +1 -0
  112. package/dist/state/loading-state.js +40 -0
  113. package/dist/state/state.extract-window.test.d.ts +2 -0
  114. package/dist/state/state.extract-window.test.d.ts.map +1 -0
  115. package/dist/state/state.extract-window.test.js +163 -0
  116. package/dist/state/state.interfaces.d.ts +44 -27
  117. package/dist/state/state.interfaces.d.ts.map +1 -1
  118. package/dist/state/state.interfaces.js +18 -5
  119. package/dist/state/state.pending-boundaries.test.d.ts +2 -0
  120. package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
  121. package/dist/state/state.pending-boundaries.test.js +189 -0
  122. package/dist/state/state.post-state.test.d.ts +2 -0
  123. package/dist/state/state.post-state.test.d.ts.map +1 -0
  124. package/dist/state/state.post-state.test.js +78 -0
  125. package/dist/state/state.test.js +135 -88
  126. package/dist/state/state.time-value-resolution.test.d.ts +2 -0
  127. package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
  128. package/dist/state/state.time-value-resolution.test.js +175 -0
  129. package/dist/tsdoc-metadata.json +11 -0
  130. package/dist/types/common.d.ts +0 -25
  131. package/dist/types/common.d.ts.map +1 -1
  132. package/dist/types/common.js +1 -11
  133. package/dist/types/external-domain-metadata.d.ts +20 -1
  134. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  135. package/dist/types/extraction.d.ts +92 -221
  136. package/dist/types/extraction.d.ts.map +1 -1
  137. package/dist/types/extraction.js +56 -124
  138. package/dist/types/extraction.test.js +69 -43
  139. package/dist/types/index.d.ts +3 -3
  140. package/dist/types/index.d.ts.map +1 -1
  141. package/dist/types/index.js +6 -3
  142. package/dist/types/loading.d.ts +4 -24
  143. package/dist/types/loading.d.ts.map +1 -1
  144. package/dist/types/loading.js +1 -22
  145. package/dist/types/workers.d.ts +47 -29
  146. package/dist/types/workers.d.ts.map +1 -1
  147. package/dist/types/workers.js +1 -0
  148. package/dist/uploader/uploader.d.ts +1 -0
  149. package/dist/uploader/uploader.d.ts.map +1 -1
  150. package/dist/uploader/uploader.helpers.test.js +0 -11
  151. package/dist/uploader/uploader.interfaces.d.ts +3 -3
  152. package/dist/uploader/uploader.js +53 -20
  153. package/dist/uploader/uploader.test.js +22 -10
  154. package/package.json +10 -13
  155. package/dist/common/event-type-translation.d.ts +0 -24
  156. package/dist/common/event-type-translation.d.ts.map +0 -1
  157. package/dist/common/event-type-translation.js +0 -117
  158. package/dist/deprecated/adapter/index.d.ts +0 -62
  159. package/dist/deprecated/adapter/index.d.ts.map +0 -1
  160. package/dist/deprecated/adapter/index.js +0 -151
  161. package/dist/deprecated/common/helpers.d.ts +0 -7
  162. package/dist/deprecated/common/helpers.d.ts.map +0 -1
  163. package/dist/deprecated/common/helpers.js +0 -47
  164. package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
  165. package/dist/deprecated/demo-extractor/index.d.ts +0 -18
  166. package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
  167. package/dist/deprecated/demo-extractor/index.js +0 -161
  168. package/dist/deprecated/http/client.d.ts +0 -22
  169. package/dist/deprecated/http/client.d.ts.map +0 -1
  170. package/dist/deprecated/http/client.js +0 -161
  171. package/dist/deprecated/uploader/index.d.ts +0 -35
  172. package/dist/deprecated/uploader/index.d.ts.map +0 -1
  173. package/dist/deprecated/uploader/index.js +0 -161
  174. package/dist/http/types.d.ts +0 -17
  175. package/dist/http/types.d.ts.map +0 -1
  176. package/dist/http/types.js +0 -2
  177. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +0 -2
  178. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +0 -1
  179. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +0 -91
  180. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +0 -1
  181. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +0 -1
  182. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +0 -2
  183. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +0 -1
  184. package/dist/multithreading/worker-adapter/worker-adapter.js +0 -745
  185. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
  186. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
  187. package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
  188. package/dist/state/state.d.ts +0 -30
  189. package/dist/state/state.d.ts.map +0 -1
  190. package/dist/state/state.js +0 -166
  191. /package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js} +0 -0
@@ -1,161 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.HTTPClient = exports.defaultResponse = void 0;
37
- const axios_1 = __importStar(require("axios"));
38
- const constants_1 = require("../../http/constants");
39
- exports.defaultResponse = {
40
- data: {
41
- delay: 0,
42
- nextPage: 1,
43
- records: [],
44
- },
45
- message: '',
46
- success: false,
47
- };
48
- /**
49
- * HTTPClient class to make HTTP requests
50
- * @deprecated
51
- */
52
- class HTTPClient {
53
- constructor() {
54
- this.retryAfter = 0;
55
- this.retryAt = 0;
56
- this.axiosInstance = axios_1.default.create();
57
- // Add request interceptor to check for retryAfter before making a request
58
- this.axiosInstance.interceptors.request.use((config) => {
59
- // Check if retryAfter is not 0 and return a LIMIT_EXCEEDED error
60
- if (this.retryAfter !== 0) {
61
- // check if the current time is greater than the retryAt time
62
- const currentTime = new Date().getTime();
63
- if (currentTime < this.retryAt) {
64
- console.error('Rate limit exceeded. Interceptor has retryAfter: ' +
65
- this.retryAfter);
66
- // Rate limit exceeded.
67
- return Promise.reject(constants_1.RATE_LIMIT_EXCEEDED);
68
- }
69
- else {
70
- // Reset the retryAfter
71
- this.retryAfter = 0;
72
- }
73
- }
74
- return config;
75
- }, (error) => {
76
- return Promise.reject(error);
77
- });
78
- }
79
- /**
80
- *
81
- * Function to make a GET call to the endpoint.
82
- * There is special handling for rate limit exceeded error.
83
- * In case of rate limit exceeded, the function returns success as true and the delay time in seconds
84
- * In case of any other error, the function returns success as false and the error message
85
- */
86
- async getCall(endpoint, headers,
87
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
88
- params) {
89
- var _a;
90
- // Return the LIMIT_EXCEEDED error if the retryAfter is not 0
91
- try {
92
- const res = await this.axiosInstance.get(endpoint, {
93
- headers: headers,
94
- params: params,
95
- });
96
- return Object.assign(Object.assign({}, exports.defaultResponse), { data: {
97
- delay: 0,
98
- records: res.data,
99
- }, success: true });
100
- }
101
- catch (error) {
102
- console.log('Error in getCall: ' + JSON.stringify(error));
103
- // send error to adapter
104
- if ((0, axios_1.isAxiosError)(error)) {
105
- if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === constants_1.RATE_LIMIT_EXCEEDED_STATUS_CODE) {
106
- this.retryAfter = error.response.headers['retry-after']
107
- ? error.response.headers['retry-after']
108
- : 0;
109
- this.retryAt = new Date().getTime() + this.retryAfter * 1000;
110
- console.warn('Rate limit exceeded. Error code: ' +
111
- error.response.status +
112
- ' RetryAfter: ' +
113
- this.retryAfter +
114
- ' RetryAt: ' +
115
- this.retryAt);
116
- return {
117
- data: {
118
- delay: this.retryAfter,
119
- records: [],
120
- },
121
- message: constants_1.RATE_LIMIT_EXCEEDED,
122
- success: true,
123
- };
124
- }
125
- if (error.response) {
126
- return Object.assign(Object.assign({}, exports.defaultResponse), { message: error.response.data });
127
- }
128
- else {
129
- return Object.assign(Object.assign({}, exports.defaultResponse), { message: error.message });
130
- }
131
- }
132
- else {
133
- if (this.retryAfter !== 0) {
134
- console.warn('Rate limit exceeded. Going to return the following response: ' +
135
- JSON.stringify(error));
136
- return {
137
- data: {
138
- delay: this.retryAfter,
139
- records: [],
140
- },
141
- message: typeof error === 'string'
142
- ? error
143
- : JSON.stringify(error, Object.getOwnPropertyNames(error)),
144
- success: true,
145
- };
146
- }
147
- return {
148
- data: {
149
- delay: this.retryAfter,
150
- records: [],
151
- },
152
- message: typeof error === 'string'
153
- ? error
154
- : JSON.stringify(error, Object.getOwnPropertyNames(error)),
155
- success: false,
156
- };
157
- }
158
- }
159
- }
160
- }
161
- exports.HTTPClient = HTTPClient;
@@ -1,35 +0,0 @@
1
- import { UploadResponse } from '../../uploader/uploader.interfaces';
2
- /**
3
- * Uploader class is used to upload files to the DevRev platform.
4
- * The class provides utilities to
5
- * - prepare artifact
6
- * - upload artifact
7
- * - return the artifact information to the platform
8
- *
9
- * @class Uploader
10
- * @constructor
11
- * @param {string} endpoint - The endpoint of the DevRev platform
12
- * @param {string} token - The token to authenticate with the DevRev platform
13
- * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
14
- */
15
- export declare class Uploader {
16
- private betaDevrevSdk;
17
- private local;
18
- constructor(endpoint: string, token: string, local?: boolean);
19
- /**
20
- *
21
- * Uploads the file to the DevRev platform. The file is uploaded to the platform
22
- * and the artifact information is returned.
23
- *
24
- * @param {string} filename - The name of the file to be uploaded
25
- * @param {string} entity - The entity type of the file to be uploaded
26
- * @param {object[] | object} fetchedObjects - The objects to be uploaded
27
- * @param filetype - The type of the file to be uploaded
28
- * @returns {Promise<UploadResponse>} - The response object containing the artifact information
29
- */
30
- upload(filename: string, entity: string, fetchedObjects: object[] | object, filetype?: string): Promise<UploadResponse>;
31
- private prepareArtifact;
32
- private uploadToArtifact;
33
- private downloadToLocal;
34
- }
35
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/deprecated/uploader/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAY,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAG9E;;;;;;;;;;;;GAYG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,KAAK,CAAU;gBACX,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,UAAQ;IAQ1D;;;;;;;;;;OAUG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,EACjC,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAAC,cAAc,CAAC;YAuCZ,eAAe;YAiBf,gBAAgB;YAqBhB,eAAe;CAgC9B"}
@@ -1,161 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.Uploader = void 0;
37
- const typescript_sdk_1 = require("@devrev/typescript-sdk");
38
- const fs_1 = __importStar(require("fs"));
39
- const axios_client_internal_1 = require("../../http/axios-client-internal");
40
- const helpers_1 = require("../common/helpers");
41
- /**
42
- * Uploader class is used to upload files to the DevRev platform.
43
- * The class provides utilities to
44
- * - prepare artifact
45
- * - upload artifact
46
- * - return the artifact information to the platform
47
- *
48
- * @class Uploader
49
- * @constructor
50
- * @param {string} endpoint - The endpoint of the DevRev platform
51
- * @param {string} token - The token to authenticate with the DevRev platform
52
- * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
53
- */
54
- class Uploader {
55
- constructor(endpoint, token, local = false) {
56
- this.betaDevrevSdk = typescript_sdk_1.client.setupBeta({
57
- endpoint,
58
- token,
59
- });
60
- this.local = local;
61
- }
62
- /**
63
- *
64
- * Uploads the file to the DevRev platform. The file is uploaded to the platform
65
- * and the artifact information is returned.
66
- *
67
- * @param {string} filename - The name of the file to be uploaded
68
- * @param {string} entity - The entity type of the file to be uploaded
69
- * @param {object[] | object} fetchedObjects - The objects to be uploaded
70
- * @param filetype - The type of the file to be uploaded
71
- * @returns {Promise<UploadResponse>} - The response object containing the artifact information
72
- */
73
- async upload(filename, entity, fetchedObjects, filetype = 'application/jsonl+json') {
74
- if (this.local) {
75
- await this.downloadToLocal(filename, fetchedObjects);
76
- }
77
- const preparedArtifact = await this.prepareArtifact(filename, filetype);
78
- if (!preparedArtifact) {
79
- return {
80
- artifact: undefined,
81
- error: { message: 'Error while preparing artifact' },
82
- };
83
- }
84
- const uploadedArtifact = await this.uploadToArtifact(preparedArtifact, fetchedObjects);
85
- if (!uploadedArtifact) {
86
- return {
87
- artifact: undefined,
88
- error: { message: 'Error while uploading artifact' },
89
- };
90
- }
91
- // If file was successfully uploaded we want to post data about that file when emitting
92
- const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1;
93
- const artifact = {
94
- id: preparedArtifact.id,
95
- item_type: entity,
96
- item_count: itemCount,
97
- };
98
- console.log(`Artifact uploaded successfully: ${artifact.id}`);
99
- return { artifact, error: undefined };
100
- }
101
- async prepareArtifact(filename, filetype) {
102
- try {
103
- const response = await this.betaDevrevSdk.artifactsPrepare({
104
- file_name: filename,
105
- file_type: filetype,
106
- });
107
- return response.data;
108
- }
109
- catch (error) {
110
- console.error('Error while preparing artifact: ' + error);
111
- return null;
112
- }
113
- }
114
- async uploadToArtifact(
115
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
116
- preparedArtifact, fetchedObjects
117
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
118
- ) {
119
- const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
120
- try {
121
- const response = await axios_client_internal_1.axiosClient.post(preparedArtifact.url, formData, {
122
- headers: {
123
- 'Content-Type': 'multipart/form-data',
124
- },
125
- });
126
- return response;
127
- }
128
- catch (error) {
129
- console.error('Error while uploading artifact: ' + error);
130
- return null;
131
- }
132
- }
133
- async downloadToLocal(filePath, fetchedObjects) {
134
- console.log(`Uploading ${filePath} to local file system`);
135
- try {
136
- if (!fs_1.default.existsSync('extracted_files')) {
137
- fs_1.default.mkdirSync('extracted_files');
138
- }
139
- const timestamp = new Date().getTime();
140
- const fileHandle = await fs_1.promises.open(`extracted_files/${timestamp}_${filePath}`, 'w');
141
- let objArray = [];
142
- if (!Array.isArray(fetchedObjects)) {
143
- objArray.push(fetchedObjects);
144
- }
145
- else {
146
- objArray = fetchedObjects;
147
- }
148
- for (const jsonObject of objArray) {
149
- const jsonLine = JSON.stringify(jsonObject) + '\n';
150
- await fileHandle.write(jsonLine);
151
- }
152
- await fileHandle.close();
153
- console.log('Data successfully written to', filePath);
154
- }
155
- catch (error) {
156
- console.error('Error writing data to file:', error);
157
- return Promise.reject(error);
158
- }
159
- }
160
- }
161
- exports.Uploader = Uploader;
@@ -1,17 +0,0 @@
1
- /**
2
- * HTTP Response type
3
- * @deprecated
4
- */
5
- export type HTTPResponse = {
6
- success: boolean;
7
- message: string;
8
- data: Data;
9
- };
10
- interface Data {
11
- records: object[];
12
- delay: number;
13
- nextPage?: number;
14
- metadata?: object;
15
- }
16
- export {};
17
- //# sourceMappingURL=types.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/http/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,UAAU,IAAI;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB"}
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=worker-adapter.artifacts.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.artifacts.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts"],"names":[],"mappings":""}
@@ -1,91 +0,0 @@
1
- import { Mappers } from '../../mappers/mappers';
2
- import { Repo } from '../../repo/repo';
3
- import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
4
- import { AdapterState } from '../../state/state.interfaces';
5
- import { AirdropEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../../types/extraction';
6
- import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../../types/loading';
7
- import { WorkerAdapterInterface, WorkerAdapterOptions } from '../../types/workers';
8
- import { Artifact } from '../../uploader/uploader.interfaces';
9
- export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
10
- /**
11
- * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
12
- * as parameter in processTask and onTimeout functions. The class provides
13
- * utilities to emit control events to the platform, update the state of the connector,
14
- * and upload artifacts to the platform.
15
- * @class WorkerAdapter
16
- * @constructor
17
- * @param options - The options to create a new instance of WorkerAdapter class
18
- * @param event - The event object received from the platform
19
- * @param initialState - The initial state of the adapter
20
- * @param isLocalDevelopment - A flag to indicate if the adapter is being used in local development
21
- * @param workerPath - The path to the worker file
22
- *
23
- * @public
24
- */
25
- export declare class WorkerAdapter<ConnectorState> {
26
- readonly event: AirdropEvent;
27
- readonly options?: WorkerAdapterOptions;
28
- isTimeout: boolean;
29
- private adapterState;
30
- private _artifacts;
31
- private hasWorkerEmitted;
32
- private repos;
33
- private loaderReports;
34
- private _processedFiles;
35
- private _mappers;
36
- private uploader;
37
- constructor({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>);
38
- get state(): AdapterState<ConnectorState>;
39
- set state(value: AdapterState<ConnectorState>);
40
- get reports(): LoaderReport[];
41
- get processedFiles(): string[];
42
- get mappers(): Mappers;
43
- initializeRepos(repos: RepoInterface[]): void;
44
- getRepo(itemType: string): Repo | undefined;
45
- postState(): Promise<void>;
46
- get artifacts(): Artifact[];
47
- set artifacts(artifacts: Artifact[]);
48
- /**
49
- * Emits an event to the platform.
50
- *
51
- * @param newEventType - The event type to be emitted
52
- * @param data - The data to be sent with the event
53
- */
54
- emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
55
- uploadAllRepos(): Promise<void>;
56
- handleTimeout(): void;
57
- loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
58
- getLoaderBatches({ supportedItemTypes, }: {
59
- supportedItemTypes: string[];
60
- }): Promise<FileToLoad[]>;
61
- loadAttachments({ create, }: {
62
- create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
63
- }): Promise<LoadItemTypesResponse>;
64
- loadItem({ item, itemTypeToLoad, }: {
65
- item: ExternalSystemItem;
66
- itemTypeToLoad: ItemTypeToLoad;
67
- }): Promise<LoadItemResponse>;
68
- processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
69
- /**
70
- * Destroys a stream to prevent memory leaks.
71
- * @param httpStream - The axios response stream to destroy
72
- */
73
- private destroyHttpStream;
74
- loadAttachment({ item, create, }: {
75
- item: ExternalSystemAttachment;
76
- create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
77
- }): Promise<LoadItemResponse>;
78
- /**
79
- * Streams the attachments to the DevRev platform.
80
- * The attachments are streamed to the platform and the artifact information is returned.
81
- * @param params - The parameters to stream the attachments
82
- * @returns The response object containing the ssorAttachment artifact information
83
- * or error information if there was an error
84
- */
85
- streamAttachments<NewBatch>({ stream, processors, batchSize, }: {
86
- stream: ExternalSystemAttachmentStreamingFunction;
87
- processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
88
- batchSize?: number;
89
- }): Promise<StreamAttachmentsReturnType>;
90
- }
91
- //# sourceMappingURL=worker-adapter.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAqFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAqHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA6E5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAoGvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAqD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAyHzC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=worker-adapter.helpers.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.helpers.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.test.ts"],"names":[],"mappings":""}