@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/MIGRATION.md +1028 -0
  2. package/README.md +5 -11
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -9
  4. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
  6. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  7. package/dist/attachments-streaming/attachments-streaming-pool.js +54 -56
  8. package/dist/common/constants.d.ts +4 -5
  9. package/dist/common/constants.d.ts.map +1 -1
  10. package/dist/common/constants.js +6 -7
  11. package/dist/common/helpers.d.ts +3 -35
  12. package/dist/common/helpers.d.ts.map +1 -1
  13. package/dist/common/helpers.js +5 -50
  14. package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
  15. package/dist/http/client.d.ts.map +1 -0
  16. package/dist/http/{axios-client-internal.js → client.js} +4 -8
  17. package/dist/index.d.ts +24 -13
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +43 -32
  20. package/dist/logger/logger.d.ts +12 -53
  21. package/dist/logger/logger.d.ts.map +1 -1
  22. package/dist/logger/logger.interfaces.d.ts +0 -1
  23. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  24. package/dist/logger/logger.js +36 -68
  25. package/dist/mappers/mappers.d.ts +7 -46
  26. package/dist/mappers/mappers.d.ts.map +1 -1
  27. package/dist/mappers/mappers.interfaces.d.ts +175 -0
  28. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  29. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +4 -13
  30. package/dist/mappers/mappers.js +11 -45
  31. package/dist/multithreading/adapters/base-adapter.d.ts +30 -32
  32. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  33. package/dist/multithreading/adapters/base-adapter.js +97 -62
  34. package/dist/multithreading/adapters/extraction-adapter.d.ts +16 -29
  35. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  36. package/dist/multithreading/adapters/extraction-adapter.js +203 -206
  37. package/dist/multithreading/adapters/loading-adapter.d.ts +8 -13
  38. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  39. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
  40. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
  41. package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
  42. package/dist/multithreading/adapters/loading-adapter.js +294 -346
  43. package/dist/multithreading/emit.d.ts +11 -0
  44. package/dist/multithreading/emit.d.ts.map +1 -0
  45. package/dist/{common/control-protocol.js → multithreading/emit.js} +5 -7
  46. package/dist/multithreading/process-task.d.ts +2 -12
  47. package/dist/multithreading/process-task.d.ts.map +1 -1
  48. package/dist/multithreading/process-task.js +46 -43
  49. package/dist/multithreading/spawn/spawn.d.ts +3 -9
  50. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  51. package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
  52. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  53. package/dist/multithreading/spawn/spawn.helpers.js +128 -20
  54. package/dist/multithreading/spawn/spawn.js +24 -39
  55. package/dist/multithreading/worker.js +1 -4
  56. package/dist/repo/repo.d.ts +10 -0
  57. package/dist/repo/repo.d.ts.map +1 -1
  58. package/dist/repo/repo.helpers.d.ts +6 -0
  59. package/dist/repo/repo.helpers.d.ts.map +1 -0
  60. package/dist/repo/repo.helpers.js +16 -0
  61. package/dist/repo/repo.interfaces.d.ts +4 -16
  62. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  63. package/dist/repo/repo.js +23 -8
  64. package/dist/state/base-state.d.ts +16 -38
  65. package/dist/state/base-state.d.ts.map +1 -1
  66. package/dist/state/base-state.js +19 -41
  67. package/dist/state/extraction-state.d.ts +6 -21
  68. package/dist/state/extraction-state.d.ts.map +1 -1
  69. package/dist/state/extraction-state.js +8 -25
  70. package/dist/{common → state}/install-initial-domain-mapping.d.ts +1 -1
  71. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  72. package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
  73. package/dist/state/loading-state.d.ts +3 -14
  74. package/dist/state/loading-state.d.ts.map +1 -1
  75. package/dist/state/loading-state.js +3 -14
  76. package/dist/state/state.d.ts +4 -0
  77. package/dist/state/state.d.ts.map +1 -0
  78. package/dist/state/state.interfaces.d.ts +33 -37
  79. package/dist/state/state.interfaces.d.ts.map +1 -1
  80. package/dist/state/state.interfaces.js +16 -11
  81. package/dist/state/state.js +11 -0
  82. package/dist/state/time-value-resolver.d.ts +17 -0
  83. package/dist/state/time-value-resolver.d.ts.map +1 -0
  84. package/dist/{common → state}/time-value-resolver.js +9 -41
  85. package/dist/testing/mock-event.d.ts +10 -0
  86. package/dist/testing/mock-event.d.ts.map +1 -0
  87. package/dist/{common/test-utils.js → testing/mock-event.js} +17 -14
  88. package/dist/testing/mock-server.d.ts +29 -0
  89. package/dist/testing/mock-server.d.ts.map +1 -0
  90. package/dist/testing/mock-server.interfaces.d.ts +48 -0
  91. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  92. package/dist/{mock-server → testing}/mock-server.js +21 -33
  93. package/dist/types/common.d.ts +1 -12
  94. package/dist/types/common.d.ts.map +1 -1
  95. package/dist/types/common.js +1 -6
  96. package/dist/types/errors.d.ts +9 -0
  97. package/dist/types/errors.d.ts.map +1 -0
  98. package/dist/types/external-domain-metadata.d.ts +30 -171
  99. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  100. package/dist/types/extraction.d.ts +64 -94
  101. package/dist/types/extraction.d.ts.map +1 -1
  102. package/dist/types/extraction.js +6 -34
  103. package/dist/types/loading.d.ts +1 -2
  104. package/dist/types/loading.d.ts.map +1 -1
  105. package/dist/types/loading.js +0 -1
  106. package/dist/types/workers.d.ts +39 -94
  107. package/dist/types/workers.d.ts.map +1 -1
  108. package/dist/types/workers.js +0 -6
  109. package/dist/uploader/uploader.d.ts +8 -59
  110. package/dist/uploader/uploader.d.ts.map +1 -1
  111. package/dist/uploader/uploader.helpers.d.ts +4 -27
  112. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  113. package/dist/uploader/uploader.helpers.js +64 -30
  114. package/dist/uploader/uploader.interfaces.d.ts +15 -25
  115. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  116. package/dist/uploader/uploader.interfaces.js +8 -0
  117. package/dist/uploader/uploader.js +23 -89
  118. package/package.json +8 -6
  119. package/dist/airsync-sdk.d.ts +0 -2059
  120. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  121. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  122. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
  123. package/dist/common/control-protocol.d.ts +0 -10
  124. package/dist/common/control-protocol.d.ts.map +0 -1
  125. package/dist/common/errors.d.ts +0 -6
  126. package/dist/common/errors.d.ts.map +0 -1
  127. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  128. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  129. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  130. package/dist/common/install-initial-domain-mapping.test.js +0 -211
  131. package/dist/common/test-utils.d.ts +0 -19
  132. package/dist/common/test-utils.d.ts.map +0 -1
  133. package/dist/common/time-value-resolver.d.ts +0 -47
  134. package/dist/common/time-value-resolver.d.ts.map +0 -1
  135. package/dist/common/time-value-resolver.test.d.ts +0 -2
  136. package/dist/common/time-value-resolver.test.d.ts.map +0 -1
  137. package/dist/common/time-value-resolver.test.js +0 -268
  138. package/dist/http/axios-client-internal.d.ts.map +0 -1
  139. package/dist/http/axios-client-internal.test.d.ts +0 -2
  140. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  141. package/dist/http/axios-client-internal.test.js +0 -175
  142. package/dist/http/axios-client.d.ts +0 -27
  143. package/dist/http/axios-client.d.ts.map +0 -1
  144. package/dist/http/axios-client.js +0 -60
  145. package/dist/http/constants.d.ts +0 -4
  146. package/dist/http/constants.d.ts.map +0 -1
  147. package/dist/http/constants.js +0 -6
  148. package/dist/http/index.d.ts +0 -2
  149. package/dist/http/index.d.ts.map +0 -1
  150. package/dist/http/index.js +0 -17
  151. package/dist/logger/logger.constants.d.ts +0 -6
  152. package/dist/logger/logger.constants.d.ts.map +0 -1
  153. package/dist/logger/logger.constants.js +0 -13
  154. package/dist/logger/logger.context.d.ts +0 -58
  155. package/dist/logger/logger.context.d.ts.map +0 -1
  156. package/dist/logger/logger.context.js +0 -86
  157. package/dist/logger/logger.test.d.ts +0 -2
  158. package/dist/logger/logger.test.d.ts.map +0 -1
  159. package/dist/logger/logger.test.js +0 -537
  160. package/dist/mappers/mappers.interface.d.ts +0 -294
  161. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  162. package/dist/mappers/mappers.test.d.ts +0 -2
  163. package/dist/mappers/mappers.test.d.ts.map +0 -1
  164. package/dist/mappers/mappers.test.js +0 -105
  165. package/dist/mock-server/mock-server.d.ts +0 -47
  166. package/dist/mock-server/mock-server.d.ts.map +0 -1
  167. package/dist/mock-server/mock-server.interfaces.d.ts +0 -73
  168. package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
  169. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
  170. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
  171. package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
  172. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
  173. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
  174. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
  175. package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
  176. package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
  177. package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
  178. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
  179. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
  180. package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
  181. package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
  182. package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
  183. package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
  184. package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
  185. package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
  186. package/dist/multithreading/adapters/serialization.test.js +0 -71
  187. package/dist/multithreading/create-worker.test.d.ts +0 -2
  188. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  189. package/dist/multithreading/create-worker.test.js +0 -108
  190. package/dist/multithreading/process-task.test.d.ts +0 -2
  191. package/dist/multithreading/process-task.test.d.ts.map +0 -1
  192. package/dist/multithreading/process-task.test.js +0 -146
  193. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  194. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  195. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  196. package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
  197. package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
  198. package/dist/multithreading/spawn/spawn.test.js +0 -232
  199. package/dist/repo/repo.test.d.ts +0 -2
  200. package/dist/repo/repo.test.d.ts.map +0 -1
  201. package/dist/repo/repo.test.js +0 -184
  202. package/dist/state/state.extract-window.test.d.ts +0 -2
  203. package/dist/state/state.extract-window.test.d.ts.map +0 -1
  204. package/dist/state/state.extract-window.test.js +0 -163
  205. package/dist/state/state.pending-boundaries.test.d.ts +0 -2
  206. package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
  207. package/dist/state/state.pending-boundaries.test.js +0 -189
  208. package/dist/state/state.post-state.test.d.ts +0 -2
  209. package/dist/state/state.post-state.test.d.ts.map +0 -1
  210. package/dist/state/state.post-state.test.js +0 -78
  211. package/dist/state/state.test.d.ts +0 -2
  212. package/dist/state/state.test.d.ts.map +0 -1
  213. package/dist/state/state.test.js +0 -271
  214. package/dist/state/state.time-value-resolution.test.d.ts +0 -2
  215. package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
  216. package/dist/state/state.time-value-resolution.test.js +0 -175
  217. package/dist/tsdoc-metadata.json +0 -11
  218. package/dist/types/extraction.test.d.ts +0 -2
  219. package/dist/types/extraction.test.d.ts.map +0 -1
  220. package/dist/types/extraction.test.js +0 -96
  221. package/dist/types/index.d.ts +0 -10
  222. package/dist/types/index.d.ts.map +0 -1
  223. package/dist/types/index.js +0 -21
  224. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  225. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  226. package/dist/uploader/uploader.helpers.test.js +0 -256
  227. package/dist/uploader/uploader.test.d.ts +0 -2
  228. package/dist/uploader/uploader.test.d.ts.map +0 -1
  229. package/dist/uploader/uploader.test.js +0 -601
  230. /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
  231. /package/dist/{common → types}/errors.js +0 -0
@@ -6,19 +6,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.LoadingAdapter = void 0;
7
7
  const axios_1 = __importDefault(require("axios"));
8
8
  const logger_1 = require("../../logger/logger");
9
- const logger_context_1 = require("../../logger/logger.context");
10
9
  const mappers_1 = require("../../mappers/mappers");
11
- const mappers_interface_1 = require("../../mappers/mappers.interface");
10
+ const mappers_interfaces_1 = require("../../mappers/mappers.interfaces");
12
11
  const extraction_1 = require("../../types/extraction");
13
12
  const loading_1 = require("../../types/loading");
14
13
  const base_adapter_1 = require("./base-adapter");
15
14
  const loading_adapter_helpers_1 = require("./loading-adapter.helpers");
16
- /**
17
- * LoadingAdapter is the adapter passed to loading tasks. It exposes the loading
18
- * surface (item/attachment loading, mappers, loader reports).
19
- *
20
- * @public
21
- */
15
+ /** Adapter passed to loading tasks (item/attachment loading, mappers, loader reports). @public */
22
16
  class LoadingAdapter extends base_adapter_1.BaseAdapter {
23
17
  constructor(params) {
24
18
  super(params);
@@ -39,7 +33,7 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
39
33
  return this._mappers;
40
34
  }
41
35
  async beforeEmit() {
42
- // Loading has no pre-emit work (no repos, no extraction boundaries).
36
+ // Loading has no pre-emit work.
43
37
  }
44
38
  buildEmitPayload(newEventType) {
45
39
  const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
@@ -54,408 +48,362 @@ class LoadingAdapter extends base_adapter_1.BaseAdapter {
54
48
  // Loading keeps its accumulated reports/processed files across emits.
55
49
  }
56
50
  async loadItemTypes({ itemTypesToLoad, }) {
57
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
58
- var _a;
59
- if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
60
- const itemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
61
- if (!itemTypes.length) {
62
- console.warn('No item types to load, returning.');
51
+ var _a;
52
+ if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
53
+ const itemTypes = itemTypesToLoad.map((itemTypeToLoad) => itemTypeToLoad.itemType);
54
+ if (!itemTypes.length) {
55
+ console.warn('No item types to load, returning.');
56
+ return { status: 'success' };
57
+ }
58
+ const filesToLoad = await this.getLoaderBatches({
59
+ supportedItemTypes: itemTypes,
60
+ });
61
+ this.sdkState.fromDevRev = {
62
+ filesToLoad,
63
+ };
64
+ }
65
+ if (!this.sdkState.fromDevRev ||
66
+ !this.sdkState.fromDevRev.filesToLoad.length) {
67
+ console.warn('No files to load, returning.');
68
+ return { status: 'success' };
69
+ }
70
+ console.log('Files to load in state', (_a = this.sdkState.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
71
+ try {
72
+ for (const fileToLoad of this.sdkState.fromDevRev.filesToLoad) {
73
+ const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
74
+ if (!itemTypeToLoad) {
75
+ console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
63
76
  return {
64
- reports: this.reports,
65
- processed_files: this.processedFiles,
77
+ status: 'error',
78
+ error: {
79
+ message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
80
+ },
66
81
  };
67
82
  }
68
- const filesToLoad = await this.getLoaderBatches({
69
- supportedItemTypes: itemTypes,
70
- });
71
- this.adapterState.sdkState.fromDevRev = {
72
- filesToLoad,
73
- };
74
- }
75
- if (!this.adapterState.sdkState.fromDevRev ||
76
- !this.adapterState.sdkState.fromDevRev.filesToLoad.length) {
77
- console.warn('No files to load, returning.');
78
- return {
79
- reports: this.reports,
80
- processed_files: this.processedFiles,
81
- };
82
- }
83
- console.log('Files to load in state', (_a = this.adapterState.sdkState.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad);
84
- try {
85
- outerloop: for (const fileToLoad of this.adapterState.sdkState
86
- .fromDevRev.filesToLoad) {
87
- const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
88
- if (!itemTypeToLoad) {
89
- console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
90
- await this.emit(loading_1.LoaderEventType.DataLoadingError, {
83
+ if (!fileToLoad.completed) {
84
+ const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
85
+ artifactId: fileToLoad.id,
86
+ isGzipped: true,
87
+ });
88
+ if (transformerFileError) {
89
+ console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
90
+ return {
91
+ status: 'error',
91
92
  error: {
92
- message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
93
+ message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
93
94
  },
94
- });
95
- break;
95
+ };
96
96
  }
97
- if (!fileToLoad.completed) {
98
- const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
99
- artifactId: fileToLoad.id,
100
- isGzipped: true,
97
+ const transformerFile = response;
98
+ for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
99
+ if (this.isTimeout) {
100
+ console.log('Timeout detected during data loading. Returning progress to allow continuation.');
101
+ return { status: 'progress' };
102
+ }
103
+ const { report, rateLimit } = await this.loadItem({
104
+ item: transformerFile[i],
105
+ itemTypeToLoad,
101
106
  });
102
- if (transformerFileError) {
103
- console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
104
- await this.emit(loading_1.LoaderEventType.DataLoadingError, {
105
- error: {
106
- message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
107
- },
108
- });
109
- break outerloop;
107
+ if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
108
+ return { status: 'delay', delaySeconds: rateLimit.delay };
110
109
  }
111
- const transformerFile = response;
112
- for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
113
- if (this.isTimeout) {
114
- console.log('Timeout detected during data loading. Emitting progress to allow continuation.');
115
- await this.emit(loading_1.LoaderEventType.DataLoadingProgress);
116
- process.exit(0);
117
- }
118
- const { report, rateLimit } = await this.loadItem({
119
- item: transformerFile[i],
120
- itemTypeToLoad,
110
+ if (report) {
111
+ (0, loading_adapter_helpers_1.addReportToLoaderReport)({
112
+ loaderReports: this.loaderReports,
113
+ report,
121
114
  });
122
- if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
123
- await this.emit(loading_1.LoaderEventType.DataLoadingDelayed, {
124
- delay: rateLimit.delay,
125
- reports: this.reports,
126
- processed_files: this.processedFiles,
127
- });
128
- break outerloop;
129
- }
130
- if (report) {
131
- (0, loading_adapter_helpers_1.addReportToLoaderReport)({
132
- loaderReports: this.loaderReports,
133
- report,
134
- });
135
- fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
136
- }
115
+ fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
137
116
  }
138
- fileToLoad.completed = true;
139
- this._processedFiles.push(fileToLoad.id);
140
117
  }
118
+ fileToLoad.completed = true;
119
+ this._processedFiles.push(fileToLoad.id);
141
120
  }
142
121
  }
143
- catch (error) {
144
- console.error('Error during data loading.', (0, logger_1.serializeError)(error));
145
- await this.emit(loading_1.LoaderEventType.DataLoadingError, {
146
- error: {
147
- message: `Error during data loading. ${(0, logger_1.serializeError)(error)}`,
148
- },
149
- });
150
- process.exit(1);
151
- }
122
+ }
123
+ catch (error) {
124
+ console.error('Error during data loading.', (0, logger_1.serializeError)(error));
152
125
  return {
153
- reports: this.reports,
154
- processed_files: this.processedFiles,
126
+ status: 'error',
127
+ error: {
128
+ message: `Error during data loading. ${(0, logger_1.serializeError)(error)}`,
129
+ },
155
130
  };
156
- });
131
+ }
132
+ return { status: 'success' };
157
133
  }
158
134
  async getLoaderBatches({ supportedItemTypes, }) {
159
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
160
- var _a;
161
- const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
162
- if (statsFileArtifactId) {
163
- const { response, error: statsFileError } = await this.uploader.getJsonObjectByArtifactId({
164
- artifactId: statsFileArtifactId,
165
- });
166
- const statsFile = response;
167
- if (statsFileError || statsFile.length === 0) {
168
- return [];
169
- }
170
- const filesToLoad = (0, loading_adapter_helpers_1.getFilesToLoad)({
171
- supportedItemTypes,
172
- statsFile,
173
- });
174
- return filesToLoad;
135
+ var _a;
136
+ const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
137
+ if (statsFileArtifactId) {
138
+ const { response, error: statsFileError } = await this.uploader.getJsonObjectByArtifactId({
139
+ artifactId: statsFileArtifactId,
140
+ });
141
+ const statsFile = response;
142
+ if (statsFileError || statsFile.length === 0) {
143
+ return [];
175
144
  }
176
- return [];
177
- });
145
+ const filesToLoad = (0, loading_adapter_helpers_1.getFilesToLoad)({
146
+ supportedItemTypes,
147
+ statsFile,
148
+ });
149
+ return filesToLoad;
150
+ }
151
+ return [];
178
152
  }
179
153
  async loadAttachments({ create, }) {
180
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
181
- var _a, _b;
182
- if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
183
- this.adapterState.sdkState.fromDevRev = {
184
- filesToLoad: await this.getLoaderBatches({
185
- supportedItemTypes: ['attachment'],
186
- }),
187
- };
188
- }
189
- if (!this.adapterState.sdkState.fromDevRev ||
190
- ((_a = this.adapterState.sdkState.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
191
- console.log('No files to load, returning.');
192
- return {
193
- reports: this.reports,
194
- processed_files: this.processedFiles,
195
- };
196
- }
197
- const filesToLoad = (_b = this.adapterState.sdkState.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
198
- try {
199
- outerloop: for (const fileToLoad of filesToLoad) {
200
- if (!fileToLoad.completed) {
201
- const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
202
- artifactId: fileToLoad.id,
203
- isGzipped: true,
154
+ var _a, _b;
155
+ if (this.event.payload.event_type === extraction_1.EventType.StartLoadingAttachments) {
156
+ this.sdkState.fromDevRev = {
157
+ filesToLoad: await this.getLoaderBatches({
158
+ supportedItemTypes: ['attachment'],
159
+ }),
160
+ };
161
+ }
162
+ if (!this.sdkState.fromDevRev ||
163
+ ((_a = this.sdkState.fromDevRev) === null || _a === void 0 ? void 0 : _a.filesToLoad.length) === 0) {
164
+ console.log('No files to load, returning.');
165
+ return { status: 'success' };
166
+ }
167
+ const filesToLoad = (_b = this.sdkState.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad;
168
+ try {
169
+ for (const fileToLoad of filesToLoad) {
170
+ if (!fileToLoad.completed) {
171
+ const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({
172
+ artifactId: fileToLoad.id,
173
+ isGzipped: true,
174
+ });
175
+ const transformerFile = response;
176
+ if (transformerFileError) {
177
+ console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
178
+ return {
179
+ status: 'error',
180
+ error: {
181
+ message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
182
+ },
183
+ };
184
+ }
185
+ for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
186
+ if (this.isTimeout) {
187
+ console.log('Timeout detected during attachment loading. Returning progress to allow continuation.');
188
+ return { status: 'progress' };
189
+ }
190
+ const { report, rateLimit } = await this.loadAttachment({
191
+ item: transformerFile[i],
192
+ create,
204
193
  });
205
- const transformerFile = response;
206
- if (transformerFileError) {
207
- console.error(`Transformer file not found for artifact ID: ${fileToLoad.id}.`);
208
- await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
209
- error: {
210
- message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`,
211
- },
212
- });
213
- break outerloop;
194
+ if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
195
+ return { status: 'delay', delaySeconds: rateLimit.delay };
214
196
  }
215
- for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
216
- if (this.isTimeout) {
217
- console.log('Timeout detected during attachment loading. Emitting progress to allow continuation.');
218
- await this.emit(loading_1.LoaderEventType.AttachmentLoadingProgress);
219
- process.exit(0);
220
- }
221
- const { report, rateLimit } = await this.loadAttachment({
222
- item: transformerFile[i],
223
- create,
197
+ if (report) {
198
+ (0, loading_adapter_helpers_1.addReportToLoaderReport)({
199
+ loaderReports: this.loaderReports,
200
+ report,
224
201
  });
225
- if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
226
- await this.emit(loading_1.LoaderEventType.AttachmentLoadingDelayed, {
227
- delay: rateLimit.delay,
228
- reports: this.reports,
229
- processed_files: this.processedFiles,
230
- });
231
- break outerloop;
232
- }
233
- if (report) {
234
- (0, loading_adapter_helpers_1.addReportToLoaderReport)({
235
- loaderReports: this.loaderReports,
236
- report,
237
- });
238
- fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
239
- }
202
+ fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
240
203
  }
241
- fileToLoad.completed = true;
242
- this._processedFiles.push(fileToLoad.id);
243
204
  }
205
+ fileToLoad.completed = true;
206
+ this._processedFiles.push(fileToLoad.id);
244
207
  }
245
208
  }
246
- catch (error) {
247
- console.error('Error during attachment loading.', (0, logger_1.serializeError)(error));
248
- await this.emit(loading_1.LoaderEventType.AttachmentLoadingError, {
249
- error: {
250
- message: `Error during attachment loading. ${(0, logger_1.serializeError)(error)}`,
251
- },
252
- });
253
- process.exit(1);
254
- }
209
+ }
210
+ catch (error) {
211
+ console.error('Error during attachment loading.', (0, logger_1.serializeError)(error));
255
212
  return {
256
- reports: this.reports,
257
- processed_files: this.processedFiles,
213
+ status: 'error',
214
+ error: {
215
+ message: `Error during attachment loading. ${(0, logger_1.serializeError)(error)}`,
216
+ },
258
217
  };
259
- });
218
+ }
219
+ return { status: 'success' };
260
220
  }
261
221
  async loadItem({ item, itemTypeToLoad, }) {
262
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
263
- var _a;
264
- const devrevId = item.id.devrev;
265
- try {
266
- const syncMapperRecordResponse = await this._mappers.getByTargetId({
267
- sync_unit: this.event.payload.event_context.sync_unit,
268
- target: devrevId,
269
- });
270
- const syncMapperRecord = syncMapperRecordResponse.data;
271
- if (!syncMapperRecord) {
272
- console.warn('Failed to get sync mapper record from response.');
222
+ var _a;
223
+ const devrevId = item.id.devrev;
224
+ try {
225
+ const syncMapperRecord = await this._mappers.getByTargetId({
226
+ sync_unit: this.event.payload.event_context.sync_unit,
227
+ target: devrevId,
228
+ });
229
+ if (!syncMapperRecord) {
230
+ console.warn('Failed to get sync mapper record from response.');
231
+ return {
232
+ error: {
233
+ message: 'Failed to get sync mapper record from response.',
234
+ },
235
+ };
236
+ }
237
+ const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
238
+ item,
239
+ mappers: this._mappers,
240
+ event: this.event,
241
+ });
242
+ if (id) {
243
+ try {
244
+ const syncMapperRecordUpdate = await this._mappers.update(Object.assign(Object.assign({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL }, (modifiedDate && {
245
+ external_versions: {
246
+ add: [
247
+ {
248
+ modified_date: modifiedDate,
249
+ recipe_version: 0,
250
+ },
251
+ ],
252
+ },
253
+ })), { external_ids: {
254
+ add: [id],
255
+ }, targets: {
256
+ add: [devrevId],
257
+ } }));
258
+ console.log('Successfully updated sync mapper record.', syncMapperRecordUpdate);
259
+ }
260
+ catch (error) {
261
+ console.warn('Failed to update sync mapper record.', (0, logger_1.serializeError)(error));
273
262
  return {
274
263
  error: {
275
- message: 'Failed to get sync mapper record from response.',
264
+ message: 'Failed to update sync mapper record' + (0, logger_1.serializeError)(error),
276
265
  },
277
266
  };
278
267
  }
279
- // Update item in external system
280
- const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
281
- return await itemTypeToLoad.update({
268
+ return {
269
+ report: {
270
+ item_type: itemTypeToLoad.itemType,
271
+ [loading_1.ActionType.UPDATED]: 1,
272
+ },
273
+ };
274
+ }
275
+ else if (delay) {
276
+ console.log(`Rate limited while updating item in external system, delaying for ${delay} seconds.`);
277
+ return {
278
+ rateLimit: {
279
+ delay,
280
+ },
281
+ };
282
+ }
283
+ else {
284
+ console.warn('Failed to update item in external system', error);
285
+ return {
286
+ report: {
287
+ item_type: itemTypeToLoad.itemType,
288
+ [loading_1.ActionType.FAILED]: 1,
289
+ },
290
+ };
291
+ }
292
+ // TODO: Update mapper (optional)
293
+ }
294
+ catch (error) {
295
+ if (axios_1.default.isAxiosError(error)) {
296
+ if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
297
+ // Create item in external system if mapper record not found
298
+ const { id, modifiedDate, delay, error } = await itemTypeToLoad.create({
282
299
  item,
283
300
  mappers: this._mappers,
284
301
  event: this.event,
285
302
  });
286
- });
287
- if (id) {
288
- try {
289
- const syncMapperRecordUpdateResponse = await this._mappers.update(Object.assign(Object.assign({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL }, (modifiedDate && {
290
- external_versions: {
291
- add: [
303
+ if (id) {
304
+ try {
305
+ const syncMapperRecordCreate = await this._mappers.create(Object.assign({ sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [id], targets: [devrevId] }, (modifiedDate && {
306
+ external_versions: [
292
307
  {
293
308
  modified_date: modifiedDate,
294
309
  recipe_version: 0,
295
310
  },
296
311
  ],
297
- },
298
- })), { external_ids: {
299
- add: [id],
300
- }, targets: {
301
- add: [devrevId],
302
- } }));
303
- console.log('Successfully updated sync mapper record.', syncMapperRecordUpdateResponse.data);
304
- }
305
- catch (error) {
306
- console.warn('Failed to update sync mapper record.', (0, logger_1.serializeError)(error));
307
- return {
308
- error: {
309
- message: 'Failed to update sync mapper record' + (0, logger_1.serializeError)(error),
310
- },
311
- };
312
- }
313
- return {
314
- report: {
315
- item_type: itemTypeToLoad.itemType,
316
- [loading_1.ActionType.UPDATED]: 1,
317
- },
318
- };
319
- }
320
- else if (delay) {
321
- console.log(`Rate limited while updating item in external system, delaying for ${delay} seconds.`);
322
- return {
323
- rateLimit: {
324
- delay,
325
- },
326
- };
327
- }
328
- else {
329
- console.warn('Failed to update item in external system', error);
330
- return {
331
- report: {
332
- item_type: itemTypeToLoad.itemType,
333
- [loading_1.ActionType.FAILED]: 1,
334
- },
335
- };
336
- }
337
- // TODO: Update mapper (optional)
338
- }
339
- catch (error) {
340
- if (axios_1.default.isAxiosError(error)) {
341
- if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
342
- // Create item in external system if mapper record not found
343
- const { id, modifiedDate, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => {
344
- return await itemTypeToLoad.create({
345
- item,
346
- mappers: this._mappers,
347
- event: this.event,
348
- });
349
- });
350
- if (id) {
351
- // Create mapper
352
- try {
353
- const syncMapperRecordCreateResponse = await this._mappers.create(Object.assign({ sync_unit: this.event.payload.event_context.sync_unit, status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL, external_ids: [id], targets: [devrevId] }, (modifiedDate && {
354
- external_versions: [
355
- {
356
- modified_date: modifiedDate,
357
- recipe_version: 0,
358
- },
359
- ],
360
- })));
361
- console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
362
- return {
363
- report: {
364
- item_type: itemTypeToLoad.itemType,
365
- [loading_1.ActionType.CREATED]: 1,
366
- },
367
- };
368
- }
369
- catch (error) {
370
- console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
371
- return {
372
- error: {
373
- message: 'Failed to create sync mapper record. ' +
374
- (0, logger_1.serializeError)(error),
375
- },
376
- };
377
- }
378
- }
379
- else if (delay) {
312
+ })));
313
+ console.log('Successfully created sync mapper record.', syncMapperRecordCreate);
380
314
  return {
381
- rateLimit: {
382
- delay,
315
+ report: {
316
+ item_type: itemTypeToLoad.itemType,
317
+ [loading_1.ActionType.CREATED]: 1,
383
318
  },
384
319
  };
385
320
  }
386
- else {
387
- console.warn('Failed to create item in external system.', (0, logger_1.serializeError)(error));
321
+ catch (error) {
322
+ console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
388
323
  return {
389
- report: {
390
- item_type: itemTypeToLoad.itemType,
391
- [loading_1.ActionType.FAILED]: 1,
324
+ error: {
325
+ message: 'Failed to create sync mapper record. ' +
326
+ (0, logger_1.serializeError)(error),
392
327
  },
393
328
  };
394
329
  }
395
330
  }
331
+ else if (delay) {
332
+ return {
333
+ rateLimit: {
334
+ delay,
335
+ },
336
+ };
337
+ }
396
338
  else {
397
- console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
339
+ console.warn('Failed to create item in external system.', (0, logger_1.serializeError)(error));
398
340
  return {
399
- error: {
400
- message: error.message,
341
+ report: {
342
+ item_type: itemTypeToLoad.itemType,
343
+ [loading_1.ActionType.FAILED]: 1,
401
344
  },
402
345
  };
403
346
  }
404
347
  }
405
- console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
406
- return {
407
- error: {
408
- message: 'Failed to get sync mapper record. ' + (0, logger_1.serializeError)(error),
409
- },
410
- };
348
+ else {
349
+ console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
350
+ return {
351
+ error: {
352
+ message: error.message,
353
+ },
354
+ };
355
+ }
411
356
  }
412
- });
357
+ console.warn('Failed to get sync mapper record.', (0, logger_1.serializeError)(error));
358
+ return {
359
+ error: {
360
+ message: 'Failed to get sync mapper record. ' + (0, logger_1.serializeError)(error),
361
+ },
362
+ };
363
+ }
413
364
  }
414
365
  async loadAttachment({ item, create, }) {
415
- return (0, logger_context_1.runWithSdkLogContext)(async () => {
416
- // Create item
417
- const { id, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => create({
418
- item,
419
- mappers: this._mappers,
420
- event: this.event,
421
- }));
422
- if (delay) {
423
- return {
424
- rateLimit: {
425
- delay,
426
- },
427
- };
428
- }
429
- else if (id) {
430
- try {
431
- const syncMapperRecordCreateResponse = await this._mappers.create({
432
- sync_unit: this.event.payload.event_context.sync_unit,
433
- external_ids: [id],
434
- targets: [item.reference_id],
435
- status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
436
- });
437
- console.log('Successfully created sync mapper record.', syncMapperRecordCreateResponse.data);
438
- }
439
- catch (error) {
440
- console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
441
- }
442
- return {
443
- report: {
444
- item_type: 'attachment',
445
- [loading_1.ActionType.CREATED]: 1,
446
- },
447
- };
366
+ const { id, delay, error } = await create({
367
+ item,
368
+ mappers: this._mappers,
369
+ event: this.event,
370
+ });
371
+ if (delay) {
372
+ return {
373
+ rateLimit: {
374
+ delay,
375
+ },
376
+ };
377
+ }
378
+ else if (id) {
379
+ try {
380
+ const syncMapperRecordCreate = await this._mappers.create({
381
+ sync_unit: this.event.payload.event_context.sync_unit,
382
+ external_ids: [id],
383
+ targets: [item.reference_id],
384
+ status: mappers_interfaces_1.SyncMapperRecordStatus.OPERATIONAL,
385
+ });
386
+ console.log('Successfully created sync mapper record.', syncMapperRecordCreate);
448
387
  }
449
- else {
450
- console.warn('Failed to create attachment in external system', error);
451
- return {
452
- report: {
453
- item_type: 'attachment',
454
- [loading_1.ActionType.FAILED]: 1,
455
- },
456
- };
388
+ catch (error) {
389
+ console.warn('Failed to create sync mapper record.', (0, logger_1.serializeError)(error));
457
390
  }
458
- });
391
+ return {
392
+ report: {
393
+ item_type: 'attachments',
394
+ [loading_1.ActionType.CREATED]: 1,
395
+ },
396
+ };
397
+ }
398
+ else {
399
+ console.warn('Failed to create attachment in external system', error);
400
+ return {
401
+ report: {
402
+ item_type: 'attachments',
403
+ [loading_1.ActionType.FAILED]: 1,
404
+ },
405
+ };
406
+ }
459
407
  }
460
408
  }
461
409
  exports.LoadingAdapter = LoadingAdapter;