@devrev/airsync-sdk 2.0.0-beta.3 → 2.0.0-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/MIGRATION.md +1028 -0
  2. package/README.md +5 -11
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -9
  4. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -7
  6. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  7. package/dist/attachments-streaming/attachments-streaming-pool.js +54 -56
  8. package/dist/common/constants.d.ts +4 -5
  9. package/dist/common/constants.d.ts.map +1 -1
  10. package/dist/common/constants.js +6 -7
  11. package/dist/common/helpers.d.ts +3 -35
  12. package/dist/common/helpers.d.ts.map +1 -1
  13. package/dist/common/helpers.js +5 -50
  14. package/dist/http/{axios-client-internal.d.ts → client.d.ts} +1 -1
  15. package/dist/http/client.d.ts.map +1 -0
  16. package/dist/http/{axios-client-internal.js → client.js} +4 -8
  17. package/dist/index.d.ts +24 -13
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +43 -32
  20. package/dist/logger/logger.d.ts +12 -53
  21. package/dist/logger/logger.d.ts.map +1 -1
  22. package/dist/logger/logger.interfaces.d.ts +0 -1
  23. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  24. package/dist/logger/logger.js +36 -68
  25. package/dist/mappers/mappers.d.ts +7 -46
  26. package/dist/mappers/mappers.d.ts.map +1 -1
  27. package/dist/mappers/mappers.interfaces.d.ts +175 -0
  28. package/dist/mappers/mappers.interfaces.d.ts.map +1 -0
  29. package/dist/mappers/{mappers.interface.js → mappers.interfaces.js} +4 -13
  30. package/dist/mappers/mappers.js +11 -45
  31. package/dist/multithreading/adapters/base-adapter.d.ts +30 -32
  32. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  33. package/dist/multithreading/adapters/base-adapter.js +97 -62
  34. package/dist/multithreading/adapters/extraction-adapter.d.ts +16 -29
  35. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  36. package/dist/multithreading/adapters/extraction-adapter.js +203 -206
  37. package/dist/multithreading/adapters/loading-adapter.d.ts +8 -13
  38. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  39. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
  40. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
  41. package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
  42. package/dist/multithreading/adapters/loading-adapter.js +294 -346
  43. package/dist/multithreading/emit.d.ts +11 -0
  44. package/dist/multithreading/emit.d.ts.map +1 -0
  45. package/dist/{common/control-protocol.js → multithreading/emit.js} +5 -7
  46. package/dist/multithreading/process-task.d.ts +2 -12
  47. package/dist/multithreading/process-task.d.ts.map +1 -1
  48. package/dist/multithreading/process-task.js +46 -43
  49. package/dist/multithreading/spawn/spawn.d.ts +3 -9
  50. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  51. package/dist/multithreading/spawn/spawn.helpers.d.ts +12 -11
  52. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  53. package/dist/multithreading/spawn/spawn.helpers.js +128 -20
  54. package/dist/multithreading/spawn/spawn.js +24 -39
  55. package/dist/multithreading/worker.js +1 -4
  56. package/dist/repo/repo.d.ts +10 -0
  57. package/dist/repo/repo.d.ts.map +1 -1
  58. package/dist/repo/repo.helpers.d.ts +6 -0
  59. package/dist/repo/repo.helpers.d.ts.map +1 -0
  60. package/dist/repo/repo.helpers.js +16 -0
  61. package/dist/repo/repo.interfaces.d.ts +4 -16
  62. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  63. package/dist/repo/repo.js +23 -8
  64. package/dist/state/base-state.d.ts +16 -38
  65. package/dist/state/base-state.d.ts.map +1 -1
  66. package/dist/state/base-state.js +19 -41
  67. package/dist/state/extraction-state.d.ts +6 -21
  68. package/dist/state/extraction-state.d.ts.map +1 -1
  69. package/dist/state/extraction-state.js +8 -25
  70. package/dist/{common → state}/install-initial-domain-mapping.d.ts +1 -1
  71. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -0
  72. package/dist/{common → state}/install-initial-domain-mapping.js +5 -7
  73. package/dist/state/loading-state.d.ts +3 -14
  74. package/dist/state/loading-state.d.ts.map +1 -1
  75. package/dist/state/loading-state.js +3 -14
  76. package/dist/state/state.d.ts +4 -0
  77. package/dist/state/state.d.ts.map +1 -0
  78. package/dist/state/state.interfaces.d.ts +33 -37
  79. package/dist/state/state.interfaces.d.ts.map +1 -1
  80. package/dist/state/state.interfaces.js +16 -11
  81. package/dist/state/state.js +11 -0
  82. package/dist/state/time-value-resolver.d.ts +17 -0
  83. package/dist/state/time-value-resolver.d.ts.map +1 -0
  84. package/dist/{common → state}/time-value-resolver.js +9 -41
  85. package/dist/testing/mock-event.d.ts +10 -0
  86. package/dist/testing/mock-event.d.ts.map +1 -0
  87. package/dist/{common/test-utils.js → testing/mock-event.js} +17 -14
  88. package/dist/testing/mock-server.d.ts +29 -0
  89. package/dist/testing/mock-server.d.ts.map +1 -0
  90. package/dist/testing/mock-server.interfaces.d.ts +48 -0
  91. package/dist/testing/mock-server.interfaces.d.ts.map +1 -0
  92. package/dist/{mock-server → testing}/mock-server.js +21 -33
  93. package/dist/types/common.d.ts +1 -12
  94. package/dist/types/common.d.ts.map +1 -1
  95. package/dist/types/common.js +1 -6
  96. package/dist/types/errors.d.ts +9 -0
  97. package/dist/types/errors.d.ts.map +1 -0
  98. package/dist/types/external-domain-metadata.d.ts +30 -171
  99. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  100. package/dist/types/extraction.d.ts +64 -94
  101. package/dist/types/extraction.d.ts.map +1 -1
  102. package/dist/types/extraction.js +6 -34
  103. package/dist/types/loading.d.ts +1 -2
  104. package/dist/types/loading.d.ts.map +1 -1
  105. package/dist/types/loading.js +0 -1
  106. package/dist/types/workers.d.ts +39 -94
  107. package/dist/types/workers.d.ts.map +1 -1
  108. package/dist/types/workers.js +0 -6
  109. package/dist/uploader/uploader.d.ts +8 -59
  110. package/dist/uploader/uploader.d.ts.map +1 -1
  111. package/dist/uploader/uploader.helpers.d.ts +4 -27
  112. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  113. package/dist/uploader/uploader.helpers.js +64 -30
  114. package/dist/uploader/uploader.interfaces.d.ts +15 -25
  115. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  116. package/dist/uploader/uploader.interfaces.js +8 -0
  117. package/dist/uploader/uploader.js +23 -89
  118. package/package.json +8 -6
  119. package/dist/airsync-sdk.d.ts +0 -2059
  120. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +0 -2
  121. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +0 -1
  122. package/dist/attachments-streaming/attachments-streaming-pool.test.js +0 -540
  123. package/dist/common/control-protocol.d.ts +0 -10
  124. package/dist/common/control-protocol.d.ts.map +0 -1
  125. package/dist/common/errors.d.ts +0 -6
  126. package/dist/common/errors.d.ts.map +0 -1
  127. package/dist/common/install-initial-domain-mapping.d.ts.map +0 -1
  128. package/dist/common/install-initial-domain-mapping.test.d.ts +0 -2
  129. package/dist/common/install-initial-domain-mapping.test.d.ts.map +0 -1
  130. package/dist/common/install-initial-domain-mapping.test.js +0 -211
  131. package/dist/common/test-utils.d.ts +0 -19
  132. package/dist/common/test-utils.d.ts.map +0 -1
  133. package/dist/common/time-value-resolver.d.ts +0 -47
  134. package/dist/common/time-value-resolver.d.ts.map +0 -1
  135. package/dist/common/time-value-resolver.test.d.ts +0 -2
  136. package/dist/common/time-value-resolver.test.d.ts.map +0 -1
  137. package/dist/common/time-value-resolver.test.js +0 -268
  138. package/dist/http/axios-client-internal.d.ts.map +0 -1
  139. package/dist/http/axios-client-internal.test.d.ts +0 -2
  140. package/dist/http/axios-client-internal.test.d.ts.map +0 -1
  141. package/dist/http/axios-client-internal.test.js +0 -175
  142. package/dist/http/axios-client.d.ts +0 -27
  143. package/dist/http/axios-client.d.ts.map +0 -1
  144. package/dist/http/axios-client.js +0 -60
  145. package/dist/http/constants.d.ts +0 -4
  146. package/dist/http/constants.d.ts.map +0 -1
  147. package/dist/http/constants.js +0 -6
  148. package/dist/http/index.d.ts +0 -2
  149. package/dist/http/index.d.ts.map +0 -1
  150. package/dist/http/index.js +0 -17
  151. package/dist/logger/logger.constants.d.ts +0 -6
  152. package/dist/logger/logger.constants.d.ts.map +0 -1
  153. package/dist/logger/logger.constants.js +0 -13
  154. package/dist/logger/logger.context.d.ts +0 -58
  155. package/dist/logger/logger.context.d.ts.map +0 -1
  156. package/dist/logger/logger.context.js +0 -86
  157. package/dist/logger/logger.test.d.ts +0 -2
  158. package/dist/logger/logger.test.d.ts.map +0 -1
  159. package/dist/logger/logger.test.js +0 -537
  160. package/dist/mappers/mappers.interface.d.ts +0 -294
  161. package/dist/mappers/mappers.interface.d.ts.map +0 -1
  162. package/dist/mappers/mappers.test.d.ts +0 -2
  163. package/dist/mappers/mappers.test.d.ts.map +0 -1
  164. package/dist/mappers/mappers.test.js +0 -105
  165. package/dist/mock-server/mock-server.d.ts +0 -47
  166. package/dist/mock-server/mock-server.d.ts.map +0 -1
  167. package/dist/mock-server/mock-server.interfaces.d.ts +0 -73
  168. package/dist/mock-server/mock-server.interfaces.d.ts.map +0 -1
  169. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts +0 -2
  170. package/dist/multithreading/adapters/base-adapter.emit.test.d.ts.map +0 -1
  171. package/dist/multithreading/adapters/base-adapter.emit.test.js +0 -403
  172. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts +0 -2
  173. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.d.ts.map +0 -1
  174. package/dist/multithreading/adapters/extraction-adapter.artifacts.test.js +0 -175
  175. package/dist/multithreading/adapters/extraction-adapter.test.d.ts +0 -2
  176. package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map +0 -1
  177. package/dist/multithreading/adapters/extraction-adapter.test.js +0 -789
  178. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts +0 -2
  179. package/dist/multithreading/adapters/loading-adapter.helpers.test.d.ts.map +0 -1
  180. package/dist/multithreading/adapters/loading-adapter.helpers.test.js +0 -514
  181. package/dist/multithreading/adapters/loading-adapter.test.d.ts +0 -2
  182. package/dist/multithreading/adapters/loading-adapter.test.d.ts.map +0 -1
  183. package/dist/multithreading/adapters/loading-adapter.test.js +0 -589
  184. package/dist/multithreading/adapters/serialization.test.d.ts +0 -2
  185. package/dist/multithreading/adapters/serialization.test.d.ts.map +0 -1
  186. package/dist/multithreading/adapters/serialization.test.js +0 -71
  187. package/dist/multithreading/create-worker.test.d.ts +0 -2
  188. package/dist/multithreading/create-worker.test.d.ts.map +0 -1
  189. package/dist/multithreading/create-worker.test.js +0 -108
  190. package/dist/multithreading/process-task.test.d.ts +0 -2
  191. package/dist/multithreading/process-task.test.d.ts.map +0 -1
  192. package/dist/multithreading/process-task.test.js +0 -146
  193. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +0 -2
  194. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +0 -1
  195. package/dist/multithreading/spawn/spawn.helpers.test.js +0 -293
  196. package/dist/multithreading/spawn/spawn.test.d.ts +0 -2
  197. package/dist/multithreading/spawn/spawn.test.d.ts.map +0 -1
  198. package/dist/multithreading/spawn/spawn.test.js +0 -232
  199. package/dist/repo/repo.test.d.ts +0 -2
  200. package/dist/repo/repo.test.d.ts.map +0 -1
  201. package/dist/repo/repo.test.js +0 -184
  202. package/dist/state/state.extract-window.test.d.ts +0 -2
  203. package/dist/state/state.extract-window.test.d.ts.map +0 -1
  204. package/dist/state/state.extract-window.test.js +0 -163
  205. package/dist/state/state.pending-boundaries.test.d.ts +0 -2
  206. package/dist/state/state.pending-boundaries.test.d.ts.map +0 -1
  207. package/dist/state/state.pending-boundaries.test.js +0 -189
  208. package/dist/state/state.post-state.test.d.ts +0 -2
  209. package/dist/state/state.post-state.test.d.ts.map +0 -1
  210. package/dist/state/state.post-state.test.js +0 -78
  211. package/dist/state/state.test.d.ts +0 -2
  212. package/dist/state/state.test.d.ts.map +0 -1
  213. package/dist/state/state.test.js +0 -271
  214. package/dist/state/state.time-value-resolution.test.d.ts +0 -2
  215. package/dist/state/state.time-value-resolution.test.d.ts.map +0 -1
  216. package/dist/state/state.time-value-resolution.test.js +0 -175
  217. package/dist/tsdoc-metadata.json +0 -11
  218. package/dist/types/extraction.test.d.ts +0 -2
  219. package/dist/types/extraction.test.d.ts.map +0 -1
  220. package/dist/types/extraction.test.js +0 -96
  221. package/dist/types/index.d.ts +0 -10
  222. package/dist/types/index.d.ts.map +0 -1
  223. package/dist/types/index.js +0 -21
  224. package/dist/uploader/uploader.helpers.test.d.ts +0 -2
  225. package/dist/uploader/uploader.helpers.test.d.ts.map +0 -1
  226. package/dist/uploader/uploader.helpers.test.js +0 -256
  227. package/dist/uploader/uploader.test.d.ts +0 -2
  228. package/dist/uploader/uploader.test.d.ts.map +0 -1
  229. package/dist/uploader/uploader.test.js +0 -601
  230. /package/dist/{mock-server → testing}/mock-server.interfaces.js +0 -0
  231. /package/dist/{common → types}/errors.js +0 -0
package/README.md CHANGED
@@ -1,16 +1,14 @@
1
- # AirSync SDK
1
+ # Airdrop SDK
2
2
 
3
- [![Coverage Status](https://coveralls.io/repos/github/devrev/adaas-sdk/badge.svg?branch=v2&t=s4Otlm)](https://coveralls.io/github/devrev/adaas-sdk?branch=v2)
4
-
5
- > **Note:** This is the v2 beta of the AirSync SDK (formerly Airdrop SDK). The package has been renamed from `@devrev/ts-adaas` to `@devrev/airsync-sdk`.
3
+ [![Coverage Status](https://coveralls.io/repos/github/devrev/adaas-sdk/badge.svg?branch=main&t=s4Otlm)](https://coveralls.io/github/devrev/adaas-sdk?branch=main)
6
4
 
7
5
  ## Overview
8
6
 
9
- The AirSync SDK for TypeScript helps developers build snap-ins that integrate with DevRev's AirSync platform. This SDK simplifies the workflow for handling data extraction and loading, event-driven actions, state management, and artifact handling.
7
+ The Airdrop SDK for TypeScript helps developers build snap-ins that integrate with DevRev’s Airdrop platform. This SDK simplifies the workflow for handling data extraction and loading, event-driven actions, state management, and artifact handling.
10
8
 
11
9
  It provides features such as:
12
10
 
13
- - Type Definitions: Structured types for AirSync control protocol
11
+ - Type Definitions: Structured types for Airdrop control protocol
14
12
  - Event Management: Easily emit events for different extraction or loading phases
15
13
  - State Handling: Update and access state in real-time within tasks
16
14
  - Artifact Management: Supports batched storage of artifacts
@@ -19,9 +17,5 @@ It provides features such as:
19
17
  ## Installation
20
18
 
21
19
  ```bash
22
- npm install @devrev/airsync-sdk@beta
20
+ npm install @devrev/airsync-sdk
23
21
  ```
24
-
25
- ## Changelog
26
-
27
- See [CHANGELOG.md](./CHANGELOG.md) for release notes and migration guides.
@@ -1,23 +1,21 @@
1
- import { ProcessAttachmentReturnType } from '../types';
1
+ import { ProcessAttachmentReturnType } from '../types/extraction';
2
2
  import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces';
3
3
  export declare class AttachmentsStreamingPool<ConnectorState> {
4
4
  private adapter;
5
5
  private attachments;
6
- private attachmentsMetadata;
7
6
  private batchSize;
8
7
  private delay;
9
8
  private stream;
10
9
  private totalProcessedCount;
11
10
  private readonly PROGRESS_REPORT_INTERVAL;
12
- constructor({ adapter, attachments, attachmentsMetadata, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
13
- private updateProgress;
11
+ constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
12
+ private recordProcessedAttachment;
14
13
  /**
15
- * Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format.
16
- *
17
- * @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[])
18
- * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
14
+ * Backfills `status` on entries recorded before the field existed (those
15
+ * only ever held successes); state from >= v1.15.2 needs no shape migration.
19
16
  */
20
- private migrateProcessedAttachments;
17
+ private backfillProcessedAttachmentStatus;
18
+ private updateProgress;
21
19
  streamAll(): Promise<ProcessAttachmentReturnType>;
22
20
  startPoolStreaming(): Promise<void>;
23
21
  }
@@ -1 +1 @@
1
- {"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAGA,OAAO,EAGL,2BAA2B,EAC5B,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,mBAAmB,CAA8C;IACzE,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,mBAAmB,EACnB,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;YASnC,cAAc;IAS5B;;;;;OAKG;IAEH,OAAO,CAAC,2BAA2B;IAsB7B,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IAyCjD,kBAAkB;CAuFzB"}
1
+ {"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAOA,OAAO,EAEL,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD,OAAO,CAAC,yBAAyB;IAyBjC;;;OAGG;IACH,OAAO,CAAC,iCAAiC;YAS3B,cAAc;IAUtB,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IA6CjD,kBAAkB;CAyFzB"}
@@ -1,14 +1,9 @@
1
- import { ExternalSystemAttachmentStreamingFunction, NormalizedAttachment } from '../types';
2
1
  import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter';
3
- import { ToDevRev } from '../state/state.interfaces';
2
+ import { NormalizedAttachment } from '../repo/repo.interfaces';
3
+ import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction';
4
4
  export interface AttachmentsStreamingPoolParams<ConnectorState> {
5
5
  adapter: ExtractionAdapter<ConnectorState>;
6
6
  attachments: NormalizedAttachment[];
7
- /**
8
- * SDK-owned attachments bookkeeping. Passed in directly so the pool does not
9
- * reach into the (now-encapsulated) adapter state.
10
- */
11
- attachmentsMetadata: ToDevRev['attachmentsMetadata'] | undefined;
12
7
  batchSize?: number;
13
8
  stream: ExternalSystemAttachmentStreamingFunction;
14
9
  }
@@ -1 +1 @@
1
- {"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yCAAyC,EACzC,oBAAoB,EACrB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAErD,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC;;;OAGG;IACH,mBAAmB,EAAE,QAAQ,CAAC,qBAAqB,CAAC,GAAG,SAAS,CAAC;IACjE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}
1
+ {"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,yCAAyC,EAAE,MAAM,qBAAqB,CAAC;AAEhF,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAC3C,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}
@@ -2,104 +2,105 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.AttachmentsStreamingPool = void 0;
4
4
  const helpers_1 = require("../common/helpers");
5
+ const state_interfaces_1 = require("../state/state.interfaces");
5
6
  class AttachmentsStreamingPool {
6
- constructor({ adapter, attachments, attachmentsMetadata, batchSize = 10, stream, }) {
7
+ constructor({ adapter, attachments, batchSize = 10, stream, }) {
7
8
  this.totalProcessedCount = 0;
8
9
  this.PROGRESS_REPORT_INTERVAL = 50;
9
10
  this.adapter = adapter;
10
11
  this.attachments = [...attachments]; // Create a copy we can mutate
11
- this.attachmentsMetadata = attachmentsMetadata;
12
12
  this.batchSize = batchSize;
13
13
  this.delay = undefined;
14
14
  this.stream = stream;
15
15
  }
16
- async updateProgress() {
17
- this.totalProcessedCount++;
18
- if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
19
- console.info(`Processed ${this.totalProcessedCount} attachments so far.`);
20
- // Sleep for 100ms to avoid blocking the event loop
21
- await (0, helpers_1.sleep)(100);
16
+ recordProcessedAttachment(attachment, status) {
17
+ var _a;
18
+ const attachmentsMetadata = (_a = this.adapter.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
19
+ if (!(attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.lastProcessedAttachmentsIdsList)) {
20
+ return;
21
+ }
22
+ const processedAttachmentsIdsList = attachmentsMetadata.lastProcessedAttachmentsIdsList;
23
+ const alreadyRecorded = processedAttachmentsIdsList.some((it) => it.id === attachment.id && it.parent_id === attachment.parent_id);
24
+ if (!alreadyRecorded) {
25
+ processedAttachmentsIdsList.push({
26
+ id: attachment.id,
27
+ parent_id: attachment.parent_id,
28
+ status,
29
+ });
22
30
  }
23
31
  }
24
32
  /**
25
- * Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format.
26
- *
27
- * @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[])
28
- * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid
33
+ * Backfills `status` on entries recorded before the field existed (those
34
+ * only ever held successes); state from >= v1.15.2 needs no shape migration.
29
35
  */
30
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
31
- migrateProcessedAttachments(attachments) {
32
- // Handle null/undefined
33
- if (!attachments || !Array.isArray(attachments)) {
34
- return [];
35
- }
36
- // If already migrated (first element is an object), return as-is
37
- if (attachments.length > 0 && typeof attachments[0] === 'object') {
38
- return attachments;
39
- }
40
- // Migrate old string[] format
41
- if (attachments.length > 0 && typeof attachments[0] === 'string') {
42
- return attachments.map((it) => ({
43
- id: it,
44
- parent_id: '',
45
- }));
36
+ backfillProcessedAttachmentStatus(attachments) {
37
+ return attachments.map((it) => {
38
+ var _a;
39
+ return (Object.assign(Object.assign({}, it), { status: (_a = it.status) !== null && _a !== void 0 ? _a : state_interfaces_1.ProcessedAttachmentStatus.Success }));
40
+ });
41
+ }
42
+ async updateProgress() {
43
+ this.totalProcessedCount++;
44
+ if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
45
+ console.info(`Processed ${this.totalProcessedCount} attachments so far.`);
46
+ // Let a pending soft-timeout message (WorkerMessageExit) be delivered so
47
+ // adapter.isTimeout can flip before the next batch is processed.
48
+ await (0, helpers_1.yieldToEventLoop)();
46
49
  }
47
- return [];
48
50
  }
49
51
  async streamAll() {
50
52
  console.log(`Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.`);
51
- if (!this.attachmentsMetadata) {
53
+ if (!this.adapter.sdkState.toDevRev) {
52
54
  const error = new Error('toDevRev state is not initialized');
53
55
  console.error(error);
54
56
  return { error };
55
57
  }
56
- // Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction.
57
- // If no such list exists, create an empty one.
58
- if (!this.attachmentsMetadata.lastProcessedAttachmentsIdsList) {
59
- this.attachmentsMetadata.lastProcessedAttachmentsIdsList = [];
58
+ // Attachments processed (successfully or not) by a previous, possibly incomplete, run.
59
+ if (!this.adapter.sdkState.toDevRev.attachmentsMetadata
60
+ .lastProcessedAttachmentsIdsList) {
61
+ this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
62
+ [];
60
63
  }
61
- // Migrate old processed attachments to the new format.
62
- this.attachmentsMetadata.lastProcessedAttachmentsIdsList =
63
- this.migrateProcessedAttachments(this.attachmentsMetadata.lastProcessedAttachmentsIdsList);
64
- // Start initial batch of promises up to batchSize limit
64
+ this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
65
+ this.backfillProcessedAttachmentStatus(this.adapter.sdkState.toDevRev.attachmentsMetadata
66
+ .lastProcessedAttachmentsIdsList);
65
67
  const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
66
68
  const initialPromises = [];
67
69
  for (let i = 0; i < initialBatchSize; i++) {
68
70
  initialPromises.push(this.startPoolStreaming());
69
71
  }
70
- // Wait for all promises to complete
71
- await Promise.all(initialPromises);
72
+ await Promise.race([
73
+ Promise.all(initialPromises),
74
+ this.adapter.timeoutSignal,
75
+ ]);
72
76
  if (this.delay) {
73
77
  return { delay: this.delay };
74
78
  }
75
79
  return {};
76
80
  }
77
81
  async startPoolStreaming() {
78
- var _a, _b, _c;
79
- // Process attachments until the attachments array is empty
82
+ var _a;
80
83
  while (this.attachments.length > 0) {
81
- // If delay is set, stop streaming
82
84
  if (this.delay) {
83
85
  break;
84
86
  }
85
- // If timeout is set, stop streaming
86
87
  if (this.adapter.isTimeout) {
87
88
  console.log('Timeout detected while streaming attachments. Stopping streaming.');
88
89
  break;
89
90
  }
90
- // Check if we can process next attachment
91
91
  const attachment = this.attachments.shift();
92
92
  if (!attachment) {
93
- break; // Exit if no more attachments
93
+ break;
94
94
  }
95
- if ((_b = (_a = this.attachmentsMetadata) === null || _a === void 0 ? void 0 : _a.lastProcessedAttachmentsIdsList) === null || _b === void 0 ? void 0 : _b.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id)) {
96
- continue; // Skip if the attachment ID is already processed
95
+ if (this.adapter.sdkState.toDevRev &&
96
+ ((_a = this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.some((it) => it.id == attachment.id && it.parent_id == attachment.parent_id))) {
97
+ continue; // Already processed in a previous run
97
98
  }
98
99
  try {
99
100
  const response = await this.adapter.processAttachment(attachment, this.stream);
100
- // Check if rate limit was hit
101
+ // Rate limit hit
101
102
  if (response === null || response === void 0 ? void 0 : response.delay) {
102
- this.delay = response.delay; // Set the delay for rate limiting
103
+ this.delay = response.delay;
103
104
  return;
104
105
  }
105
106
  if (response === null || response === void 0 ? void 0 : response.error) {
@@ -111,15 +112,12 @@ class AttachmentsStreamingPool {
111
112
  ? `and content_type ${attachment.content_type} `
112
113
  : '';
113
114
  console.warn(`Skipping attachment with ID ${attachment.id} with extension ${fileExtension} ${fileSizeInfo}${contentTypeInfo}due to error returned by the stream function`, response.error.message);
115
+ this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Failed);
114
116
  await this.updateProgress();
115
117
  continue;
116
118
  }
117
- // No rate limiting, process normally
118
- if ((_c = this.attachmentsMetadata) === null || _c === void 0 ? void 0 : _c.lastProcessedAttachmentsIdsList) {
119
- this.attachmentsMetadata.lastProcessedAttachmentsIdsList.push({
120
- id: attachment.id,
121
- parent_id: attachment.parent_id,
122
- });
119
+ if (!this.adapter.isTimeout) {
120
+ this.recordProcessedAttachment(attachment, state_interfaces_1.ProcessedAttachmentStatus.Success);
123
121
  }
124
122
  await this.updateProgress();
125
123
  }
@@ -15,6 +15,7 @@ export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
15
15
  export declare const MAX_EVENT_SIZE_BYTES = 200000;
16
16
  export declare const EVENT_SIZE_THRESHOLD_BYTES: number;
17
17
  export declare const SSOR_ATTACHMENT: string;
18
+ export declare const UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE";
18
19
  export declare enum AirSyncDefaultItemTypes {
19
20
  EXTERNAL_DOMAIN_METADATA = "external_domain_metadata",
20
21
  ATTACHMENTS = "attachments",
@@ -26,11 +27,9 @@ export declare const HARD_TIMEOUT_MULTIPLIER = 1.3;
26
27
  export declare const MEMORY_LOG_INTERVAL: number;
27
28
  export declare const DEFAULT_SLEEP_DELAY_MS: number;
28
29
  /**
29
- * Sentinel value representing an unbounded (no limit) extraction timestamp.
30
- * Used as the resolved value for TimeValueType.UNBOUNDED, stored as workersOldest
31
- * when the initial import has no lower time bound. The Unix epoch ensures that
32
- * no real extraction timestamp can be earlier, preventing accidental overwrites
33
- * of the boundary by subsequent syncs (e.g. reconciliation with absolute dates).
30
+ * Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED),
31
+ * stored as workersOldest. The Unix epoch guarantees no real timestamp can be
32
+ * earlier, so subsequent syncs cannot accidentally overwrite the boundary.
34
33
  */
35
34
  export declare const UNBOUNDED_DATE_TIME_VALUE = "1970-01-01T00:00:00.000Z";
36
35
  //# sourceMappingURL=constants.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AAEzD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC;AAEpD;;;;;;GAMG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
1
+ {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAGvD,eAAO,MAAM,oBAAoB,SAAU,CAAC;AAE5C,eAAO,MAAM,0BAA0B,QAEtC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAA0B,CAAC;AACzD,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AAEvD,oBAAY,uBAAuB;IACjC,wBAAwB,6BAA6B;IACrD,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC;AAEpD;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,6BAA6B,CAAC"}
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.UNBOUNDED_DATE_TIME_VALUE = exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
3
+ exports.UNBOUNDED_DATE_TIME_VALUE = exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AirSyncDefaultItemTypes = exports.UNKNOWN_EVENT_TYPE = exports.SSOR_ATTACHMENT = exports.EVENT_SIZE_THRESHOLD_BYTES = exports.MAX_EVENT_SIZE_BYTES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
4
4
  const extraction_1 = require("../types/extraction");
5
5
  const helpers_1 = require("./helpers");
6
6
  exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
@@ -45,12 +45,13 @@ exports.STATEFUL_EVENT_TYPES = [
45
45
  exports.ARTIFACT_BATCH_SIZE = 2000;
46
46
  exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
47
47
  exports.MAX_DEVREV_FILENAME_LENGTH = 256;
48
- exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20; // 20 characters for the file extension
48
+ exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
49
49
  // Max SQS message size is 250KB, we want to leave some room for the other data in the message
50
50
  exports.MAX_EVENT_SIZE_BYTES = 200000;
51
51
  // We want to leave some room for the other data in the message and process the rest of queued messages
52
52
  exports.EVENT_SIZE_THRESHOLD_BYTES = Math.floor(exports.MAX_EVENT_SIZE_BYTES * 0.8);
53
53
  exports.SSOR_ATTACHMENT = 'ssor_attachment';
54
+ exports.UNKNOWN_EVENT_TYPE = 'UNKNOWN_EVENT_TYPE';
54
55
  var AirSyncDefaultItemTypes;
55
56
  (function (AirSyncDefaultItemTypes) {
56
57
  AirSyncDefaultItemTypes["EXTERNAL_DOMAIN_METADATA"] = "external_domain_metadata";
@@ -63,10 +64,8 @@ exports.HARD_TIMEOUT_MULTIPLIER = 1.3;
63
64
  exports.MEMORY_LOG_INTERVAL = 30 * 1000; // 30 seconds
64
65
  exports.DEFAULT_SLEEP_DELAY_MS = 3 * 60 * 1000; // 3 minutes
65
66
  /**
66
- * Sentinel value representing an unbounded (no limit) extraction timestamp.
67
- * Used as the resolved value for TimeValueType.UNBOUNDED, stored as workersOldest
68
- * when the initial import has no lower time bound. The Unix epoch ensures that
69
- * no real extraction timestamp can be earlier, preventing accidental overwrites
70
- * of the boundary by subsequent syncs (e.g. reconciliation with absolute dates).
67
+ * Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED),
68
+ * stored as workersOldest. The Unix epoch guarantees no real timestamp can be
69
+ * earlier, so subsequent syncs cannot accidentally overwrite the boundary.
71
70
  */
72
71
  exports.UNBOUNDED_DATE_TIME_VALUE = '1970-01-01T00:00:00.000Z';
@@ -1,30 +1,10 @@
1
- /**
2
- * Gets the library version from the package.json file.
3
- * @returns {string} The library version
4
- */
5
1
  export declare function getLibraryVersion(): any;
6
- /**
7
- * Sleeps for a given number of milliseconds.
8
- * @param {number} ms - The number of milliseconds to sleep
9
- * @returns {Promise<void>} A promise that resolves after the given number of milliseconds
10
- */
11
2
  export declare function sleep(ms: number): Promise<unknown>;
12
3
  /**
13
- * Truncates a filename if it exceeds the maximum allowed length.
14
- * @param {string} filename - The filename to truncate
15
- * @returns {string} The truncated filename
16
- */
17
- export declare function truncateFilename(filename: string): string;
18
- /**
19
- * MemoryInfo is an interface that represents the memory usage information.
20
- * @interface MemoryInfo
21
- * @property {string} rssUsedMB - The RSS used in MB
22
- * @property {string} rssUsedPercent - The RSS used percentage
23
- * @property {string} heapUsedPercent - The heap used percentage
24
- * @property {string} externalMB - The external memory used in MB
25
- * @property {string} arrayBuffersMB - The array buffers memory used in MB
26
- * @property {string} formattedMessage - The formatted message
4
+ * Yields once to the event loop so pending events (e.g. worker messages) can
5
+ * be delivered before continuing.
27
6
  */
7
+ export declare function yieldToEventLoop(): Promise<void>;
28
8
  export interface MemoryInfo {
29
9
  rssUsedMB: string;
30
10
  rssUsedPercent: string;
@@ -33,17 +13,5 @@ export interface MemoryInfo {
33
13
  arrayBuffersMB: string;
34
14
  formattedMessage: string;
35
15
  }
36
- /**
37
- * Gets the memory usage information.
38
- * @returns {MemoryInfo} The memory usage information
39
- */
40
16
  export declare function getMemoryUsage(): MemoryInfo;
41
- /**
42
- * Truncates a message if it exceeds the maximum allowed length.
43
- * Adds a suffix indicating how many characters were omitted.
44
- *
45
- * @param message - The message to truncate
46
- * @returns Truncated message or original if within limits
47
- */
48
- export declare function truncateMessage(message: string): string;
49
17
  //# sourceMappingURL=helpers.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AAUA;;;GAGG;AACH,wBAAgB,iBAAiB,QAiBhC;AAED;;;;GAIG;AACH,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,oBAGrC;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,UAAU,CAmD3C;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAOvD"}
1
+ {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AAIA,wBAAgB,iBAAiB,QAiBhC;AAED,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,oBAErC;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAEtD;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,cAAc,IAAI,UAAU,CA+C3C"}
@@ -35,18 +35,11 @@ var __importStar = (this && this.__importStar) || (function () {
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.getLibraryVersion = getLibraryVersion;
37
37
  exports.sleep = sleep;
38
- exports.truncateFilename = truncateFilename;
38
+ exports.yieldToEventLoop = yieldToEventLoop;
39
39
  exports.getMemoryUsage = getMemoryUsage;
40
- exports.truncateMessage = truncateMessage;
41
40
  const fs_1 = require("fs");
42
41
  const path = __importStar(require("path"));
43
42
  const v8 = __importStar(require("v8"));
44
- const constants_1 = require("./constants");
45
- const logger_constants_1 = require("../logger/logger.constants");
46
- /**
47
- * Gets the library version from the package.json file.
48
- * @returns {string} The library version
49
- */
50
43
  function getLibraryVersion() {
51
44
  var _a;
52
45
  try {
@@ -61,37 +54,16 @@ function getLibraryVersion() {
61
54
  return '';
62
55
  }
63
56
  }
64
- /**
65
- * Sleeps for a given number of milliseconds.
66
- * @param {number} ms - The number of milliseconds to sleep
67
- * @returns {Promise<void>} A promise that resolves after the given number of milliseconds
68
- */
69
57
  async function sleep(ms) {
70
- console.log(`Sleeping for ${ms}ms.`);
71
58
  return new Promise((resolve) => setTimeout(resolve, ms));
72
59
  }
73
60
  /**
74
- * Truncates a filename if it exceeds the maximum allowed length.
75
- * @param {string} filename - The filename to truncate
76
- * @returns {string} The truncated filename
61
+ * Yields once to the event loop so pending events (e.g. worker messages) can
62
+ * be delivered before continuing.
77
63
  */
78
- function truncateFilename(filename) {
79
- // If the filename is already within the limit, return it as is.
80
- if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
81
- return filename;
82
- }
83
- console.warn(`Filename length exceeds the maximum limit of ${constants_1.MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.`);
84
- const extension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
85
- // Calculate how many characters are available for the name part after accounting for the extension and "..."
86
- const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..."
87
- // Truncate the name part and add an ellipsis
88
- const truncatedFilename = filename.slice(0, availableNameLength);
89
- return `${truncatedFilename}...${extension}`;
64
+ async function yieldToEventLoop() {
65
+ return new Promise((resolve) => setImmediate(resolve));
90
66
  }
91
- /**
92
- * Gets the memory usage information.
93
- * @returns {MemoryInfo} The memory usage information
94
- */
95
67
  function getMemoryUsage() {
96
68
  try {
97
69
  const memUsage = process.memoryUsage();
@@ -99,16 +71,12 @@ function getMemoryUsage() {
99
71
  const rssUsedMB = memUsage.rss / 1024 / 1024;
100
72
  const heapLimitMB = heapStats.heap_size_limit / 1024 / 1024;
101
73
  const effectiveMemoryLimitMB = heapLimitMB;
102
- // Calculate heap values for consistent format
103
74
  const heapUsedMB = heapStats.used_heap_size / 1024 / 1024;
104
75
  const heapTotalMB = heapStats.heap_size_limit / 1024 / 1024;
105
- // Calculate external and buffer values (critical for detecting stream leaks)
106
76
  const externalMB = memUsage.external / 1024 / 1024;
107
77
  const arrayBuffersMB = memUsage.arrayBuffers / 1024 / 1024;
108
- // Critical percentages for OOM detection
109
78
  const rssUsedPercent = ((rssUsedMB / effectiveMemoryLimitMB) * 100).toFixed(2) + '%';
110
79
  const heapUsedPercent = ((heapStats.used_heap_size / heapStats.heap_size_limit) * 100).toFixed(2) + '%';
111
- // Detailed message showing RSS breakdown for leak detection
112
80
  const formattedMessage = `Memory: RSS ${rssUsedMB.toFixed(2)}/${effectiveMemoryLimitMB.toFixed(2)}MB (${rssUsedPercent}) [Heap ${heapUsedMB.toFixed(2)}/${heapTotalMB.toFixed(2)}MB (${heapUsedPercent}) + External ${externalMB.toFixed(2)}MB + Buffers ${arrayBuffersMB.toFixed(2)}MB].`;
113
81
  return {
114
82
  rssUsedMB: rssUsedMB.toFixed(2),
@@ -124,16 +92,3 @@ function getMemoryUsage() {
124
92
  throw err;
125
93
  }
126
94
  }
127
- /**
128
- * Truncates a message if it exceeds the maximum allowed length.
129
- * Adds a suffix indicating how many characters were omitted.
130
- *
131
- * @param message - The message to truncate
132
- * @returns Truncated message or original if within limits
133
- */
134
- function truncateMessage(message) {
135
- if (message.length > logger_constants_1.MAX_LOG_STRING_LENGTH) {
136
- return `${message.substring(0, logger_constants_1.MAX_LOG_STRING_LENGTH)}... ${message.length - logger_constants_1.MAX_LOG_STRING_LENGTH} more characters`;
137
- }
138
- return message;
139
- }
@@ -1,3 +1,3 @@
1
1
  declare const axiosClient: import("axios").AxiosInstance;
2
2
  export { axiosClient };
3
- //# sourceMappingURL=axios-client-internal.d.ts.map
3
+ //# sourceMappingURL=client.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/http/client.ts"],"names":[],"mappings":"AAGA,QAAA,MAAM,WAAW,+BAEf,CAAC;AAmFH,OAAO,EAAE,WAAW,EAAE,CAAC"}
@@ -34,11 +34,10 @@ exports.axiosClient = axiosClient;
34
34
  var _a, _b, _c, _d, _e, _f, _g;
35
35
  const retryAfter = ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['retry-after']) ||
36
36
  ((_d = (_c = error.response) === null || _c === void 0 ? void 0 : _c.headers) === null || _d === void 0 ? void 0 : _d['Retry-After']);
37
- // 5xx errors
38
37
  if (((_e = error.response) === null || _e === void 0 ? void 0 : _e.status) && error.response.status >= 500) {
39
38
  return true;
40
39
  }
41
- // 429 errors when retry-after header is present
40
+ // 429 only when a valid non-negative Retry-After header is present
42
41
  else if (((_f = error.response) === null || _f === void 0 ? void 0 : _f.status) &&
43
42
  error.response.status === 429 &&
44
43
  retryAfter &&
@@ -46,19 +45,16 @@ exports.axiosClient = axiosClient;
46
45
  Number(retryAfter) >= 0) {
47
46
  return true;
48
47
  }
49
- // Network errors for idempotent requests if not 429, because 429 is handled above
48
+ // Network errors for idempotent requests; 429 already handled above
50
49
  else if (axios_retry_1.default.isNetworkOrIdempotentRequestError(error) &&
51
50
  ((_g = error.response) === null || _g === void 0 ? void 0 : _g.status) !== 429) {
52
51
  return true;
53
52
  }
54
- // Request timeout errors (ECONNABORTED) — axios-retry explicitly excludes
55
- // ECONNABORTED from isNetworkError, so we handle it here separately.
56
- // Axios only produces ECONNABORTED on client-side timeouts and browser
57
- // cancellations, never on server responses, so no response guard is needed.
53
+ // axios-retry excludes ECONNABORTED (client-side timeout only, never a
54
+ // server response) from isNetworkError, so handle it here separately.
58
55
  else if (error.code === 'ECONNABORTED') {
59
56
  return true;
60
57
  }
61
- // all other errors
62
58
  else {
63
59
  return false;
64
60
  }
package/dist/index.d.ts CHANGED
@@ -1,16 +1,27 @@
1
- export { AirSyncDefaultItemTypes } from './common/constants';
2
- export { ExtractionCommonError } from './common/errors';
3
- export * from './common/install-initial-domain-mapping';
4
- export * from './http';
5
- export { serializeAxiosError } from './logger/logger';
6
- export { MockServer } from './mock-server/mock-server';
7
- export type { RequestInfo, RetryConfig, RouteConfig, } from './mock-server/mock-server.interfaces';
8
- export { processExtractionTask, processLoadingTask, } from './multithreading/process-task';
9
- export { spawn } from './multithreading/spawn/spawn';
10
1
  export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter';
11
2
  export { LoadingAdapter } from './multithreading/adapters/loading-adapter';
12
- export { createMockEvent, MOCK_SERVER_DEFAULT_URL } from './common/test-utils';
13
- export type { DeepPartial } from './common/test-utils';
14
- export * from './types';
15
- export * from './types/workers';
3
+ export { processExtractionTask, processLoadingTask, } from './multithreading/process-task';
4
+ export { spawn } from './multithreading/spawn/spawn';
5
+ export type { ExtractionScope, ProcessTaskInterface, SpawnFactoryInterface, SpawnInterface, TaskAdapterInterface, TaskResult, TaskStatus, WorkerAdapterInterface, WorkerAdapterOptions, WorkerPathOverrides, } from './types/workers';
6
+ export { AirSyncDefaultItemTypes } from './common/constants';
7
+ export { UNBOUNDED_DATE_TIME_VALUE } from './common/constants';
8
+ export { ExtractionCommonError } from './types/errors';
9
+ export { installInitialDomainMapping } from './state/install-initial-domain-mapping';
10
+ export { serializeError } from './logger/logger';
11
+ export type { ErrorRecord, InitialDomainMapping } from './types/common';
12
+ export { SyncMode } from './types/common';
13
+ export type { AirSyncEvent, AirSyncMessage, ConnectionData, EventContext, EventData, ExternalProcessAttachmentFunction, ExternalSyncUnit, ExternalSystemAttachmentIteratorFunction, ExternalSystemAttachmentReducerFunction, ExternalSystemAttachmentStreamingFunction, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExtractorEvent, HttpStreamResponse, ProcessAttachmentReturnType, TimeValue, } from './types/extraction';
14
+ export { EventType, ExtractorEventType, InitialSyncScope, TimeUnit, TimeValueType, } from './types/extraction';
15
+ export type { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemItemLoadingParams, ExternalSystemItemLoadingResponse, ItemTypeToLoad, } from './types/loading';
16
+ export { LoaderEventType } from './types/loading';
17
+ export type { Item, NormalizedAttachment, NormalizedItem, RepoInterface, } from './repo/repo.interfaces';
18
+ export { Mappers } from './mappers/mappers';
19
+ export type { MappersCreateParams, MappersGetByExternalIdParams, MappersGetByTargetIdParams, MappersUpdateParams, } from './mappers/mappers.interfaces';
20
+ export { SyncMapperRecordStatus, SyncMapperRecordTargetType, } from './mappers/mappers.interfaces';
21
+ export type { Artifact, ArtifactsPrepareResponse, SsorAttachment, StreamAttachmentsResponse, StreamResponse, UploadResponse, } from './uploader/uploader.interfaces';
22
+ export type { CollectionData, ConditionalPrivilegeData, CustomLinkData, CustomLinkNames, CustomStage, CustomState, EnumData, EnumValue, EnumValueKey, ExternalDomainMetadata, Field, FieldCondition, FieldConditionComparator, FieldConditionEffect, FieldConditions, FieldKey, FieldPrivilegeData, FieldReferenceData, FieldType, FloatData, IntData, PermissionData, RecordType, RecordTypeCategory, RecordTypeCategoryKey, RecordTypeKey, RecordTypePrivilegeData, RecordTypeScope, ReferenceData, ReferenceDetail, ReferenceType, SchemaVersion, StageDiagram, StageKey, StateKey, StructData, StructType, StructTypeKey, TargetTypeKeyData, TextData, TypedReferenceData, } from './types/external-domain-metadata';
23
+ export type { DeepPartial } from './testing/mock-event';
24
+ export { createMockEvent } from './testing/mock-event';
25
+ export { MOCK_SERVER_DEFAULT_URL, MockServer } from './testing/mock-server';
26
+ export type { RequestInfo, RetryConfig, RouteConfig, } from './testing/mock-server.interfaces';
16
27
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,cAAc,yCAAyC,CAAC;AACxD,cAAc,QAAQ,CAAC;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,YAAY,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,cAAc,SAAS,CAAC;AACxB,cAAc,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EACL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AAGrD,YAAY,EACV,eAAe,EACf,oBAAoB,EACpB,qBAAqB,EACrB,cAAc,EACd,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAGvD,OAAO,EAAE,2BAA2B,EAAE,MAAM,wCAAwC,CAAC;AAGrF,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,SAAS,EACT,iCAAiC,EACjC,gBAAgB,EAChB,wCAAwC,EACxC,uCAAuC,EACvC,yCAAyC,EACzC,uCAAuC,EACvC,yCAAyC,EACzC,cAAc,EACd,kBAAkB,EAClB,2BAA2B,EAC3B,SAAS,GACV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,gBAAgB,EAChB,QAAQ,EACR,aAAa,GACd,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,wBAAwB,EACxB,kBAAkB,EAClB,+BAA+B,EAC/B,iCAAiC,EACjC,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,YAAY,EACV,IAAI,EACJ,oBAAoB,EACpB,cAAc,EACd,aAAa,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,4BAA4B,EAC5B,0BAA0B,EAC1B,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,QAAQ,EACR,wBAAwB,EACxB,cAAc,EACd,yBAAyB,EACzB,cAAc,EACd,cAAc,GACf,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,eAAe,EACf,WAAW,EACX,WAAW,EACX,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,sBAAsB,EACtB,KAAK,EACL,cAAc,EACd,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,kBAAkB,EAClB,kBAAkB,EAClB,SAAS,EACT,SAAS,EACT,OAAO,EACP,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAG1C,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EACV,WAAW,EACX,WAAW,EACX,WAAW,GACZ,MAAM,kCAAkC,CAAC"}