@devrev/airsync-sdk 2.0.0-beta.4 → 2.0.0-beta.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/MIGRATION.md +1028 -0
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +6 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +2 -2
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
  6. package/dist/attachments-streaming/attachments-streaming-pool.js +47 -21
  7. package/dist/common/constants.d.ts +3 -5
  8. package/dist/common/constants.d.ts.map +1 -1
  9. package/dist/common/constants.js +4 -6
  10. package/dist/common/helpers.d.ts +3 -21
  11. package/dist/common/helpers.d.ts.map +1 -1
  12. package/dist/common/helpers.js +6 -15
  13. package/dist/http/client.d.ts.map +1 -1
  14. package/dist/http/client.js +4 -8
  15. package/dist/index.d.ts +11 -11
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +11 -18
  18. package/dist/logger/logger.d.ts +8 -63
  19. package/dist/logger/logger.d.ts.map +1 -1
  20. package/dist/logger/logger.interfaces.d.ts +0 -1
  21. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  22. package/dist/logger/logger.js +14 -93
  23. package/dist/mappers/mappers.d.ts +2 -40
  24. package/dist/mappers/mappers.d.ts.map +1 -1
  25. package/dist/mappers/mappers.interfaces.d.ts +35 -154
  26. package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
  27. package/dist/mappers/mappers.interfaces.js +1 -10
  28. package/dist/mappers/mappers.js +2 -40
  29. package/dist/multithreading/adapters/base-adapter.d.ts +19 -36
  30. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  31. package/dist/multithreading/adapters/base-adapter.js +73 -71
  32. package/dist/multithreading/adapters/extraction-adapter.d.ts +12 -24
  33. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  34. package/dist/multithreading/adapters/extraction-adapter.js +199 -200
  35. package/dist/multithreading/adapters/loading-adapter.d.ts +1 -6
  36. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  37. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts +2 -12
  38. package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map +1 -1
  39. package/dist/multithreading/adapters/loading-adapter.helpers.js +2 -12
  40. package/dist/multithreading/adapters/loading-adapter.js +292 -314
  41. package/dist/multithreading/emit.d.ts +3 -2
  42. package/dist/multithreading/emit.d.ts.map +1 -1
  43. package/dist/multithreading/emit.js +3 -5
  44. package/dist/multithreading/process-task.d.ts +2 -12
  45. package/dist/multithreading/process-task.d.ts.map +1 -1
  46. package/dist/multithreading/process-task.js +45 -59
  47. package/dist/multithreading/spawn/spawn.d.ts +3 -9
  48. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  49. package/dist/multithreading/spawn/spawn.helpers.d.ts +6 -25
  50. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  51. package/dist/multithreading/spawn/spawn.helpers.js +7 -49
  52. package/dist/multithreading/spawn/spawn.js +18 -33
  53. package/dist/multithreading/worker.js +2 -4
  54. package/dist/repo/repo.d.ts +10 -0
  55. package/dist/repo/repo.d.ts.map +1 -1
  56. package/dist/repo/repo.helpers.d.ts +6 -0
  57. package/dist/repo/repo.helpers.d.ts.map +1 -0
  58. package/dist/repo/repo.helpers.js +16 -0
  59. package/dist/repo/repo.interfaces.d.ts +4 -16
  60. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  61. package/dist/repo/repo.js +23 -8
  62. package/dist/state/base-state.d.ts +14 -36
  63. package/dist/state/base-state.d.ts.map +1 -1
  64. package/dist/state/base-state.js +16 -38
  65. package/dist/state/extraction-state.d.ts +4 -19
  66. package/dist/state/extraction-state.d.ts.map +1 -1
  67. package/dist/state/extraction-state.js +6 -23
  68. package/dist/state/install-initial-domain-mapping.d.ts +1 -1
  69. package/dist/state/install-initial-domain-mapping.d.ts.map +1 -1
  70. package/dist/state/install-initial-domain-mapping.js +1 -3
  71. package/dist/state/loading-state.d.ts +1 -11
  72. package/dist/state/loading-state.d.ts.map +1 -1
  73. package/dist/state/loading-state.js +2 -12
  74. package/dist/state/state.d.ts +2 -2
  75. package/dist/state/state.d.ts.map +1 -1
  76. package/dist/state/state.interfaces.d.ts +10 -24
  77. package/dist/state/state.interfaces.d.ts.map +1 -1
  78. package/dist/state/state.interfaces.js +9 -11
  79. package/dist/state/state.js +3 -3
  80. package/dist/state/time-value-resolver.d.ts +7 -36
  81. package/dist/state/time-value-resolver.d.ts.map +1 -1
  82. package/dist/state/time-value-resolver.js +7 -40
  83. package/dist/testing/mock-event.d.ts +2 -10
  84. package/dist/testing/mock-event.d.ts.map +1 -1
  85. package/dist/testing/mock-event.js +3 -11
  86. package/dist/testing/mock-server.d.ts +4 -27
  87. package/dist/testing/mock-server.d.ts.map +1 -1
  88. package/dist/testing/mock-server.interfaces.d.ts +10 -35
  89. package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
  90. package/dist/testing/mock-server.js +19 -36
  91. package/dist/types/common.d.ts +1 -12
  92. package/dist/types/common.d.ts.map +1 -1
  93. package/dist/types/common.js +1 -6
  94. package/dist/types/external-domain-metadata.d.ts +30 -171
  95. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  96. package/dist/types/extraction.d.ts +35 -120
  97. package/dist/types/extraction.d.ts.map +1 -1
  98. package/dist/types/extraction.js +5 -31
  99. package/dist/types/loading.d.ts.map +1 -1
  100. package/dist/types/workers.d.ts +18 -116
  101. package/dist/types/workers.d.ts.map +1 -1
  102. package/dist/types/workers.js +0 -6
  103. package/dist/uploader/uploader.d.ts +6 -58
  104. package/dist/uploader/uploader.d.ts.map +1 -1
  105. package/dist/uploader/uploader.helpers.d.ts +4 -27
  106. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  107. package/dist/uploader/uploader.helpers.js +64 -30
  108. package/dist/uploader/uploader.interfaces.d.ts +15 -25
  109. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  110. package/dist/uploader/uploader.interfaces.js +8 -0
  111. package/dist/uploader/uploader.js +18 -84
  112. package/package.json +6 -4
@@ -6,8 +6,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.Uploader = void 0;
7
7
  const form_data_1 = __importDefault(require("form-data"));
8
8
  const js_jsonl_1 = require("js-jsonl");
9
- const client_1 = require("../http/client");
10
9
  const constants_1 = require("../common/constants");
10
+ const client_1 = require("../http/client");
11
11
  const logger_1 = require("../logger/logger");
12
12
  const uploader_helpers_1 = require("./uploader.helpers");
13
13
  class Uploader {
@@ -22,17 +22,11 @@ class Uploader {
22
22
  Authorization: `Bearer ${this.devrevApiToken}`,
23
23
  };
24
24
  }
25
- /**
26
- * Uploads the fetched objects to the DevRev platform. Fetched objects are compressed to a gzipped jsonl object and uploaded to the platform.
27
- * @param {string} itemType - The type of the item to be uploaded
28
- * @param {object[] | object} fetchedObjects - The objects to be uploaded
29
- * @returns {Promise<UploadResponse>} - The response object containing the artifact information or error information if there was an error
30
- */
25
+ /** Compresses fetched objects to gzipped JSONL and uploads them as an artifact. */
31
26
  async upload(itemType, fetchedObjects) {
32
27
  if (this.isLocalDevelopment) {
33
28
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
34
29
  }
35
- // Compress the fetched objects to a gzipped jsonl object
36
30
  const { response: file, error: fileError } = (0, uploader_helpers_1.compressGzip)(js_jsonl_1.jsonl.stringify(fetchedObjects));
37
31
  if (fileError) {
38
32
  return {
@@ -44,7 +38,6 @@ class Uploader {
44
38
  }
45
39
  const filename = itemType + '.jsonl.gz';
46
40
  const fileType = 'application/x-gzip';
47
- // Get upload url
48
41
  const { error: preparedArtifactError, response: preparedArtifact } = await this.getArtifactUploadUrl(filename, fileType);
49
42
  if (preparedArtifactError) {
50
43
  return {
@@ -54,7 +47,6 @@ class Uploader {
54
47
  },
55
48
  };
56
49
  }
57
- // Upload prepared artifact to the given url
58
50
  const { error: uploadItemError } = await this.uploadArtifact(preparedArtifact, file);
59
51
  if (uploadItemError) {
60
52
  return {
@@ -64,10 +56,9 @@ class Uploader {
64
56
  },
65
57
  };
66
58
  }
67
- // Skip confirmation for External Sync Units, as this confirmation attachments
68
- // uploads to the sync, which we haven't created yet when extracting External Sync Units.
59
+ // Skip confirmation for External Sync Units: confirmation attaches the upload
60
+ // to the sync, which doesn't exist yet when extracting External Sync Units.
69
61
  if (!this.skipConfirmation) {
70
- // Confirm upload
71
62
  const { error: confirmArtifactUploadError } = await this.confirmArtifactUpload(preparedArtifact.artifact_id);
72
63
  if (confirmArtifactUploadError) {
73
64
  return {
@@ -78,21 +69,10 @@ class Uploader {
78
69
  };
79
70
  }
80
71
  }
81
- // Return the artifact information to the platform
82
- const artifact = {
83
- id: preparedArtifact.artifact_id,
84
- item_type: itemType,
85
- item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1,
86
- };
72
+ const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
73
+ const artifact = Object.assign({ id: preparedArtifact.artifact_id, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
87
74
  return { artifact };
88
75
  }
89
- /**
90
- * Gets the upload URL for an artifact from the DevRev API.
91
- * @param {string} filename - The name of the file to upload
92
- * @param {string} fileType - The MIME type of the file
93
- * @param {number} [fileSize] - Optional file size in bytes
94
- * @returns {Promise<ArtifactToUpload | void>} The artifact upload information or undefined on error
95
- */
96
76
  async getArtifactUploadUrl(filename, fileType, fileSize) {
97
77
  const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.upload-url`;
98
78
  if (fileSize !== undefined && fileSize <= 0) {
@@ -116,12 +96,7 @@ class Uploader {
116
96
  return { error };
117
97
  }
118
98
  }
119
- /**
120
- * Uploads an artifact file to the provided upload URL using multipart form data.
121
- * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
122
- * @param {Buffer} file - The file buffer to upload
123
- * @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
124
- */
99
+ /** Uploads the file buffer to the artifact's upload URL as multipart form data. */
125
100
  async uploadArtifact(artifact, file) {
126
101
  const formData = new form_data_1.default();
127
102
  for (const field in artifact.form_data) {
@@ -138,30 +113,24 @@ class Uploader {
138
113
  return { error };
139
114
  }
140
115
  }
141
- /**
142
- * Streams an artifact file from an HTTP streaming response to the upload URL.
143
- * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data
144
- * @param {HttpStreamResponse} fileStream - The HTTP streaming response containing the file data
145
- * @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
146
- */
116
+ /** Streams a file from an HTTP response directly to the artifact's upload URL. */
147
117
  async streamArtifact(artifact, fileStream) {
148
118
  const formData = new form_data_1.default();
149
119
  for (const field in artifact.form_data) {
150
120
  formData.append(field, artifact.form_data[field]);
151
121
  }
152
122
  formData.append('file', fileStream.data);
123
+ const hasContentLength = !!fileStream.headers['content-length'];
153
124
  try {
154
- const response = await client_1.axiosClient.post(artifact.upload_url, formData, {
155
- headers: Object.assign(Object.assign({}, formData.getHeaders()), (!fileStream.headers['content-length']
125
+ const response = await client_1.axiosClient.post(artifact.upload_url, formData, Object.assign({ headers: Object.assign(Object.assign({}, formData.getHeaders()), (!hasContentLength
156
126
  ? {
157
127
  'Content-Length': constants_1.MAX_DEVREV_ARTIFACT_SIZE,
158
128
  }
159
- : {})),
160
- // Prevents buffering of the response in the memory
161
- maxRedirects: 0,
162
- // Allow 2xx and 3xx (redirects) to be considered successful, 4xx and 5xx will throw errors and be caught in the catch block
163
- validateStatus: (status) => status >= 200 && status < 400,
164
- });
129
+ : {})),
130
+ // Prevents buffering of the response in memory
131
+ maxRedirects: 0,
132
+ // 2xx and 3xx are success; 4xx/5xx throw into the catch block
133
+ validateStatus: (status) => status >= 200 && status < 400 }, (!hasContentLength ? { 'axios-retry': { retries: 0 } } : {})));
165
134
  this.destroyStream(fileStream);
166
135
  return { response };
167
136
  }
@@ -170,11 +139,6 @@ class Uploader {
170
139
  return { error };
171
140
  }
172
141
  }
173
- /**
174
- * Confirms that an artifact upload has been completed successfully.
175
- * @param {string} artifactId - The ID of the artifact to confirm
176
- * @returns {Promise<AxiosResponse | void>} The axios response or undefined on error
177
- */
178
142
  async confirmArtifactUpload(artifactId) {
179
143
  const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.confirm-upload`;
180
144
  try {
@@ -184,7 +148,6 @@ class Uploader {
184
148
  }, {
185
149
  headers: Object.assign({}, this.defaultHeaders),
186
150
  });
187
- // If response exists and the status is 2xx, return the response
188
151
  if ((response === null || response === void 0 ? void 0 : response.status) >= 200 && (response === null || response === void 0 ? void 0 : response.status) < 300) {
189
152
  return { response };
190
153
  }
@@ -201,14 +164,10 @@ class Uploader {
201
164
  return { error: { message: (0, logger_1.serializeError)(error) } };
202
165
  }
203
166
  }
204
- /**
205
- * Destroys a stream to prevent resource leaks.
206
- * @param {HttpStreamResponse} fileStream - The HTTP streaming response to destroy
207
- */
208
167
  destroyStream(fileStream) {
209
168
  try {
210
169
  if (fileStream && fileStream.data) {
211
- // For axios response streams, the data property contains the actual stream
170
+ // For axios response streams, `data` holds the actual stream
212
171
  if (typeof fileStream.data.destroy === 'function') {
213
172
  fileStream.data.destroy();
214
173
  }
@@ -221,14 +180,8 @@ class Uploader {
221
180
  console.warn('Error while destroying stream:', (0, logger_1.serializeError)(error));
222
181
  }
223
182
  }
224
- /**
225
- * Retrieves attachment metadata from an artifact by downloading and parsing it.
226
- * @param {object} param0 - Configuration object
227
- * @param {string} param0.artifact - The artifact ID to download attachments from
228
- * @returns {Promise<{attachments?: NormalizedAttachment[], error?: {message: string}}>} The attachments array or error object
229
- */
183
+ /** Downloads an artifact and parses it into attachment metadata. */
230
184
  async getAttachmentsFromArtifactId({ artifact, }) {
231
- // Get the URL of the attachments metadata artifact
232
185
  const { response: artifactUrl, error: artifactUrlError } = await this.getArtifactDownloadUrl(artifact);
233
186
  if (artifactUrlError) {
234
187
  return {
@@ -238,7 +191,6 @@ class Uploader {
238
191
  },
239
192
  };
240
193
  }
241
- // Download artifact from the URL
242
194
  const { response: gzippedJsonlObject, error: gzippedJsonlObjectError } = await this.downloadArtifact(artifactUrl);
243
195
  if (gzippedJsonlObjectError) {
244
196
  return {
@@ -248,7 +200,6 @@ class Uploader {
248
200
  },
249
201
  };
250
202
  }
251
- // Decompress the gzipped jsonl object
252
203
  const { response: jsonlObject, error: jsonlObjectError } = (0, uploader_helpers_1.decompressGzip)(gzippedJsonlObject);
253
204
  if (jsonlObjectError) {
254
205
  return {
@@ -258,7 +209,6 @@ class Uploader {
258
209
  },
259
210
  };
260
211
  }
261
- // Parse the jsonl object to get the attachment metadata
262
212
  const { response: jsonObject, error: jsonObjectError } = (0, uploader_helpers_1.parseJsonl)(jsonlObject);
263
213
  if (jsonObjectError) {
264
214
  return {
@@ -270,11 +220,6 @@ class Uploader {
270
220
  }
271
221
  return { attachments: jsonObject };
272
222
  }
273
- /**
274
- * Gets the download URL for an artifact from the DevRev API.
275
- * @param {string} artifactId - The ID of the artifact to download
276
- * @returns {Promise<string | void>} The download URL or undefined on error
277
- */
278
223
  async getArtifactDownloadUrl(artifactId) {
279
224
  const url = `${this.devrevApiEndpoint}/internal/airdrop.artifacts.download-url`;
280
225
  try {
@@ -291,11 +236,6 @@ class Uploader {
291
236
  return { error };
292
237
  }
293
238
  }
294
- /**
295
- * Downloads an artifact file from the given URL.
296
- * @param {string} artifactUrl - The URL to download the artifact from
297
- * @returns {Promise<Buffer | void>} The artifact file buffer or undefined on error
298
- */
299
239
  async downloadArtifact(artifactUrl) {
300
240
  try {
301
241
  const response = await client_1.axiosClient.get(artifactUrl, {
@@ -307,13 +247,7 @@ class Uploader {
307
247
  return { error };
308
248
  }
309
249
  }
310
- /**
311
- * Retrieves and parses JSON objects from an artifact by artifact ID.
312
- * @param {object} param0 - Configuration object
313
- * @param {string} param0.artifactId - The artifact ID to download and parse
314
- * @param {boolean} [param0.isGzipped=false] - Whether the artifact is gzipped
315
- * @returns {Promise<object[] | object | void>} The parsed JSON objects or undefined on error
316
- */
250
+ /** Downloads an artifact (optionally gzipped) and parses it as JSONL. */
317
251
  async getJsonObjectByArtifactId({ artifactId, isGzipped = false, }) {
318
252
  const { response: artifactUrl, error: artifactUrlError } = await this.getArtifactDownloadUrl(artifactId);
319
253
  if (artifactUrlError) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@devrev/airsync-sdk",
3
- "version": "2.0.0-beta.4",
3
+ "version": "2.0.0-beta.6",
4
4
  "description": "Typescript SDK for building AirSync snap-ins on the DevRev platform.",
5
5
  "type": "commonjs",
6
6
  "main": "./dist/index.js",
@@ -38,6 +38,7 @@
38
38
  "eslint": "9.32.0",
39
39
  "eslint-config-prettier": "^9.1.2",
40
40
  "eslint-plugin-prettier": "4.0.0",
41
+ "eslint-plugin-simple-import-sort": "^13.0.0",
41
42
  "jest": "^29.7.0",
42
43
  "jiti": "^2.6.1",
43
44
  "prettier": "^2.8.3",
@@ -46,8 +47,8 @@
46
47
  "typescript-eslint": "^8.46.1"
47
48
  },
48
49
  "dependencies": {
49
- "@devrev/typescript-sdk": "^1.1.74",
50
- "axios": "^1.15.2",
50
+ "@devrev/typescript-sdk": "^1.1.76",
51
+ "axios": "^1.17.0",
51
52
  "axios-retry": "^4.5.0",
52
53
  "form-data": "^4.0.4",
53
54
  "js-jsonl": "^1.1.1",
@@ -56,6 +57,7 @@
56
57
  },
57
58
  "files": [
58
59
  "dist/**/*",
59
- "!dist/tests/**/*"
60
+ "!dist/tests/**/*",
61
+ "MIGRATION.md"
60
62
  ]
61
63
  }