@devrev/ts-adaas 1.19.9-beta.0 → 1.19.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  2. package/dist/attachments-streaming/attachments-streaming-pool.js +6 -4
  3. package/dist/attachments-streaming/attachments-streaming-pool.test.js +65 -0
  4. package/dist/common/control-protocol.d.ts +2 -3
  5. package/dist/common/control-protocol.d.ts.map +1 -1
  6. package/dist/common/control-protocol.js +4 -2
  7. package/dist/http/axios-client-internal.d.ts.map +1 -1
  8. package/dist/http/axios-client-internal.js +1 -19
  9. package/dist/mock-server/mock-server.d.ts +0 -4
  10. package/dist/mock-server/mock-server.d.ts.map +1 -1
  11. package/dist/mock-server/mock-server.interfaces.d.ts +4 -9
  12. package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -1
  13. package/dist/mock-server/mock-server.js +18 -30
  14. package/dist/multithreading/spawn/spawn.d.ts +1 -1
  15. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  16. package/dist/multithreading/spawn/spawn.js +6 -11
  17. package/dist/multithreading/spawn/spawn.test.js +0 -1
  18. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +1 -1
  19. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +5 -11
  20. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
  21. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +9 -90
  22. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +1 -1
  23. package/dist/multithreading/worker-adapter/worker-adapter.js +64 -93
  24. package/dist/repo/repo.d.ts +2 -11
  25. package/dist/repo/repo.d.ts.map +1 -1
  26. package/dist/repo/repo.interfaces.d.ts +0 -2
  27. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  28. package/dist/repo/repo.js +11 -37
  29. package/dist/repo/repo.test.js +0 -184
  30. package/dist/types/extraction.d.ts +1 -8
  31. package/dist/types/extraction.d.ts.map +1 -1
  32. package/dist/types/workers.d.ts +0 -4
  33. package/dist/types/workers.d.ts.map +1 -1
  34. package/dist/uploader/uploader.d.ts.map +1 -1
  35. package/dist/uploader/uploader.helpers.d.ts +1 -6
  36. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  37. package/dist/uploader/uploader.helpers.js +0 -65
  38. package/dist/uploader/uploader.helpers.test.js +7 -148
  39. package/dist/uploader/uploader.interfaces.d.ts +0 -11
  40. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  41. package/dist/uploader/uploader.interfaces.js +0 -11
  42. package/dist/uploader/uploader.js +5 -2
  43. package/dist/uploader/uploader.test.js +3 -54
  44. package/package.json +2 -2
  45. package/dist/common/control-protocol.test.d.ts +0 -2
  46. package/dist/common/control-protocol.test.d.ts.map +0 -1
  47. package/dist/common/control-protocol.test.js +0 -126
  48. package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts +0 -2
  49. package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts.map +0 -1
  50. package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.js +0 -135
@@ -1,9 +1,4 @@
1
- import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
2
- /**
3
- * Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
4
- * @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
5
- */
6
- export declare function computeArtifactDateRanges(fetchedObjects: object[] | object): ArtifactDateRanges;
1
+ import { UploaderResult } from './uploader.interfaces';
7
2
  /**
8
3
  * Compresses a JSONL string using gzip compression.
9
4
  * @param {string} jsonlObject - The JSONL string to compress
@@ -1 +1 @@
1
- {"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
1
+ {"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
@@ -36,7 +36,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
36
36
  return (mod && mod.__esModule) ? mod : { "default": mod };
37
37
  };
38
38
  Object.defineProperty(exports, "__esModule", { value: true });
39
- exports.computeArtifactDateRanges = computeArtifactDateRanges;
40
39
  exports.compressGzip = compressGzip;
41
40
  exports.decompressGzip = decompressGzip;
42
41
  exports.parseJsonl = parseJsonl;
@@ -46,70 +45,6 @@ const fs_1 = __importStar(require("fs"));
46
45
  const js_jsonl_1 = require("js-jsonl");
47
46
  const zlib_1 = __importDefault(require("zlib"));
48
47
  const constants_1 = require("../common/constants");
49
- const uploader_interfaces_1 = require("./uploader.interfaces");
50
- /**
51
- * Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
52
- * @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
53
- */
54
- function computeArtifactDateRanges(fetchedObjects) {
55
- const items = Array.isArray(fetchedObjects)
56
- ? fetchedObjects
57
- : [fetchedObjects];
58
- const created = {
59
- min: Number.POSITIVE_INFINITY,
60
- max: Number.NEGATIVE_INFINITY,
61
- };
62
- const modified = {
63
- min: Number.POSITIVE_INFINITY,
64
- max: Number.NEGATIVE_INFINITY,
65
- };
66
- let hasCreated = false;
67
- let hasModified = false;
68
- const parseTimestamp = (value) => {
69
- const ts = new Date(value).getTime();
70
- return Number.isNaN(ts) ? undefined : ts;
71
- };
72
- for (const obj of items) {
73
- if (!obj || typeof obj !== 'object') {
74
- continue;
75
- }
76
- const item = obj;
77
- if (item.created_date != undefined) {
78
- const ts = parseTimestamp(item.created_date);
79
- if (ts != undefined) {
80
- if (ts < created.min) {
81
- created.min = ts;
82
- }
83
- if (ts > created.max) {
84
- created.max = ts;
85
- }
86
- hasCreated = true;
87
- }
88
- }
89
- if (item.modified_date != undefined) {
90
- const ts = parseTimestamp(item.modified_date);
91
- if (ts != undefined) {
92
- if (ts < modified.min) {
93
- modified.min = ts;
94
- }
95
- if (ts > modified.max) {
96
- modified.max = ts;
97
- }
98
- hasModified = true;
99
- }
100
- }
101
- }
102
- const result = {};
103
- if (hasCreated) {
104
- result[uploader_interfaces_1.ArtifactDateField.OldestCreatedDate] = new Date(created.min).toISOString();
105
- result[uploader_interfaces_1.ArtifactDateField.NewestCreatedDate] = new Date(created.max).toISOString();
106
- }
107
- if (hasModified) {
108
- result[uploader_interfaces_1.ArtifactDateField.OldestModifiedDate] = new Date(modified.min).toISOString();
109
- result[uploader_interfaces_1.ArtifactDateField.NewestModifiedDate] = new Date(modified.max).toISOString();
110
- }
111
- return result;
112
- }
113
48
  /**
114
49
  * Compresses a JSONL string using gzip compression.
115
50
  * @param {string} jsonlObject - The JSONL string to compress
@@ -1,42 +1,10 @@
1
1
  "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
2
  var __importDefault = (this && this.__importDefault) || function (mod) {
36
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
37
4
  };
38
5
  Object.defineProperty(exports, "__esModule", { value: true });
39
- const fs_1 = __importStar(require("fs"));
6
+ const fs_1 = __importDefault(require("fs"));
7
+ const fs_2 = require("fs");
40
8
  const js_jsonl_1 = require("js-jsonl");
41
9
  const zlib_1 = __importDefault(require("zlib"));
42
10
  const constants_1 = require("../common/constants");
@@ -131,7 +99,7 @@ describe('uploader.helpers', () => {
131
99
  close: jest.fn().mockResolvedValue(undefined),
132
100
  };
133
101
  const fsPromisesOpenSpy = jest
134
- .spyOn(fs_1.promises, 'open')
102
+ .spyOn(fs_2.promises, 'open')
135
103
  .mockResolvedValueOnce(mockFileHandle);
136
104
  // Act
137
105
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
@@ -150,7 +118,7 @@ describe('uploader.helpers', () => {
150
118
  close: jest.fn().mockResolvedValue(undefined),
151
119
  };
152
120
  const fsPromisesOpenSpy = jest
153
- .spyOn(fs_1.promises, 'open')
121
+ .spyOn(fs_2.promises, 'open')
154
122
  .mockResolvedValueOnce(mockFileHandle);
155
123
  // Act
156
124
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
@@ -171,7 +139,7 @@ describe('uploader.helpers', () => {
171
139
  close: jest.fn().mockResolvedValue(undefined),
172
140
  };
173
141
  const fsPromisesOpenSpy = jest
174
- .spyOn(fs_1.promises, 'open')
142
+ .spyOn(fs_2.promises, 'open')
175
143
  .mockResolvedValueOnce(mockFileHandle);
176
144
  // Act
177
145
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
@@ -190,7 +158,7 @@ describe('uploader.helpers', () => {
190
158
  close: jest.fn().mockResolvedValue(undefined),
191
159
  };
192
160
  const fsPromisesOpenSpy = jest
193
- .spyOn(fs_1.promises, 'open')
161
+ .spyOn(fs_2.promises, 'open')
194
162
  .mockResolvedValueOnce(mockFileHandle);
195
163
  // Act
196
164
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
@@ -205,122 +173,13 @@ describe('uploader.helpers', () => {
205
173
  const fileError = new Error('File write failed');
206
174
  mockExistsSync.mockReturnValueOnce(true);
207
175
  const fsPromisesOpenSpy = jest
208
- .spyOn(fs_1.promises, 'open')
176
+ .spyOn(fs_2.promises, 'open')
209
177
  .mockRejectedValueOnce(fileError);
210
178
  // Act & Assert
211
179
  await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
212
180
  fsPromisesOpenSpy.mockRestore();
213
181
  });
214
182
  });
215
- describe(uploader_helpers_1.computeArtifactDateRanges.name, () => {
216
- it('should compute min and max across multiple items', () => {
217
- // Arrange
218
- const items = [
219
- {
220
- id: '1',
221
- created_date: '2020-01-01T00:00:00.000Z',
222
- modified_date: '2021-06-01T00:00:00.000Z',
223
- data: {},
224
- },
225
- {
226
- id: '2',
227
- created_date: '2022-03-15T12:00:00.000Z',
228
- modified_date: '2020-12-31T23:59:59.000Z',
229
- data: {},
230
- },
231
- ];
232
- // Act
233
- const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
234
- // Assert
235
- expect(result.oldest_created_date).toBe('2020-01-01T00:00:00.000Z');
236
- expect(result.newest_created_date).toBe('2022-03-15T12:00:00.000Z');
237
- expect(result.oldest_modified_date).toBe('2020-12-31T23:59:59.000Z');
238
- expect(result.newest_modified_date).toBe('2021-06-01T00:00:00.000Z');
239
- });
240
- it('should return zeros when no items have date fields', () => {
241
- // Arrange
242
- const items = [
243
- { id: 1, name: 'a' },
244
- { id: 2, name: 'b' },
245
- ];
246
- // Act
247
- const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
248
- // Assert
249
- expect(result).toEqual({});
250
- });
251
- it('should aggregate only fields that are present on items', () => {
252
- // Arrange
253
- const items = [
254
- {
255
- id: '1',
256
- created_date: '2021-01-01T00:00:00.000Z',
257
- data: {},
258
- },
259
- {
260
- id: '2',
261
- modified_date: '2023-01-01T00:00:00.000Z',
262
- data: {},
263
- },
264
- ];
265
- // Act
266
- const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
267
- // Assert
268
- expect(result.oldest_created_date).toBe('2021-01-01T00:00:00.000Z');
269
- expect(result.newest_created_date).toBe('2021-01-01T00:00:00.000Z');
270
- expect(result.oldest_modified_date).toBe('2023-01-01T00:00:00.000Z');
271
- expect(result.newest_modified_date).toBe('2023-01-01T00:00:00.000Z');
272
- });
273
- it('should handle single object input', () => {
274
- // Arrange
275
- const item = {
276
- id: '1',
277
- created_date: '2019-05-10T08:30:00.000Z',
278
- modified_date: '2019-05-10T08:30:00.000Z',
279
- data: {},
280
- };
281
- // Act
282
- const result = (0, uploader_helpers_1.computeArtifactDateRanges)(item);
283
- // Assert
284
- const ts = '2019-05-10T08:30:00.000Z';
285
- expect(result.oldest_created_date).toBe(ts);
286
- expect(result.newest_created_date).toBe(ts);
287
- expect(result.oldest_modified_date).toBe(ts);
288
- expect(result.newest_modified_date).toBe(ts);
289
- });
290
- it('should skip invalid date values', () => {
291
- const items = [
292
- {
293
- id: '1',
294
- created_date: 'not-a-date',
295
- modified_date: '2024-01-01T00:00:00.000Z',
296
- data: {},
297
- },
298
- ];
299
- const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
300
- expect(result).toEqual({
301
- oldest_modified_date: '2024-01-01T00:00:00.000Z',
302
- newest_modified_date: '2024-01-01T00:00:00.000Z',
303
- });
304
- });
305
- it('[edge] should skip non-object entries in an array', () => {
306
- // Arrange
307
- const items = [
308
- null,
309
- {
310
- id: '1',
311
- created_date: '2024-01-01T00:00:00.000Z',
312
- modified_date: '2024-06-01T00:00:00.000Z',
313
- data: {},
314
- },
315
- 'not-an-object',
316
- ];
317
- // Act
318
- const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
319
- // Assert
320
- expect(result.oldest_created_date).toBe('2024-01-01T00:00:00.000Z');
321
- expect(result.newest_created_date).toBe('2024-01-01T00:00:00.000Z');
322
- });
323
- });
324
183
  describe(uploader_helpers_1.truncateFilename.name, () => {
325
184
  it('should return filename unchanged when within the limit', () => {
326
185
  // Arrange
@@ -20,22 +20,11 @@ export type UploaderResult<T> = {
20
20
  /**
21
21
  * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
22
22
  */
23
- export declare enum ArtifactDateField {
24
- OldestCreatedDate = "oldest_created_date",
25
- NewestCreatedDate = "newest_created_date",
26
- OldestModifiedDate = "oldest_modified_date",
27
- NewestModifiedDate = "newest_modified_date"
28
- }
29
23
  export interface Artifact {
30
24
  id: string;
31
25
  item_type: string;
32
26
  item_count: number;
33
- [ArtifactDateField.OldestCreatedDate]?: string;
34
- [ArtifactDateField.NewestCreatedDate]?: string;
35
- [ArtifactDateField.OldestModifiedDate]?: string;
36
- [ArtifactDateField.NewestModifiedDate]?: string;
37
27
  }
38
- export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
39
28
  /**
40
29
  * ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
41
30
  */
@@ -1 +1 @@
1
- {"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
1
+ {"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
@@ -1,13 +1,2 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ArtifactDateField = void 0;
4
- /**
5
- * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
6
- */
7
- var ArtifactDateField;
8
- (function (ArtifactDateField) {
9
- ArtifactDateField["OldestCreatedDate"] = "oldest_created_date";
10
- ArtifactDateField["NewestCreatedDate"] = "newest_created_date";
11
- ArtifactDateField["OldestModifiedDate"] = "oldest_modified_date";
12
- ArtifactDateField["NewestModifiedDate"] = "newest_modified_date";
13
- })(ArtifactDateField || (exports.ArtifactDateField = ArtifactDateField = {}));
@@ -78,9 +78,12 @@ class Uploader {
78
78
  };
79
79
  }
80
80
  }
81
- const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
82
81
  // Return the artifact information to the platform
83
- const artifact = Object.assign({ id: preparedArtifact.artifact_id, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
82
+ const artifact = {
83
+ id: preparedArtifact.artifact_id,
84
+ item_type: itemType,
85
+ item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1,
86
+ };
84
87
  return { artifact };
85
88
  }
86
89
  /**
@@ -6,12 +6,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
6
6
  const form_data_1 = __importDefault(require("form-data"));
7
7
  const js_jsonl_1 = require("js-jsonl");
8
8
  const zlib_1 = __importDefault(require("zlib"));
9
- const test_utils_1 = require("../common/test-utils");
10
9
  const axios_client_internal_1 = require("../http/axios-client-internal");
11
- const jest_setup_1 = require("../tests/jest.setup");
12
10
  const test_helpers_1 = require("../tests/test-helpers");
13
- const uploader_1 = require("./uploader");
11
+ const jest_setup_1 = require("../tests/jest.setup");
12
+ const test_utils_1 = require("../common/test-utils");
14
13
  const uploader_helpers_1 = require("./uploader.helpers");
14
+ const uploader_1 = require("./uploader");
15
15
  jest.mock('../http/axios-client-internal');
16
16
  jest.mock('./uploader.helpers', () => (Object.assign(Object.assign({}, jest.requireActual('./uploader.helpers')), { downloadToLocal: jest.fn(), compressGzip: jest.fn(jest.requireActual('./uploader.helpers').compressGzip) })));
17
17
  const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
@@ -51,57 +51,6 @@ describe(uploader_1.Uploader.name, () => {
51
51
  });
52
52
  expect(result.error).toBeUndefined();
53
53
  });
54
- it('should compute oldest/newest created and modified dates from normalized items', async () => {
55
- var _a, _b, _c, _d;
56
- // Arrange
57
- const itemType = 'tasks';
58
- const fetchedObjects = [
59
- {
60
- id: '1',
61
- created_date: '2020-06-15T10:00:00.000Z',
62
- modified_date: '2021-01-20T10:00:00.000Z',
63
- data: { name: 'Task 1' },
64
- },
65
- {
66
- id: '2',
67
- created_date: '2019-03-01T08:00:00.000Z',
68
- modified_date: '2022-11-30T18:00:00.000Z',
69
- data: { name: 'Task 2' },
70
- },
71
- ];
72
- mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
73
- mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
74
- // Act
75
- const result = await uploader.upload(itemType, fetchedObjects);
76
- // Assert
77
- expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe('2019-03-01T08:00:00.000Z');
78
- expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe('2020-06-15T10:00:00.000Z');
79
- expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe('2021-01-20T10:00:00.000Z');
80
- expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe('2022-11-30T18:00:00.000Z');
81
- expect(result.error).toBeUndefined();
82
- });
83
- it('should compute date ranges for single object upload', async () => {
84
- var _a, _b, _c, _d;
85
- // Arrange
86
- const itemType = 'metadata';
87
- const fetchedObject = {
88
- id: '1',
89
- created_date: '2018-12-25T00:00:00.000Z',
90
- modified_date: '2018-12-26T00:00:00.000Z',
91
- data: { key: 'value' },
92
- };
93
- mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
94
- mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
95
- // Act
96
- const result = await uploader.upload(itemType, fetchedObject);
97
- // Assert
98
- const createdTs = '2018-12-25T00:00:00.000Z';
99
- const modifiedTs = '2018-12-26T00:00:00.000Z';
100
- expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe(createdTs);
101
- expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe(createdTs);
102
- expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe(modifiedTs);
103
- expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe(modifiedTs);
104
- });
105
54
  it('should report item_count as 1 when uploading single object', async () => {
106
55
  var _a;
107
56
  // Arrange
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@devrev/ts-adaas",
3
- "version": "1.19.9-beta.0",
3
+ "version": "1.19.9",
4
4
  "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
5
5
  "type": "commonjs",
6
6
  "main": "./dist/index.js",
@@ -46,7 +46,7 @@
46
46
  "typescript-eslint": "^8.46.1"
47
47
  },
48
48
  "dependencies": {
49
- "@devrev/typescript-sdk": "^1.1.74",
49
+ "@devrev/typescript-sdk": "^1.1.75",
50
50
  "axios": "^1.15.2",
51
51
  "axios-retry": "^4.5.0",
52
52
  "form-data": "^4.0.4",
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=control-protocol.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"control-protocol.test.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.test.ts"],"names":[],"mappings":""}
@@ -1,126 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const axios_client_internal_1 = require("../http/axios-client-internal");
4
- const test_helpers_1 = require("../tests/test-helpers");
5
- const test_utils_1 = require("./test-utils");
6
- const control_protocol_1 = require("./control-protocol");
7
- const extraction_1 = require("../types/extraction");
8
- const loading_1 = require("../types/loading");
9
- jest.mock('../http/axios-client-internal');
10
- const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
11
- describe('control-protocol.emit', () => {
12
- beforeEach(() => {
13
- jest.clearAllMocks();
14
- mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
15
- });
16
- it.each([
17
- {
18
- title: 'extractor events',
19
- inputEventType: extraction_1.EventType.StartExtractingData,
20
- outputEventType: extraction_1.ExtractorEventType.DataExtractionProgress,
21
- },
22
- {
23
- title: 'loader events',
24
- inputEventType: extraction_1.EventType.StartLoadingData,
25
- outputEventType: loading_1.LoaderEventType.DataLoadingProgress,
26
- },
27
- {
28
- title: 'unknown events',
29
- inputEventType: extraction_1.EventType.StartExtractingData,
30
- outputEventType: 'SOME_UNKNOWN_EVENT',
31
- },
32
- ])('sets state dates from event context for $title', async ({ inputEventType, outputEventType }) => {
33
- const event = (0, test_utils_1.createMockEvent)(undefined, {
34
- payload: {
35
- event_type: inputEventType,
36
- event_context: {
37
- extract_from: '2024-01-01T00:00:00.000Z',
38
- extract_to: '2024-06-01T00:00:00.000Z',
39
- },
40
- },
41
- });
42
- await (0, control_protocol_1.emit)({
43
- event,
44
- eventType: outputEventType,
45
- });
46
- expect(mockedAxiosClient.post).toHaveBeenCalledTimes(1);
47
- const [, body, config] = mockedAxiosClient.post.mock.calls[0];
48
- expect(body).toMatchObject({
49
- event_type: outputEventType,
50
- event_context: expect.objectContaining({
51
- extract_from: '2024-01-01T00:00:00.000Z',
52
- extract_to: '2024-06-01T00:00:00.000Z',
53
- }),
54
- worker_metadata: expect.objectContaining({
55
- oldest_state_date: '2024-01-01T00:00:00.000Z',
56
- newest_state_date: '2024-06-01T00:00:00.000Z',
57
- }),
58
- });
59
- expect(config).toEqual(expect.objectContaining({
60
- headers: expect.objectContaining({
61
- 'X-DevRev-Client-Version': expect.any(String),
62
- }),
63
- }));
64
- });
65
- it.each([
66
- {
67
- title: 'only extract_from is set',
68
- extractFrom: '2024-01-01T00:00:00.000Z',
69
- extractTo: undefined,
70
- },
71
- {
72
- title: 'only extract_to is set',
73
- extractFrom: undefined,
74
- extractTo: '2024-06-01T00:00:00.000Z',
75
- },
76
- {
77
- title: 'neither extract_from nor extract_to is set',
78
- extractFrom: undefined,
79
- extractTo: undefined,
80
- },
81
- ])('handles state-date absence when $title', async ({ extractFrom, extractTo }) => {
82
- const event = (0, test_utils_1.createMockEvent)(undefined, {
83
- payload: {
84
- event_type: extraction_1.EventType.StartExtractingData,
85
- event_context: {
86
- extract_from: extractFrom,
87
- extract_to: extractTo,
88
- },
89
- },
90
- });
91
- await (0, control_protocol_1.emit)({
92
- event,
93
- eventType: extraction_1.ExtractorEventType.DataExtractionProgress,
94
- });
95
- const [, body] = mockedAxiosClient.post.mock.calls[0];
96
- const workerMetadata = body.worker_metadata;
97
- expect(workerMetadata.oldest_state_date).toBe(extractFrom);
98
- expect(workerMetadata.newest_state_date).toBe(extractTo);
99
- });
100
- it('overrides caller-provided worker_metadata state dates', async () => {
101
- const event = (0, test_utils_1.createMockEvent)(undefined, {
102
- payload: {
103
- event_type: extraction_1.EventType.StartExtractingData,
104
- event_context: {
105
- extract_from: '2024-01-01T00:00:00.000Z',
106
- extract_to: '2024-06-01T00:00:00.000Z',
107
- },
108
- },
109
- });
110
- await (0, control_protocol_1.emit)({
111
- event,
112
- eventType: extraction_1.ExtractorEventType.DataExtractionProgress,
113
- worker_metadata: {
114
- item_type: 'tasks',
115
- oldest_state_date: 'should-be-overwritten',
116
- newest_state_date: 'should-be-overwritten',
117
- },
118
- });
119
- const [, body] = mockedAxiosClient.post.mock.calls[0];
120
- expect(body.worker_metadata).toEqual(expect.objectContaining({
121
- item_type: 'tasks',
122
- oldest_state_date: '2024-01-01T00:00:00.000Z',
123
- newest_state_date: '2024-06-01T00:00:00.000Z',
124
- }));
125
- });
126
- });
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=worker-adapter.upload-failure.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.upload-failure.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.upload-failure.test.ts"],"names":[],"mappings":""}