@devrev/ts-adaas 1.19.10 → 1.19.11-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/control-protocol.d.ts +3 -2
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +2 -4
- package/dist/common/control-protocol.test.d.ts +2 -0
- package/dist/common/control-protocol.test.d.ts.map +1 -0
- package/dist/common/control-protocol.test.js +126 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +94 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +7 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +28 -0
- package/dist/multithreading/worker-adapter/worker-adapter.interfaces.d.ts +14 -0
- package/dist/multithreading/worker-adapter/worker-adapter.interfaces.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.interfaces.js +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +21 -2
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +4 -3
- package/dist/repo/repo.d.ts +10 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.helpers.d.ts +6 -0
- package/dist/repo/repo.helpers.d.ts.map +1 -0
- package/dist/repo/repo.helpers.js +16 -0
- package/dist/repo/repo.helpers.test.d.ts +2 -0
- package/dist/repo/repo.helpers.test.d.ts.map +1 -0
- package/dist/repo/repo.helpers.test.js +71 -0
- package/dist/repo/repo.interfaces.d.ts +2 -0
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +21 -0
- package/dist/repo/repo.test.js +162 -0
- package/dist/types/extraction.d.ts +8 -1
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +6 -1
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +65 -0
- package/dist/uploader/uploader.helpers.test.js +148 -7
- package/dist/uploader/uploader.interfaces.d.ts +11 -0
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +11 -0
- package/dist/uploader/uploader.js +7 -11
- package/dist/uploader/uploader.test.js +78 -3
- package/package.json +3 -3
package/dist/repo/repo.js
CHANGED
|
@@ -4,8 +4,13 @@ exports.Repo = void 0;
|
|
|
4
4
|
const constants_1 = require("../common/constants");
|
|
5
5
|
const uploader_1 = require("../uploader/uploader");
|
|
6
6
|
const logger_context_1 = require("../logger/logger.context");
|
|
7
|
+
const repo_helpers_1 = require("./repo.helpers");
|
|
7
8
|
class Repo {
|
|
8
9
|
constructor({ event, itemType, normalize, onUpload, options, }) {
|
|
10
|
+
this.dateRanges = {
|
|
11
|
+
creationDate: {},
|
|
12
|
+
modifiedDate: {},
|
|
13
|
+
};
|
|
9
14
|
this.items = [];
|
|
10
15
|
this.itemType = itemType;
|
|
11
16
|
this.normalize = normalize;
|
|
@@ -20,6 +25,22 @@ class Repo {
|
|
|
20
25
|
async upload(batch) {
|
|
21
26
|
const itemsToUpload = batch || this.items;
|
|
22
27
|
if (itemsToUpload.length > 0) {
|
|
28
|
+
for (const item of itemsToUpload) {
|
|
29
|
+
const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
|
|
30
|
+
if (createdDate != null) {
|
|
31
|
+
const createdMs = (0, repo_helpers_1.toValidTimestamp)(createdDate);
|
|
32
|
+
if (createdMs !== undefined) {
|
|
33
|
+
(0, repo_helpers_1.updateRange)(this.dateRanges.creationDate, createdMs);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const modifiedDate = item === null || item === void 0 ? void 0 : item.modified_date;
|
|
37
|
+
if (modifiedDate != null && modifiedDate !== '') {
|
|
38
|
+
const modifiedMs = (0, repo_helpers_1.toValidTimestamp)(modifiedDate);
|
|
39
|
+
if (modifiedMs !== undefined) {
|
|
40
|
+
(0, repo_helpers_1.updateRange)(this.dateRanges.modifiedDate, modifiedMs);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
23
44
|
console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
|
|
24
45
|
const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
|
|
25
46
|
if (error || !artifact) {
|
package/dist/repo/repo.test.js
CHANGED
|
@@ -7,6 +7,22 @@ const test_utils_1 = require("../common/test-utils");
|
|
|
7
7
|
const types_1 = require("../types");
|
|
8
8
|
const repo_1 = require("./repo");
|
|
9
9
|
jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
|
|
10
|
+
const mockUploadFn = jest.fn().mockResolvedValue({
|
|
11
|
+
error: null,
|
|
12
|
+
artifact: { id: 'art-1', item_type: 'test', item_count: 0 },
|
|
13
|
+
});
|
|
14
|
+
jest.mock('../uploader/uploader', () => ({
|
|
15
|
+
Uploader: jest.fn().mockImplementation(() => ({
|
|
16
|
+
upload: mockUploadFn,
|
|
17
|
+
})),
|
|
18
|
+
}));
|
|
19
|
+
function itemWithDate(id, created_date) {
|
|
20
|
+
return { id, created_date, modified_date: created_date, data: {} };
|
|
21
|
+
}
|
|
22
|
+
function itemWithDates(id, created_date, modified_date) {
|
|
23
|
+
return { id, created_date, modified_date, data: {} };
|
|
24
|
+
}
|
|
25
|
+
const ts = (iso) => new Date(iso).getTime();
|
|
10
26
|
describe(repo_1.Repo.name, () => {
|
|
11
27
|
let repo;
|
|
12
28
|
let normalize;
|
|
@@ -181,4 +197,150 @@ describe(repo_1.Repo.name, () => {
|
|
|
181
197
|
expect(repo.getItems().length).toBe(5);
|
|
182
198
|
});
|
|
183
199
|
});
|
|
200
|
+
describe('dateRanges', () => {
|
|
201
|
+
beforeEach(() => {
|
|
202
|
+
mockUploadFn.mockResolvedValue({
|
|
203
|
+
error: null,
|
|
204
|
+
artifact: { id: 'art-1', item_type: 'test', item_count: 0 },
|
|
205
|
+
});
|
|
206
|
+
});
|
|
207
|
+
it('should track min and max created_date from a single upload batch', async () => {
|
|
208
|
+
await repo.upload([
|
|
209
|
+
itemWithDate('1', '2023-06-15T12:00:00.000Z'),
|
|
210
|
+
itemWithDate('2', '2020-01-01T00:00:00.000Z'),
|
|
211
|
+
itemWithDate('3', '2021-03-01T00:00:00.000Z'),
|
|
212
|
+
]);
|
|
213
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
214
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2023-06-15T12:00:00.000Z'));
|
|
215
|
+
expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
216
|
+
expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-06-15T12:00:00.000Z'));
|
|
217
|
+
});
|
|
218
|
+
it('should skip items without created_date', async () => {
|
|
219
|
+
const attachmentWithoutDate = {
|
|
220
|
+
id: 'att-1',
|
|
221
|
+
url: 'https://example.com/file',
|
|
222
|
+
file_name: 'file.txt',
|
|
223
|
+
parent_id: 'parent-1',
|
|
224
|
+
};
|
|
225
|
+
await repo.upload([
|
|
226
|
+
itemWithDate('1', '2022-06-01T00:00:00.000Z'),
|
|
227
|
+
{ id: '2', created_date: null, modified_date: '', data: {} },
|
|
228
|
+
{ id: '3', modified_date: '', data: {} },
|
|
229
|
+
attachmentWithoutDate,
|
|
230
|
+
]);
|
|
231
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-06-01T00:00:00.000Z'));
|
|
232
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-06-01T00:00:00.000Z'));
|
|
233
|
+
});
|
|
234
|
+
it('should leave date ranges unset when no items have created_date', async () => {
|
|
235
|
+
await repo.upload([
|
|
236
|
+
{ id: '1', modified_date: '', data: {} },
|
|
237
|
+
{
|
|
238
|
+
id: 'att-1',
|
|
239
|
+
url: 'https://example.com/file',
|
|
240
|
+
file_name: 'file.txt',
|
|
241
|
+
parent_id: 'parent-1',
|
|
242
|
+
},
|
|
243
|
+
]);
|
|
244
|
+
expect(repo.dateRanges).toEqual({
|
|
245
|
+
creationDate: {},
|
|
246
|
+
modifiedDate: {},
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
it('should not update timestamps or call uploader on empty upload', async () => {
|
|
250
|
+
await repo.upload([]);
|
|
251
|
+
expect(repo.dateRanges).toEqual({
|
|
252
|
+
creationDate: {},
|
|
253
|
+
modifiedDate: {},
|
|
254
|
+
});
|
|
255
|
+
expect(mockUploadFn).not.toHaveBeenCalled();
|
|
256
|
+
});
|
|
257
|
+
it('should track an item dated exactly at the Unix epoch instead of treating it as unset', async () => {
|
|
258
|
+
await repo.upload([itemWithDate('1', '1970-01-01T00:00:00.000Z')]);
|
|
259
|
+
expect(repo.dateRanges.creationDate).toEqual({ oldest: 0, newest: 0 });
|
|
260
|
+
await repo.upload([itemWithDate('2', '2020-01-01T00:00:00.000Z')]);
|
|
261
|
+
expect(repo.dateRanges.creationDate).toEqual({
|
|
262
|
+
oldest: 0,
|
|
263
|
+
newest: ts('2020-01-01T00:00:00.000Z'),
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
it('should accumulate min and max across multiple upload batches via push', async () => {
|
|
267
|
+
repo = new repo_1.Repo({
|
|
268
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
269
|
+
payload: { event_type: types_1.EventType.ExtractionDataStart },
|
|
270
|
+
}),
|
|
271
|
+
itemType: 'test_item_type',
|
|
272
|
+
onUpload: jest.fn(),
|
|
273
|
+
options: { batchSize: 3 },
|
|
274
|
+
});
|
|
275
|
+
const dates = [
|
|
276
|
+
'2020-01-01T00:00:00.000Z',
|
|
277
|
+
'2021-01-01T00:00:00.000Z',
|
|
278
|
+
'2022-01-01T00:00:00.000Z',
|
|
279
|
+
'2023-01-01T00:00:00.000Z',
|
|
280
|
+
'2024-06-01T00:00:00.000Z',
|
|
281
|
+
'2024-12-01T00:00:00.000Z',
|
|
282
|
+
'2024-12-31T00:00:00.000Z',
|
|
283
|
+
];
|
|
284
|
+
const items = dates.map((created_date, index) => itemWithDate(String(index), created_date));
|
|
285
|
+
await repo.push(items);
|
|
286
|
+
expect(repo.getItems()).toHaveLength(1);
|
|
287
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
288
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-12-01T00:00:00.000Z'));
|
|
289
|
+
await repo.push([
|
|
290
|
+
itemWithDate('7', '2019-01-01T00:00:00.000Z'),
|
|
291
|
+
itemWithDate('8', '2025-01-01T00:00:00.000Z'),
|
|
292
|
+
]);
|
|
293
|
+
expect(repo.getItems()).toHaveLength(0);
|
|
294
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2019-01-01T00:00:00.000Z'));
|
|
295
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2025-01-01T00:00:00.000Z'));
|
|
296
|
+
});
|
|
297
|
+
it('should extend min and max when subsequent batches have wider date range', async () => {
|
|
298
|
+
await repo.upload([
|
|
299
|
+
itemWithDate('1', '2022-06-01T00:00:00.000Z'),
|
|
300
|
+
itemWithDate('2', '2023-06-01T00:00:00.000Z'),
|
|
301
|
+
]);
|
|
302
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-06-01T00:00:00.000Z'));
|
|
303
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2023-06-01T00:00:00.000Z'));
|
|
304
|
+
await repo.upload([
|
|
305
|
+
itemWithDate('3', '2020-01-01T00:00:00.000Z'),
|
|
306
|
+
itemWithDate('4', '2024-01-01T00:00:00.000Z'),
|
|
307
|
+
]);
|
|
308
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
309
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-01-01T00:00:00.000Z'));
|
|
310
|
+
});
|
|
311
|
+
it('should update timestamps even when upload fails', async () => {
|
|
312
|
+
mockUploadFn.mockResolvedValueOnce({
|
|
313
|
+
error: new Error('fail'),
|
|
314
|
+
artifact: null,
|
|
315
|
+
});
|
|
316
|
+
await repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')]);
|
|
317
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
318
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
319
|
+
});
|
|
320
|
+
it('should ignore invalid created_date and modified_date values', async () => {
|
|
321
|
+
await repo.upload([
|
|
322
|
+
{
|
|
323
|
+
id: '1',
|
|
324
|
+
created_date: 'not-a-date',
|
|
325
|
+
modified_date: 'still-not-a-date',
|
|
326
|
+
data: {},
|
|
327
|
+
},
|
|
328
|
+
itemWithDates('2', '2022-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'),
|
|
329
|
+
]);
|
|
330
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
331
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
332
|
+
expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2023-01-01T00:00:00.000Z'));
|
|
333
|
+
expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
|
|
334
|
+
});
|
|
335
|
+
it('should track modified_date independently from created_date', async () => {
|
|
336
|
+
await repo.upload([
|
|
337
|
+
itemWithDates('1', '2020-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'),
|
|
338
|
+
itemWithDates('2', '2024-01-01T00:00:00.000Z', '2021-06-01T00:00:00.000Z'),
|
|
339
|
+
]);
|
|
340
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
341
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-01-01T00:00:00.000Z'));
|
|
342
|
+
expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2021-06-01T00:00:00.000Z'));
|
|
343
|
+
expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
|
|
344
|
+
});
|
|
345
|
+
});
|
|
184
346
|
});
|
|
@@ -388,7 +388,14 @@ export interface EventData {
|
|
|
388
388
|
* WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS.
|
|
389
389
|
*/
|
|
390
390
|
export interface WorkerMetadata {
|
|
391
|
-
adaas_library_version
|
|
391
|
+
adaas_library_version?: string;
|
|
392
|
+
item_type?: string;
|
|
393
|
+
oldest_created_date?: string;
|
|
394
|
+
newest_created_date?: string;
|
|
395
|
+
oldest_modified_date?: string;
|
|
396
|
+
newest_modified_date?: string;
|
|
397
|
+
oldest_state_date?: string;
|
|
398
|
+
newest_state_date?: string;
|
|
392
399
|
}
|
|
393
400
|
/**
|
|
394
401
|
* DomainObject is an interface that defines the structure of a domain object that can be extracted.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAI9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAChD,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAW/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA+E1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAsCzC;;;;OAIG;IACG,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAiCF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IA2DF;;;;OAIG;YACW,sBAAsB;IAsBpC;;;;OAIG;YACW,gBAAgB;IAc9B;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import { UploaderResult } from './uploader.interfaces';
|
|
1
|
+
import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
|
|
2
|
+
/**
|
|
3
|
+
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
4
|
+
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
5
|
+
*/
|
|
6
|
+
export declare function computeArtifactDateRanges(fetchedObjects: object[] | object): ArtifactDateRanges;
|
|
2
7
|
/**
|
|
3
8
|
* Compresses a JSONL string using gzip compression.
|
|
4
9
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
|
|
@@ -36,6 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.computeArtifactDateRanges = computeArtifactDateRanges;
|
|
39
40
|
exports.compressGzip = compressGzip;
|
|
40
41
|
exports.decompressGzip = decompressGzip;
|
|
41
42
|
exports.parseJsonl = parseJsonl;
|
|
@@ -45,6 +46,70 @@ const fs_1 = __importStar(require("fs"));
|
|
|
45
46
|
const js_jsonl_1 = require("js-jsonl");
|
|
46
47
|
const zlib_1 = __importDefault(require("zlib"));
|
|
47
48
|
const constants_1 = require("../common/constants");
|
|
49
|
+
const uploader_interfaces_1 = require("./uploader.interfaces");
|
|
50
|
+
/**
|
|
51
|
+
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
52
|
+
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
53
|
+
*/
|
|
54
|
+
function computeArtifactDateRanges(fetchedObjects) {
|
|
55
|
+
const items = Array.isArray(fetchedObjects)
|
|
56
|
+
? fetchedObjects
|
|
57
|
+
: [fetchedObjects];
|
|
58
|
+
const created = {
|
|
59
|
+
min: Number.POSITIVE_INFINITY,
|
|
60
|
+
max: Number.NEGATIVE_INFINITY,
|
|
61
|
+
};
|
|
62
|
+
const modified = {
|
|
63
|
+
min: Number.POSITIVE_INFINITY,
|
|
64
|
+
max: Number.NEGATIVE_INFINITY,
|
|
65
|
+
};
|
|
66
|
+
let hasCreated = false;
|
|
67
|
+
let hasModified = false;
|
|
68
|
+
const parseTimestamp = (value) => {
|
|
69
|
+
const ts = new Date(value).getTime();
|
|
70
|
+
return Number.isNaN(ts) ? undefined : ts;
|
|
71
|
+
};
|
|
72
|
+
for (const obj of items) {
|
|
73
|
+
if (!obj || typeof obj !== 'object') {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
const item = obj;
|
|
77
|
+
if (item.created_date != undefined) {
|
|
78
|
+
const ts = parseTimestamp(item.created_date);
|
|
79
|
+
if (ts != undefined) {
|
|
80
|
+
if (ts < created.min) {
|
|
81
|
+
created.min = ts;
|
|
82
|
+
}
|
|
83
|
+
if (ts > created.max) {
|
|
84
|
+
created.max = ts;
|
|
85
|
+
}
|
|
86
|
+
hasCreated = true;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (item.modified_date != undefined) {
|
|
90
|
+
const ts = parseTimestamp(item.modified_date);
|
|
91
|
+
if (ts != undefined) {
|
|
92
|
+
if (ts < modified.min) {
|
|
93
|
+
modified.min = ts;
|
|
94
|
+
}
|
|
95
|
+
if (ts > modified.max) {
|
|
96
|
+
modified.max = ts;
|
|
97
|
+
}
|
|
98
|
+
hasModified = true;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const result = {};
|
|
103
|
+
if (hasCreated) {
|
|
104
|
+
result[uploader_interfaces_1.ArtifactDateField.OldestCreatedDate] = new Date(created.min).toISOString();
|
|
105
|
+
result[uploader_interfaces_1.ArtifactDateField.NewestCreatedDate] = new Date(created.max).toISOString();
|
|
106
|
+
}
|
|
107
|
+
if (hasModified) {
|
|
108
|
+
result[uploader_interfaces_1.ArtifactDateField.OldestModifiedDate] = new Date(modified.min).toISOString();
|
|
109
|
+
result[uploader_interfaces_1.ArtifactDateField.NewestModifiedDate] = new Date(modified.max).toISOString();
|
|
110
|
+
}
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
48
113
|
/**
|
|
49
114
|
* Compresses a JSONL string using gzip compression.
|
|
50
115
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
@@ -1,10 +1,42 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
37
|
};
|
|
5
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const fs_1 =
|
|
7
|
-
const fs_2 = require("fs");
|
|
39
|
+
const fs_1 = __importStar(require("fs"));
|
|
8
40
|
const js_jsonl_1 = require("js-jsonl");
|
|
9
41
|
const zlib_1 = __importDefault(require("zlib"));
|
|
10
42
|
const constants_1 = require("../common/constants");
|
|
@@ -99,7 +131,7 @@ describe('uploader.helpers', () => {
|
|
|
99
131
|
close: jest.fn().mockResolvedValue(undefined),
|
|
100
132
|
};
|
|
101
133
|
const fsPromisesOpenSpy = jest
|
|
102
|
-
.spyOn(
|
|
134
|
+
.spyOn(fs_1.promises, 'open')
|
|
103
135
|
.mockResolvedValueOnce(mockFileHandle);
|
|
104
136
|
// Act
|
|
105
137
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
@@ -118,7 +150,7 @@ describe('uploader.helpers', () => {
|
|
|
118
150
|
close: jest.fn().mockResolvedValue(undefined),
|
|
119
151
|
};
|
|
120
152
|
const fsPromisesOpenSpy = jest
|
|
121
|
-
.spyOn(
|
|
153
|
+
.spyOn(fs_1.promises, 'open')
|
|
122
154
|
.mockResolvedValueOnce(mockFileHandle);
|
|
123
155
|
// Act
|
|
124
156
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
@@ -139,7 +171,7 @@ describe('uploader.helpers', () => {
|
|
|
139
171
|
close: jest.fn().mockResolvedValue(undefined),
|
|
140
172
|
};
|
|
141
173
|
const fsPromisesOpenSpy = jest
|
|
142
|
-
.spyOn(
|
|
174
|
+
.spyOn(fs_1.promises, 'open')
|
|
143
175
|
.mockResolvedValueOnce(mockFileHandle);
|
|
144
176
|
// Act
|
|
145
177
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
@@ -158,7 +190,7 @@ describe('uploader.helpers', () => {
|
|
|
158
190
|
close: jest.fn().mockResolvedValue(undefined),
|
|
159
191
|
};
|
|
160
192
|
const fsPromisesOpenSpy = jest
|
|
161
|
-
.spyOn(
|
|
193
|
+
.spyOn(fs_1.promises, 'open')
|
|
162
194
|
.mockResolvedValueOnce(mockFileHandle);
|
|
163
195
|
// Act
|
|
164
196
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
@@ -173,13 +205,122 @@ describe('uploader.helpers', () => {
|
|
|
173
205
|
const fileError = new Error('File write failed');
|
|
174
206
|
mockExistsSync.mockReturnValueOnce(true);
|
|
175
207
|
const fsPromisesOpenSpy = jest
|
|
176
|
-
.spyOn(
|
|
208
|
+
.spyOn(fs_1.promises, 'open')
|
|
177
209
|
.mockRejectedValueOnce(fileError);
|
|
178
210
|
// Act & Assert
|
|
179
211
|
await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
|
|
180
212
|
fsPromisesOpenSpy.mockRestore();
|
|
181
213
|
});
|
|
182
214
|
});
|
|
215
|
+
describe(uploader_helpers_1.computeArtifactDateRanges.name, () => {
|
|
216
|
+
it('should compute min and max across multiple items', () => {
|
|
217
|
+
// Arrange
|
|
218
|
+
const items = [
|
|
219
|
+
{
|
|
220
|
+
id: '1',
|
|
221
|
+
created_date: '2020-01-01T00:00:00.000Z',
|
|
222
|
+
modified_date: '2021-06-01T00:00:00.000Z',
|
|
223
|
+
data: {},
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
id: '2',
|
|
227
|
+
created_date: '2022-03-15T12:00:00.000Z',
|
|
228
|
+
modified_date: '2020-12-31T23:59:59.000Z',
|
|
229
|
+
data: {},
|
|
230
|
+
},
|
|
231
|
+
];
|
|
232
|
+
// Act
|
|
233
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
234
|
+
// Assert
|
|
235
|
+
expect(result.oldest_created_date).toBe('2020-01-01T00:00:00.000Z');
|
|
236
|
+
expect(result.newest_created_date).toBe('2022-03-15T12:00:00.000Z');
|
|
237
|
+
expect(result.oldest_modified_date).toBe('2020-12-31T23:59:59.000Z');
|
|
238
|
+
expect(result.newest_modified_date).toBe('2021-06-01T00:00:00.000Z');
|
|
239
|
+
});
|
|
240
|
+
it('should return zeros when no items have date fields', () => {
|
|
241
|
+
// Arrange
|
|
242
|
+
const items = [
|
|
243
|
+
{ id: 1, name: 'a' },
|
|
244
|
+
{ id: 2, name: 'b' },
|
|
245
|
+
];
|
|
246
|
+
// Act
|
|
247
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
248
|
+
// Assert
|
|
249
|
+
expect(result).toEqual({});
|
|
250
|
+
});
|
|
251
|
+
it('should aggregate only fields that are present on items', () => {
|
|
252
|
+
// Arrange
|
|
253
|
+
const items = [
|
|
254
|
+
{
|
|
255
|
+
id: '1',
|
|
256
|
+
created_date: '2021-01-01T00:00:00.000Z',
|
|
257
|
+
data: {},
|
|
258
|
+
},
|
|
259
|
+
{
|
|
260
|
+
id: '2',
|
|
261
|
+
modified_date: '2023-01-01T00:00:00.000Z',
|
|
262
|
+
data: {},
|
|
263
|
+
},
|
|
264
|
+
];
|
|
265
|
+
// Act
|
|
266
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
267
|
+
// Assert
|
|
268
|
+
expect(result.oldest_created_date).toBe('2021-01-01T00:00:00.000Z');
|
|
269
|
+
expect(result.newest_created_date).toBe('2021-01-01T00:00:00.000Z');
|
|
270
|
+
expect(result.oldest_modified_date).toBe('2023-01-01T00:00:00.000Z');
|
|
271
|
+
expect(result.newest_modified_date).toBe('2023-01-01T00:00:00.000Z');
|
|
272
|
+
});
|
|
273
|
+
it('should handle single object input', () => {
|
|
274
|
+
// Arrange
|
|
275
|
+
const item = {
|
|
276
|
+
id: '1',
|
|
277
|
+
created_date: '2019-05-10T08:30:00.000Z',
|
|
278
|
+
modified_date: '2019-05-10T08:30:00.000Z',
|
|
279
|
+
data: {},
|
|
280
|
+
};
|
|
281
|
+
// Act
|
|
282
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(item);
|
|
283
|
+
// Assert
|
|
284
|
+
const ts = '2019-05-10T08:30:00.000Z';
|
|
285
|
+
expect(result.oldest_created_date).toBe(ts);
|
|
286
|
+
expect(result.newest_created_date).toBe(ts);
|
|
287
|
+
expect(result.oldest_modified_date).toBe(ts);
|
|
288
|
+
expect(result.newest_modified_date).toBe(ts);
|
|
289
|
+
});
|
|
290
|
+
it('should skip invalid date values', () => {
|
|
291
|
+
const items = [
|
|
292
|
+
{
|
|
293
|
+
id: '1',
|
|
294
|
+
created_date: 'not-a-date',
|
|
295
|
+
modified_date: '2024-01-01T00:00:00.000Z',
|
|
296
|
+
data: {},
|
|
297
|
+
},
|
|
298
|
+
];
|
|
299
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
300
|
+
expect(result).toEqual({
|
|
301
|
+
oldest_modified_date: '2024-01-01T00:00:00.000Z',
|
|
302
|
+
newest_modified_date: '2024-01-01T00:00:00.000Z',
|
|
303
|
+
});
|
|
304
|
+
});
|
|
305
|
+
it('[edge] should skip non-object entries in an array', () => {
|
|
306
|
+
// Arrange
|
|
307
|
+
const items = [
|
|
308
|
+
null,
|
|
309
|
+
{
|
|
310
|
+
id: '1',
|
|
311
|
+
created_date: '2024-01-01T00:00:00.000Z',
|
|
312
|
+
modified_date: '2024-06-01T00:00:00.000Z',
|
|
313
|
+
data: {},
|
|
314
|
+
},
|
|
315
|
+
'not-an-object',
|
|
316
|
+
];
|
|
317
|
+
// Act
|
|
318
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
319
|
+
// Assert
|
|
320
|
+
expect(result.oldest_created_date).toBe('2024-01-01T00:00:00.000Z');
|
|
321
|
+
expect(result.newest_created_date).toBe('2024-01-01T00:00:00.000Z');
|
|
322
|
+
});
|
|
323
|
+
});
|
|
183
324
|
describe(uploader_helpers_1.truncateFilename.name, () => {
|
|
184
325
|
it('should return filename unchanged when within the limit', () => {
|
|
185
326
|
// Arrange
|
|
@@ -20,11 +20,22 @@ export type UploaderResult<T> = {
|
|
|
20
20
|
/**
|
|
21
21
|
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
22
22
|
*/
|
|
23
|
+
export declare enum ArtifactDateField {
|
|
24
|
+
OldestCreatedDate = "oldest_created_date",
|
|
25
|
+
NewestCreatedDate = "newest_created_date",
|
|
26
|
+
OldestModifiedDate = "oldest_modified_date",
|
|
27
|
+
NewestModifiedDate = "newest_modified_date"
|
|
28
|
+
}
|
|
23
29
|
export interface Artifact {
|
|
24
30
|
id: string;
|
|
25
31
|
item_type: string;
|
|
26
32
|
item_count: number;
|
|
33
|
+
[ArtifactDateField.OldestCreatedDate]?: string;
|
|
34
|
+
[ArtifactDateField.NewestCreatedDate]?: string;
|
|
35
|
+
[ArtifactDateField.OldestModifiedDate]?: string;
|
|
36
|
+
[ArtifactDateField.NewestModifiedDate]?: string;
|
|
27
37
|
}
|
|
38
|
+
export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
|
|
28
39
|
/**
|
|
29
40
|
* ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
|
|
30
41
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
|
|
@@ -1,2 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ArtifactDateField = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
6
|
+
*/
|
|
7
|
+
var ArtifactDateField;
|
|
8
|
+
(function (ArtifactDateField) {
|
|
9
|
+
ArtifactDateField["OldestCreatedDate"] = "oldest_created_date";
|
|
10
|
+
ArtifactDateField["NewestCreatedDate"] = "newest_created_date";
|
|
11
|
+
ArtifactDateField["OldestModifiedDate"] = "oldest_modified_date";
|
|
12
|
+
ArtifactDateField["NewestModifiedDate"] = "newest_modified_date";
|
|
13
|
+
})(ArtifactDateField || (exports.ArtifactDateField = ArtifactDateField = {}));
|