@devrev/ts-adaas 1.19.8 → 1.19.9-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/control-protocol.d.ts +3 -2
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +2 -4
- package/dist/common/control-protocol.test.d.ts +2 -0
- package/dist/common/control-protocol.test.d.ts.map +1 -0
- package/dist/common/control-protocol.test.js +126 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +75 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +7 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +19 -0
- package/dist/repo/repo.d.ts +10 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.helpers.d.ts +6 -0
- package/dist/repo/repo.helpers.d.ts.map +1 -0
- package/dist/repo/repo.helpers.js +16 -0
- package/dist/repo/repo.interfaces.d.ts +2 -0
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +21 -0
- package/dist/repo/repo.test.js +153 -0
- package/dist/types/extraction.d.ts +8 -1
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +6 -1
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +65 -0
- package/dist/uploader/uploader.helpers.test.js +148 -7
- package/dist/uploader/uploader.interfaces.d.ts +11 -0
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +11 -0
- package/dist/uploader/uploader.js +2 -5
- package/dist/uploader/uploader.test.js +54 -3
- package/package.json +1 -1
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { AxiosResponse } from 'axios';
|
|
2
|
-
import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
|
|
2
|
+
import { AirdropEvent, EventData, ExtractorEventType, WorkerMetadata } from '../types/extraction';
|
|
3
3
|
import { LoaderEventType } from '../types/loading';
|
|
4
4
|
export interface EmitInterface {
|
|
5
5
|
event: AirdropEvent;
|
|
6
6
|
eventType: ExtractorEventType | LoaderEventType;
|
|
7
7
|
data?: EventData;
|
|
8
|
+
worker_metadata?: WorkerMetadata;
|
|
8
9
|
}
|
|
9
|
-
export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<AxiosResponse>;
|
|
10
|
+
export declare const emit: ({ event, eventType, data, worker_metadata, }: EmitInterface) => Promise<AxiosResponse>;
|
|
10
11
|
//# sourceMappingURL=control-protocol.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"control-protocol.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,
|
|
1
|
+
{"version":3,"file":"control-protocol.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EAElB,cAAc,EACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,eAAO,MAAM,IAAI,GAAU,8CAKxB,aAAa,KAAG,OAAO,CAAC,aAAa,CAmCvC,CAAC"}
|
|
@@ -4,7 +4,7 @@ exports.emit = void 0;
|
|
|
4
4
|
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
5
5
|
const constants_1 = require("./constants");
|
|
6
6
|
const event_type_translation_1 = require("./event-type-translation");
|
|
7
|
-
const emit = async ({ event, eventType, data, }) => {
|
|
7
|
+
const emit = async ({ event, eventType, data, worker_metadata, }) => {
|
|
8
8
|
// Translate outgoing event type to ensure we always send new event types
|
|
9
9
|
// TODO: Remove when the old types are completely phased out
|
|
10
10
|
const translatedEventType = (0, event_type_translation_1.translateOutgoingEventType)(eventType);
|
|
@@ -12,9 +12,7 @@ const emit = async ({ event, eventType, data, }) => {
|
|
|
12
12
|
event_type: translatedEventType,
|
|
13
13
|
event_context: event.payload.event_context,
|
|
14
14
|
event_data: Object.assign({}, data),
|
|
15
|
-
worker_metadata: {
|
|
16
|
-
adaas_library_version: constants_1.LIBRARY_VERSION,
|
|
17
|
-
},
|
|
15
|
+
worker_metadata: Object.assign(Object.assign({}, worker_metadata), { newest_state_date: event.payload.event_context.extract_to, oldest_state_date: event.payload.event_context.extract_from, adaas_library_version: constants_1.LIBRARY_VERSION }),
|
|
18
16
|
};
|
|
19
17
|
console.info('Emitting event', newEvent);
|
|
20
18
|
return axios_client_internal_1.axiosClient.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"control-protocol.test.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
4
|
+
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
|
+
const test_utils_1 = require("./test-utils");
|
|
6
|
+
const control_protocol_1 = require("./control-protocol");
|
|
7
|
+
const extraction_1 = require("../types/extraction");
|
|
8
|
+
const loading_1 = require("../types/loading");
|
|
9
|
+
jest.mock('../http/axios-client-internal');
|
|
10
|
+
const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
|
|
11
|
+
describe('control-protocol.emit', () => {
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
jest.clearAllMocks();
|
|
14
|
+
mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
|
|
15
|
+
});
|
|
16
|
+
it.each([
|
|
17
|
+
{
|
|
18
|
+
title: 'extractor events',
|
|
19
|
+
inputEventType: extraction_1.EventType.StartExtractingData,
|
|
20
|
+
outputEventType: extraction_1.ExtractorEventType.DataExtractionProgress,
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
title: 'loader events',
|
|
24
|
+
inputEventType: extraction_1.EventType.StartLoadingData,
|
|
25
|
+
outputEventType: loading_1.LoaderEventType.DataLoadingProgress,
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
title: 'unknown events',
|
|
29
|
+
inputEventType: extraction_1.EventType.StartExtractingData,
|
|
30
|
+
outputEventType: 'SOME_UNKNOWN_EVENT',
|
|
31
|
+
},
|
|
32
|
+
])('sets state dates from event context for $title', async ({ inputEventType, outputEventType }) => {
|
|
33
|
+
const event = (0, test_utils_1.createMockEvent)(undefined, {
|
|
34
|
+
payload: {
|
|
35
|
+
event_type: inputEventType,
|
|
36
|
+
event_context: {
|
|
37
|
+
extract_from: '2024-01-01T00:00:00.000Z',
|
|
38
|
+
extract_to: '2024-06-01T00:00:00.000Z',
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
await (0, control_protocol_1.emit)({
|
|
43
|
+
event,
|
|
44
|
+
eventType: outputEventType,
|
|
45
|
+
});
|
|
46
|
+
expect(mockedAxiosClient.post).toHaveBeenCalledTimes(1);
|
|
47
|
+
const [, body, config] = mockedAxiosClient.post.mock.calls[0];
|
|
48
|
+
expect(body).toMatchObject({
|
|
49
|
+
event_type: outputEventType,
|
|
50
|
+
event_context: expect.objectContaining({
|
|
51
|
+
extract_from: '2024-01-01T00:00:00.000Z',
|
|
52
|
+
extract_to: '2024-06-01T00:00:00.000Z',
|
|
53
|
+
}),
|
|
54
|
+
worker_metadata: expect.objectContaining({
|
|
55
|
+
oldest_state_date: '2024-01-01T00:00:00.000Z',
|
|
56
|
+
newest_state_date: '2024-06-01T00:00:00.000Z',
|
|
57
|
+
}),
|
|
58
|
+
});
|
|
59
|
+
expect(config).toEqual(expect.objectContaining({
|
|
60
|
+
headers: expect.objectContaining({
|
|
61
|
+
'X-DevRev-Client-Version': expect.any(String),
|
|
62
|
+
}),
|
|
63
|
+
}));
|
|
64
|
+
});
|
|
65
|
+
it.each([
|
|
66
|
+
{
|
|
67
|
+
title: 'only extract_from is set',
|
|
68
|
+
extractFrom: '2024-01-01T00:00:00.000Z',
|
|
69
|
+
extractTo: undefined,
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
title: 'only extract_to is set',
|
|
73
|
+
extractFrom: undefined,
|
|
74
|
+
extractTo: '2024-06-01T00:00:00.000Z',
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
title: 'neither extract_from nor extract_to is set',
|
|
78
|
+
extractFrom: undefined,
|
|
79
|
+
extractTo: undefined,
|
|
80
|
+
},
|
|
81
|
+
])('handles state-date absence when $title', async ({ extractFrom, extractTo }) => {
|
|
82
|
+
const event = (0, test_utils_1.createMockEvent)(undefined, {
|
|
83
|
+
payload: {
|
|
84
|
+
event_type: extraction_1.EventType.StartExtractingData,
|
|
85
|
+
event_context: {
|
|
86
|
+
extract_from: extractFrom,
|
|
87
|
+
extract_to: extractTo,
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
await (0, control_protocol_1.emit)({
|
|
92
|
+
event,
|
|
93
|
+
eventType: extraction_1.ExtractorEventType.DataExtractionProgress,
|
|
94
|
+
});
|
|
95
|
+
const [, body] = mockedAxiosClient.post.mock.calls[0];
|
|
96
|
+
const workerMetadata = body.worker_metadata;
|
|
97
|
+
expect(workerMetadata.oldest_state_date).toBe(extractFrom);
|
|
98
|
+
expect(workerMetadata.newest_state_date).toBe(extractTo);
|
|
99
|
+
});
|
|
100
|
+
it('overrides caller-provided worker_metadata state dates', async () => {
|
|
101
|
+
const event = (0, test_utils_1.createMockEvent)(undefined, {
|
|
102
|
+
payload: {
|
|
103
|
+
event_type: extraction_1.EventType.StartExtractingData,
|
|
104
|
+
event_context: {
|
|
105
|
+
extract_from: '2024-01-01T00:00:00.000Z',
|
|
106
|
+
extract_to: '2024-06-01T00:00:00.000Z',
|
|
107
|
+
},
|
|
108
|
+
},
|
|
109
|
+
});
|
|
110
|
+
await (0, control_protocol_1.emit)({
|
|
111
|
+
event,
|
|
112
|
+
eventType: extraction_1.ExtractorEventType.DataExtractionProgress,
|
|
113
|
+
worker_metadata: {
|
|
114
|
+
item_type: 'tasks',
|
|
115
|
+
oldest_state_date: 'should-be-overwritten',
|
|
116
|
+
newest_state_date: 'should-be-overwritten',
|
|
117
|
+
},
|
|
118
|
+
});
|
|
119
|
+
const [, body] = mockedAxiosClient.post.mock.calls[0];
|
|
120
|
+
expect(body.worker_metadata).toEqual(expect.objectContaining({
|
|
121
|
+
item_type: 'tasks',
|
|
122
|
+
oldest_state_date: '2024-01-01T00:00:00.000Z',
|
|
123
|
+
newest_state_date: '2024-06-01T00:00:00.000Z',
|
|
124
|
+
}));
|
|
125
|
+
});
|
|
126
|
+
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAoBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAI9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAC1C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,CAElC;IAEH,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAoBzC,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,EAK3B;IAED,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAyCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAmMV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAoHvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAuIzC"}
|
|
@@ -43,6 +43,7 @@ function makeAdapter(eventType = types_1.EventType.StartExtractingData) {
|
|
|
43
43
|
const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
|
|
44
44
|
return { adapter, event, adapterState };
|
|
45
45
|
}
|
|
46
|
+
const iso = (ms) => new Date(ms).toISOString();
|
|
46
47
|
describe(`${worker_adapter_1.WorkerAdapter.name}.emit`, () => {
|
|
47
48
|
let adapter;
|
|
48
49
|
let mockPostMessage;
|
|
@@ -262,6 +263,80 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.emit`, () => {
|
|
|
262
263
|
expect(emittedMessage.startsWith('E'.repeat(100))).toBe(true);
|
|
263
264
|
});
|
|
264
265
|
});
|
|
266
|
+
describe(`${worker_adapter_1.WorkerAdapter.name}.emit — worker_metadata`, () => {
|
|
267
|
+
let adapter;
|
|
268
|
+
beforeEach(() => {
|
|
269
|
+
jest.clearAllMocks();
|
|
270
|
+
({ adapter } = makeAdapter());
|
|
271
|
+
adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
|
|
272
|
+
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
273
|
+
});
|
|
274
|
+
function setupRepos(adapterInstance, lastExtractedItemType) {
|
|
275
|
+
adapterInstance['repos'] = [
|
|
276
|
+
{
|
|
277
|
+
itemType: 'issues',
|
|
278
|
+
dateRanges: {
|
|
279
|
+
creationDate: { oldest: 100, newest: 200 },
|
|
280
|
+
modifiedDate: { oldest: 150, newest: 250 },
|
|
281
|
+
},
|
|
282
|
+
},
|
|
283
|
+
{
|
|
284
|
+
itemType: 'tasks',
|
|
285
|
+
dateRanges: {
|
|
286
|
+
creationDate: { oldest: 300, newest: 400 },
|
|
287
|
+
modifiedDate: { oldest: 350, newest: 450 },
|
|
288
|
+
},
|
|
289
|
+
},
|
|
290
|
+
];
|
|
291
|
+
adapterInstance['lastExtractedItemType'] = lastExtractedItemType;
|
|
292
|
+
}
|
|
293
|
+
it('should emit flat worker_metadata for the latest extracted item type only', async () => {
|
|
294
|
+
setupRepos(adapter, 'tasks');
|
|
295
|
+
await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
|
|
296
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
297
|
+
expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({
|
|
298
|
+
item_type: 'tasks',
|
|
299
|
+
oldest_created_date: iso(300),
|
|
300
|
+
newest_created_date: iso(400),
|
|
301
|
+
oldest_modified_date: iso(350),
|
|
302
|
+
newest_modified_date: iso(450),
|
|
303
|
+
});
|
|
304
|
+
});
|
|
305
|
+
it('should omit unset RFC3339 bounds from worker_metadata', async () => {
|
|
306
|
+
adapter['repos'] = [
|
|
307
|
+
{
|
|
308
|
+
itemType: 'tasks',
|
|
309
|
+
dateRanges: {
|
|
310
|
+
creationDate: { oldest: 0, newest: 0 },
|
|
311
|
+
modifiedDate: { oldest: 0, newest: 0 },
|
|
312
|
+
},
|
|
313
|
+
},
|
|
314
|
+
];
|
|
315
|
+
adapter['lastExtractedItemType'] = 'tasks';
|
|
316
|
+
await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
|
|
317
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
318
|
+
expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({
|
|
319
|
+
item_type: 'tasks',
|
|
320
|
+
});
|
|
321
|
+
});
|
|
322
|
+
it('should send empty worker_metadata when no item type has been extracted', async () => {
|
|
323
|
+
await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
|
|
324
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
325
|
+
expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
|
|
326
|
+
});
|
|
327
|
+
it('should send empty worker_metadata for loader events', async () => {
|
|
328
|
+
setupRepos(adapter, 'tasks');
|
|
329
|
+
await adapter.emit(types_1.LoaderEventType.DataLoadingProgress);
|
|
330
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
331
|
+
expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
|
|
332
|
+
});
|
|
333
|
+
it('should send empty worker_metadata for non-progress extraction events', async () => {
|
|
334
|
+
setupRepos(adapter, 'tasks');
|
|
335
|
+
await adapter.emit(types_1.ExtractorEventType.DataExtractionError);
|
|
336
|
+
const { emit: mockEmit } = require('../../common/control-protocol');
|
|
337
|
+
expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
|
|
338
|
+
});
|
|
339
|
+
});
|
|
265
340
|
describe(`${worker_adapter_1.WorkerAdapter.name}.emit — ExternalSyncUnitExtractionDone legacy path`, () => {
|
|
266
341
|
it('should upload ESUs via a repo and strip external_sync_units from the emitted payload', async () => {
|
|
267
342
|
// Arrange
|
|
@@ -19,4 +19,5 @@ export declare function addReportToLoaderReport({ loaderReports, report, }: {
|
|
|
19
19
|
loaderReports: LoaderReport[];
|
|
20
20
|
report: LoaderReport;
|
|
21
21
|
}): LoaderReport[];
|
|
22
|
+
export declare function toRfc3339Timestamp(ms: number): string | undefined;
|
|
22
23
|
//# sourceMappingURL=worker-adapter.helpers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}
|
|
1
|
+
{"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMjE"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.getFilesToLoad = getFilesToLoad;
|
|
4
4
|
exports.addReportToLoaderReport = addReportToLoaderReport;
|
|
5
|
+
exports.toRfc3339Timestamp = toRfc3339Timestamp;
|
|
5
6
|
const loading_1 = require("../../types/loading");
|
|
6
7
|
/**
|
|
7
8
|
* Gets the files to load for the loader.
|
|
@@ -62,3 +63,9 @@ function addReportToLoaderReport({ loaderReports, report, }) {
|
|
|
62
63
|
}
|
|
63
64
|
return loaderReports;
|
|
64
65
|
}
|
|
66
|
+
function toRfc3339Timestamp(ms) {
|
|
67
|
+
if (!Number.isFinite(ms) || ms === 0) {
|
|
68
|
+
return undefined;
|
|
69
|
+
}
|
|
70
|
+
return new Date(ms).toISOString();
|
|
71
|
+
}
|
|
@@ -114,6 +114,7 @@ class WorkerAdapter {
|
|
|
114
114
|
repo.itemType !== constants_1.SSOR_ATTACHMENT;
|
|
115
115
|
return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
|
|
116
116
|
var _a;
|
|
117
|
+
this.lastExtractedItemType = repo.itemType;
|
|
117
118
|
// We need to store artifacts ids in state for later use when streaming attachments
|
|
118
119
|
if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
|
|
119
120
|
(_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
|
|
@@ -239,12 +240,30 @@ class WorkerAdapter {
|
|
|
239
240
|
}
|
|
240
241
|
const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
|
|
241
242
|
const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
|
|
243
|
+
const progressData = {};
|
|
244
|
+
if (isExtractionEvent &&
|
|
245
|
+
(newEventType == extraction_1.ExtractorEventType.DataExtractionDone ||
|
|
246
|
+
newEventType == extraction_1.ExtractorEventType.DataExtractionProgress ||
|
|
247
|
+
newEventType == extraction_1.ExtractorEventType.AttachmentExtractionDone ||
|
|
248
|
+
newEventType == extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
|
|
249
|
+
const repo = this.lastExtractedItemType
|
|
250
|
+
? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
|
|
251
|
+
: undefined;
|
|
252
|
+
if (repo) {
|
|
253
|
+
progressData.item_type = repo.itemType;
|
|
254
|
+
progressData.newest_created_date = (0, worker_adapter_helpers_1.toRfc3339Timestamp)(repo.dateRanges.creationDate.newest);
|
|
255
|
+
progressData.oldest_created_date = (0, worker_adapter_helpers_1.toRfc3339Timestamp)(repo.dateRanges.creationDate.oldest);
|
|
256
|
+
progressData.newest_modified_date = (0, worker_adapter_helpers_1.toRfc3339Timestamp)(repo.dateRanges.modifiedDate.newest);
|
|
257
|
+
progressData.oldest_modified_date = (0, worker_adapter_helpers_1.toRfc3339Timestamp)(repo.dateRanges.modifiedDate.oldest);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
242
260
|
await (0, control_protocol_1.emit)({
|
|
243
261
|
eventType: newEventType,
|
|
244
262
|
event: this.event,
|
|
245
263
|
data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
|
|
246
264
|
? { reports: this.reports, processed_files: this.processedFiles }
|
|
247
265
|
: {})),
|
|
266
|
+
worker_metadata: Object.assign({}, progressData),
|
|
248
267
|
});
|
|
249
268
|
const message = {
|
|
250
269
|
subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
|
package/dist/repo/repo.d.ts
CHANGED
|
@@ -10,6 +10,16 @@ export declare class Repo {
|
|
|
10
10
|
private onUpload;
|
|
11
11
|
private options?;
|
|
12
12
|
uploadedArtifacts: Artifact[];
|
|
13
|
+
dateRanges: {
|
|
14
|
+
creationDate: {
|
|
15
|
+
oldest: number;
|
|
16
|
+
newest: number;
|
|
17
|
+
};
|
|
18
|
+
modifiedDate: {
|
|
19
|
+
oldest: number;
|
|
20
|
+
newest: number;
|
|
21
|
+
};
|
|
22
|
+
};
|
|
13
23
|
constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
|
|
14
24
|
getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
|
|
15
25
|
upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
|
package/dist/repo/repo.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAM3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU;;;;;;;;;MAGf;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAsDxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAyC5C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"repo.helpers.d.ts","sourceRoot":"","sources":["../../src/repo/repo.helpers.ts"],"names":[],"mappings":"AAAA,wBAAgB,WAAW,CACzB,KAAK,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EACzC,EAAE,EAAE,MAAM,GACT,IAAI,CAON;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGlE"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.updateRange = updateRange;
|
|
4
|
+
exports.toValidTimestamp = toValidTimestamp;
|
|
5
|
+
function updateRange(range, ms) {
|
|
6
|
+
if (range.oldest === 0 || ms < range.oldest) {
|
|
7
|
+
range.oldest = ms;
|
|
8
|
+
}
|
|
9
|
+
if (range.newest === 0 || ms > range.newest) {
|
|
10
|
+
range.newest = ms;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function toValidTimestamp(value) {
|
|
14
|
+
const ms = new Date(value).getTime();
|
|
15
|
+
return Number.isFinite(ms) ? ms : undefined;
|
|
16
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IAIvB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
|
package/dist/repo/repo.js
CHANGED
|
@@ -4,8 +4,13 @@ exports.Repo = void 0;
|
|
|
4
4
|
const constants_1 = require("../common/constants");
|
|
5
5
|
const uploader_1 = require("../uploader/uploader");
|
|
6
6
|
const logger_context_1 = require("../logger/logger.context");
|
|
7
|
+
const repo_helpers_1 = require("./repo.helpers");
|
|
7
8
|
class Repo {
|
|
8
9
|
constructor({ event, itemType, normalize, onUpload, options, }) {
|
|
10
|
+
this.dateRanges = {
|
|
11
|
+
creationDate: { oldest: 0, newest: 0 },
|
|
12
|
+
modifiedDate: { oldest: 0, newest: 0 },
|
|
13
|
+
};
|
|
9
14
|
this.items = [];
|
|
10
15
|
this.itemType = itemType;
|
|
11
16
|
this.normalize = normalize;
|
|
@@ -20,6 +25,22 @@ class Repo {
|
|
|
20
25
|
async upload(batch) {
|
|
21
26
|
const itemsToUpload = batch || this.items;
|
|
22
27
|
if (itemsToUpload.length > 0) {
|
|
28
|
+
for (const item of itemsToUpload) {
|
|
29
|
+
const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
|
|
30
|
+
if (createdDate != null) {
|
|
31
|
+
const createdMs = (0, repo_helpers_1.toValidTimestamp)(createdDate);
|
|
32
|
+
if (createdMs !== undefined) {
|
|
33
|
+
(0, repo_helpers_1.updateRange)(this.dateRanges.creationDate, createdMs);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const modifiedDate = item === null || item === void 0 ? void 0 : item.modified_date;
|
|
37
|
+
if (modifiedDate != null && modifiedDate !== '') {
|
|
38
|
+
const modifiedMs = (0, repo_helpers_1.toValidTimestamp)(modifiedDate);
|
|
39
|
+
if (modifiedMs !== undefined) {
|
|
40
|
+
(0, repo_helpers_1.updateRange)(this.dateRanges.modifiedDate, modifiedMs);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
23
44
|
console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
|
|
24
45
|
const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
|
|
25
46
|
if (error || !artifact) {
|
package/dist/repo/repo.test.js
CHANGED
|
@@ -7,6 +7,22 @@ const test_utils_1 = require("../common/test-utils");
|
|
|
7
7
|
const types_1 = require("../types");
|
|
8
8
|
const repo_1 = require("./repo");
|
|
9
9
|
jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
|
|
10
|
+
const mockUploadFn = jest.fn().mockResolvedValue({
|
|
11
|
+
error: null,
|
|
12
|
+
artifact: { id: 'art-1', item_type: 'test', item_count: 0 },
|
|
13
|
+
});
|
|
14
|
+
jest.mock('../uploader/uploader', () => ({
|
|
15
|
+
Uploader: jest.fn().mockImplementation(() => ({
|
|
16
|
+
upload: mockUploadFn,
|
|
17
|
+
})),
|
|
18
|
+
}));
|
|
19
|
+
function itemWithDate(id, created_date) {
|
|
20
|
+
return { id, created_date, modified_date: created_date, data: {} };
|
|
21
|
+
}
|
|
22
|
+
function itemWithDates(id, created_date, modified_date) {
|
|
23
|
+
return { id, created_date, modified_date, data: {} };
|
|
24
|
+
}
|
|
25
|
+
const ts = (iso) => new Date(iso).getTime();
|
|
10
26
|
describe(repo_1.Repo.name, () => {
|
|
11
27
|
let repo;
|
|
12
28
|
let normalize;
|
|
@@ -181,4 +197,141 @@ describe(repo_1.Repo.name, () => {
|
|
|
181
197
|
expect(repo.getItems().length).toBe(5);
|
|
182
198
|
});
|
|
183
199
|
});
|
|
200
|
+
describe('dateRanges', () => {
|
|
201
|
+
beforeEach(() => {
|
|
202
|
+
mockUploadFn.mockResolvedValue({
|
|
203
|
+
error: null,
|
|
204
|
+
artifact: { id: 'art-1', item_type: 'test', item_count: 0 },
|
|
205
|
+
});
|
|
206
|
+
});
|
|
207
|
+
it('should track min and max created_date from a single upload batch', async () => {
|
|
208
|
+
await repo.upload([
|
|
209
|
+
itemWithDate('1', '2023-06-15T12:00:00.000Z'),
|
|
210
|
+
itemWithDate('2', '2020-01-01T00:00:00.000Z'),
|
|
211
|
+
itemWithDate('3', '2021-03-01T00:00:00.000Z'),
|
|
212
|
+
]);
|
|
213
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
214
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2023-06-15T12:00:00.000Z'));
|
|
215
|
+
expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
216
|
+
expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-06-15T12:00:00.000Z'));
|
|
217
|
+
});
|
|
218
|
+
it('should skip items without created_date', async () => {
|
|
219
|
+
const attachmentWithoutDate = {
|
|
220
|
+
id: 'att-1',
|
|
221
|
+
url: 'https://example.com/file',
|
|
222
|
+
file_name: 'file.txt',
|
|
223
|
+
parent_id: 'parent-1',
|
|
224
|
+
};
|
|
225
|
+
await repo.upload([
|
|
226
|
+
itemWithDate('1', '2022-06-01T00:00:00.000Z'),
|
|
227
|
+
{ id: '2', created_date: null, modified_date: '', data: {} },
|
|
228
|
+
{ id: '3', modified_date: '', data: {} },
|
|
229
|
+
attachmentWithoutDate,
|
|
230
|
+
]);
|
|
231
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-06-01T00:00:00.000Z'));
|
|
232
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-06-01T00:00:00.000Z'));
|
|
233
|
+
});
|
|
234
|
+
it('should leave timestamps at zero when no items have created_date', async () => {
|
|
235
|
+
await repo.upload([
|
|
236
|
+
{ id: '1', modified_date: '', data: {} },
|
|
237
|
+
{
|
|
238
|
+
id: 'att-1',
|
|
239
|
+
url: 'https://example.com/file',
|
|
240
|
+
file_name: 'file.txt',
|
|
241
|
+
parent_id: 'parent-1',
|
|
242
|
+
},
|
|
243
|
+
]);
|
|
244
|
+
expect(repo.dateRanges).toEqual({
|
|
245
|
+
creationDate: { oldest: 0, newest: 0 },
|
|
246
|
+
modifiedDate: { oldest: 0, newest: 0 },
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
it('should not update timestamps or call uploader on empty upload', async () => {
|
|
250
|
+
await repo.upload([]);
|
|
251
|
+
expect(repo.dateRanges).toEqual({
|
|
252
|
+
creationDate: { oldest: 0, newest: 0 },
|
|
253
|
+
modifiedDate: { oldest: 0, newest: 0 },
|
|
254
|
+
});
|
|
255
|
+
expect(mockUploadFn).not.toHaveBeenCalled();
|
|
256
|
+
});
|
|
257
|
+
it('should accumulate min and max across multiple upload batches via push', async () => {
|
|
258
|
+
repo = new repo_1.Repo({
|
|
259
|
+
event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
|
|
260
|
+
payload: { event_type: types_1.EventType.ExtractionDataStart },
|
|
261
|
+
}),
|
|
262
|
+
itemType: 'test_item_type',
|
|
263
|
+
onUpload: jest.fn(),
|
|
264
|
+
options: { batchSize: 3 },
|
|
265
|
+
});
|
|
266
|
+
const dates = [
|
|
267
|
+
'2020-01-01T00:00:00.000Z',
|
|
268
|
+
'2021-01-01T00:00:00.000Z',
|
|
269
|
+
'2022-01-01T00:00:00.000Z',
|
|
270
|
+
'2023-01-01T00:00:00.000Z',
|
|
271
|
+
'2024-06-01T00:00:00.000Z',
|
|
272
|
+
'2024-12-01T00:00:00.000Z',
|
|
273
|
+
'2024-12-31T00:00:00.000Z',
|
|
274
|
+
];
|
|
275
|
+
const items = dates.map((created_date, index) => itemWithDate(String(index), created_date));
|
|
276
|
+
await repo.push(items);
|
|
277
|
+
expect(repo.getItems()).toHaveLength(1);
|
|
278
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
279
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-12-01T00:00:00.000Z'));
|
|
280
|
+
await repo.push([
|
|
281
|
+
itemWithDate('7', '2019-01-01T00:00:00.000Z'),
|
|
282
|
+
itemWithDate('8', '2025-01-01T00:00:00.000Z'),
|
|
283
|
+
]);
|
|
284
|
+
expect(repo.getItems()).toHaveLength(0);
|
|
285
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2019-01-01T00:00:00.000Z'));
|
|
286
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2025-01-01T00:00:00.000Z'));
|
|
287
|
+
});
|
|
288
|
+
it('should extend min and max when subsequent batches have wider date range', async () => {
|
|
289
|
+
await repo.upload([
|
|
290
|
+
itemWithDate('1', '2022-06-01T00:00:00.000Z'),
|
|
291
|
+
itemWithDate('2', '2023-06-01T00:00:00.000Z'),
|
|
292
|
+
]);
|
|
293
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-06-01T00:00:00.000Z'));
|
|
294
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2023-06-01T00:00:00.000Z'));
|
|
295
|
+
await repo.upload([
|
|
296
|
+
itemWithDate('3', '2020-01-01T00:00:00.000Z'),
|
|
297
|
+
itemWithDate('4', '2024-01-01T00:00:00.000Z'),
|
|
298
|
+
]);
|
|
299
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
300
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-01-01T00:00:00.000Z'));
|
|
301
|
+
});
|
|
302
|
+
it('should update timestamps even when upload fails', async () => {
|
|
303
|
+
mockUploadFn.mockResolvedValueOnce({
|
|
304
|
+
error: new Error('fail'),
|
|
305
|
+
artifact: null,
|
|
306
|
+
});
|
|
307
|
+
await repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')]);
|
|
308
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
309
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
310
|
+
});
|
|
311
|
+
it('should ignore invalid created_date and modified_date values', async () => {
|
|
312
|
+
await repo.upload([
|
|
313
|
+
{
|
|
314
|
+
id: '1',
|
|
315
|
+
created_date: 'not-a-date',
|
|
316
|
+
modified_date: 'still-not-a-date',
|
|
317
|
+
data: {},
|
|
318
|
+
},
|
|
319
|
+
itemWithDates('2', '2022-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'),
|
|
320
|
+
]);
|
|
321
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
322
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
|
|
323
|
+
expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2023-01-01T00:00:00.000Z'));
|
|
324
|
+
expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
|
|
325
|
+
});
|
|
326
|
+
it('should track modified_date independently from created_date', async () => {
|
|
327
|
+
await repo.upload([
|
|
328
|
+
itemWithDates('1', '2020-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'),
|
|
329
|
+
itemWithDates('2', '2024-01-01T00:00:00.000Z', '2021-06-01T00:00:00.000Z'),
|
|
330
|
+
]);
|
|
331
|
+
expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
|
|
332
|
+
expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-01-01T00:00:00.000Z'));
|
|
333
|
+
expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2021-06-01T00:00:00.000Z'));
|
|
334
|
+
expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
|
|
335
|
+
});
|
|
336
|
+
});
|
|
184
337
|
});
|
|
@@ -388,7 +388,14 @@ export interface EventData {
|
|
|
388
388
|
* WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS.
|
|
389
389
|
*/
|
|
390
390
|
export interface WorkerMetadata {
|
|
391
|
-
adaas_library_version
|
|
391
|
+
adaas_library_version?: string;
|
|
392
|
+
item_type?: string;
|
|
393
|
+
oldest_created_date?: string;
|
|
394
|
+
newest_created_date?: string;
|
|
395
|
+
oldest_modified_date?: string;
|
|
396
|
+
newest_modified_date?: string;
|
|
397
|
+
oldest_state_date?: string;
|
|
398
|
+
newest_state_date?: string;
|
|
392
399
|
}
|
|
393
400
|
/**
|
|
394
401
|
* DomainObject is an interface that defines the structure of a domain object that can be extracted.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAI9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAChD,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAW/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA+E1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IA8BzC;;;;OAIG;IACG,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAiCF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IA2DF;;;;OAIG;YACW,sBAAsB;IAsBpC;;;;OAIG;YACW,gBAAgB;IAc9B;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import { UploaderResult } from './uploader.interfaces';
|
|
1
|
+
import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
|
|
2
|
+
/**
|
|
3
|
+
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
4
|
+
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
5
|
+
*/
|
|
6
|
+
export declare function computeArtifactDateRanges(fetchedObjects: object[] | object): ArtifactDateRanges;
|
|
2
7
|
/**
|
|
3
8
|
* Compresses a JSONL string using gzip compression.
|
|
4
9
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
|
|
@@ -36,6 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.computeArtifactDateRanges = computeArtifactDateRanges;
|
|
39
40
|
exports.compressGzip = compressGzip;
|
|
40
41
|
exports.decompressGzip = decompressGzip;
|
|
41
42
|
exports.parseJsonl = parseJsonl;
|
|
@@ -45,6 +46,70 @@ const fs_1 = __importStar(require("fs"));
|
|
|
45
46
|
const js_jsonl_1 = require("js-jsonl");
|
|
46
47
|
const zlib_1 = __importDefault(require("zlib"));
|
|
47
48
|
const constants_1 = require("../common/constants");
|
|
49
|
+
const uploader_interfaces_1 = require("./uploader.interfaces");
|
|
50
|
+
/**
|
|
51
|
+
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
52
|
+
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
53
|
+
*/
|
|
54
|
+
function computeArtifactDateRanges(fetchedObjects) {
|
|
55
|
+
const items = Array.isArray(fetchedObjects)
|
|
56
|
+
? fetchedObjects
|
|
57
|
+
: [fetchedObjects];
|
|
58
|
+
const created = {
|
|
59
|
+
min: Number.POSITIVE_INFINITY,
|
|
60
|
+
max: Number.NEGATIVE_INFINITY,
|
|
61
|
+
};
|
|
62
|
+
const modified = {
|
|
63
|
+
min: Number.POSITIVE_INFINITY,
|
|
64
|
+
max: Number.NEGATIVE_INFINITY,
|
|
65
|
+
};
|
|
66
|
+
let hasCreated = false;
|
|
67
|
+
let hasModified = false;
|
|
68
|
+
const parseTimestamp = (value) => {
|
|
69
|
+
const ts = new Date(value).getTime();
|
|
70
|
+
return Number.isNaN(ts) ? undefined : ts;
|
|
71
|
+
};
|
|
72
|
+
for (const obj of items) {
|
|
73
|
+
if (!obj || typeof obj !== 'object') {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
const item = obj;
|
|
77
|
+
if (item.created_date != undefined) {
|
|
78
|
+
const ts = parseTimestamp(item.created_date);
|
|
79
|
+
if (ts != undefined) {
|
|
80
|
+
if (ts < created.min) {
|
|
81
|
+
created.min = ts;
|
|
82
|
+
}
|
|
83
|
+
if (ts > created.max) {
|
|
84
|
+
created.max = ts;
|
|
85
|
+
}
|
|
86
|
+
hasCreated = true;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (item.modified_date != undefined) {
|
|
90
|
+
const ts = parseTimestamp(item.modified_date);
|
|
91
|
+
if (ts != undefined) {
|
|
92
|
+
if (ts < modified.min) {
|
|
93
|
+
modified.min = ts;
|
|
94
|
+
}
|
|
95
|
+
if (ts > modified.max) {
|
|
96
|
+
modified.max = ts;
|
|
97
|
+
}
|
|
98
|
+
hasModified = true;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const result = {};
|
|
103
|
+
if (hasCreated) {
|
|
104
|
+
result[uploader_interfaces_1.ArtifactDateField.OldestCreatedDate] = new Date(created.min).toISOString();
|
|
105
|
+
result[uploader_interfaces_1.ArtifactDateField.NewestCreatedDate] = new Date(created.max).toISOString();
|
|
106
|
+
}
|
|
107
|
+
if (hasModified) {
|
|
108
|
+
result[uploader_interfaces_1.ArtifactDateField.OldestModifiedDate] = new Date(modified.min).toISOString();
|
|
109
|
+
result[uploader_interfaces_1.ArtifactDateField.NewestModifiedDate] = new Date(modified.max).toISOString();
|
|
110
|
+
}
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
48
113
|
/**
|
|
49
114
|
* Compresses a JSONL string using gzip compression.
|
|
50
115
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
@@ -1,10 +1,42 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
37
|
};
|
|
5
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const fs_1 =
|
|
7
|
-
const fs_2 = require("fs");
|
|
39
|
+
const fs_1 = __importStar(require("fs"));
|
|
8
40
|
const js_jsonl_1 = require("js-jsonl");
|
|
9
41
|
const zlib_1 = __importDefault(require("zlib"));
|
|
10
42
|
const constants_1 = require("../common/constants");
|
|
@@ -99,7 +131,7 @@ describe('uploader.helpers', () => {
|
|
|
99
131
|
close: jest.fn().mockResolvedValue(undefined),
|
|
100
132
|
};
|
|
101
133
|
const fsPromisesOpenSpy = jest
|
|
102
|
-
.spyOn(
|
|
134
|
+
.spyOn(fs_1.promises, 'open')
|
|
103
135
|
.mockResolvedValueOnce(mockFileHandle);
|
|
104
136
|
// Act
|
|
105
137
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
@@ -118,7 +150,7 @@ describe('uploader.helpers', () => {
|
|
|
118
150
|
close: jest.fn().mockResolvedValue(undefined),
|
|
119
151
|
};
|
|
120
152
|
const fsPromisesOpenSpy = jest
|
|
121
|
-
.spyOn(
|
|
153
|
+
.spyOn(fs_1.promises, 'open')
|
|
122
154
|
.mockResolvedValueOnce(mockFileHandle);
|
|
123
155
|
// Act
|
|
124
156
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
@@ -139,7 +171,7 @@ describe('uploader.helpers', () => {
|
|
|
139
171
|
close: jest.fn().mockResolvedValue(undefined),
|
|
140
172
|
};
|
|
141
173
|
const fsPromisesOpenSpy = jest
|
|
142
|
-
.spyOn(
|
|
174
|
+
.spyOn(fs_1.promises, 'open')
|
|
143
175
|
.mockResolvedValueOnce(mockFileHandle);
|
|
144
176
|
// Act
|
|
145
177
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
@@ -158,7 +190,7 @@ describe('uploader.helpers', () => {
|
|
|
158
190
|
close: jest.fn().mockResolvedValue(undefined),
|
|
159
191
|
};
|
|
160
192
|
const fsPromisesOpenSpy = jest
|
|
161
|
-
.spyOn(
|
|
193
|
+
.spyOn(fs_1.promises, 'open')
|
|
162
194
|
.mockResolvedValueOnce(mockFileHandle);
|
|
163
195
|
// Act
|
|
164
196
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
@@ -173,13 +205,122 @@ describe('uploader.helpers', () => {
|
|
|
173
205
|
const fileError = new Error('File write failed');
|
|
174
206
|
mockExistsSync.mockReturnValueOnce(true);
|
|
175
207
|
const fsPromisesOpenSpy = jest
|
|
176
|
-
.spyOn(
|
|
208
|
+
.spyOn(fs_1.promises, 'open')
|
|
177
209
|
.mockRejectedValueOnce(fileError);
|
|
178
210
|
// Act & Assert
|
|
179
211
|
await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
|
|
180
212
|
fsPromisesOpenSpy.mockRestore();
|
|
181
213
|
});
|
|
182
214
|
});
|
|
215
|
+
describe(uploader_helpers_1.computeArtifactDateRanges.name, () => {
|
|
216
|
+
it('should compute min and max across multiple items', () => {
|
|
217
|
+
// Arrange
|
|
218
|
+
const items = [
|
|
219
|
+
{
|
|
220
|
+
id: '1',
|
|
221
|
+
created_date: '2020-01-01T00:00:00.000Z',
|
|
222
|
+
modified_date: '2021-06-01T00:00:00.000Z',
|
|
223
|
+
data: {},
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
id: '2',
|
|
227
|
+
created_date: '2022-03-15T12:00:00.000Z',
|
|
228
|
+
modified_date: '2020-12-31T23:59:59.000Z',
|
|
229
|
+
data: {},
|
|
230
|
+
},
|
|
231
|
+
];
|
|
232
|
+
// Act
|
|
233
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
234
|
+
// Assert
|
|
235
|
+
expect(result.oldest_created_date).toBe('2020-01-01T00:00:00.000Z');
|
|
236
|
+
expect(result.newest_created_date).toBe('2022-03-15T12:00:00.000Z');
|
|
237
|
+
expect(result.oldest_modified_date).toBe('2020-12-31T23:59:59.000Z');
|
|
238
|
+
expect(result.newest_modified_date).toBe('2021-06-01T00:00:00.000Z');
|
|
239
|
+
});
|
|
240
|
+
it('should return zeros when no items have date fields', () => {
|
|
241
|
+
// Arrange
|
|
242
|
+
const items = [
|
|
243
|
+
{ id: 1, name: 'a' },
|
|
244
|
+
{ id: 2, name: 'b' },
|
|
245
|
+
];
|
|
246
|
+
// Act
|
|
247
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
248
|
+
// Assert
|
|
249
|
+
expect(result).toEqual({});
|
|
250
|
+
});
|
|
251
|
+
it('should aggregate only fields that are present on items', () => {
|
|
252
|
+
// Arrange
|
|
253
|
+
const items = [
|
|
254
|
+
{
|
|
255
|
+
id: '1',
|
|
256
|
+
created_date: '2021-01-01T00:00:00.000Z',
|
|
257
|
+
data: {},
|
|
258
|
+
},
|
|
259
|
+
{
|
|
260
|
+
id: '2',
|
|
261
|
+
modified_date: '2023-01-01T00:00:00.000Z',
|
|
262
|
+
data: {},
|
|
263
|
+
},
|
|
264
|
+
];
|
|
265
|
+
// Act
|
|
266
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
267
|
+
// Assert
|
|
268
|
+
expect(result.oldest_created_date).toBe('2021-01-01T00:00:00.000Z');
|
|
269
|
+
expect(result.newest_created_date).toBe('2021-01-01T00:00:00.000Z');
|
|
270
|
+
expect(result.oldest_modified_date).toBe('2023-01-01T00:00:00.000Z');
|
|
271
|
+
expect(result.newest_modified_date).toBe('2023-01-01T00:00:00.000Z');
|
|
272
|
+
});
|
|
273
|
+
it('should handle single object input', () => {
|
|
274
|
+
// Arrange
|
|
275
|
+
const item = {
|
|
276
|
+
id: '1',
|
|
277
|
+
created_date: '2019-05-10T08:30:00.000Z',
|
|
278
|
+
modified_date: '2019-05-10T08:30:00.000Z',
|
|
279
|
+
data: {},
|
|
280
|
+
};
|
|
281
|
+
// Act
|
|
282
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(item);
|
|
283
|
+
// Assert
|
|
284
|
+
const ts = '2019-05-10T08:30:00.000Z';
|
|
285
|
+
expect(result.oldest_created_date).toBe(ts);
|
|
286
|
+
expect(result.newest_created_date).toBe(ts);
|
|
287
|
+
expect(result.oldest_modified_date).toBe(ts);
|
|
288
|
+
expect(result.newest_modified_date).toBe(ts);
|
|
289
|
+
});
|
|
290
|
+
it('should skip invalid date values', () => {
|
|
291
|
+
const items = [
|
|
292
|
+
{
|
|
293
|
+
id: '1',
|
|
294
|
+
created_date: 'not-a-date',
|
|
295
|
+
modified_date: '2024-01-01T00:00:00.000Z',
|
|
296
|
+
data: {},
|
|
297
|
+
},
|
|
298
|
+
];
|
|
299
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
300
|
+
expect(result).toEqual({
|
|
301
|
+
oldest_modified_date: '2024-01-01T00:00:00.000Z',
|
|
302
|
+
newest_modified_date: '2024-01-01T00:00:00.000Z',
|
|
303
|
+
});
|
|
304
|
+
});
|
|
305
|
+
it('[edge] should skip non-object entries in an array', () => {
|
|
306
|
+
// Arrange
|
|
307
|
+
const items = [
|
|
308
|
+
null,
|
|
309
|
+
{
|
|
310
|
+
id: '1',
|
|
311
|
+
created_date: '2024-01-01T00:00:00.000Z',
|
|
312
|
+
modified_date: '2024-06-01T00:00:00.000Z',
|
|
313
|
+
data: {},
|
|
314
|
+
},
|
|
315
|
+
'not-an-object',
|
|
316
|
+
];
|
|
317
|
+
// Act
|
|
318
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
319
|
+
// Assert
|
|
320
|
+
expect(result.oldest_created_date).toBe('2024-01-01T00:00:00.000Z');
|
|
321
|
+
expect(result.newest_created_date).toBe('2024-01-01T00:00:00.000Z');
|
|
322
|
+
});
|
|
323
|
+
});
|
|
183
324
|
describe(uploader_helpers_1.truncateFilename.name, () => {
|
|
184
325
|
it('should return filename unchanged when within the limit', () => {
|
|
185
326
|
// Arrange
|
|
@@ -20,11 +20,22 @@ export type UploaderResult<T> = {
|
|
|
20
20
|
/**
|
|
21
21
|
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
22
22
|
*/
|
|
23
|
+
export declare enum ArtifactDateField {
|
|
24
|
+
OldestCreatedDate = "oldest_created_date",
|
|
25
|
+
NewestCreatedDate = "newest_created_date",
|
|
26
|
+
OldestModifiedDate = "oldest_modified_date",
|
|
27
|
+
NewestModifiedDate = "newest_modified_date"
|
|
28
|
+
}
|
|
23
29
|
export interface Artifact {
|
|
24
30
|
id: string;
|
|
25
31
|
item_type: string;
|
|
26
32
|
item_count: number;
|
|
33
|
+
[ArtifactDateField.OldestCreatedDate]?: string;
|
|
34
|
+
[ArtifactDateField.NewestCreatedDate]?: string;
|
|
35
|
+
[ArtifactDateField.OldestModifiedDate]?: string;
|
|
36
|
+
[ArtifactDateField.NewestModifiedDate]?: string;
|
|
27
37
|
}
|
|
38
|
+
export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
|
|
28
39
|
/**
|
|
29
40
|
* ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
|
|
30
41
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
|
|
@@ -1,2 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ArtifactDateField = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
6
|
+
*/
|
|
7
|
+
var ArtifactDateField;
|
|
8
|
+
(function (ArtifactDateField) {
|
|
9
|
+
ArtifactDateField["OldestCreatedDate"] = "oldest_created_date";
|
|
10
|
+
ArtifactDateField["NewestCreatedDate"] = "newest_created_date";
|
|
11
|
+
ArtifactDateField["OldestModifiedDate"] = "oldest_modified_date";
|
|
12
|
+
ArtifactDateField["NewestModifiedDate"] = "newest_modified_date";
|
|
13
|
+
})(ArtifactDateField || (exports.ArtifactDateField = ArtifactDateField = {}));
|
|
@@ -78,12 +78,9 @@ class Uploader {
|
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
|
+
const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
|
|
81
82
|
// Return the artifact information to the platform
|
|
82
|
-
const artifact = {
|
|
83
|
-
id: preparedArtifact.artifact_id,
|
|
84
|
-
item_type: itemType,
|
|
85
|
-
item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1,
|
|
86
|
-
};
|
|
83
|
+
const artifact = Object.assign({ id: preparedArtifact.artifact_id, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
|
|
87
84
|
return { artifact };
|
|
88
85
|
}
|
|
89
86
|
/**
|
|
@@ -6,12 +6,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
const form_data_1 = __importDefault(require("form-data"));
|
|
7
7
|
const js_jsonl_1 = require("js-jsonl");
|
|
8
8
|
const zlib_1 = __importDefault(require("zlib"));
|
|
9
|
+
const test_utils_1 = require("../common/test-utils");
|
|
9
10
|
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
10
|
-
const test_helpers_1 = require("../tests/test-helpers");
|
|
11
11
|
const jest_setup_1 = require("../tests/jest.setup");
|
|
12
|
-
const
|
|
13
|
-
const uploader_helpers_1 = require("./uploader.helpers");
|
|
12
|
+
const test_helpers_1 = require("../tests/test-helpers");
|
|
14
13
|
const uploader_1 = require("./uploader");
|
|
14
|
+
const uploader_helpers_1 = require("./uploader.helpers");
|
|
15
15
|
jest.mock('../http/axios-client-internal');
|
|
16
16
|
jest.mock('./uploader.helpers', () => (Object.assign(Object.assign({}, jest.requireActual('./uploader.helpers')), { downloadToLocal: jest.fn(), compressGzip: jest.fn(jest.requireActual('./uploader.helpers').compressGzip) })));
|
|
17
17
|
const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
|
|
@@ -51,6 +51,57 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
51
51
|
});
|
|
52
52
|
expect(result.error).toBeUndefined();
|
|
53
53
|
});
|
|
54
|
+
it('should compute oldest/newest created and modified dates from normalized items', async () => {
|
|
55
|
+
var _a, _b, _c, _d;
|
|
56
|
+
// Arrange
|
|
57
|
+
const itemType = 'tasks';
|
|
58
|
+
const fetchedObjects = [
|
|
59
|
+
{
|
|
60
|
+
id: '1',
|
|
61
|
+
created_date: '2020-06-15T10:00:00.000Z',
|
|
62
|
+
modified_date: '2021-01-20T10:00:00.000Z',
|
|
63
|
+
data: { name: 'Task 1' },
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
id: '2',
|
|
67
|
+
created_date: '2019-03-01T08:00:00.000Z',
|
|
68
|
+
modified_date: '2022-11-30T18:00:00.000Z',
|
|
69
|
+
data: { name: 'Task 2' },
|
|
70
|
+
},
|
|
71
|
+
];
|
|
72
|
+
mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
|
|
73
|
+
mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
|
|
74
|
+
// Act
|
|
75
|
+
const result = await uploader.upload(itemType, fetchedObjects);
|
|
76
|
+
// Assert
|
|
77
|
+
expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe('2019-03-01T08:00:00.000Z');
|
|
78
|
+
expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe('2020-06-15T10:00:00.000Z');
|
|
79
|
+
expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe('2021-01-20T10:00:00.000Z');
|
|
80
|
+
expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe('2022-11-30T18:00:00.000Z');
|
|
81
|
+
expect(result.error).toBeUndefined();
|
|
82
|
+
});
|
|
83
|
+
it('should compute date ranges for single object upload', async () => {
|
|
84
|
+
var _a, _b, _c, _d;
|
|
85
|
+
// Arrange
|
|
86
|
+
const itemType = 'metadata';
|
|
87
|
+
const fetchedObject = {
|
|
88
|
+
id: '1',
|
|
89
|
+
created_date: '2018-12-25T00:00:00.000Z',
|
|
90
|
+
modified_date: '2018-12-26T00:00:00.000Z',
|
|
91
|
+
data: { key: 'value' },
|
|
92
|
+
};
|
|
93
|
+
mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
|
|
94
|
+
mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
|
|
95
|
+
// Act
|
|
96
|
+
const result = await uploader.upload(itemType, fetchedObject);
|
|
97
|
+
// Assert
|
|
98
|
+
const createdTs = '2018-12-25T00:00:00.000Z';
|
|
99
|
+
const modifiedTs = '2018-12-26T00:00:00.000Z';
|
|
100
|
+
expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe(createdTs);
|
|
101
|
+
expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe(createdTs);
|
|
102
|
+
expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe(modifiedTs);
|
|
103
|
+
expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe(modifiedTs);
|
|
104
|
+
});
|
|
54
105
|
it('should report item_count as 1 when uploading single object', async () => {
|
|
55
106
|
var _a;
|
|
56
107
|
// Arrange
|