@devrev/ts-adaas 1.19.8 → 1.19.9-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/common/control-protocol.d.ts +3 -2
  2. package/dist/common/control-protocol.d.ts.map +1 -1
  3. package/dist/common/control-protocol.js +2 -4
  4. package/dist/common/control-protocol.test.d.ts +2 -0
  5. package/dist/common/control-protocol.test.d.ts.map +1 -0
  6. package/dist/common/control-protocol.test.js +126 -0
  7. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +1 -0
  8. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
  9. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +75 -0
  10. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +1 -0
  11. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -1
  12. package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +7 -0
  13. package/dist/multithreading/worker-adapter/worker-adapter.js +19 -0
  14. package/dist/repo/repo.d.ts +10 -0
  15. package/dist/repo/repo.d.ts.map +1 -1
  16. package/dist/repo/repo.helpers.d.ts +6 -0
  17. package/dist/repo/repo.helpers.d.ts.map +1 -0
  18. package/dist/repo/repo.helpers.js +16 -0
  19. package/dist/repo/repo.interfaces.d.ts +2 -0
  20. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  21. package/dist/repo/repo.js +21 -0
  22. package/dist/repo/repo.test.js +153 -0
  23. package/dist/types/extraction.d.ts +8 -1
  24. package/dist/types/extraction.d.ts.map +1 -1
  25. package/dist/uploader/uploader.d.ts.map +1 -1
  26. package/dist/uploader/uploader.helpers.d.ts +6 -1
  27. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  28. package/dist/uploader/uploader.helpers.js +65 -0
  29. package/dist/uploader/uploader.helpers.test.js +148 -7
  30. package/dist/uploader/uploader.interfaces.d.ts +11 -0
  31. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  32. package/dist/uploader/uploader.interfaces.js +11 -0
  33. package/dist/uploader/uploader.js +2 -5
  34. package/dist/uploader/uploader.test.js +54 -3
  35. package/package.json +1 -1
@@ -1,10 +1,11 @@
1
1
  import { AxiosResponse } from 'axios';
2
- import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
2
+ import { AirdropEvent, EventData, ExtractorEventType, WorkerMetadata } from '../types/extraction';
3
3
  import { LoaderEventType } from '../types/loading';
4
4
  export interface EmitInterface {
5
5
  event: AirdropEvent;
6
6
  eventType: ExtractorEventType | LoaderEventType;
7
7
  data?: EventData;
8
+ worker_metadata?: WorkerMetadata;
8
9
  }
9
- export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<AxiosResponse>;
10
+ export declare const emit: ({ event, eventType, data, worker_metadata, }: EmitInterface) => Promise<AxiosResponse>;
10
11
  //# sourceMappingURL=control-protocol.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"control-protocol.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EAEnB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,CAAC,EAAE,SAAS,CAAC;CAClB;AAED,eAAO,MAAM,IAAI,GAAU,6BAIxB,aAAa,KAAG,OAAO,CAAC,aAAa,CAgCvC,CAAC"}
1
+ {"version":3,"file":"control-protocol.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EAElB,cAAc,EACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,eAAO,MAAM,IAAI,GAAU,8CAKxB,aAAa,KAAG,OAAO,CAAC,aAAa,CAmCvC,CAAC"}
@@ -4,7 +4,7 @@ exports.emit = void 0;
4
4
  const axios_client_internal_1 = require("../http/axios-client-internal");
5
5
  const constants_1 = require("./constants");
6
6
  const event_type_translation_1 = require("./event-type-translation");
7
- const emit = async ({ event, eventType, data, }) => {
7
+ const emit = async ({ event, eventType, data, worker_metadata, }) => {
8
8
  // Translate outgoing event type to ensure we always send new event types
9
9
  // TODO: Remove when the old types are completely phased out
10
10
  const translatedEventType = (0, event_type_translation_1.translateOutgoingEventType)(eventType);
@@ -12,9 +12,7 @@ const emit = async ({ event, eventType, data, }) => {
12
12
  event_type: translatedEventType,
13
13
  event_context: event.payload.event_context,
14
14
  event_data: Object.assign({}, data),
15
- worker_metadata: {
16
- adaas_library_version: constants_1.LIBRARY_VERSION,
17
- },
15
+ worker_metadata: Object.assign(Object.assign({}, worker_metadata), { newest_state_date: event.payload.event_context.extract_to, oldest_state_date: event.payload.event_context.extract_from, adaas_library_version: constants_1.LIBRARY_VERSION }),
18
16
  };
19
17
  console.info('Emitting event', newEvent);
20
18
  return axios_client_internal_1.axiosClient.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=control-protocol.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"control-protocol.test.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,126 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const axios_client_internal_1 = require("../http/axios-client-internal");
4
+ const test_helpers_1 = require("../tests/test-helpers");
5
+ const test_utils_1 = require("./test-utils");
6
+ const control_protocol_1 = require("./control-protocol");
7
+ const extraction_1 = require("../types/extraction");
8
+ const loading_1 = require("../types/loading");
9
+ jest.mock('../http/axios-client-internal');
10
+ const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
11
+ describe('control-protocol.emit', () => {
12
+ beforeEach(() => {
13
+ jest.clearAllMocks();
14
+ mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
15
+ });
16
+ it.each([
17
+ {
18
+ title: 'extractor events',
19
+ inputEventType: extraction_1.EventType.StartExtractingData,
20
+ outputEventType: extraction_1.ExtractorEventType.DataExtractionProgress,
21
+ },
22
+ {
23
+ title: 'loader events',
24
+ inputEventType: extraction_1.EventType.StartLoadingData,
25
+ outputEventType: loading_1.LoaderEventType.DataLoadingProgress,
26
+ },
27
+ {
28
+ title: 'unknown events',
29
+ inputEventType: extraction_1.EventType.StartExtractingData,
30
+ outputEventType: 'SOME_UNKNOWN_EVENT',
31
+ },
32
+ ])('sets state dates from event context for $title', async ({ inputEventType, outputEventType }) => {
33
+ const event = (0, test_utils_1.createMockEvent)(undefined, {
34
+ payload: {
35
+ event_type: inputEventType,
36
+ event_context: {
37
+ extract_from: '2024-01-01T00:00:00.000Z',
38
+ extract_to: '2024-06-01T00:00:00.000Z',
39
+ },
40
+ },
41
+ });
42
+ await (0, control_protocol_1.emit)({
43
+ event,
44
+ eventType: outputEventType,
45
+ });
46
+ expect(mockedAxiosClient.post).toHaveBeenCalledTimes(1);
47
+ const [, body, config] = mockedAxiosClient.post.mock.calls[0];
48
+ expect(body).toMatchObject({
49
+ event_type: outputEventType,
50
+ event_context: expect.objectContaining({
51
+ extract_from: '2024-01-01T00:00:00.000Z',
52
+ extract_to: '2024-06-01T00:00:00.000Z',
53
+ }),
54
+ worker_metadata: expect.objectContaining({
55
+ oldest_state_date: '2024-01-01T00:00:00.000Z',
56
+ newest_state_date: '2024-06-01T00:00:00.000Z',
57
+ }),
58
+ });
59
+ expect(config).toEqual(expect.objectContaining({
60
+ headers: expect.objectContaining({
61
+ 'X-DevRev-Client-Version': expect.any(String),
62
+ }),
63
+ }));
64
+ });
65
+ it.each([
66
+ {
67
+ title: 'only extract_from is set',
68
+ extractFrom: '2024-01-01T00:00:00.000Z',
69
+ extractTo: undefined,
70
+ },
71
+ {
72
+ title: 'only extract_to is set',
73
+ extractFrom: undefined,
74
+ extractTo: '2024-06-01T00:00:00.000Z',
75
+ },
76
+ {
77
+ title: 'neither extract_from nor extract_to is set',
78
+ extractFrom: undefined,
79
+ extractTo: undefined,
80
+ },
81
+ ])('handles state-date absence when $title', async ({ extractFrom, extractTo }) => {
82
+ const event = (0, test_utils_1.createMockEvent)(undefined, {
83
+ payload: {
84
+ event_type: extraction_1.EventType.StartExtractingData,
85
+ event_context: {
86
+ extract_from: extractFrom,
87
+ extract_to: extractTo,
88
+ },
89
+ },
90
+ });
91
+ await (0, control_protocol_1.emit)({
92
+ event,
93
+ eventType: extraction_1.ExtractorEventType.DataExtractionProgress,
94
+ });
95
+ const [, body] = mockedAxiosClient.post.mock.calls[0];
96
+ const workerMetadata = body.worker_metadata;
97
+ expect(workerMetadata.oldest_state_date).toBe(extractFrom);
98
+ expect(workerMetadata.newest_state_date).toBe(extractTo);
99
+ });
100
+ it('overrides caller-provided worker_metadata state dates', async () => {
101
+ const event = (0, test_utils_1.createMockEvent)(undefined, {
102
+ payload: {
103
+ event_type: extraction_1.EventType.StartExtractingData,
104
+ event_context: {
105
+ extract_from: '2024-01-01T00:00:00.000Z',
106
+ extract_to: '2024-06-01T00:00:00.000Z',
107
+ },
108
+ },
109
+ });
110
+ await (0, control_protocol_1.emit)({
111
+ event,
112
+ eventType: extraction_1.ExtractorEventType.DataExtractionProgress,
113
+ worker_metadata: {
114
+ item_type: 'tasks',
115
+ oldest_state_date: 'should-be-overwritten',
116
+ newest_state_date: 'should-be-overwritten',
117
+ },
118
+ });
119
+ const [, body] = mockedAxiosClient.post.mock.calls[0];
120
+ expect(body.worker_metadata).toEqual(expect.objectContaining({
121
+ item_type: 'tasks',
122
+ oldest_state_date: '2024-01-01T00:00:00.000Z',
123
+ newest_state_date: '2024-06-01T00:00:00.000Z',
124
+ }));
125
+ });
126
+ });
@@ -32,6 +32,7 @@ export declare class WorkerAdapter<ConnectorState> {
32
32
  private adapterState;
33
33
  private _artifacts;
34
34
  private repos;
35
+ private lastExtractedItemType?;
35
36
  private currentEventDataLength;
36
37
  private loaderReports;
37
38
  private _processedFiles;
@@ -1 +1 @@
1
- {"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAI9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAC1C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,CAElC;IAEH,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAoBzC,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,EAK3B;IAED,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IA0JV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAoHvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAuIzC"}
1
+ {"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAoBA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAI9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAC1C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,CAElC;IAEH,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAoBzC,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,EAK3B;IAED,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,kDAElB;IAED;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAyCtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAmMV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAoHvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAuIzC"}
@@ -43,6 +43,7 @@ function makeAdapter(eventType = types_1.EventType.StartExtractingData) {
43
43
  const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
44
44
  return { adapter, event, adapterState };
45
45
  }
46
+ const iso = (ms) => new Date(ms).toISOString();
46
47
  describe(`${worker_adapter_1.WorkerAdapter.name}.emit`, () => {
47
48
  let adapter;
48
49
  let mockPostMessage;
@@ -262,6 +263,80 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.emit`, () => {
262
263
  expect(emittedMessage.startsWith('E'.repeat(100))).toBe(true);
263
264
  });
264
265
  });
266
+ describe(`${worker_adapter_1.WorkerAdapter.name}.emit — worker_metadata`, () => {
267
+ let adapter;
268
+ beforeEach(() => {
269
+ jest.clearAllMocks();
270
+ ({ adapter } = makeAdapter());
271
+ adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
272
+ adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
273
+ });
274
+ function setupRepos(adapterInstance, lastExtractedItemType) {
275
+ adapterInstance['repos'] = [
276
+ {
277
+ itemType: 'issues',
278
+ dateRanges: {
279
+ creationDate: { oldest: 100, newest: 200 },
280
+ modifiedDate: { oldest: 150, newest: 250 },
281
+ },
282
+ },
283
+ {
284
+ itemType: 'tasks',
285
+ dateRanges: {
286
+ creationDate: { oldest: 300, newest: 400 },
287
+ modifiedDate: { oldest: 350, newest: 450 },
288
+ },
289
+ },
290
+ ];
291
+ adapterInstance['lastExtractedItemType'] = lastExtractedItemType;
292
+ }
293
+ it('should emit flat worker_metadata for the latest extracted item type only', async () => {
294
+ setupRepos(adapter, 'tasks');
295
+ await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
296
+ const { emit: mockEmit } = require('../../common/control-protocol');
297
+ expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({
298
+ item_type: 'tasks',
299
+ oldest_created_date: iso(300),
300
+ newest_created_date: iso(400),
301
+ oldest_modified_date: iso(350),
302
+ newest_modified_date: iso(450),
303
+ });
304
+ });
305
+ it('should omit unset RFC3339 bounds from worker_metadata', async () => {
306
+ adapter['repos'] = [
307
+ {
308
+ itemType: 'tasks',
309
+ dateRanges: {
310
+ creationDate: { oldest: 0, newest: 0 },
311
+ modifiedDate: { oldest: 0, newest: 0 },
312
+ },
313
+ },
314
+ ];
315
+ adapter['lastExtractedItemType'] = 'tasks';
316
+ await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
317
+ const { emit: mockEmit } = require('../../common/control-protocol');
318
+ expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({
319
+ item_type: 'tasks',
320
+ });
321
+ });
322
+ it('should send empty worker_metadata when no item type has been extracted', async () => {
323
+ await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
324
+ const { emit: mockEmit } = require('../../common/control-protocol');
325
+ expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
326
+ });
327
+ it('should send empty worker_metadata for loader events', async () => {
328
+ setupRepos(adapter, 'tasks');
329
+ await adapter.emit(types_1.LoaderEventType.DataLoadingProgress);
330
+ const { emit: mockEmit } = require('../../common/control-protocol');
331
+ expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
332
+ });
333
+ it('should send empty worker_metadata for non-progress extraction events', async () => {
334
+ setupRepos(adapter, 'tasks');
335
+ await adapter.emit(types_1.ExtractorEventType.DataExtractionError);
336
+ const { emit: mockEmit } = require('../../common/control-protocol');
337
+ expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
338
+ });
339
+ });
265
340
  describe(`${worker_adapter_1.WorkerAdapter.name}.emit — ExternalSyncUnitExtractionDone legacy path`, () => {
266
341
  it('should upload ESUs via a repo and strip external_sync_units from the emitted payload', async () => {
267
342
  // Arrange
@@ -19,4 +19,5 @@ export declare function addReportToLoaderReport({ loaderReports, report, }: {
19
19
  loaderReports: LoaderReport[];
20
20
  report: LoaderReport;
21
21
  }): LoaderReport[];
22
+ export declare function toRfc3339Timestamp(ms: number): string | undefined;
22
23
  //# sourceMappingURL=worker-adapter.helpers.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}
1
+ {"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMjE"}
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.getFilesToLoad = getFilesToLoad;
4
4
  exports.addReportToLoaderReport = addReportToLoaderReport;
5
+ exports.toRfc3339Timestamp = toRfc3339Timestamp;
5
6
  const loading_1 = require("../../types/loading");
6
7
  /**
7
8
  * Gets the files to load for the loader.
@@ -62,3 +63,9 @@ function addReportToLoaderReport({ loaderReports, report, }) {
62
63
  }
63
64
  return loaderReports;
64
65
  }
66
+ function toRfc3339Timestamp(ms) {
67
+ if (!Number.isFinite(ms) || ms === 0) {
68
+ return undefined;
69
+ }
70
+ return new Date(ms).toISOString();
71
+ }
@@ -114,6 +114,7 @@ class WorkerAdapter {
114
114
  repo.itemType !== constants_1.SSOR_ATTACHMENT;
115
115
  return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
116
116
  var _a;
117
+ this.lastExtractedItemType = repo.itemType;
117
118
  // We need to store artifacts ids in state for later use when streaming attachments
118
119
  if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
119
120
  (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
@@ -239,12 +240,30 @@ class WorkerAdapter {
239
240
  }
240
241
  const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
241
242
  const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
243
+ const progressData = {};
244
+ if (isExtractionEvent &&
245
+ (newEventType == extraction_1.ExtractorEventType.DataExtractionDone ||
246
+ newEventType == extraction_1.ExtractorEventType.DataExtractionProgress ||
247
+ newEventType == extraction_1.ExtractorEventType.AttachmentExtractionDone ||
248
+ newEventType == extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
249
+ const repo = this.lastExtractedItemType
250
+ ? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
251
+ : undefined;
252
+ if (repo) {
253
+ progressData.item_type = repo.itemType;
254
+ progressData.newest_created_date = (0, worker_adapter_helpers_1.toRfc3339Timestamp)(repo.dateRanges.creationDate.newest);
255
+ progressData.oldest_created_date = (0, worker_adapter_helpers_1.toRfc3339Timestamp)(repo.dateRanges.creationDate.oldest);
256
+ progressData.newest_modified_date = (0, worker_adapter_helpers_1.toRfc3339Timestamp)(repo.dateRanges.modifiedDate.newest);
257
+ progressData.oldest_modified_date = (0, worker_adapter_helpers_1.toRfc3339Timestamp)(repo.dateRanges.modifiedDate.oldest);
258
+ }
259
+ }
242
260
  await (0, control_protocol_1.emit)({
243
261
  eventType: newEventType,
244
262
  event: this.event,
245
263
  data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
246
264
  ? { reports: this.reports, processed_files: this.processedFiles }
247
265
  : {})),
266
+ worker_metadata: Object.assign({}, progressData),
248
267
  });
249
268
  const message = {
250
269
  subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
@@ -10,6 +10,16 @@ export declare class Repo {
10
10
  private onUpload;
11
11
  private options?;
12
12
  uploadedArtifacts: Artifact[];
13
+ dateRanges: {
14
+ creationDate: {
15
+ oldest: number;
16
+ newest: number;
17
+ };
18
+ modifiedDate: {
19
+ oldest: number;
20
+ newest: number;
21
+ };
22
+ };
13
23
  constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
14
24
  getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
15
25
  upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
@@ -1 +1 @@
1
- {"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;gBAEzB,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAqCxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAyC5C"}
1
+ {"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAM3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU;;;;;;;;;MAGf;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAsDxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAyC5C"}
@@ -0,0 +1,6 @@
1
+ export declare function updateRange(range: {
2
+ oldest: number;
3
+ newest: number;
4
+ }, ms: number): void;
5
+ export declare function toValidTimestamp(value: string): number | undefined;
6
+ //# sourceMappingURL=repo.helpers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"repo.helpers.d.ts","sourceRoot":"","sources":["../../src/repo/repo.helpers.ts"],"names":[],"mappings":"AAAA,wBAAgB,WAAW,CACzB,KAAK,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EACzC,EAAE,EAAE,MAAM,GACT,IAAI,CAON;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGlE"}
@@ -0,0 +1,16 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.updateRange = updateRange;
4
+ exports.toValidTimestamp = toValidTimestamp;
5
+ function updateRange(range, ms) {
6
+ if (range.oldest === 0 || ms < range.oldest) {
7
+ range.oldest = ms;
8
+ }
9
+ if (range.newest === 0 || ms > range.newest) {
10
+ range.newest = ms;
11
+ }
12
+ }
13
+ function toValidTimestamp(value) {
14
+ const ms = new Date(value).getTime();
15
+ return Number.isFinite(ms) ? ms : undefined;
16
+ }
@@ -39,6 +39,8 @@ export interface NormalizedAttachment {
39
39
  author_id?: string;
40
40
  inline?: boolean;
41
41
  content_type?: string;
42
+ created_date?: string;
43
+ modified_date?: string;
42
44
  grand_parent_id?: number | string;
43
45
  }
44
46
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IAItB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
1
+ {"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IAIvB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
package/dist/repo/repo.js CHANGED
@@ -4,8 +4,13 @@ exports.Repo = void 0;
4
4
  const constants_1 = require("../common/constants");
5
5
  const uploader_1 = require("../uploader/uploader");
6
6
  const logger_context_1 = require("../logger/logger.context");
7
+ const repo_helpers_1 = require("./repo.helpers");
7
8
  class Repo {
8
9
  constructor({ event, itemType, normalize, onUpload, options, }) {
10
+ this.dateRanges = {
11
+ creationDate: { oldest: 0, newest: 0 },
12
+ modifiedDate: { oldest: 0, newest: 0 },
13
+ };
9
14
  this.items = [];
10
15
  this.itemType = itemType;
11
16
  this.normalize = normalize;
@@ -20,6 +25,22 @@ class Repo {
20
25
  async upload(batch) {
21
26
  const itemsToUpload = batch || this.items;
22
27
  if (itemsToUpload.length > 0) {
28
+ for (const item of itemsToUpload) {
29
+ const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
30
+ if (createdDate != null) {
31
+ const createdMs = (0, repo_helpers_1.toValidTimestamp)(createdDate);
32
+ if (createdMs !== undefined) {
33
+ (0, repo_helpers_1.updateRange)(this.dateRanges.creationDate, createdMs);
34
+ }
35
+ }
36
+ const modifiedDate = item === null || item === void 0 ? void 0 : item.modified_date;
37
+ if (modifiedDate != null && modifiedDate !== '') {
38
+ const modifiedMs = (0, repo_helpers_1.toValidTimestamp)(modifiedDate);
39
+ if (modifiedMs !== undefined) {
40
+ (0, repo_helpers_1.updateRange)(this.dateRanges.modifiedDate, modifiedMs);
41
+ }
42
+ }
43
+ }
23
44
  console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
24
45
  const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
25
46
  if (error || !artifact) {
@@ -7,6 +7,22 @@ const test_utils_1 = require("../common/test-utils");
7
7
  const types_1 = require("../types");
8
8
  const repo_1 = require("./repo");
9
9
  jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
10
+ const mockUploadFn = jest.fn().mockResolvedValue({
11
+ error: null,
12
+ artifact: { id: 'art-1', item_type: 'test', item_count: 0 },
13
+ });
14
+ jest.mock('../uploader/uploader', () => ({
15
+ Uploader: jest.fn().mockImplementation(() => ({
16
+ upload: mockUploadFn,
17
+ })),
18
+ }));
19
+ function itemWithDate(id, created_date) {
20
+ return { id, created_date, modified_date: created_date, data: {} };
21
+ }
22
+ function itemWithDates(id, created_date, modified_date) {
23
+ return { id, created_date, modified_date, data: {} };
24
+ }
25
+ const ts = (iso) => new Date(iso).getTime();
10
26
  describe(repo_1.Repo.name, () => {
11
27
  let repo;
12
28
  let normalize;
@@ -181,4 +197,141 @@ describe(repo_1.Repo.name, () => {
181
197
  expect(repo.getItems().length).toBe(5);
182
198
  });
183
199
  });
200
+ describe('dateRanges', () => {
201
+ beforeEach(() => {
202
+ mockUploadFn.mockResolvedValue({
203
+ error: null,
204
+ artifact: { id: 'art-1', item_type: 'test', item_count: 0 },
205
+ });
206
+ });
207
+ it('should track min and max created_date from a single upload batch', async () => {
208
+ await repo.upload([
209
+ itemWithDate('1', '2023-06-15T12:00:00.000Z'),
210
+ itemWithDate('2', '2020-01-01T00:00:00.000Z'),
211
+ itemWithDate('3', '2021-03-01T00:00:00.000Z'),
212
+ ]);
213
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
214
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2023-06-15T12:00:00.000Z'));
215
+ expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
216
+ expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-06-15T12:00:00.000Z'));
217
+ });
218
+ it('should skip items without created_date', async () => {
219
+ const attachmentWithoutDate = {
220
+ id: 'att-1',
221
+ url: 'https://example.com/file',
222
+ file_name: 'file.txt',
223
+ parent_id: 'parent-1',
224
+ };
225
+ await repo.upload([
226
+ itemWithDate('1', '2022-06-01T00:00:00.000Z'),
227
+ { id: '2', created_date: null, modified_date: '', data: {} },
228
+ { id: '3', modified_date: '', data: {} },
229
+ attachmentWithoutDate,
230
+ ]);
231
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-06-01T00:00:00.000Z'));
232
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-06-01T00:00:00.000Z'));
233
+ });
234
+ it('should leave timestamps at zero when no items have created_date', async () => {
235
+ await repo.upload([
236
+ { id: '1', modified_date: '', data: {} },
237
+ {
238
+ id: 'att-1',
239
+ url: 'https://example.com/file',
240
+ file_name: 'file.txt',
241
+ parent_id: 'parent-1',
242
+ },
243
+ ]);
244
+ expect(repo.dateRanges).toEqual({
245
+ creationDate: { oldest: 0, newest: 0 },
246
+ modifiedDate: { oldest: 0, newest: 0 },
247
+ });
248
+ });
249
+ it('should not update timestamps or call uploader on empty upload', async () => {
250
+ await repo.upload([]);
251
+ expect(repo.dateRanges).toEqual({
252
+ creationDate: { oldest: 0, newest: 0 },
253
+ modifiedDate: { oldest: 0, newest: 0 },
254
+ });
255
+ expect(mockUploadFn).not.toHaveBeenCalled();
256
+ });
257
+ it('should accumulate min and max across multiple upload batches via push', async () => {
258
+ repo = new repo_1.Repo({
259
+ event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
260
+ payload: { event_type: types_1.EventType.ExtractionDataStart },
261
+ }),
262
+ itemType: 'test_item_type',
263
+ onUpload: jest.fn(),
264
+ options: { batchSize: 3 },
265
+ });
266
+ const dates = [
267
+ '2020-01-01T00:00:00.000Z',
268
+ '2021-01-01T00:00:00.000Z',
269
+ '2022-01-01T00:00:00.000Z',
270
+ '2023-01-01T00:00:00.000Z',
271
+ '2024-06-01T00:00:00.000Z',
272
+ '2024-12-01T00:00:00.000Z',
273
+ '2024-12-31T00:00:00.000Z',
274
+ ];
275
+ const items = dates.map((created_date, index) => itemWithDate(String(index), created_date));
276
+ await repo.push(items);
277
+ expect(repo.getItems()).toHaveLength(1);
278
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
279
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-12-01T00:00:00.000Z'));
280
+ await repo.push([
281
+ itemWithDate('7', '2019-01-01T00:00:00.000Z'),
282
+ itemWithDate('8', '2025-01-01T00:00:00.000Z'),
283
+ ]);
284
+ expect(repo.getItems()).toHaveLength(0);
285
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2019-01-01T00:00:00.000Z'));
286
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2025-01-01T00:00:00.000Z'));
287
+ });
288
+ it('should extend min and max when subsequent batches have wider date range', async () => {
289
+ await repo.upload([
290
+ itemWithDate('1', '2022-06-01T00:00:00.000Z'),
291
+ itemWithDate('2', '2023-06-01T00:00:00.000Z'),
292
+ ]);
293
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-06-01T00:00:00.000Z'));
294
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2023-06-01T00:00:00.000Z'));
295
+ await repo.upload([
296
+ itemWithDate('3', '2020-01-01T00:00:00.000Z'),
297
+ itemWithDate('4', '2024-01-01T00:00:00.000Z'),
298
+ ]);
299
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
300
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-01-01T00:00:00.000Z'));
301
+ });
302
+ it('should update timestamps even when upload fails', async () => {
303
+ mockUploadFn.mockResolvedValueOnce({
304
+ error: new Error('fail'),
305
+ artifact: null,
306
+ });
307
+ await repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')]);
308
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
309
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
310
+ });
311
+ it('should ignore invalid created_date and modified_date values', async () => {
312
+ await repo.upload([
313
+ {
314
+ id: '1',
315
+ created_date: 'not-a-date',
316
+ modified_date: 'still-not-a-date',
317
+ data: {},
318
+ },
319
+ itemWithDates('2', '2022-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'),
320
+ ]);
321
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
322
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
323
+ expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2023-01-01T00:00:00.000Z'));
324
+ expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
325
+ });
326
+ it('should track modified_date independently from created_date', async () => {
327
+ await repo.upload([
328
+ itemWithDates('1', '2020-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'),
329
+ itemWithDates('2', '2024-01-01T00:00:00.000Z', '2021-06-01T00:00:00.000Z'),
330
+ ]);
331
+ expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
332
+ expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-01-01T00:00:00.000Z'));
333
+ expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2021-06-01T00:00:00.000Z'));
334
+ expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
335
+ });
336
+ });
184
337
  });
@@ -388,7 +388,14 @@ export interface EventData {
388
388
  * WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS.
389
389
  */
390
390
  export interface WorkerMetadata {
391
- adaas_library_version: string;
391
+ adaas_library_version?: string;
392
+ item_type?: string;
393
+ oldest_created_date?: string;
394
+ newest_created_date?: string;
395
+ oldest_modified_date?: string;
396
+ newest_modified_date?: string;
397
+ oldest_state_date?: string;
398
+ newest_state_date?: string;
392
399
  }
393
400
  /**
394
401
  * DomainObject is an interface that defines the structure of a domain object that can be extracted.
@@ -1 +1 @@
1
- {"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAChD,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
1
+ {"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAI9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAChD,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
@@ -1 +1 @@
1
- {"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAU/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA4E1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IA8BzC;;;;OAIG;IACG,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAiCF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IA2DF;;;;OAIG;YACW,sBAAsB;IAsBpC;;;;OAIG;YACW,gBAAgB;IAc9B;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
1
+ {"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAW/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA+E1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IA8BzC;;;;OAIG;IACG,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAiCF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IA2DF;;;;OAIG;YACW,sBAAsB;IAsBpC;;;;OAIG;YACW,gBAAgB;IAc9B;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
@@ -1,4 +1,9 @@
1
- import { UploaderResult } from './uploader.interfaces';
1
+ import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
2
+ /**
3
+ * Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
4
+ * @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
5
+ */
6
+ export declare function computeArtifactDateRanges(fetchedObjects: object[] | object): ArtifactDateRanges;
2
7
  /**
3
8
  * Compresses a JSONL string using gzip compression.
4
9
  * @param {string} jsonlObject - The JSONL string to compress
@@ -1 +1 @@
1
- {"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
1
+ {"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
@@ -36,6 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
36
36
  return (mod && mod.__esModule) ? mod : { "default": mod };
37
37
  };
38
38
  Object.defineProperty(exports, "__esModule", { value: true });
39
+ exports.computeArtifactDateRanges = computeArtifactDateRanges;
39
40
  exports.compressGzip = compressGzip;
40
41
  exports.decompressGzip = decompressGzip;
41
42
  exports.parseJsonl = parseJsonl;
@@ -45,6 +46,70 @@ const fs_1 = __importStar(require("fs"));
45
46
  const js_jsonl_1 = require("js-jsonl");
46
47
  const zlib_1 = __importDefault(require("zlib"));
47
48
  const constants_1 = require("../common/constants");
49
+ const uploader_interfaces_1 = require("./uploader.interfaces");
50
+ /**
51
+ * Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
52
+ * @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
53
+ */
54
+ function computeArtifactDateRanges(fetchedObjects) {
55
+ const items = Array.isArray(fetchedObjects)
56
+ ? fetchedObjects
57
+ : [fetchedObjects];
58
+ const created = {
59
+ min: Number.POSITIVE_INFINITY,
60
+ max: Number.NEGATIVE_INFINITY,
61
+ };
62
+ const modified = {
63
+ min: Number.POSITIVE_INFINITY,
64
+ max: Number.NEGATIVE_INFINITY,
65
+ };
66
+ let hasCreated = false;
67
+ let hasModified = false;
68
+ const parseTimestamp = (value) => {
69
+ const ts = new Date(value).getTime();
70
+ return Number.isNaN(ts) ? undefined : ts;
71
+ };
72
+ for (const obj of items) {
73
+ if (!obj || typeof obj !== 'object') {
74
+ continue;
75
+ }
76
+ const item = obj;
77
+ if (item.created_date != undefined) {
78
+ const ts = parseTimestamp(item.created_date);
79
+ if (ts != undefined) {
80
+ if (ts < created.min) {
81
+ created.min = ts;
82
+ }
83
+ if (ts > created.max) {
84
+ created.max = ts;
85
+ }
86
+ hasCreated = true;
87
+ }
88
+ }
89
+ if (item.modified_date != undefined) {
90
+ const ts = parseTimestamp(item.modified_date);
91
+ if (ts != undefined) {
92
+ if (ts < modified.min) {
93
+ modified.min = ts;
94
+ }
95
+ if (ts > modified.max) {
96
+ modified.max = ts;
97
+ }
98
+ hasModified = true;
99
+ }
100
+ }
101
+ }
102
+ const result = {};
103
+ if (hasCreated) {
104
+ result[uploader_interfaces_1.ArtifactDateField.OldestCreatedDate] = new Date(created.min).toISOString();
105
+ result[uploader_interfaces_1.ArtifactDateField.NewestCreatedDate] = new Date(created.max).toISOString();
106
+ }
107
+ if (hasModified) {
108
+ result[uploader_interfaces_1.ArtifactDateField.OldestModifiedDate] = new Date(modified.min).toISOString();
109
+ result[uploader_interfaces_1.ArtifactDateField.NewestModifiedDate] = new Date(modified.max).toISOString();
110
+ }
111
+ return result;
112
+ }
48
113
  /**
49
114
  * Compresses a JSONL string using gzip compression.
50
115
  * @param {string} jsonlObject - The JSONL string to compress
@@ -1,10 +1,42 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
2
35
  var __importDefault = (this && this.__importDefault) || function (mod) {
3
36
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
37
  };
5
38
  Object.defineProperty(exports, "__esModule", { value: true });
6
- const fs_1 = __importDefault(require("fs"));
7
- const fs_2 = require("fs");
39
+ const fs_1 = __importStar(require("fs"));
8
40
  const js_jsonl_1 = require("js-jsonl");
9
41
  const zlib_1 = __importDefault(require("zlib"));
10
42
  const constants_1 = require("../common/constants");
@@ -99,7 +131,7 @@ describe('uploader.helpers', () => {
99
131
  close: jest.fn().mockResolvedValue(undefined),
100
132
  };
101
133
  const fsPromisesOpenSpy = jest
102
- .spyOn(fs_2.promises, 'open')
134
+ .spyOn(fs_1.promises, 'open')
103
135
  .mockResolvedValueOnce(mockFileHandle);
104
136
  // Act
105
137
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
@@ -118,7 +150,7 @@ describe('uploader.helpers', () => {
118
150
  close: jest.fn().mockResolvedValue(undefined),
119
151
  };
120
152
  const fsPromisesOpenSpy = jest
121
- .spyOn(fs_2.promises, 'open')
153
+ .spyOn(fs_1.promises, 'open')
122
154
  .mockResolvedValueOnce(mockFileHandle);
123
155
  // Act
124
156
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
@@ -139,7 +171,7 @@ describe('uploader.helpers', () => {
139
171
  close: jest.fn().mockResolvedValue(undefined),
140
172
  };
141
173
  const fsPromisesOpenSpy = jest
142
- .spyOn(fs_2.promises, 'open')
174
+ .spyOn(fs_1.promises, 'open')
143
175
  .mockResolvedValueOnce(mockFileHandle);
144
176
  // Act
145
177
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
@@ -158,7 +190,7 @@ describe('uploader.helpers', () => {
158
190
  close: jest.fn().mockResolvedValue(undefined),
159
191
  };
160
192
  const fsPromisesOpenSpy = jest
161
- .spyOn(fs_2.promises, 'open')
193
+ .spyOn(fs_1.promises, 'open')
162
194
  .mockResolvedValueOnce(mockFileHandle);
163
195
  // Act
164
196
  await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
@@ -173,13 +205,122 @@ describe('uploader.helpers', () => {
173
205
  const fileError = new Error('File write failed');
174
206
  mockExistsSync.mockReturnValueOnce(true);
175
207
  const fsPromisesOpenSpy = jest
176
- .spyOn(fs_2.promises, 'open')
208
+ .spyOn(fs_1.promises, 'open')
177
209
  .mockRejectedValueOnce(fileError);
178
210
  // Act & Assert
179
211
  await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
180
212
  fsPromisesOpenSpy.mockRestore();
181
213
  });
182
214
  });
215
+ describe(uploader_helpers_1.computeArtifactDateRanges.name, () => {
216
+ it('should compute min and max across multiple items', () => {
217
+ // Arrange
218
+ const items = [
219
+ {
220
+ id: '1',
221
+ created_date: '2020-01-01T00:00:00.000Z',
222
+ modified_date: '2021-06-01T00:00:00.000Z',
223
+ data: {},
224
+ },
225
+ {
226
+ id: '2',
227
+ created_date: '2022-03-15T12:00:00.000Z',
228
+ modified_date: '2020-12-31T23:59:59.000Z',
229
+ data: {},
230
+ },
231
+ ];
232
+ // Act
233
+ const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
234
+ // Assert
235
+ expect(result.oldest_created_date).toBe('2020-01-01T00:00:00.000Z');
236
+ expect(result.newest_created_date).toBe('2022-03-15T12:00:00.000Z');
237
+ expect(result.oldest_modified_date).toBe('2020-12-31T23:59:59.000Z');
238
+ expect(result.newest_modified_date).toBe('2021-06-01T00:00:00.000Z');
239
+ });
240
+ it('should return zeros when no items have date fields', () => {
241
+ // Arrange
242
+ const items = [
243
+ { id: 1, name: 'a' },
244
+ { id: 2, name: 'b' },
245
+ ];
246
+ // Act
247
+ const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
248
+ // Assert
249
+ expect(result).toEqual({});
250
+ });
251
+ it('should aggregate only fields that are present on items', () => {
252
+ // Arrange
253
+ const items = [
254
+ {
255
+ id: '1',
256
+ created_date: '2021-01-01T00:00:00.000Z',
257
+ data: {},
258
+ },
259
+ {
260
+ id: '2',
261
+ modified_date: '2023-01-01T00:00:00.000Z',
262
+ data: {},
263
+ },
264
+ ];
265
+ // Act
266
+ const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
267
+ // Assert
268
+ expect(result.oldest_created_date).toBe('2021-01-01T00:00:00.000Z');
269
+ expect(result.newest_created_date).toBe('2021-01-01T00:00:00.000Z');
270
+ expect(result.oldest_modified_date).toBe('2023-01-01T00:00:00.000Z');
271
+ expect(result.newest_modified_date).toBe('2023-01-01T00:00:00.000Z');
272
+ });
273
+ it('should handle single object input', () => {
274
+ // Arrange
275
+ const item = {
276
+ id: '1',
277
+ created_date: '2019-05-10T08:30:00.000Z',
278
+ modified_date: '2019-05-10T08:30:00.000Z',
279
+ data: {},
280
+ };
281
+ // Act
282
+ const result = (0, uploader_helpers_1.computeArtifactDateRanges)(item);
283
+ // Assert
284
+ const ts = '2019-05-10T08:30:00.000Z';
285
+ expect(result.oldest_created_date).toBe(ts);
286
+ expect(result.newest_created_date).toBe(ts);
287
+ expect(result.oldest_modified_date).toBe(ts);
288
+ expect(result.newest_modified_date).toBe(ts);
289
+ });
290
+ it('should skip invalid date values', () => {
291
+ const items = [
292
+ {
293
+ id: '1',
294
+ created_date: 'not-a-date',
295
+ modified_date: '2024-01-01T00:00:00.000Z',
296
+ data: {},
297
+ },
298
+ ];
299
+ const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
300
+ expect(result).toEqual({
301
+ oldest_modified_date: '2024-01-01T00:00:00.000Z',
302
+ newest_modified_date: '2024-01-01T00:00:00.000Z',
303
+ });
304
+ });
305
+ it('[edge] should skip non-object entries in an array', () => {
306
+ // Arrange
307
+ const items = [
308
+ null,
309
+ {
310
+ id: '1',
311
+ created_date: '2024-01-01T00:00:00.000Z',
312
+ modified_date: '2024-06-01T00:00:00.000Z',
313
+ data: {},
314
+ },
315
+ 'not-an-object',
316
+ ];
317
+ // Act
318
+ const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
319
+ // Assert
320
+ expect(result.oldest_created_date).toBe('2024-01-01T00:00:00.000Z');
321
+ expect(result.newest_created_date).toBe('2024-01-01T00:00:00.000Z');
322
+ });
323
+ });
183
324
  describe(uploader_helpers_1.truncateFilename.name, () => {
184
325
  it('should return filename unchanged when within the limit', () => {
185
326
  // Arrange
@@ -20,11 +20,22 @@ export type UploaderResult<T> = {
20
20
  /**
21
21
  * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
22
22
  */
23
+ export declare enum ArtifactDateField {
24
+ OldestCreatedDate = "oldest_created_date",
25
+ NewestCreatedDate = "newest_created_date",
26
+ OldestModifiedDate = "oldest_modified_date",
27
+ NewestModifiedDate = "newest_modified_date"
28
+ }
23
29
  export interface Artifact {
24
30
  id: string;
25
31
  item_type: string;
26
32
  item_count: number;
33
+ [ArtifactDateField.OldestCreatedDate]?: string;
34
+ [ArtifactDateField.NewestCreatedDate]?: string;
35
+ [ArtifactDateField.OldestModifiedDate]?: string;
36
+ [ArtifactDateField.NewestModifiedDate]?: string;
27
37
  }
38
+ export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
28
39
  /**
29
40
  * ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
30
41
  */
@@ -1 +1 @@
1
- {"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
1
+ {"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
@@ -1,2 +1,13 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ArtifactDateField = void 0;
4
+ /**
5
+ * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
6
+ */
7
+ var ArtifactDateField;
8
+ (function (ArtifactDateField) {
9
+ ArtifactDateField["OldestCreatedDate"] = "oldest_created_date";
10
+ ArtifactDateField["NewestCreatedDate"] = "newest_created_date";
11
+ ArtifactDateField["OldestModifiedDate"] = "oldest_modified_date";
12
+ ArtifactDateField["NewestModifiedDate"] = "newest_modified_date";
13
+ })(ArtifactDateField || (exports.ArtifactDateField = ArtifactDateField = {}));
@@ -78,12 +78,9 @@ class Uploader {
78
78
  };
79
79
  }
80
80
  }
81
+ const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
81
82
  // Return the artifact information to the platform
82
- const artifact = {
83
- id: preparedArtifact.artifact_id,
84
- item_type: itemType,
85
- item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1,
86
- };
83
+ const artifact = Object.assign({ id: preparedArtifact.artifact_id, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
87
84
  return { artifact };
88
85
  }
89
86
  /**
@@ -6,12 +6,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
6
6
  const form_data_1 = __importDefault(require("form-data"));
7
7
  const js_jsonl_1 = require("js-jsonl");
8
8
  const zlib_1 = __importDefault(require("zlib"));
9
+ const test_utils_1 = require("../common/test-utils");
9
10
  const axios_client_internal_1 = require("../http/axios-client-internal");
10
- const test_helpers_1 = require("../tests/test-helpers");
11
11
  const jest_setup_1 = require("../tests/jest.setup");
12
- const test_utils_1 = require("../common/test-utils");
13
- const uploader_helpers_1 = require("./uploader.helpers");
12
+ const test_helpers_1 = require("../tests/test-helpers");
14
13
  const uploader_1 = require("./uploader");
14
+ const uploader_helpers_1 = require("./uploader.helpers");
15
15
  jest.mock('../http/axios-client-internal');
16
16
  jest.mock('./uploader.helpers', () => (Object.assign(Object.assign({}, jest.requireActual('./uploader.helpers')), { downloadToLocal: jest.fn(), compressGzip: jest.fn(jest.requireActual('./uploader.helpers').compressGzip) })));
17
17
  const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
@@ -51,6 +51,57 @@ describe(uploader_1.Uploader.name, () => {
51
51
  });
52
52
  expect(result.error).toBeUndefined();
53
53
  });
54
+ it('should compute oldest/newest created and modified dates from normalized items', async () => {
55
+ var _a, _b, _c, _d;
56
+ // Arrange
57
+ const itemType = 'tasks';
58
+ const fetchedObjects = [
59
+ {
60
+ id: '1',
61
+ created_date: '2020-06-15T10:00:00.000Z',
62
+ modified_date: '2021-01-20T10:00:00.000Z',
63
+ data: { name: 'Task 1' },
64
+ },
65
+ {
66
+ id: '2',
67
+ created_date: '2019-03-01T08:00:00.000Z',
68
+ modified_date: '2022-11-30T18:00:00.000Z',
69
+ data: { name: 'Task 2' },
70
+ },
71
+ ];
72
+ mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
73
+ mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
74
+ // Act
75
+ const result = await uploader.upload(itemType, fetchedObjects);
76
+ // Assert
77
+ expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe('2019-03-01T08:00:00.000Z');
78
+ expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe('2020-06-15T10:00:00.000Z');
79
+ expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe('2021-01-20T10:00:00.000Z');
80
+ expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe('2022-11-30T18:00:00.000Z');
81
+ expect(result.error).toBeUndefined();
82
+ });
83
+ it('should compute date ranges for single object upload', async () => {
84
+ var _a, _b, _c, _d;
85
+ // Arrange
86
+ const itemType = 'metadata';
87
+ const fetchedObject = {
88
+ id: '1',
89
+ created_date: '2018-12-25T00:00:00.000Z',
90
+ modified_date: '2018-12-26T00:00:00.000Z',
91
+ data: { key: 'value' },
92
+ };
93
+ mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
94
+ mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
95
+ // Act
96
+ const result = await uploader.upload(itemType, fetchedObject);
97
+ // Assert
98
+ const createdTs = '2018-12-25T00:00:00.000Z';
99
+ const modifiedTs = '2018-12-26T00:00:00.000Z';
100
+ expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe(createdTs);
101
+ expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe(createdTs);
102
+ expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe(modifiedTs);
103
+ expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe(modifiedTs);
104
+ });
54
105
  it('should report item_count as 1 when uploading single object', async () => {
55
106
  var _a;
56
107
  // Arrange
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@devrev/ts-adaas",
3
- "version": "1.19.8",
3
+ "version": "1.19.9-beta.1",
4
4
  "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
5
5
  "type": "commonjs",
6
6
  "main": "./dist/index.js",