@devrev/ts-adaas 1.19.9-beta.0 → 1.19.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +6 -4
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +65 -0
- package/dist/common/control-protocol.d.ts +2 -3
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +4 -2
- package/dist/http/axios-client-internal.d.ts.map +1 -1
- package/dist/http/axios-client-internal.js +1 -19
- package/dist/mock-server/mock-server.d.ts +0 -4
- package/dist/mock-server/mock-server.d.ts.map +1 -1
- package/dist/mock-server/mock-server.interfaces.d.ts +4 -9
- package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -1
- package/dist/mock-server/mock-server.js +18 -30
- package/dist/multithreading/spawn/spawn.d.ts +1 -1
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.js +6 -11
- package/dist/multithreading/spawn/spawn.test.js +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +5 -11
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +9 -90
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.js +64 -93
- package/dist/repo/repo.d.ts +2 -11
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +0 -2
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +11 -37
- package/dist/repo/repo.test.js +0 -184
- package/dist/types/extraction.d.ts +1 -8
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/workers.d.ts +0 -4
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +1 -6
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +0 -65
- package/dist/uploader/uploader.helpers.test.js +7 -148
- package/dist/uploader/uploader.interfaces.d.ts +0 -11
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +0 -11
- package/dist/uploader/uploader.js +5 -2
- package/dist/uploader/uploader.test.js +3 -54
- package/package.json +2 -2
- package/dist/common/control-protocol.test.d.ts +0 -2
- package/dist/common/control-protocol.test.d.ts.map +0 -1
- package/dist/common/control-protocol.test.js +0 -126
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.js +0 -135
|
@@ -1,9 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
/**
|
|
3
|
-
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
4
|
-
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
5
|
-
*/
|
|
6
|
-
export declare function computeArtifactDateRanges(fetchedObjects: object[] | object): ArtifactDateRanges;
|
|
1
|
+
import { UploaderResult } from './uploader.interfaces';
|
|
7
2
|
/**
|
|
8
3
|
* Compresses a JSONL string using gzip compression.
|
|
9
4
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
|
|
@@ -36,7 +36,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.computeArtifactDateRanges = computeArtifactDateRanges;
|
|
40
39
|
exports.compressGzip = compressGzip;
|
|
41
40
|
exports.decompressGzip = decompressGzip;
|
|
42
41
|
exports.parseJsonl = parseJsonl;
|
|
@@ -46,70 +45,6 @@ const fs_1 = __importStar(require("fs"));
|
|
|
46
45
|
const js_jsonl_1 = require("js-jsonl");
|
|
47
46
|
const zlib_1 = __importDefault(require("zlib"));
|
|
48
47
|
const constants_1 = require("../common/constants");
|
|
49
|
-
const uploader_interfaces_1 = require("./uploader.interfaces");
|
|
50
|
-
/**
|
|
51
|
-
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
52
|
-
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
53
|
-
*/
|
|
54
|
-
function computeArtifactDateRanges(fetchedObjects) {
|
|
55
|
-
const items = Array.isArray(fetchedObjects)
|
|
56
|
-
? fetchedObjects
|
|
57
|
-
: [fetchedObjects];
|
|
58
|
-
const created = {
|
|
59
|
-
min: Number.POSITIVE_INFINITY,
|
|
60
|
-
max: Number.NEGATIVE_INFINITY,
|
|
61
|
-
};
|
|
62
|
-
const modified = {
|
|
63
|
-
min: Number.POSITIVE_INFINITY,
|
|
64
|
-
max: Number.NEGATIVE_INFINITY,
|
|
65
|
-
};
|
|
66
|
-
let hasCreated = false;
|
|
67
|
-
let hasModified = false;
|
|
68
|
-
const parseTimestamp = (value) => {
|
|
69
|
-
const ts = new Date(value).getTime();
|
|
70
|
-
return Number.isNaN(ts) ? undefined : ts;
|
|
71
|
-
};
|
|
72
|
-
for (const obj of items) {
|
|
73
|
-
if (!obj || typeof obj !== 'object') {
|
|
74
|
-
continue;
|
|
75
|
-
}
|
|
76
|
-
const item = obj;
|
|
77
|
-
if (item.created_date != undefined) {
|
|
78
|
-
const ts = parseTimestamp(item.created_date);
|
|
79
|
-
if (ts != undefined) {
|
|
80
|
-
if (ts < created.min) {
|
|
81
|
-
created.min = ts;
|
|
82
|
-
}
|
|
83
|
-
if (ts > created.max) {
|
|
84
|
-
created.max = ts;
|
|
85
|
-
}
|
|
86
|
-
hasCreated = true;
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
if (item.modified_date != undefined) {
|
|
90
|
-
const ts = parseTimestamp(item.modified_date);
|
|
91
|
-
if (ts != undefined) {
|
|
92
|
-
if (ts < modified.min) {
|
|
93
|
-
modified.min = ts;
|
|
94
|
-
}
|
|
95
|
-
if (ts > modified.max) {
|
|
96
|
-
modified.max = ts;
|
|
97
|
-
}
|
|
98
|
-
hasModified = true;
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
const result = {};
|
|
103
|
-
if (hasCreated) {
|
|
104
|
-
result[uploader_interfaces_1.ArtifactDateField.OldestCreatedDate] = new Date(created.min).toISOString();
|
|
105
|
-
result[uploader_interfaces_1.ArtifactDateField.NewestCreatedDate] = new Date(created.max).toISOString();
|
|
106
|
-
}
|
|
107
|
-
if (hasModified) {
|
|
108
|
-
result[uploader_interfaces_1.ArtifactDateField.OldestModifiedDate] = new Date(modified.min).toISOString();
|
|
109
|
-
result[uploader_interfaces_1.ArtifactDateField.NewestModifiedDate] = new Date(modified.max).toISOString();
|
|
110
|
-
}
|
|
111
|
-
return result;
|
|
112
|
-
}
|
|
113
48
|
/**
|
|
114
49
|
* Compresses a JSONL string using gzip compression.
|
|
115
50
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
@@ -1,42 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
4
|
};
|
|
38
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
const fs_1 =
|
|
6
|
+
const fs_1 = __importDefault(require("fs"));
|
|
7
|
+
const fs_2 = require("fs");
|
|
40
8
|
const js_jsonl_1 = require("js-jsonl");
|
|
41
9
|
const zlib_1 = __importDefault(require("zlib"));
|
|
42
10
|
const constants_1 = require("../common/constants");
|
|
@@ -131,7 +99,7 @@ describe('uploader.helpers', () => {
|
|
|
131
99
|
close: jest.fn().mockResolvedValue(undefined),
|
|
132
100
|
};
|
|
133
101
|
const fsPromisesOpenSpy = jest
|
|
134
|
-
.spyOn(
|
|
102
|
+
.spyOn(fs_2.promises, 'open')
|
|
135
103
|
.mockResolvedValueOnce(mockFileHandle);
|
|
136
104
|
// Act
|
|
137
105
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
@@ -150,7 +118,7 @@ describe('uploader.helpers', () => {
|
|
|
150
118
|
close: jest.fn().mockResolvedValue(undefined),
|
|
151
119
|
};
|
|
152
120
|
const fsPromisesOpenSpy = jest
|
|
153
|
-
.spyOn(
|
|
121
|
+
.spyOn(fs_2.promises, 'open')
|
|
154
122
|
.mockResolvedValueOnce(mockFileHandle);
|
|
155
123
|
// Act
|
|
156
124
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
@@ -171,7 +139,7 @@ describe('uploader.helpers', () => {
|
|
|
171
139
|
close: jest.fn().mockResolvedValue(undefined),
|
|
172
140
|
};
|
|
173
141
|
const fsPromisesOpenSpy = jest
|
|
174
|
-
.spyOn(
|
|
142
|
+
.spyOn(fs_2.promises, 'open')
|
|
175
143
|
.mockResolvedValueOnce(mockFileHandle);
|
|
176
144
|
// Act
|
|
177
145
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
@@ -190,7 +158,7 @@ describe('uploader.helpers', () => {
|
|
|
190
158
|
close: jest.fn().mockResolvedValue(undefined),
|
|
191
159
|
};
|
|
192
160
|
const fsPromisesOpenSpy = jest
|
|
193
|
-
.spyOn(
|
|
161
|
+
.spyOn(fs_2.promises, 'open')
|
|
194
162
|
.mockResolvedValueOnce(mockFileHandle);
|
|
195
163
|
// Act
|
|
196
164
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
@@ -205,122 +173,13 @@ describe('uploader.helpers', () => {
|
|
|
205
173
|
const fileError = new Error('File write failed');
|
|
206
174
|
mockExistsSync.mockReturnValueOnce(true);
|
|
207
175
|
const fsPromisesOpenSpy = jest
|
|
208
|
-
.spyOn(
|
|
176
|
+
.spyOn(fs_2.promises, 'open')
|
|
209
177
|
.mockRejectedValueOnce(fileError);
|
|
210
178
|
// Act & Assert
|
|
211
179
|
await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
|
|
212
180
|
fsPromisesOpenSpy.mockRestore();
|
|
213
181
|
});
|
|
214
182
|
});
|
|
215
|
-
describe(uploader_helpers_1.computeArtifactDateRanges.name, () => {
|
|
216
|
-
it('should compute min and max across multiple items', () => {
|
|
217
|
-
// Arrange
|
|
218
|
-
const items = [
|
|
219
|
-
{
|
|
220
|
-
id: '1',
|
|
221
|
-
created_date: '2020-01-01T00:00:00.000Z',
|
|
222
|
-
modified_date: '2021-06-01T00:00:00.000Z',
|
|
223
|
-
data: {},
|
|
224
|
-
},
|
|
225
|
-
{
|
|
226
|
-
id: '2',
|
|
227
|
-
created_date: '2022-03-15T12:00:00.000Z',
|
|
228
|
-
modified_date: '2020-12-31T23:59:59.000Z',
|
|
229
|
-
data: {},
|
|
230
|
-
},
|
|
231
|
-
];
|
|
232
|
-
// Act
|
|
233
|
-
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
234
|
-
// Assert
|
|
235
|
-
expect(result.oldest_created_date).toBe('2020-01-01T00:00:00.000Z');
|
|
236
|
-
expect(result.newest_created_date).toBe('2022-03-15T12:00:00.000Z');
|
|
237
|
-
expect(result.oldest_modified_date).toBe('2020-12-31T23:59:59.000Z');
|
|
238
|
-
expect(result.newest_modified_date).toBe('2021-06-01T00:00:00.000Z');
|
|
239
|
-
});
|
|
240
|
-
it('should return zeros when no items have date fields', () => {
|
|
241
|
-
// Arrange
|
|
242
|
-
const items = [
|
|
243
|
-
{ id: 1, name: 'a' },
|
|
244
|
-
{ id: 2, name: 'b' },
|
|
245
|
-
];
|
|
246
|
-
// Act
|
|
247
|
-
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
248
|
-
// Assert
|
|
249
|
-
expect(result).toEqual({});
|
|
250
|
-
});
|
|
251
|
-
it('should aggregate only fields that are present on items', () => {
|
|
252
|
-
// Arrange
|
|
253
|
-
const items = [
|
|
254
|
-
{
|
|
255
|
-
id: '1',
|
|
256
|
-
created_date: '2021-01-01T00:00:00.000Z',
|
|
257
|
-
data: {},
|
|
258
|
-
},
|
|
259
|
-
{
|
|
260
|
-
id: '2',
|
|
261
|
-
modified_date: '2023-01-01T00:00:00.000Z',
|
|
262
|
-
data: {},
|
|
263
|
-
},
|
|
264
|
-
];
|
|
265
|
-
// Act
|
|
266
|
-
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
267
|
-
// Assert
|
|
268
|
-
expect(result.oldest_created_date).toBe('2021-01-01T00:00:00.000Z');
|
|
269
|
-
expect(result.newest_created_date).toBe('2021-01-01T00:00:00.000Z');
|
|
270
|
-
expect(result.oldest_modified_date).toBe('2023-01-01T00:00:00.000Z');
|
|
271
|
-
expect(result.newest_modified_date).toBe('2023-01-01T00:00:00.000Z');
|
|
272
|
-
});
|
|
273
|
-
it('should handle single object input', () => {
|
|
274
|
-
// Arrange
|
|
275
|
-
const item = {
|
|
276
|
-
id: '1',
|
|
277
|
-
created_date: '2019-05-10T08:30:00.000Z',
|
|
278
|
-
modified_date: '2019-05-10T08:30:00.000Z',
|
|
279
|
-
data: {},
|
|
280
|
-
};
|
|
281
|
-
// Act
|
|
282
|
-
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(item);
|
|
283
|
-
// Assert
|
|
284
|
-
const ts = '2019-05-10T08:30:00.000Z';
|
|
285
|
-
expect(result.oldest_created_date).toBe(ts);
|
|
286
|
-
expect(result.newest_created_date).toBe(ts);
|
|
287
|
-
expect(result.oldest_modified_date).toBe(ts);
|
|
288
|
-
expect(result.newest_modified_date).toBe(ts);
|
|
289
|
-
});
|
|
290
|
-
it('should skip invalid date values', () => {
|
|
291
|
-
const items = [
|
|
292
|
-
{
|
|
293
|
-
id: '1',
|
|
294
|
-
created_date: 'not-a-date',
|
|
295
|
-
modified_date: '2024-01-01T00:00:00.000Z',
|
|
296
|
-
data: {},
|
|
297
|
-
},
|
|
298
|
-
];
|
|
299
|
-
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
300
|
-
expect(result).toEqual({
|
|
301
|
-
oldest_modified_date: '2024-01-01T00:00:00.000Z',
|
|
302
|
-
newest_modified_date: '2024-01-01T00:00:00.000Z',
|
|
303
|
-
});
|
|
304
|
-
});
|
|
305
|
-
it('[edge] should skip non-object entries in an array', () => {
|
|
306
|
-
// Arrange
|
|
307
|
-
const items = [
|
|
308
|
-
null,
|
|
309
|
-
{
|
|
310
|
-
id: '1',
|
|
311
|
-
created_date: '2024-01-01T00:00:00.000Z',
|
|
312
|
-
modified_date: '2024-06-01T00:00:00.000Z',
|
|
313
|
-
data: {},
|
|
314
|
-
},
|
|
315
|
-
'not-an-object',
|
|
316
|
-
];
|
|
317
|
-
// Act
|
|
318
|
-
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
319
|
-
// Assert
|
|
320
|
-
expect(result.oldest_created_date).toBe('2024-01-01T00:00:00.000Z');
|
|
321
|
-
expect(result.newest_created_date).toBe('2024-01-01T00:00:00.000Z');
|
|
322
|
-
});
|
|
323
|
-
});
|
|
324
183
|
describe(uploader_helpers_1.truncateFilename.name, () => {
|
|
325
184
|
it('should return filename unchanged when within the limit', () => {
|
|
326
185
|
// Arrange
|
|
@@ -20,22 +20,11 @@ export type UploaderResult<T> = {
|
|
|
20
20
|
/**
|
|
21
21
|
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
22
22
|
*/
|
|
23
|
-
export declare enum ArtifactDateField {
|
|
24
|
-
OldestCreatedDate = "oldest_created_date",
|
|
25
|
-
NewestCreatedDate = "newest_created_date",
|
|
26
|
-
OldestModifiedDate = "oldest_modified_date",
|
|
27
|
-
NewestModifiedDate = "newest_modified_date"
|
|
28
|
-
}
|
|
29
23
|
export interface Artifact {
|
|
30
24
|
id: string;
|
|
31
25
|
item_type: string;
|
|
32
26
|
item_count: number;
|
|
33
|
-
[ArtifactDateField.OldestCreatedDate]?: string;
|
|
34
|
-
[ArtifactDateField.NewestCreatedDate]?: string;
|
|
35
|
-
[ArtifactDateField.OldestModifiedDate]?: string;
|
|
36
|
-
[ArtifactDateField.NewestModifiedDate]?: string;
|
|
37
27
|
}
|
|
38
|
-
export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
|
|
39
28
|
/**
|
|
40
29
|
* ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
|
|
41
30
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
|
|
@@ -1,13 +1,2 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ArtifactDateField = void 0;
|
|
4
|
-
/**
|
|
5
|
-
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
6
|
-
*/
|
|
7
|
-
var ArtifactDateField;
|
|
8
|
-
(function (ArtifactDateField) {
|
|
9
|
-
ArtifactDateField["OldestCreatedDate"] = "oldest_created_date";
|
|
10
|
-
ArtifactDateField["NewestCreatedDate"] = "newest_created_date";
|
|
11
|
-
ArtifactDateField["OldestModifiedDate"] = "oldest_modified_date";
|
|
12
|
-
ArtifactDateField["NewestModifiedDate"] = "newest_modified_date";
|
|
13
|
-
})(ArtifactDateField || (exports.ArtifactDateField = ArtifactDateField = {}));
|
|
@@ -78,9 +78,12 @@ class Uploader {
|
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
|
-
const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
|
|
82
81
|
// Return the artifact information to the platform
|
|
83
|
-
const artifact =
|
|
82
|
+
const artifact = {
|
|
83
|
+
id: preparedArtifact.artifact_id,
|
|
84
|
+
item_type: itemType,
|
|
85
|
+
item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1,
|
|
86
|
+
};
|
|
84
87
|
return { artifact };
|
|
85
88
|
}
|
|
86
89
|
/**
|
|
@@ -6,12 +6,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
const form_data_1 = __importDefault(require("form-data"));
|
|
7
7
|
const js_jsonl_1 = require("js-jsonl");
|
|
8
8
|
const zlib_1 = __importDefault(require("zlib"));
|
|
9
|
-
const test_utils_1 = require("../common/test-utils");
|
|
10
9
|
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
11
|
-
const jest_setup_1 = require("../tests/jest.setup");
|
|
12
10
|
const test_helpers_1 = require("../tests/test-helpers");
|
|
13
|
-
const
|
|
11
|
+
const jest_setup_1 = require("../tests/jest.setup");
|
|
12
|
+
const test_utils_1 = require("../common/test-utils");
|
|
14
13
|
const uploader_helpers_1 = require("./uploader.helpers");
|
|
14
|
+
const uploader_1 = require("./uploader");
|
|
15
15
|
jest.mock('../http/axios-client-internal');
|
|
16
16
|
jest.mock('./uploader.helpers', () => (Object.assign(Object.assign({}, jest.requireActual('./uploader.helpers')), { downloadToLocal: jest.fn(), compressGzip: jest.fn(jest.requireActual('./uploader.helpers').compressGzip) })));
|
|
17
17
|
const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
|
|
@@ -51,57 +51,6 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
51
51
|
});
|
|
52
52
|
expect(result.error).toBeUndefined();
|
|
53
53
|
});
|
|
54
|
-
it('should compute oldest/newest created and modified dates from normalized items', async () => {
|
|
55
|
-
var _a, _b, _c, _d;
|
|
56
|
-
// Arrange
|
|
57
|
-
const itemType = 'tasks';
|
|
58
|
-
const fetchedObjects = [
|
|
59
|
-
{
|
|
60
|
-
id: '1',
|
|
61
|
-
created_date: '2020-06-15T10:00:00.000Z',
|
|
62
|
-
modified_date: '2021-01-20T10:00:00.000Z',
|
|
63
|
-
data: { name: 'Task 1' },
|
|
64
|
-
},
|
|
65
|
-
{
|
|
66
|
-
id: '2',
|
|
67
|
-
created_date: '2019-03-01T08:00:00.000Z',
|
|
68
|
-
modified_date: '2022-11-30T18:00:00.000Z',
|
|
69
|
-
data: { name: 'Task 2' },
|
|
70
|
-
},
|
|
71
|
-
];
|
|
72
|
-
mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
|
|
73
|
-
mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
|
|
74
|
-
// Act
|
|
75
|
-
const result = await uploader.upload(itemType, fetchedObjects);
|
|
76
|
-
// Assert
|
|
77
|
-
expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe('2019-03-01T08:00:00.000Z');
|
|
78
|
-
expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe('2020-06-15T10:00:00.000Z');
|
|
79
|
-
expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe('2021-01-20T10:00:00.000Z');
|
|
80
|
-
expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe('2022-11-30T18:00:00.000Z');
|
|
81
|
-
expect(result.error).toBeUndefined();
|
|
82
|
-
});
|
|
83
|
-
it('should compute date ranges for single object upload', async () => {
|
|
84
|
-
var _a, _b, _c, _d;
|
|
85
|
-
// Arrange
|
|
86
|
-
const itemType = 'metadata';
|
|
87
|
-
const fetchedObject = {
|
|
88
|
-
id: '1',
|
|
89
|
-
created_date: '2018-12-25T00:00:00.000Z',
|
|
90
|
-
modified_date: '2018-12-26T00:00:00.000Z',
|
|
91
|
-
data: { key: 'value' },
|
|
92
|
-
};
|
|
93
|
-
mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
|
|
94
|
-
mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
|
|
95
|
-
// Act
|
|
96
|
-
const result = await uploader.upload(itemType, fetchedObject);
|
|
97
|
-
// Assert
|
|
98
|
-
const createdTs = '2018-12-25T00:00:00.000Z';
|
|
99
|
-
const modifiedTs = '2018-12-26T00:00:00.000Z';
|
|
100
|
-
expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe(createdTs);
|
|
101
|
-
expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe(createdTs);
|
|
102
|
-
expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe(modifiedTs);
|
|
103
|
-
expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe(modifiedTs);
|
|
104
|
-
});
|
|
105
54
|
it('should report item_count as 1 when uploading single object', async () => {
|
|
106
55
|
var _a;
|
|
107
56
|
// Arrange
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devrev/ts-adaas",
|
|
3
|
-
"version": "1.19.9
|
|
3
|
+
"version": "1.19.9",
|
|
4
4
|
"description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"typescript-eslint": "^8.46.1"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@devrev/typescript-sdk": "^1.1.
|
|
49
|
+
"@devrev/typescript-sdk": "^1.1.75",
|
|
50
50
|
"axios": "^1.15.2",
|
|
51
51
|
"axios-retry": "^4.5.0",
|
|
52
52
|
"form-data": "^4.0.4",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"control-protocol.test.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,126 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
4
|
-
const test_helpers_1 = require("../tests/test-helpers");
|
|
5
|
-
const test_utils_1 = require("./test-utils");
|
|
6
|
-
const control_protocol_1 = require("./control-protocol");
|
|
7
|
-
const extraction_1 = require("../types/extraction");
|
|
8
|
-
const loading_1 = require("../types/loading");
|
|
9
|
-
jest.mock('../http/axios-client-internal');
|
|
10
|
-
const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
|
|
11
|
-
describe('control-protocol.emit', () => {
|
|
12
|
-
beforeEach(() => {
|
|
13
|
-
jest.clearAllMocks();
|
|
14
|
-
mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
|
|
15
|
-
});
|
|
16
|
-
it.each([
|
|
17
|
-
{
|
|
18
|
-
title: 'extractor events',
|
|
19
|
-
inputEventType: extraction_1.EventType.StartExtractingData,
|
|
20
|
-
outputEventType: extraction_1.ExtractorEventType.DataExtractionProgress,
|
|
21
|
-
},
|
|
22
|
-
{
|
|
23
|
-
title: 'loader events',
|
|
24
|
-
inputEventType: extraction_1.EventType.StartLoadingData,
|
|
25
|
-
outputEventType: loading_1.LoaderEventType.DataLoadingProgress,
|
|
26
|
-
},
|
|
27
|
-
{
|
|
28
|
-
title: 'unknown events',
|
|
29
|
-
inputEventType: extraction_1.EventType.StartExtractingData,
|
|
30
|
-
outputEventType: 'SOME_UNKNOWN_EVENT',
|
|
31
|
-
},
|
|
32
|
-
])('sets state dates from event context for $title', async ({ inputEventType, outputEventType }) => {
|
|
33
|
-
const event = (0, test_utils_1.createMockEvent)(undefined, {
|
|
34
|
-
payload: {
|
|
35
|
-
event_type: inputEventType,
|
|
36
|
-
event_context: {
|
|
37
|
-
extract_from: '2024-01-01T00:00:00.000Z',
|
|
38
|
-
extract_to: '2024-06-01T00:00:00.000Z',
|
|
39
|
-
},
|
|
40
|
-
},
|
|
41
|
-
});
|
|
42
|
-
await (0, control_protocol_1.emit)({
|
|
43
|
-
event,
|
|
44
|
-
eventType: outputEventType,
|
|
45
|
-
});
|
|
46
|
-
expect(mockedAxiosClient.post).toHaveBeenCalledTimes(1);
|
|
47
|
-
const [, body, config] = mockedAxiosClient.post.mock.calls[0];
|
|
48
|
-
expect(body).toMatchObject({
|
|
49
|
-
event_type: outputEventType,
|
|
50
|
-
event_context: expect.objectContaining({
|
|
51
|
-
extract_from: '2024-01-01T00:00:00.000Z',
|
|
52
|
-
extract_to: '2024-06-01T00:00:00.000Z',
|
|
53
|
-
}),
|
|
54
|
-
worker_metadata: expect.objectContaining({
|
|
55
|
-
oldest_state_date: '2024-01-01T00:00:00.000Z',
|
|
56
|
-
newest_state_date: '2024-06-01T00:00:00.000Z',
|
|
57
|
-
}),
|
|
58
|
-
});
|
|
59
|
-
expect(config).toEqual(expect.objectContaining({
|
|
60
|
-
headers: expect.objectContaining({
|
|
61
|
-
'X-DevRev-Client-Version': expect.any(String),
|
|
62
|
-
}),
|
|
63
|
-
}));
|
|
64
|
-
});
|
|
65
|
-
it.each([
|
|
66
|
-
{
|
|
67
|
-
title: 'only extract_from is set',
|
|
68
|
-
extractFrom: '2024-01-01T00:00:00.000Z',
|
|
69
|
-
extractTo: undefined,
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
title: 'only extract_to is set',
|
|
73
|
-
extractFrom: undefined,
|
|
74
|
-
extractTo: '2024-06-01T00:00:00.000Z',
|
|
75
|
-
},
|
|
76
|
-
{
|
|
77
|
-
title: 'neither extract_from nor extract_to is set',
|
|
78
|
-
extractFrom: undefined,
|
|
79
|
-
extractTo: undefined,
|
|
80
|
-
},
|
|
81
|
-
])('handles state-date absence when $title', async ({ extractFrom, extractTo }) => {
|
|
82
|
-
const event = (0, test_utils_1.createMockEvent)(undefined, {
|
|
83
|
-
payload: {
|
|
84
|
-
event_type: extraction_1.EventType.StartExtractingData,
|
|
85
|
-
event_context: {
|
|
86
|
-
extract_from: extractFrom,
|
|
87
|
-
extract_to: extractTo,
|
|
88
|
-
},
|
|
89
|
-
},
|
|
90
|
-
});
|
|
91
|
-
await (0, control_protocol_1.emit)({
|
|
92
|
-
event,
|
|
93
|
-
eventType: extraction_1.ExtractorEventType.DataExtractionProgress,
|
|
94
|
-
});
|
|
95
|
-
const [, body] = mockedAxiosClient.post.mock.calls[0];
|
|
96
|
-
const workerMetadata = body.worker_metadata;
|
|
97
|
-
expect(workerMetadata.oldest_state_date).toBe(extractFrom);
|
|
98
|
-
expect(workerMetadata.newest_state_date).toBe(extractTo);
|
|
99
|
-
});
|
|
100
|
-
it('overrides caller-provided worker_metadata state dates', async () => {
|
|
101
|
-
const event = (0, test_utils_1.createMockEvent)(undefined, {
|
|
102
|
-
payload: {
|
|
103
|
-
event_type: extraction_1.EventType.StartExtractingData,
|
|
104
|
-
event_context: {
|
|
105
|
-
extract_from: '2024-01-01T00:00:00.000Z',
|
|
106
|
-
extract_to: '2024-06-01T00:00:00.000Z',
|
|
107
|
-
},
|
|
108
|
-
},
|
|
109
|
-
});
|
|
110
|
-
await (0, control_protocol_1.emit)({
|
|
111
|
-
event,
|
|
112
|
-
eventType: extraction_1.ExtractorEventType.DataExtractionProgress,
|
|
113
|
-
worker_metadata: {
|
|
114
|
-
item_type: 'tasks',
|
|
115
|
-
oldest_state_date: 'should-be-overwritten',
|
|
116
|
-
newest_state_date: 'should-be-overwritten',
|
|
117
|
-
},
|
|
118
|
-
});
|
|
119
|
-
const [, body] = mockedAxiosClient.post.mock.calls[0];
|
|
120
|
-
expect(body.worker_metadata).toEqual(expect.objectContaining({
|
|
121
|
-
item_type: 'tasks',
|
|
122
|
-
oldest_state_date: '2024-01-01T00:00:00.000Z',
|
|
123
|
-
newest_state_date: '2024-06-01T00:00:00.000Z',
|
|
124
|
-
}));
|
|
125
|
-
});
|
|
126
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.upload-failure.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.upload-failure.test.ts"],"names":[],"mappings":""}
|