@devrev/ts-adaas 1.19.7 → 1.19.9-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/control-protocol.d.ts +3 -2
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +2 -4
- package/dist/common/control-protocol.test.d.ts +2 -0
- package/dist/common/control-protocol.test.d.ts.map +1 -0
- package/dist/common/control-protocol.test.js +126 -0
- package/dist/common/errors.d.ts +4 -1
- package/dist/common/errors.d.ts.map +1 -1
- package/dist/http/axios-client-internal.d.ts.map +1 -1
- package/dist/http/axios-client-internal.js +19 -1
- package/dist/mock-server/mock-server.d.ts +4 -0
- package/dist/mock-server/mock-server.d.ts.map +1 -1
- package/dist/mock-server/mock-server.interfaces.d.ts +9 -0
- package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -1
- package/dist/mock-server/mock-server.js +28 -6
- package/dist/multithreading/spawn/spawn.d.ts +1 -1
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.js +11 -6
- package/dist/multithreading/spawn/spawn.test.js +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +10 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +90 -9
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.js +91 -46
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.js +135 -0
- package/dist/repo/repo.d.ts +11 -2
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +2 -0
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +37 -11
- package/dist/repo/repo.test.js +184 -0
- package/dist/types/extraction.d.ts +8 -1
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/workers.d.ts +4 -0
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +6 -1
- package/dist/uploader/uploader.helpers.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.js +65 -0
- package/dist/uploader/uploader.helpers.test.js +148 -7
- package/dist/uploader/uploader.interfaces.d.ts +11 -0
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
- package/dist/uploader/uploader.interfaces.js +11 -0
- package/dist/uploader/uploader.js +2 -5
- package/dist/uploader/uploader.test.js +54 -3
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,oBAAY,QAAQ;IAClB,kBAAkB;IAClB,WAAW,OAAO;IAClB,iCAAiC;IACjC,YAAY,OAAO;IACnB,mCAAmC;IACnC,eAAe,YAAO;IACtB,mBAAmB;IACnB,YAAY,OAAO;IACnB,cAAc;IACd,OAAO,MAAM;IACb,cAAc;IACd,OAAO,MAAM;IACb,YAAY;IACZ,KAAK,MAAM;CACZ;AAED;;;GAGG;AACH,oBAAY,aAAa;IACvB,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,iEAAiE;IACjE,2BAA2B,gCAAgC;IAC3D,yCAAyC;IACzC,cAAc,mBAAmB;IACjC,gEAAgE;IAChE,0BAA0B,+BAA+B;IACzD,mBAAmB;IACnB,YAAY,iBAAiB;IAC7B,wCAAwC;IACxC,aAAa,kBAAkB;IAC/B,4CAA4C;IAC5C,SAAS,cAAc;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,SAAS,CAAC;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,SAAS,CAAC;IAChC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAG9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAChD,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
package/dist/types/workers.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Worker } from 'worker_threads';
|
|
2
|
+
import type { Logger } from '../logger/logger';
|
|
2
3
|
import type { LogLevel } from '../logger/logger.interfaces';
|
|
3
4
|
import { State } from '../state/state';
|
|
4
5
|
import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter';
|
|
@@ -40,6 +41,8 @@ export interface WorkerAdapterOptions {
|
|
|
40
41
|
batchSize?: number;
|
|
41
42
|
workerPathOverrides?: WorkerPathOverrides;
|
|
42
43
|
skipConfirmation?: boolean;
|
|
44
|
+
/** Test-only: limits axios-retry attempts inside worker threads (see axios-client-internal). */
|
|
45
|
+
httpRetries?: number;
|
|
43
46
|
}
|
|
44
47
|
/**
|
|
45
48
|
* SpawnInterface is an interface for Spawn class.
|
|
@@ -54,6 +57,7 @@ export interface SpawnInterface {
|
|
|
54
57
|
options?: WorkerAdapterOptions;
|
|
55
58
|
resolve: (value: void | PromiseLike<void>) => void;
|
|
56
59
|
originalConsole?: Console;
|
|
60
|
+
logger?: Logger;
|
|
57
61
|
}
|
|
58
62
|
/**
|
|
59
63
|
* SpawnFactoryInterface is an interface for Spawn class factory.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAEhF,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC;AAEnE;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,gBAAgB,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAEhF,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC;AAEnE;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gGAAgG;IAChG,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAE7B,wGAAwG;IACxG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACxC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,SAAS,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED;;GAEG;AACH,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;IACxB,mBAAmB,WAAW;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,oBAAoB,CAAC,gBAAgB,CAAC;IAC/C,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,KAAK,EAAE,QAAQ,CAAC;QAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;CACH;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,oBAAoB,CAAC,mBAAmB,CAAC;IAClD,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,oBAAoB,GACpB,iBAAiB,GACjB,gBAAgB,GAChB,mBAAmB,CAAC;AAExB;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAOtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAU/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,gBAAgB,CAAU;gBAEtB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IA+E1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IA2B5C;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAmBzC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IA8BzC;;;;OAIG;IACG,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACvD,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;IAiCF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IA2DF;;;;OAIG;YACW,sBAAsB;IAsBpC;;;;OAIG;YACW,gBAAgB;IAc9B;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;CA6B/C"}
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import { UploaderResult } from './uploader.interfaces';
|
|
1
|
+
import { ArtifactDateRanges, UploaderResult } from './uploader.interfaces';
|
|
2
|
+
/**
|
|
3
|
+
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
4
|
+
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
5
|
+
*/
|
|
6
|
+
export declare function computeArtifactDateRanges(fetchedObjects: object[] | object): ArtifactDateRanges;
|
|
2
7
|
/**
|
|
3
8
|
* Compresses a JSONL string using gzip compression.
|
|
4
9
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,kBAAkB,CAwEpB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,kBAAkB,EAAE,MAAM,GACzB,cAAc,CAAC,MAAM,CAAC,CAOxB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAMxE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
|
|
@@ -36,6 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.computeArtifactDateRanges = computeArtifactDateRanges;
|
|
39
40
|
exports.compressGzip = compressGzip;
|
|
40
41
|
exports.decompressGzip = decompressGzip;
|
|
41
42
|
exports.parseJsonl = parseJsonl;
|
|
@@ -45,6 +46,70 @@ const fs_1 = __importStar(require("fs"));
|
|
|
45
46
|
const js_jsonl_1 = require("js-jsonl");
|
|
46
47
|
const zlib_1 = __importDefault(require("zlib"));
|
|
47
48
|
const constants_1 = require("../common/constants");
|
|
49
|
+
const uploader_interfaces_1 = require("./uploader.interfaces");
|
|
50
|
+
/**
|
|
51
|
+
* Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items.
|
|
52
|
+
* @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[])
|
|
53
|
+
*/
|
|
54
|
+
function computeArtifactDateRanges(fetchedObjects) {
|
|
55
|
+
const items = Array.isArray(fetchedObjects)
|
|
56
|
+
? fetchedObjects
|
|
57
|
+
: [fetchedObjects];
|
|
58
|
+
const created = {
|
|
59
|
+
min: Number.POSITIVE_INFINITY,
|
|
60
|
+
max: Number.NEGATIVE_INFINITY,
|
|
61
|
+
};
|
|
62
|
+
const modified = {
|
|
63
|
+
min: Number.POSITIVE_INFINITY,
|
|
64
|
+
max: Number.NEGATIVE_INFINITY,
|
|
65
|
+
};
|
|
66
|
+
let hasCreated = false;
|
|
67
|
+
let hasModified = false;
|
|
68
|
+
const parseTimestamp = (value) => {
|
|
69
|
+
const ts = new Date(value).getTime();
|
|
70
|
+
return Number.isNaN(ts) ? undefined : ts;
|
|
71
|
+
};
|
|
72
|
+
for (const obj of items) {
|
|
73
|
+
if (!obj || typeof obj !== 'object') {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
const item = obj;
|
|
77
|
+
if (item.created_date != undefined) {
|
|
78
|
+
const ts = parseTimestamp(item.created_date);
|
|
79
|
+
if (ts != undefined) {
|
|
80
|
+
if (ts < created.min) {
|
|
81
|
+
created.min = ts;
|
|
82
|
+
}
|
|
83
|
+
if (ts > created.max) {
|
|
84
|
+
created.max = ts;
|
|
85
|
+
}
|
|
86
|
+
hasCreated = true;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (item.modified_date != undefined) {
|
|
90
|
+
const ts = parseTimestamp(item.modified_date);
|
|
91
|
+
if (ts != undefined) {
|
|
92
|
+
if (ts < modified.min) {
|
|
93
|
+
modified.min = ts;
|
|
94
|
+
}
|
|
95
|
+
if (ts > modified.max) {
|
|
96
|
+
modified.max = ts;
|
|
97
|
+
}
|
|
98
|
+
hasModified = true;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const result = {};
|
|
103
|
+
if (hasCreated) {
|
|
104
|
+
result[uploader_interfaces_1.ArtifactDateField.OldestCreatedDate] = new Date(created.min).toISOString();
|
|
105
|
+
result[uploader_interfaces_1.ArtifactDateField.NewestCreatedDate] = new Date(created.max).toISOString();
|
|
106
|
+
}
|
|
107
|
+
if (hasModified) {
|
|
108
|
+
result[uploader_interfaces_1.ArtifactDateField.OldestModifiedDate] = new Date(modified.min).toISOString();
|
|
109
|
+
result[uploader_interfaces_1.ArtifactDateField.NewestModifiedDate] = new Date(modified.max).toISOString();
|
|
110
|
+
}
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
48
113
|
/**
|
|
49
114
|
* Compresses a JSONL string using gzip compression.
|
|
50
115
|
* @param {string} jsonlObject - The JSONL string to compress
|
|
@@ -1,10 +1,42 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
37
|
};
|
|
5
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const fs_1 =
|
|
7
|
-
const fs_2 = require("fs");
|
|
39
|
+
const fs_1 = __importStar(require("fs"));
|
|
8
40
|
const js_jsonl_1 = require("js-jsonl");
|
|
9
41
|
const zlib_1 = __importDefault(require("zlib"));
|
|
10
42
|
const constants_1 = require("../common/constants");
|
|
@@ -99,7 +131,7 @@ describe('uploader.helpers', () => {
|
|
|
99
131
|
close: jest.fn().mockResolvedValue(undefined),
|
|
100
132
|
};
|
|
101
133
|
const fsPromisesOpenSpy = jest
|
|
102
|
-
.spyOn(
|
|
134
|
+
.spyOn(fs_1.promises, 'open')
|
|
103
135
|
.mockResolvedValueOnce(mockFileHandle);
|
|
104
136
|
// Act
|
|
105
137
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
@@ -118,7 +150,7 @@ describe('uploader.helpers', () => {
|
|
|
118
150
|
close: jest.fn().mockResolvedValue(undefined),
|
|
119
151
|
};
|
|
120
152
|
const fsPromisesOpenSpy = jest
|
|
121
|
-
.spyOn(
|
|
153
|
+
.spyOn(fs_1.promises, 'open')
|
|
122
154
|
.mockResolvedValueOnce(mockFileHandle);
|
|
123
155
|
// Act
|
|
124
156
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
@@ -139,7 +171,7 @@ describe('uploader.helpers', () => {
|
|
|
139
171
|
close: jest.fn().mockResolvedValue(undefined),
|
|
140
172
|
};
|
|
141
173
|
const fsPromisesOpenSpy = jest
|
|
142
|
-
.spyOn(
|
|
174
|
+
.spyOn(fs_1.promises, 'open')
|
|
143
175
|
.mockResolvedValueOnce(mockFileHandle);
|
|
144
176
|
// Act
|
|
145
177
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
@@ -158,7 +190,7 @@ describe('uploader.helpers', () => {
|
|
|
158
190
|
close: jest.fn().mockResolvedValue(undefined),
|
|
159
191
|
};
|
|
160
192
|
const fsPromisesOpenSpy = jest
|
|
161
|
-
.spyOn(
|
|
193
|
+
.spyOn(fs_1.promises, 'open')
|
|
162
194
|
.mockResolvedValueOnce(mockFileHandle);
|
|
163
195
|
// Act
|
|
164
196
|
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
@@ -173,13 +205,122 @@ describe('uploader.helpers', () => {
|
|
|
173
205
|
const fileError = new Error('File write failed');
|
|
174
206
|
mockExistsSync.mockReturnValueOnce(true);
|
|
175
207
|
const fsPromisesOpenSpy = jest
|
|
176
|
-
.spyOn(
|
|
208
|
+
.spyOn(fs_1.promises, 'open')
|
|
177
209
|
.mockRejectedValueOnce(fileError);
|
|
178
210
|
// Act & Assert
|
|
179
211
|
await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
|
|
180
212
|
fsPromisesOpenSpy.mockRestore();
|
|
181
213
|
});
|
|
182
214
|
});
|
|
215
|
+
describe(uploader_helpers_1.computeArtifactDateRanges.name, () => {
|
|
216
|
+
it('should compute min and max across multiple items', () => {
|
|
217
|
+
// Arrange
|
|
218
|
+
const items = [
|
|
219
|
+
{
|
|
220
|
+
id: '1',
|
|
221
|
+
created_date: '2020-01-01T00:00:00.000Z',
|
|
222
|
+
modified_date: '2021-06-01T00:00:00.000Z',
|
|
223
|
+
data: {},
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
id: '2',
|
|
227
|
+
created_date: '2022-03-15T12:00:00.000Z',
|
|
228
|
+
modified_date: '2020-12-31T23:59:59.000Z',
|
|
229
|
+
data: {},
|
|
230
|
+
},
|
|
231
|
+
];
|
|
232
|
+
// Act
|
|
233
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
234
|
+
// Assert
|
|
235
|
+
expect(result.oldest_created_date).toBe('2020-01-01T00:00:00.000Z');
|
|
236
|
+
expect(result.newest_created_date).toBe('2022-03-15T12:00:00.000Z');
|
|
237
|
+
expect(result.oldest_modified_date).toBe('2020-12-31T23:59:59.000Z');
|
|
238
|
+
expect(result.newest_modified_date).toBe('2021-06-01T00:00:00.000Z');
|
|
239
|
+
});
|
|
240
|
+
it('should return zeros when no items have date fields', () => {
|
|
241
|
+
// Arrange
|
|
242
|
+
const items = [
|
|
243
|
+
{ id: 1, name: 'a' },
|
|
244
|
+
{ id: 2, name: 'b' },
|
|
245
|
+
];
|
|
246
|
+
// Act
|
|
247
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
248
|
+
// Assert
|
|
249
|
+
expect(result).toEqual({});
|
|
250
|
+
});
|
|
251
|
+
it('should aggregate only fields that are present on items', () => {
|
|
252
|
+
// Arrange
|
|
253
|
+
const items = [
|
|
254
|
+
{
|
|
255
|
+
id: '1',
|
|
256
|
+
created_date: '2021-01-01T00:00:00.000Z',
|
|
257
|
+
data: {},
|
|
258
|
+
},
|
|
259
|
+
{
|
|
260
|
+
id: '2',
|
|
261
|
+
modified_date: '2023-01-01T00:00:00.000Z',
|
|
262
|
+
data: {},
|
|
263
|
+
},
|
|
264
|
+
];
|
|
265
|
+
// Act
|
|
266
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
267
|
+
// Assert
|
|
268
|
+
expect(result.oldest_created_date).toBe('2021-01-01T00:00:00.000Z');
|
|
269
|
+
expect(result.newest_created_date).toBe('2021-01-01T00:00:00.000Z');
|
|
270
|
+
expect(result.oldest_modified_date).toBe('2023-01-01T00:00:00.000Z');
|
|
271
|
+
expect(result.newest_modified_date).toBe('2023-01-01T00:00:00.000Z');
|
|
272
|
+
});
|
|
273
|
+
it('should handle single object input', () => {
|
|
274
|
+
// Arrange
|
|
275
|
+
const item = {
|
|
276
|
+
id: '1',
|
|
277
|
+
created_date: '2019-05-10T08:30:00.000Z',
|
|
278
|
+
modified_date: '2019-05-10T08:30:00.000Z',
|
|
279
|
+
data: {},
|
|
280
|
+
};
|
|
281
|
+
// Act
|
|
282
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(item);
|
|
283
|
+
// Assert
|
|
284
|
+
const ts = '2019-05-10T08:30:00.000Z';
|
|
285
|
+
expect(result.oldest_created_date).toBe(ts);
|
|
286
|
+
expect(result.newest_created_date).toBe(ts);
|
|
287
|
+
expect(result.oldest_modified_date).toBe(ts);
|
|
288
|
+
expect(result.newest_modified_date).toBe(ts);
|
|
289
|
+
});
|
|
290
|
+
it('should skip invalid date values', () => {
|
|
291
|
+
const items = [
|
|
292
|
+
{
|
|
293
|
+
id: '1',
|
|
294
|
+
created_date: 'not-a-date',
|
|
295
|
+
modified_date: '2024-01-01T00:00:00.000Z',
|
|
296
|
+
data: {},
|
|
297
|
+
},
|
|
298
|
+
];
|
|
299
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
300
|
+
expect(result).toEqual({
|
|
301
|
+
oldest_modified_date: '2024-01-01T00:00:00.000Z',
|
|
302
|
+
newest_modified_date: '2024-01-01T00:00:00.000Z',
|
|
303
|
+
});
|
|
304
|
+
});
|
|
305
|
+
it('[edge] should skip non-object entries in an array', () => {
|
|
306
|
+
// Arrange
|
|
307
|
+
const items = [
|
|
308
|
+
null,
|
|
309
|
+
{
|
|
310
|
+
id: '1',
|
|
311
|
+
created_date: '2024-01-01T00:00:00.000Z',
|
|
312
|
+
modified_date: '2024-06-01T00:00:00.000Z',
|
|
313
|
+
data: {},
|
|
314
|
+
},
|
|
315
|
+
'not-an-object',
|
|
316
|
+
];
|
|
317
|
+
// Act
|
|
318
|
+
const result = (0, uploader_helpers_1.computeArtifactDateRanges)(items);
|
|
319
|
+
// Assert
|
|
320
|
+
expect(result.oldest_created_date).toBe('2024-01-01T00:00:00.000Z');
|
|
321
|
+
expect(result.newest_created_date).toBe('2024-01-01T00:00:00.000Z');
|
|
322
|
+
});
|
|
323
|
+
});
|
|
183
324
|
describe(uploader_helpers_1.truncateFilename.name, () => {
|
|
184
325
|
it('should return filename unchanged when within the limit', () => {
|
|
185
326
|
// Arrange
|
|
@@ -20,11 +20,22 @@ export type UploaderResult<T> = {
|
|
|
20
20
|
/**
|
|
21
21
|
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
22
22
|
*/
|
|
23
|
+
export declare enum ArtifactDateField {
|
|
24
|
+
OldestCreatedDate = "oldest_created_date",
|
|
25
|
+
NewestCreatedDate = "newest_created_date",
|
|
26
|
+
OldestModifiedDate = "oldest_modified_date",
|
|
27
|
+
NewestModifiedDate = "newest_modified_date"
|
|
28
|
+
}
|
|
23
29
|
export interface Artifact {
|
|
24
30
|
id: string;
|
|
25
31
|
item_type: string;
|
|
26
32
|
item_count: number;
|
|
33
|
+
[ArtifactDateField.OldestCreatedDate]?: string;
|
|
34
|
+
[ArtifactDateField.NewestCreatedDate]?: string;
|
|
35
|
+
[ArtifactDateField.OldestModifiedDate]?: string;
|
|
36
|
+
[ArtifactDateField.NewestModifiedDate]?: string;
|
|
27
37
|
}
|
|
38
|
+
export type ArtifactDateRanges = Pick<Artifact, ArtifactDateField.OldestCreatedDate | ArtifactDateField.NewestCreatedDate | ArtifactDateField.OldestModifiedDate | ArtifactDateField.NewestModifiedDate>;
|
|
28
39
|
/**
|
|
29
40
|
* ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
|
|
30
41
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,oBAAY,iBAAiB;IAC3B,iBAAiB,wBAAwB;IACzC,iBAAiB,wBAAwB;IACzC,kBAAkB,yBAAyB;IAC3C,kBAAkB,yBAAyB;CAC5C;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC;IAC/C,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;IAChD,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,EAAE,MAAM,CAAC;CACjD;AAED,MAAM,MAAM,kBAAkB,GAAG,IAAI,CACnC,QAAQ,EACN,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,iBAAiB,GACnC,iBAAiB,CAAC,kBAAkB,GACpC,iBAAiB,CAAC,kBAAkB,CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
|
|
@@ -1,2 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ArtifactDateField = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
6
|
+
*/
|
|
7
|
+
var ArtifactDateField;
|
|
8
|
+
(function (ArtifactDateField) {
|
|
9
|
+
ArtifactDateField["OldestCreatedDate"] = "oldest_created_date";
|
|
10
|
+
ArtifactDateField["NewestCreatedDate"] = "newest_created_date";
|
|
11
|
+
ArtifactDateField["OldestModifiedDate"] = "oldest_modified_date";
|
|
12
|
+
ArtifactDateField["NewestModifiedDate"] = "newest_modified_date";
|
|
13
|
+
})(ArtifactDateField || (exports.ArtifactDateField = ArtifactDateField = {}));
|
|
@@ -78,12 +78,9 @@ class Uploader {
|
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
|
+
const artifactDateRanges = (0, uploader_helpers_1.computeArtifactDateRanges)(fetchedObjects);
|
|
81
82
|
// Return the artifact information to the platform
|
|
82
|
-
const artifact = {
|
|
83
|
-
id: preparedArtifact.artifact_id,
|
|
84
|
-
item_type: itemType,
|
|
85
|
-
item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1,
|
|
86
|
-
};
|
|
83
|
+
const artifact = Object.assign({ id: preparedArtifact.artifact_id, item_type: itemType, item_count: Array.isArray(fetchedObjects) ? fetchedObjects.length : 1 }, artifactDateRanges);
|
|
87
84
|
return { artifact };
|
|
88
85
|
}
|
|
89
86
|
/**
|
|
@@ -6,12 +6,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
const form_data_1 = __importDefault(require("form-data"));
|
|
7
7
|
const js_jsonl_1 = require("js-jsonl");
|
|
8
8
|
const zlib_1 = __importDefault(require("zlib"));
|
|
9
|
+
const test_utils_1 = require("../common/test-utils");
|
|
9
10
|
const axios_client_internal_1 = require("../http/axios-client-internal");
|
|
10
|
-
const test_helpers_1 = require("../tests/test-helpers");
|
|
11
11
|
const jest_setup_1 = require("../tests/jest.setup");
|
|
12
|
-
const
|
|
13
|
-
const uploader_helpers_1 = require("./uploader.helpers");
|
|
12
|
+
const test_helpers_1 = require("../tests/test-helpers");
|
|
14
13
|
const uploader_1 = require("./uploader");
|
|
14
|
+
const uploader_helpers_1 = require("./uploader.helpers");
|
|
15
15
|
jest.mock('../http/axios-client-internal');
|
|
16
16
|
jest.mock('./uploader.helpers', () => (Object.assign(Object.assign({}, jest.requireActual('./uploader.helpers')), { downloadToLocal: jest.fn(), compressGzip: jest.fn(jest.requireActual('./uploader.helpers').compressGzip) })));
|
|
17
17
|
const mockedAxiosClient = jest.mocked(axios_client_internal_1.axiosClient);
|
|
@@ -51,6 +51,57 @@ describe(uploader_1.Uploader.name, () => {
|
|
|
51
51
|
});
|
|
52
52
|
expect(result.error).toBeUndefined();
|
|
53
53
|
});
|
|
54
|
+
it('should compute oldest/newest created and modified dates from normalized items', async () => {
|
|
55
|
+
var _a, _b, _c, _d;
|
|
56
|
+
// Arrange
|
|
57
|
+
const itemType = 'tasks';
|
|
58
|
+
const fetchedObjects = [
|
|
59
|
+
{
|
|
60
|
+
id: '1',
|
|
61
|
+
created_date: '2020-06-15T10:00:00.000Z',
|
|
62
|
+
modified_date: '2021-01-20T10:00:00.000Z',
|
|
63
|
+
data: { name: 'Task 1' },
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
id: '2',
|
|
67
|
+
created_date: '2019-03-01T08:00:00.000Z',
|
|
68
|
+
modified_date: '2022-11-30T18:00:00.000Z',
|
|
69
|
+
data: { name: 'Task 2' },
|
|
70
|
+
},
|
|
71
|
+
];
|
|
72
|
+
mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
|
|
73
|
+
mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
|
|
74
|
+
// Act
|
|
75
|
+
const result = await uploader.upload(itemType, fetchedObjects);
|
|
76
|
+
// Assert
|
|
77
|
+
expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe('2019-03-01T08:00:00.000Z');
|
|
78
|
+
expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe('2020-06-15T10:00:00.000Z');
|
|
79
|
+
expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe('2021-01-20T10:00:00.000Z');
|
|
80
|
+
expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe('2022-11-30T18:00:00.000Z');
|
|
81
|
+
expect(result.error).toBeUndefined();
|
|
82
|
+
});
|
|
83
|
+
it('should compute date ranges for single object upload', async () => {
|
|
84
|
+
var _a, _b, _c, _d;
|
|
85
|
+
// Arrange
|
|
86
|
+
const itemType = 'metadata';
|
|
87
|
+
const fetchedObject = {
|
|
88
|
+
id: '1',
|
|
89
|
+
created_date: '2018-12-25T00:00:00.000Z',
|
|
90
|
+
modified_date: '2018-12-26T00:00:00.000Z',
|
|
91
|
+
data: { key: 'value' },
|
|
92
|
+
};
|
|
93
|
+
mockedAxiosClient.get.mockResolvedValueOnce(mockArtifactUploadUrlResponse);
|
|
94
|
+
mockedAxiosClient.post.mockResolvedValue((0, test_helpers_1.createAxiosResponse)());
|
|
95
|
+
// Act
|
|
96
|
+
const result = await uploader.upload(itemType, fetchedObject);
|
|
97
|
+
// Assert
|
|
98
|
+
const createdTs = '2018-12-25T00:00:00.000Z';
|
|
99
|
+
const modifiedTs = '2018-12-26T00:00:00.000Z';
|
|
100
|
+
expect((_a = result.artifact) === null || _a === void 0 ? void 0 : _a.oldest_created_date).toBe(createdTs);
|
|
101
|
+
expect((_b = result.artifact) === null || _b === void 0 ? void 0 : _b.newest_created_date).toBe(createdTs);
|
|
102
|
+
expect((_c = result.artifact) === null || _c === void 0 ? void 0 : _c.oldest_modified_date).toBe(modifiedTs);
|
|
103
|
+
expect((_d = result.artifact) === null || _d === void 0 ? void 0 : _d.newest_modified_date).toBe(modifiedTs);
|
|
104
|
+
});
|
|
54
105
|
it('should report item_count as 1 when uploading single object', async () => {
|
|
55
106
|
var _a;
|
|
56
107
|
// Arrange
|