@devrev/ts-adaas 1.13.3 → 1.13.4-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/common/helpers.d.ts +28 -21
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +18 -156
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/multithreading/create-worker.d.ts.map +1 -0
- package/dist/multithreading/create-worker.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.d.ts.map +1 -0
- package/dist/{workers → multithreading}/process-task.js +1 -1
- package/dist/{workers → multithreading/spawn}/spawn.d.ts +1 -1
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.d.ts +21 -0
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.js +114 -0
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +2 -0
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.test.js +293 -0
- package/dist/{workers → multithreading/spawn}/spawn.js +11 -10
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +1 -0
- package/dist/{workers → multithreading/worker-adapter}/worker-adapter.artifacts.test.js +4 -4
- package/dist/{workers → multithreading/worker-adapter}/worker-adapter.d.ts +8 -8
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +22 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +64 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +514 -0
- package/dist/{workers → multithreading/worker-adapter}/worker-adapter.js +16 -16
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +1 -0
- package/dist/{workers → multithreading/worker-adapter}/worker-adapter.test.js +10 -10
- package/dist/{workers → multithreading}/worker.d.ts.map +1 -1
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.js +1 -2
- package/dist/types/extraction.d.ts +1 -1
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/workers.d.ts +1 -1
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +1 -26
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +32 -0
- package/dist/uploader/uploader.helpers.d.ts.map +1 -0
- package/dist/uploader/uploader.helpers.js +140 -0
- package/dist/uploader/uploader.helpers.test.d.ts +2 -0
- package/dist/uploader/uploader.helpers.test.d.ts.map +1 -0
- package/dist/uploader/uploader.helpers.test.js +265 -0
- package/dist/uploader/uploader.js +7 -117
- package/dist/uploader/uploader.test.js +561 -72
- package/package.json +1 -1
- package/dist/common/helpers.test.d.ts +0 -2
- package/dist/common/helpers.test.d.ts.map +0 -1
- package/dist/common/helpers.test.js +0 -158
- package/dist/workers/create-worker.d.ts.map +0 -1
- package/dist/workers/create-worker.test.d.ts.map +0 -1
- package/dist/workers/process-task.d.ts.map +0 -1
- package/dist/workers/spawn.d.ts.map +0 -1
- package/dist/workers/worker-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/workers/worker-adapter.d.ts.map +0 -1
- package/dist/workers/worker-adapter.test.d.ts.map +0 -1
- /package/dist/{workers → multithreading}/create-worker.d.ts +0 -0
- /package/dist/{workers → multithreading}/create-worker.js +0 -0
- /package/dist/{workers → multithreading}/create-worker.test.d.ts +0 -0
- /package/dist/{workers → multithreading}/create-worker.test.js +0 -0
- /package/dist/{workers → multithreading}/process-task.d.ts +0 -0
- /package/dist/{workers → multithreading/worker-adapter}/worker-adapter.artifacts.test.d.ts +0 -0
- /package/dist/{workers → multithreading/worker-adapter}/worker-adapter.test.d.ts +0 -0
- /package/dist/{workers → multithreading}/worker.d.ts +0 -0
- /package/dist/{workers → multithreading}/worker.js +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/types/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7D;;;GAGG;AACH,oBAAY,SAAS;IAEnB;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,oBAAoB,2BAA2B;IAC/C;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,2BAA2B,kCAAkC;IAG7D,gBAAgB,uBAAuB;IACvC,mBAAmB,0BAA0B;IAC7C,uBAAuB,8BAA8B;IACrD,0BAA0B,iCAAiC;IAC3D,wBAAwB,gCAAgC;IACxD,kCAAkC,2CAA2C;IAG7E,gBAAgB,uBAAuB;IAGvC,gCAAgC,yCAAyC;IACzE,uBAAuB,8BAA8B;IACrD,mBAAmB,0BAA0B;IAC7C,sBAAsB,6BAA6B;IACnD,2BAA2B,mCAAmC;IAC9D,0BAA0B,iCAAiC;IAC3D,6BAA6B,oCAAoC;IACjE,sCAAsC,+CAA+C;CACtF;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAE5B;;OAEG;IACH,+BAA+B,wCAAwC;IACvE;;OAEG;IACH,gCAAgC,yCAAyC;IACzE;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,uBAAuB,8BAA8B;IACrD;;OAEG;IACH,sBAAsB,6BAA6B;IACnD;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,kBAAkB,yBAAyB;IAC3C;;OAEG;IACH,mBAAmB,0BAA0B;IAC7C;;OAEG;IACH,wBAAwB,gCAAgC;IACxD;;OAEG;IACH,yBAAyB,iCAAiC;IAC1D;;OAEG;IACH,6BAA6B,oCAAoC;IACjE;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,yBAAyB,gCAAgC;IACzD;;OAEG;IACH,0BAA0B,iCAAiC;IAC3D;;OAEG;IACH,+BAA+B,uCAAuC;IACtE;;OAEG;IACH,gCAAgC,wCAAwC;IAGxE,gBAAgB,uBAAuB;IAGvC,8BAA8B,uCAAuC;IACrE,+BAA+B,wCAAwC;IACvE,sBAAsB,6BAA6B;IACnD,uBAAuB,8BAA8B;IACrD,sBAAsB,6BAA6B;IACnD,qBAAqB,4BAA4B;IACjD,kBAAkB,yBAAyB;IAC3C,mBAAmB,0BAA0B;IAC7C,0BAA0B,kCAAkC;IAC5D,2BAA2B,mCAAmC;IAC9D,4BAA4B,mCAAmC;IAC/D,2BAA2B,kCAAkC;IAC7D,wBAAwB,+BAA+B;IACvD,yBAAyB,gCAAgC;IACzD,qCAAqC,8CAA8C;IACnF,sCAAsC,+CAA+C;CACtF;AAED;;;;GAIG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE;QACP,OAAO,EAAE;YACP,qBAAqB,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE;QAClB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,UAAU,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,cAAc,CAAC;IAChC,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,EAAE,SAAS,CAAC;IACtB,UAAU,CAAC,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,eAAe,CAAC,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,MAAM,yCAAyC,GAAG,CAAC,EACvD,IAAI,EACJ,KAAK,GACN,EAAE,uCAAuC,KAAK,OAAO,CAAC,yCAAyC,CAAC,CAAC;AAElG,MAAM,WAAW,uCAAuC;IACtD,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,yCAAyC;IACxD,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7B,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,2BAA2B,GACnC;IACE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,GACD,SAAS,CAAC;AAEd,MAAM,MAAM,uCAAuC,CACjD,KAAK,EACL,QAAQ,EACR,cAAc,IACZ,CAAC,EACH,WAAW,EACX,OAAO,EACP,SAAS,GACV,EAAE;IACD,WAAW,EAAE,KAAK,CAAC;IACnB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,QAAQ,CAAC;AAEf,MAAM,MAAM,iCAAiC,GAAG,CAAC,EAC/C,UAAU,EACV,MAAM,GACP,EAAE;IACD,UAAU,EAAE,oBAAoB,CAAC;IACjC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,wCAAwC,CAAC,QAAQ,EAAE,cAAc,IAC3E,CAAC,EACC,kBAAkB,EAClB,OAAO,EACP,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,MAAM,EAAE,yCAAyC,CAAC;CACnD,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE7C,MAAM,WAAW,kCAAkC,CACjD,cAAc,EACd,KAAK,EACL,QAAQ;IAER,OAAO,EAAE,uCAAuC,CAC9C,KAAK,EACL,QAAQ,EACR,cAAc,CACf,CAAC;IACF,QAAQ,EAAE,wCAAwC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;CAC9E"}
|
package/dist/types/workers.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Worker } from 'worker_threads';
|
|
2
2
|
import { State } from '../state/state';
|
|
3
|
-
import { WorkerAdapter } from '../
|
|
3
|
+
import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter';
|
|
4
4
|
import { AirdropEvent, EventType, ExtractorEventType } from './extraction';
|
|
5
5
|
import { LoaderEventType } from './loading';
|
|
6
6
|
import { InitialDomainMapping } from './common';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"workers.d.ts","sourceRoot":"","sources":["../../src/types/workers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAEhF,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB,CAAC,cAAc;IACpD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACnD,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB,CAAC,cAAc;IACnD,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAE7B,wGAAwG;IACxG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACxC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAClD,IAAI,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,SAAS,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,oBAAY,WAAW;IACrB,aAAa,YAAY;IACzB,YAAY,WAAW;IACvB,WAAW,UAAU;IACrB,UAAU,SAAS;CACpB;AAED;;GAEG;AACH,oBAAY,oBAAoB;IAC9B,oBAAoB,SAAS;IAC7B,iBAAiB,SAAS;IAC1B,gBAAgB,QAAQ;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,oBAAoB,CAAC,oBAAoB,CAAC;IACnD,OAAO,EAAE;QACP,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;KACjD,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,oBAAoB,CAAC,iBAAiB,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,oBAAoB,GAAG,iBAAiB,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,cAAc;IACxC,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,cAAc,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
2
1
|
import { AxiosResponse } from 'axios';
|
|
2
|
+
import { NormalizedAttachment } from '../repo/repo.interfaces';
|
|
3
3
|
import { ArtifactToUpload, UploadResponse, UploaderFactoryInterface } from './uploader.interfaces';
|
|
4
4
|
export declare class Uploader {
|
|
5
5
|
private isLocalDevelopment?;
|
|
@@ -74,24 +74,6 @@ export declare class Uploader {
|
|
|
74
74
|
* @returns {Promise<Buffer | void>} The artifact file buffer or undefined on error
|
|
75
75
|
*/
|
|
76
76
|
private downloadArtifact;
|
|
77
|
-
/**
|
|
78
|
-
* Compresses a JSONL string using gzip compression.
|
|
79
|
-
* @param {string} jsonlObject - The JSONL string to compress
|
|
80
|
-
* @returns {Buffer | void} The compressed buffer or undefined on error
|
|
81
|
-
*/
|
|
82
|
-
private compressGzip;
|
|
83
|
-
/**
|
|
84
|
-
* Decompresses a gzipped buffer to a JSONL string.
|
|
85
|
-
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
86
|
-
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
87
|
-
*/
|
|
88
|
-
private decompressGzip;
|
|
89
|
-
/**
|
|
90
|
-
* Parses a JSONL string into an array of objects.
|
|
91
|
-
* @param {string} jsonlObject - The JSONL string to parse
|
|
92
|
-
* @returns {object[] | null} The parsed array of objects or null on error
|
|
93
|
-
*/
|
|
94
|
-
private parseJsonl;
|
|
95
77
|
/**
|
|
96
78
|
* Retrieves and parses JSON objects from an artifact by artifact ID.
|
|
97
79
|
* @param {object} param0 - Configuration object
|
|
@@ -103,12 +85,5 @@ export declare class Uploader {
|
|
|
103
85
|
artifactId: string;
|
|
104
86
|
isGzipped?: boolean;
|
|
105
87
|
}): Promise<object[] | object | void>;
|
|
106
|
-
/**
|
|
107
|
-
* Downloads fetched objects to the local file system (for local development).
|
|
108
|
-
* @param {string} itemType - The type of items being downloaded
|
|
109
|
-
* @param {object | object[]} fetchedObjects - The objects to write to file
|
|
110
|
-
* @returns {Promise<void>} Resolves when the file is written or rejects on error
|
|
111
|
-
*/
|
|
112
|
-
private downloadToLocal;
|
|
113
88
|
}
|
|
114
89
|
//# sourceMappingURL=uploader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAMtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAU/D,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACzB,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,QAAQ;IACnB,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAUxD;;;;;OAKG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwD1B;;;;;;OAMG;IACG,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAwBnC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmBhC;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA+BhC;;;;OAIG;IACG,qBAAqB,CACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAwBhC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;IAqCF;;;;OAIG;YACW,sBAAsB;IAyBpC;;;;OAIG;YACW,gBAAgB;IAe9B;;;;;;OAMG;IACG,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC;CAwBtC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compresses a JSONL string using gzip compression.
|
|
3
|
+
* @param {string} jsonlObject - The JSONL string to compress
|
|
4
|
+
* @returns {Buffer | void} The compressed buffer or undefined on error
|
|
5
|
+
*/
|
|
6
|
+
export declare function compressGzip(jsonlObject: string): Buffer | void;
|
|
7
|
+
/**
|
|
8
|
+
* Decompresses a gzipped buffer to a JSONL string.
|
|
9
|
+
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
10
|
+
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
11
|
+
*/
|
|
12
|
+
export declare function decompressGzip(gzippedJsonlObject: Buffer): string | void;
|
|
13
|
+
/**
|
|
14
|
+
* Parses a JSONL string into an array of objects.
|
|
15
|
+
* @param {string} jsonlObject - The JSONL string to parse
|
|
16
|
+
* @returns {object[] | null} The parsed array of objects or null on error
|
|
17
|
+
*/
|
|
18
|
+
export declare function parseJsonl(jsonlObject: string): object[] | null;
|
|
19
|
+
/**
|
|
20
|
+
* Downloads fetched objects to the local file system (for local development).
|
|
21
|
+
* @param {string} itemType - The type of items being downloaded
|
|
22
|
+
* @param {object | object[]} fetchedObjects - The objects to write to file
|
|
23
|
+
* @returns {Promise<void>} Resolves when the file is written or rejects on error
|
|
24
|
+
*/
|
|
25
|
+
export declare function downloadToLocal(itemType: string, fetchedObjects: object | object[]): Promise<void>;
|
|
26
|
+
/**
|
|
27
|
+
* Truncates a filename if it exceeds the maximum allowed length.
|
|
28
|
+
* @param {string} filename - The filename to truncate
|
|
29
|
+
* @returns {string} The truncated filename
|
|
30
|
+
*/
|
|
31
|
+
export declare function truncateFilename(filename: string): string;
|
|
32
|
+
//# sourceMappingURL=uploader.helpers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"uploader.helpers.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.ts"],"names":[],"mappings":"AASA;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM/D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,kBAAkB,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAOxE;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,CAO/D;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.compressGzip = compressGzip;
|
|
40
|
+
exports.decompressGzip = decompressGzip;
|
|
41
|
+
exports.parseJsonl = parseJsonl;
|
|
42
|
+
exports.downloadToLocal = downloadToLocal;
|
|
43
|
+
exports.truncateFilename = truncateFilename;
|
|
44
|
+
const fs_1 = __importStar(require("fs"));
|
|
45
|
+
const js_jsonl_1 = require("js-jsonl");
|
|
46
|
+
const zlib_1 = __importDefault(require("zlib"));
|
|
47
|
+
const constants_1 = require("../common/constants");
|
|
48
|
+
/**
|
|
49
|
+
* Compresses a JSONL string using gzip compression.
|
|
50
|
+
* @param {string} jsonlObject - The JSONL string to compress
|
|
51
|
+
* @returns {Buffer | void} The compressed buffer or undefined on error
|
|
52
|
+
*/
|
|
53
|
+
function compressGzip(jsonlObject) {
|
|
54
|
+
try {
|
|
55
|
+
return zlib_1.default.gzipSync(jsonlObject);
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
console.error('Error while compressing jsonl object.', error);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Decompresses a gzipped buffer to a JSONL string.
|
|
63
|
+
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
64
|
+
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
65
|
+
*/
|
|
66
|
+
function decompressGzip(gzippedJsonlObject) {
|
|
67
|
+
try {
|
|
68
|
+
const jsonlObject = zlib_1.default.gunzipSync(gzippedJsonlObject);
|
|
69
|
+
return jsonlObject.toString();
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
console.error('Error while decompressing gzipped jsonl object.', error);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Parses a JSONL string into an array of objects.
|
|
77
|
+
* @param {string} jsonlObject - The JSONL string to parse
|
|
78
|
+
* @returns {object[] | null} The parsed array of objects or null on error
|
|
79
|
+
*/
|
|
80
|
+
function parseJsonl(jsonlObject) {
|
|
81
|
+
try {
|
|
82
|
+
return js_jsonl_1.jsonl.parse(jsonlObject);
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
console.error('Error while parsing jsonl object.', error);
|
|
86
|
+
}
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Downloads fetched objects to the local file system (for local development).
|
|
91
|
+
* @param {string} itemType - The type of items being downloaded
|
|
92
|
+
* @param {object | object[]} fetchedObjects - The objects to write to file
|
|
93
|
+
* @returns {Promise<void>} Resolves when the file is written or rejects on error
|
|
94
|
+
*/
|
|
95
|
+
async function downloadToLocal(itemType, fetchedObjects) {
|
|
96
|
+
console.log(`Downloading ${itemType} to local file system.`);
|
|
97
|
+
try {
|
|
98
|
+
if (!fs_1.default.existsSync('extracted_files')) {
|
|
99
|
+
fs_1.default.mkdirSync('extracted_files');
|
|
100
|
+
}
|
|
101
|
+
const timestamp = new Date().getTime();
|
|
102
|
+
const filePath = `extracted_files/extractor_${itemType}_${timestamp}.${itemType === 'external_domain_metadata' ? 'json' : 'jsonl'}`;
|
|
103
|
+
const fileHandle = await fs_1.promises.open(filePath, 'w');
|
|
104
|
+
let objArray = [];
|
|
105
|
+
if (!Array.isArray(fetchedObjects)) {
|
|
106
|
+
objArray.push(fetchedObjects);
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
objArray = fetchedObjects;
|
|
110
|
+
}
|
|
111
|
+
for (const jsonObject of objArray) {
|
|
112
|
+
const jsonLine = JSON.stringify(jsonObject) + '\n';
|
|
113
|
+
await fileHandle.write(jsonLine);
|
|
114
|
+
}
|
|
115
|
+
await fileHandle.close();
|
|
116
|
+
console.log('Data successfully written to', filePath);
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
console.error('Error writing data to file.', error);
|
|
120
|
+
return Promise.reject(error);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Truncates a filename if it exceeds the maximum allowed length.
|
|
125
|
+
* @param {string} filename - The filename to truncate
|
|
126
|
+
* @returns {string} The truncated filename
|
|
127
|
+
*/
|
|
128
|
+
function truncateFilename(filename) {
|
|
129
|
+
// If the filename is already within the limit, return it as is.
|
|
130
|
+
if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
|
|
131
|
+
return filename;
|
|
132
|
+
}
|
|
133
|
+
console.warn(`Filename length exceeds the maximum limit of ${constants_1.MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.`);
|
|
134
|
+
const extension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
135
|
+
// Calculate how many characters are available for the name part after accounting for the extension and "..."
|
|
136
|
+
const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..."
|
|
137
|
+
// Truncate the name part and add an ellipsis
|
|
138
|
+
const truncatedFilename = filename.slice(0, availableNameLength);
|
|
139
|
+
return `${truncatedFilename}...${extension}`;
|
|
140
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"uploader.helpers.test.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const fs_1 = __importDefault(require("fs"));
|
|
7
|
+
const fs_2 = require("fs");
|
|
8
|
+
const js_jsonl_1 = require("js-jsonl");
|
|
9
|
+
const zlib_1 = __importDefault(require("zlib"));
|
|
10
|
+
const constants_1 = require("../common/constants");
|
|
11
|
+
const uploader_helpers_1 = require("./uploader.helpers");
|
|
12
|
+
describe('uploader.helpers', () => {
|
|
13
|
+
let consoleErrorSpy;
|
|
14
|
+
let consoleLogSpy;
|
|
15
|
+
let consoleWarnSpy;
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
|
18
|
+
consoleLogSpy = jest.spyOn(console, 'log').mockImplementation();
|
|
19
|
+
consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
20
|
+
});
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
jest.clearAllMocks();
|
|
23
|
+
consoleErrorSpy.mockRestore();
|
|
24
|
+
consoleLogSpy.mockRestore();
|
|
25
|
+
consoleWarnSpy.mockRestore();
|
|
26
|
+
});
|
|
27
|
+
describe(uploader_helpers_1.compressGzip.name, () => {
|
|
28
|
+
it('should compress a string to gzip buffer', () => {
|
|
29
|
+
// Arrange
|
|
30
|
+
const input = '{"id":1}\n{"id":2}';
|
|
31
|
+
// Act
|
|
32
|
+
const result = (0, uploader_helpers_1.compressGzip)(input);
|
|
33
|
+
// Assert
|
|
34
|
+
expect(result).toBeInstanceOf(Buffer);
|
|
35
|
+
const decompressed = zlib_1.default.gunzipSync(result).toString();
|
|
36
|
+
expect(decompressed).toBe(input);
|
|
37
|
+
});
|
|
38
|
+
it('[edge] should return undefined and log error when compression fails', () => {
|
|
39
|
+
// Arrange
|
|
40
|
+
const gzipSyncSpy = jest
|
|
41
|
+
.spyOn(zlib_1.default, 'gzipSync')
|
|
42
|
+
.mockImplementationOnce(() => {
|
|
43
|
+
throw new Error('Compression failed');
|
|
44
|
+
});
|
|
45
|
+
// Act
|
|
46
|
+
const result = (0, uploader_helpers_1.compressGzip)('test data');
|
|
47
|
+
// Assert
|
|
48
|
+
expect(result).toBeUndefined();
|
|
49
|
+
gzipSyncSpy.mockRestore();
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
describe(uploader_helpers_1.decompressGzip.name, () => {
|
|
53
|
+
it('should decompress a gzip buffer to string', () => {
|
|
54
|
+
// Arrange
|
|
55
|
+
const originalString = '{"id":1}\n{"id":2}';
|
|
56
|
+
const compressed = zlib_1.default.gzipSync(originalString);
|
|
57
|
+
// Act
|
|
58
|
+
const result = (0, uploader_helpers_1.decompressGzip)(compressed);
|
|
59
|
+
// Assert
|
|
60
|
+
expect(result).toBe(originalString);
|
|
61
|
+
});
|
|
62
|
+
it('[edge] should return undefined and log error when decompression fails', () => {
|
|
63
|
+
// Arrange
|
|
64
|
+
const invalidGzip = Buffer.from('not valid gzip data');
|
|
65
|
+
// Act
|
|
66
|
+
const result = (0, uploader_helpers_1.decompressGzip)(invalidGzip);
|
|
67
|
+
// Assert
|
|
68
|
+
expect(result).toBeUndefined();
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
describe(uploader_helpers_1.parseJsonl.name, () => {
|
|
72
|
+
it('should parse valid JSONL string to array of objects', () => {
|
|
73
|
+
// Arrange
|
|
74
|
+
const data = [
|
|
75
|
+
{ id: 1, name: 'Item 1' },
|
|
76
|
+
{ id: 2, name: 'Item 2' },
|
|
77
|
+
];
|
|
78
|
+
const jsonlString = js_jsonl_1.jsonl.stringify(data);
|
|
79
|
+
// Act
|
|
80
|
+
const result = (0, uploader_helpers_1.parseJsonl)(jsonlString);
|
|
81
|
+
// Assert
|
|
82
|
+
expect(result).toEqual(data);
|
|
83
|
+
});
|
|
84
|
+
it('[edge] should return null and log error when parsing fails', () => {
|
|
85
|
+
// Arrange
|
|
86
|
+
const invalidJsonl = 'not valid jsonl {{{';
|
|
87
|
+
// Act
|
|
88
|
+
const result = (0, uploader_helpers_1.parseJsonl)(invalidJsonl);
|
|
89
|
+
// Assert
|
|
90
|
+
expect(result).toBeNull();
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
describe(uploader_helpers_1.downloadToLocal.name, () => {
|
|
94
|
+
const mockExistsSync = jest.spyOn(fs_1.default, 'existsSync');
|
|
95
|
+
const mockMkdirSync = jest.spyOn(fs_1.default, 'mkdirSync');
|
|
96
|
+
afterEach(() => {
|
|
97
|
+
mockExistsSync.mockReset();
|
|
98
|
+
mockMkdirSync.mockReset();
|
|
99
|
+
});
|
|
100
|
+
it('should create extracted_files directory if it does not exist', async () => {
|
|
101
|
+
// Arrange
|
|
102
|
+
const itemType = 'tasks';
|
|
103
|
+
const fetchedObjects = [{ id: 1 }];
|
|
104
|
+
mockExistsSync.mockReturnValueOnce(false);
|
|
105
|
+
mockMkdirSync.mockImplementationOnce(() => undefined);
|
|
106
|
+
const mockFileHandle = {
|
|
107
|
+
write: jest.fn().mockResolvedValue(undefined),
|
|
108
|
+
close: jest.fn().mockResolvedValue(undefined),
|
|
109
|
+
};
|
|
110
|
+
const fsPromisesOpenSpy = jest
|
|
111
|
+
.spyOn(fs_2.promises, 'open')
|
|
112
|
+
.mockResolvedValueOnce(mockFileHandle);
|
|
113
|
+
// Act
|
|
114
|
+
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
115
|
+
// Assert
|
|
116
|
+
expect(mockExistsSync).toHaveBeenCalledWith('extracted_files');
|
|
117
|
+
expect(mockMkdirSync).toHaveBeenCalledWith('extracted_files');
|
|
118
|
+
fsPromisesOpenSpy.mockRestore();
|
|
119
|
+
});
|
|
120
|
+
it('should write objects as JSONL lines to file', async () => {
|
|
121
|
+
// Arrange
|
|
122
|
+
const itemType = 'tasks';
|
|
123
|
+
const fetchedObjects = [{ id: 1 }, { id: 2 }];
|
|
124
|
+
mockExistsSync.mockReturnValueOnce(true);
|
|
125
|
+
const mockFileHandle = {
|
|
126
|
+
write: jest.fn().mockResolvedValue(undefined),
|
|
127
|
+
close: jest.fn().mockResolvedValue(undefined),
|
|
128
|
+
};
|
|
129
|
+
const fsPromisesOpenSpy = jest
|
|
130
|
+
.spyOn(fs_2.promises, 'open')
|
|
131
|
+
.mockResolvedValueOnce(mockFileHandle);
|
|
132
|
+
// Act
|
|
133
|
+
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
134
|
+
// Assert
|
|
135
|
+
expect(mockFileHandle.write).toHaveBeenCalledTimes(2);
|
|
136
|
+
expect(mockFileHandle.write).toHaveBeenCalledWith('{"id":1}\n');
|
|
137
|
+
expect(mockFileHandle.write).toHaveBeenCalledWith('{"id":2}\n');
|
|
138
|
+
expect(mockFileHandle.close).toHaveBeenCalled();
|
|
139
|
+
fsPromisesOpenSpy.mockRestore();
|
|
140
|
+
});
|
|
141
|
+
it('should handle single object (non-array) input', async () => {
|
|
142
|
+
// Arrange
|
|
143
|
+
const itemType = 'metadata';
|
|
144
|
+
const fetchedObject = { key: 'value' };
|
|
145
|
+
mockExistsSync.mockReturnValueOnce(true);
|
|
146
|
+
const mockFileHandle = {
|
|
147
|
+
write: jest.fn().mockResolvedValue(undefined),
|
|
148
|
+
close: jest.fn().mockResolvedValue(undefined),
|
|
149
|
+
};
|
|
150
|
+
const fsPromisesOpenSpy = jest
|
|
151
|
+
.spyOn(fs_2.promises, 'open')
|
|
152
|
+
.mockResolvedValueOnce(mockFileHandle);
|
|
153
|
+
// Act
|
|
154
|
+
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
155
|
+
// Assert
|
|
156
|
+
expect(mockFileHandle.write).toHaveBeenCalledTimes(1);
|
|
157
|
+
expect(mockFileHandle.write).toHaveBeenCalledWith('{"key":"value"}\n');
|
|
158
|
+
fsPromisesOpenSpy.mockRestore();
|
|
159
|
+
});
|
|
160
|
+
it('should use .json extension when itemType is external_domain_metadata', async () => {
|
|
161
|
+
// Arrange
|
|
162
|
+
const itemType = 'external_domain_metadata';
|
|
163
|
+
const fetchedObject = { domain: 'example.com' };
|
|
164
|
+
mockExistsSync.mockReturnValueOnce(true);
|
|
165
|
+
const mockFileHandle = {
|
|
166
|
+
write: jest.fn().mockResolvedValue(undefined),
|
|
167
|
+
close: jest.fn().mockResolvedValue(undefined),
|
|
168
|
+
};
|
|
169
|
+
const fsPromisesOpenSpy = jest
|
|
170
|
+
.spyOn(fs_2.promises, 'open')
|
|
171
|
+
.mockResolvedValueOnce(mockFileHandle);
|
|
172
|
+
// Act
|
|
173
|
+
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
174
|
+
// Assert
|
|
175
|
+
expect(fsPromisesOpenSpy).toHaveBeenCalledWith(expect.stringMatching(/extractor_external_domain_metadata_\d+\.json$/), 'w');
|
|
176
|
+
fsPromisesOpenSpy.mockRestore();
|
|
177
|
+
});
|
|
178
|
+
it('[edge] should reject and log error when file operations fail', async () => {
|
|
179
|
+
// Arrange
|
|
180
|
+
const itemType = 'tasks';
|
|
181
|
+
const fetchedObjects = [{ id: 1 }];
|
|
182
|
+
const fileError = new Error('File write failed');
|
|
183
|
+
mockExistsSync.mockReturnValueOnce(true);
|
|
184
|
+
const fsPromisesOpenSpy = jest
|
|
185
|
+
.spyOn(fs_2.promises, 'open')
|
|
186
|
+
.mockRejectedValueOnce(fileError);
|
|
187
|
+
// Act & Assert
|
|
188
|
+
await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
|
|
189
|
+
fsPromisesOpenSpy.mockRestore();
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
describe(uploader_helpers_1.truncateFilename.name, () => {
|
|
193
|
+
it('should return filename unchanged when within the limit', () => {
|
|
194
|
+
// Arrange
|
|
195
|
+
const filename = 'short-filename.txt';
|
|
196
|
+
// Act
|
|
197
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
198
|
+
// Assert
|
|
199
|
+
expect(result).toBe(filename);
|
|
200
|
+
});
|
|
201
|
+
it('should return filename unchanged when exactly at the limit', () => {
|
|
202
|
+
// Arrange
|
|
203
|
+
const filename = 'a'.repeat(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
204
|
+
// Act
|
|
205
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
206
|
+
// Assert
|
|
207
|
+
expect(result).toBe(filename);
|
|
208
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
209
|
+
});
|
|
210
|
+
it('should truncate filename and preserve extension when exceeding the limit', () => {
|
|
211
|
+
// Arrange
|
|
212
|
+
const longName = 'a'.repeat(300);
|
|
213
|
+
const extension = '.txt';
|
|
214
|
+
const filename = longName + extension;
|
|
215
|
+
// Act
|
|
216
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
217
|
+
// Assert
|
|
218
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
219
|
+
expect(result).toContain('...');
|
|
220
|
+
expect(result.endsWith(extension)).toBe(true);
|
|
221
|
+
});
|
|
222
|
+
it('should preserve the last MAX_DEVREV_FILENAME_EXTENSION_LENGTH characters as extension', () => {
|
|
223
|
+
// Arrange
|
|
224
|
+
const longName = 'document-'.repeat(50);
|
|
225
|
+
const extension = '.verylongextension';
|
|
226
|
+
const filename = longName + extension;
|
|
227
|
+
// Act
|
|
228
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
229
|
+
// Assert
|
|
230
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
231
|
+
const expectedExtension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
232
|
+
expect(result.endsWith(expectedExtension)).toBe(true);
|
|
233
|
+
});
|
|
234
|
+
it('should correctly format the truncated filename with ellipsis', () => {
|
|
235
|
+
// Arrange
|
|
236
|
+
const filename = 'x'.repeat(300) + '.pdf';
|
|
237
|
+
// Act
|
|
238
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
239
|
+
// Assert
|
|
240
|
+
const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3;
|
|
241
|
+
const expectedPrefix = 'x'.repeat(availableNameLength);
|
|
242
|
+
const expectedExtension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
243
|
+
expect(result).toBe(`${expectedPrefix}...${expectedExtension}`);
|
|
244
|
+
});
|
|
245
|
+
it('[edge] should handle filename with no extension', () => {
|
|
246
|
+
// Arrange
|
|
247
|
+
const filename = 'a'.repeat(300);
|
|
248
|
+
// Act
|
|
249
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
250
|
+
// Assert
|
|
251
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
252
|
+
expect(result).toContain('...');
|
|
253
|
+
// Last 20 chars are preserved as "extension"
|
|
254
|
+
expect(result.endsWith('a'.repeat(constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH))).toBe(true);
|
|
255
|
+
});
|
|
256
|
+
it('[edge] should handle filename that is just one character over the limit', () => {
|
|
257
|
+
// Arrange
|
|
258
|
+
const filename = 'a'.repeat(constants_1.MAX_DEVREV_FILENAME_LENGTH + 1);
|
|
259
|
+
// Act
|
|
260
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
261
|
+
// Assert
|
|
262
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
});
|