@coveo/push-api-client 1.6.9 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/definitions/environment.d.ts +3 -3
- package/dist/definitions/errors/privilegeError.d.ts +6 -0
- package/dist/definitions/fieldAnalyser/fieldAnalyser.d.ts +2 -10
- package/dist/definitions/fieldAnalyser/{utils.d.ts → fieldUtils.d.ts} +0 -0
- package/dist/definitions/fieldAnalyser/{utils.spec.d.ts → fieldsUtils.spec.d.ts} +0 -0
- package/dist/definitions/fieldAnalyser/inconsistencies.d.ts +1 -1
- package/dist/definitions/fieldAnalyser/typeUtils.d.ts +3 -0
- package/dist/definitions/{source.spec.d.ts → fieldAnalyser/typeUtils.spec.d.ts} +0 -0
- package/dist/definitions/help/axiosUtils.d.ts +2 -0
- package/dist/definitions/help/fileConsumer.d.ts +26 -0
- package/dist/definitions/help/fileConsumer.spec.d.ts +1 -0
- package/dist/definitions/help/fileContainer.d.ts +9 -0
- package/dist/definitions/help/fileContainer.spec.d.ts +1 -0
- package/dist/definitions/help/urlUtils.d.ts +27 -0
- package/dist/definitions/index.d.ts +6 -2
- package/dist/definitions/interfaces.d.ts +39 -0
- package/dist/definitions/source/batchUploadDocumentsFromFile.d.ts +11 -0
- package/dist/definitions/source/catalog.d.ts +69 -0
- package/dist/definitions/source/catalog.spec.d.ts +1 -0
- package/dist/definitions/source/catalog.stream.spec.d.ts +1 -0
- package/dist/definitions/source/documentUploader.d.ts +5 -0
- package/dist/definitions/{source.d.ts → source/push.d.ts} +35 -77
- package/dist/definitions/source/push.spec.d.ts +1 -0
- package/dist/definitions/source/securityIdenty.d.ts +42 -0
- package/dist/definitions/source/source.d.ts +5 -0
- package/dist/definitions/uploadStrategy/fileContainerStrategy.d.ts +23 -0
- package/dist/definitions/uploadStrategy/index.d.ts +3 -0
- package/dist/definitions/uploadStrategy/strategy.d.ts +16 -0
- package/dist/definitions/uploadStrategy/streamChunkStrategy.d.ts +27 -0
- package/dist/definitions/validation/preconditions/apiKeyPrivilege.d.ts +3 -0
- package/dist/definitions/validation/preconditions/apiKeyPrivilege.spec.d.ts +1 -0
- package/dist/definitions/validation/preconditions/platformPrivilege.d.ts +7 -0
- package/dist/docs/assets/search.js +1 -1
- package/dist/docs/classes/AnySecurityIdentityBuilder.html +3 -3
- package/dist/docs/classes/DocumentBuilder.html +17 -17
- package/dist/docs/classes/FieldAnalyser.html +3 -3
- package/dist/docs/classes/GroupSecurityIdentityBuilder.html +3 -3
- package/dist/docs/classes/PushSource.html +43 -0
- package/dist/docs/classes/UserSecurityIdentityBuilder.html +3 -3
- package/dist/docs/classes/VirtualGroupSecurityIdentityBuilder.html +3 -3
- package/dist/docs/enums/PlatformEnvironment.html +1 -1
- package/dist/docs/enums/Region.html +1 -1
- package/dist/docs/enums/SourceVisibility.html +1 -1
- package/dist/docs/index.html +2 -2
- package/dist/docs/interfaces/BatchUpdateDocuments.html +1 -1
- package/dist/docs/interfaces/Document.html +14 -14
- package/dist/docs/interfaces/SecurityIdentity.html +4 -4
- package/dist/docs/interfaces/SecurityIdentityBuilder.html +1 -1
- package/dist/docs/interfaces/UploadBatchCallbackData.html +1 -1
- package/dist/docs/modules.html +3 -2
- package/dist/environment.js.map +1 -1
- package/dist/errors/privilegeError.js +13 -0
- package/dist/errors/privilegeError.js.map +1 -0
- package/dist/fieldAnalyser/fieldAnalyser.js +24 -62
- package/dist/fieldAnalyser/fieldAnalyser.js.map +1 -1
- package/dist/fieldAnalyser/fieldAnalyser.spec.js +28 -22
- package/dist/fieldAnalyser/fieldAnalyser.spec.js.map +1 -1
- package/dist/fieldAnalyser/{utils.js → fieldUtils.js} +5 -1
- package/dist/fieldAnalyser/fieldUtils.js.map +1 -0
- package/dist/fieldAnalyser/{utils.spec.js → fieldsUtils.spec.js} +7 -4
- package/dist/fieldAnalyser/fieldsUtils.spec.js.map +1 -0
- package/dist/fieldAnalyser/inconsistencies.js.map +1 -1
- package/dist/fieldAnalyser/typeUtils.js +39 -0
- package/dist/fieldAnalyser/typeUtils.js.map +1 -0
- package/dist/fieldAnalyser/typeUtils.spec.js +48 -0
- package/dist/fieldAnalyser/typeUtils.spec.js.map +1 -0
- package/dist/help/axiosUtils.js +12 -0
- package/dist/help/axiosUtils.js.map +1 -0
- package/dist/help/fileConsumer.js +100 -0
- package/dist/help/fileConsumer.js.map +1 -0
- package/dist/help/fileConsumer.spec.js +75 -0
- package/dist/help/fileConsumer.spec.js.map +1 -0
- package/dist/help/fileContainer.js +20 -0
- package/dist/help/fileContainer.js.map +1 -0
- package/dist/help/fileContainer.spec.js +34 -0
- package/dist/help/fileContainer.spec.js.map +1 -0
- package/dist/help/urlUtils.js +43 -0
- package/dist/help/urlUtils.js.map +1 -0
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/dist/interfaces.js +3 -0
- package/dist/interfaces.js.map +1 -0
- package/dist/localtest.js +2 -2
- package/dist/localtest.js.map +1 -1
- package/dist/source/batchUploadDocumentsFromFile.js +50 -0
- package/dist/source/batchUploadDocumentsFromFile.js.map +1 -0
- package/dist/source/catalog.js +115 -0
- package/dist/source/catalog.js.map +1 -0
- package/dist/source/catalog.spec.js +239 -0
- package/dist/source/catalog.spec.js.map +1 -0
- package/dist/source/catalog.stream.spec.js +189 -0
- package/dist/source/catalog.stream.spec.js.map +1 -0
- package/dist/source/documentUploader.js +23 -0
- package/dist/source/documentUploader.js.map +1 -0
- package/dist/source/push.js +199 -0
- package/dist/source/push.js.map +1 -0
- package/dist/{source.spec.js → source/push.spec.js} +44 -27
- package/dist/source/push.spec.js.map +1 -0
- package/dist/source/securityIdenty.js +59 -0
- package/dist/source/securityIdenty.js.map +1 -0
- package/dist/source/source.js +9 -0
- package/dist/source/source.js.map +1 -0
- package/dist/uploadStrategy/fileContainerStrategy.js +34 -0
- package/dist/uploadStrategy/fileContainerStrategy.js.map +1 -0
- package/dist/uploadStrategy/index.js +7 -0
- package/dist/uploadStrategy/index.js.map +1 -0
- package/dist/uploadStrategy/strategy.js +3 -0
- package/dist/uploadStrategy/strategy.js.map +1 -0
- package/dist/uploadStrategy/streamChunkStrategy.js +51 -0
- package/dist/uploadStrategy/streamChunkStrategy.js.map +1 -0
- package/dist/validation/preconditions/apiKeyPrivilege.js +21 -0
- package/dist/validation/preconditions/apiKeyPrivilege.js.map +1 -0
- package/dist/validation/preconditions/apiKeyPrivilege.spec.js +70 -0
- package/dist/validation/preconditions/apiKeyPrivilege.spec.js.map +1 -0
- package/dist/validation/preconditions/platformPrivilege.js +35 -0
- package/dist/validation/preconditions/platformPrivilege.js.map +1 -0
- package/package.json +1 -1
- package/dist/docs/classes/Source.html +0 -45
- package/dist/docs/interfaces/BatchUpdateDocumentsFromFiles.html +0 -7
- package/dist/fieldAnalyser/utils.js.map +0 -1
- package/dist/fieldAnalyser/utils.spec.js.map +0 -1
- package/dist/source.js +0 -322
- package/dist/source.js.map +0 -1
- package/dist/source.spec.js.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"typeUtils.js","sourceRoot":"","sources":["../../src/fieldAnalyser/typeUtils.ts"],"names":[],"mappings":";;;AAAA,8DAAoD;AAEpD,MAAM,mBAAmB,GAAqD;IAC5E,EAAC,IAAI,EAAE,4BAAU,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,4BAAU,CAAC,OAAO,EAAE,4BAAU,CAAC,MAAM,CAAC,EAAC;IACpE,EAAC,IAAI,EAAE,4BAAU,CAAC,OAAO,EAAE,EAAE,EAAE,CAAC,4BAAU,CAAC,MAAM,CAAC,EAAC;CACpD,CAAC;AAEF,SAAgB,qBAAqB,CACnC,YAAwB,EACxB,SAAqB;;IAErB,IAAI,YAAY,KAAK,SAAS,EAAE;QAC9B,OAAO,IAAI,CAAC;KACb;IACD,MAAM,wBAAwB,GAAG,MAAA,mBAAmB;SACjD,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,0CACnC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC3C,CAAC;AAXD,sDAWC;AAED,SAAgB,uBAAuB,CAAC,GAAY;IAClD,QAAQ,OAAO,GAAG,EAAE;QAClB,KAAK,QAAQ;YACX,OAAO,sBAAsB,CAAC,GAAG,CAAC,CAAC;QACrC,KAAK,QAAQ;YACX,OAAO,4BAAU,CAAC,MAAM,CAAC;QAC3B;YACE,OAAO,4BAAU,CAAC,MAAM,CAAC;KAC5B;AACH,CAAC;AATD,0DASC;AAED,SAAS,sBAAsB,CAAC,MAAc;IAC5C,MAAM,cAAc,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAEpD,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;QAC5B,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,4BAAU,CAAC,IAAI,CAAC,CAAC,CAAC,4BAAU,CAAC,OAAO,CAAC;KACtE;SAAM;QACL,OAAO,4BAAU,CAAC,MAAM,CAAC;KAC1B;AACH,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const platform_client_1 = require("@coveord/platform-client");
|
|
4
|
+
const typeUtils_1 = require("./typeUtils");
|
|
5
|
+
describe('typeUtils', () => {
|
|
6
|
+
const signedIntegerLimit = 0b1111111111111111111111111111111;
|
|
7
|
+
it.each([
|
|
8
|
+
// String
|
|
9
|
+
{ value: 'over 9000', type: platform_client_1.FieldTypes.STRING },
|
|
10
|
+
// LONG
|
|
11
|
+
{ value: 0, type: platform_client_1.FieldTypes.LONG },
|
|
12
|
+
{ value: -10, type: platform_client_1.FieldTypes.LONG },
|
|
13
|
+
{ value: signedIntegerLimit, type: platform_client_1.FieldTypes.LONG },
|
|
14
|
+
{ value: 10.0, type: platform_client_1.FieldTypes.LONG },
|
|
15
|
+
// LONG 64
|
|
16
|
+
{ value: signedIntegerLimit + 1, type: platform_client_1.FieldTypes.LONG_64 },
|
|
17
|
+
{ value: signedIntegerLimit * -1 - 2, type: platform_client_1.FieldTypes.LONG_64 },
|
|
18
|
+
// DOUBLE
|
|
19
|
+
{ value: 3.14, type: platform_client_1.FieldTypes.DOUBLE },
|
|
20
|
+
{ value: -1.23, type: platform_client_1.FieldTypes.DOUBLE },
|
|
21
|
+
{ value: 9.164e-17, type: platform_client_1.FieldTypes.DOUBLE },
|
|
22
|
+
{ value: -9.1642346778e-175, type: platform_client_1.FieldTypes.DOUBLE },
|
|
23
|
+
])('value $value should be of type $type', ({ value, type }) => {
|
|
24
|
+
expect((0, typeUtils_1.getGuessedTypeFromValue)(value)).toBe(type);
|
|
25
|
+
});
|
|
26
|
+
describe('when the transition is authorized', () => {
|
|
27
|
+
it.each([
|
|
28
|
+
{ from: platform_client_1.FieldTypes.LONG, to: platform_client_1.FieldTypes.LONG },
|
|
29
|
+
{ from: platform_client_1.FieldTypes.LONG, to: platform_client_1.FieldTypes.LONG_64 },
|
|
30
|
+
{ from: platform_client_1.FieldTypes.LONG, to: platform_client_1.FieldTypes.DOUBLE },
|
|
31
|
+
{ from: platform_client_1.FieldTypes.LONG_64, to: platform_client_1.FieldTypes.DOUBLE },
|
|
32
|
+
{ from: platform_client_1.FieldTypes.DOUBLE, to: platform_client_1.FieldTypes.DOUBLE },
|
|
33
|
+
])('should return true', ({ from, to }) => {
|
|
34
|
+
expect((0, typeUtils_1.isValidTypeTransition)(from, to)).toBe(true);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
describe('when the transition is not authorized', () => {
|
|
38
|
+
it.each([
|
|
39
|
+
{ from: platform_client_1.FieldTypes.LONG_64, to: platform_client_1.FieldTypes.LONG },
|
|
40
|
+
{ from: platform_client_1.FieldTypes.DOUBLE, to: platform_client_1.FieldTypes.LONG_64 },
|
|
41
|
+
{ from: platform_client_1.FieldTypes.LONG_64, to: platform_client_1.FieldTypes.STRING },
|
|
42
|
+
{ from: platform_client_1.FieldTypes.STRING, to: platform_client_1.FieldTypes.LONG },
|
|
43
|
+
])('should return true', ({ from, to }) => {
|
|
44
|
+
expect((0, typeUtils_1.isValidTypeTransition)(from, to)).toBe(false);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
//# sourceMappingURL=typeUtils.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"typeUtils.spec.js","sourceRoot":"","sources":["../../src/fieldAnalyser/typeUtils.spec.ts"],"names":[],"mappings":";;AAAA,8DAAoD;AACpD,2CAA2E;AAE3E,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,MAAM,kBAAkB,GAAG,iCAAiC,CAAC;IAE7D,EAAE,CAAC,IAAI,CAAC;QACN,SAAS;QACT,EAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;QAC7C,OAAO;QACP,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,4BAAU,CAAC,IAAI,EAAC;QACjC,EAAC,KAAK,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,4BAAU,CAAC,IAAI,EAAC;QACnC,EAAC,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,4BAAU,CAAC,IAAI,EAAC;QAClD,EAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,4BAAU,CAAC,IAAI,EAAC;QACpC,UAAU;QACV,EAAC,KAAK,EAAE,kBAAkB,GAAG,CAAC,EAAE,IAAI,EAAE,4BAAU,CAAC,OAAO,EAAC;QACzD,EAAC,KAAK,EAAE,kBAAkB,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,4BAAU,CAAC,OAAO,EAAC;QAC9D,SAAS;QACT,EAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;QACtC,EAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;QACvC,EAAC,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;QAC3C,EAAC,KAAK,EAAE,CAAC,iBAAiB,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;KACrD,CAAC,CAAC,sCAAsC,EAAE,CAAC,EAAC,KAAK,EAAE,IAAI,EAAC,EAAE,EAAE;QAC3D,MAAM,CAAC,IAAA,mCAAuB,EAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;QACjD,EAAE,CAAC,IAAI,CAAC;YACN,EAAC,IAAI,EAAE,4BAAU,CAAC,IAAI,EAAE,EAAE,EAAE,4BAAU,CAAC,IAAI,EAAC;YAC5C,EAAC,IAAI,EAAE,4BAAU,CAAC,IAAI,EAAE,EAAE,EAAE,4BAAU,CAAC,OAAO,EAAC;YAC/C,EAAC,IAAI,EAAE,4BAAU,CAAC,IAAI,EAAE,EAAE,EAAE,4BAAU,CAAC,MAAM,EAAC;YAC9C,EAAC,IAAI,EAAE,4BAAU,CAAC,OAAO,EAAE,EAAE,EAAE,4BAAU,CAAC,MAAM,EAAC;YACjD,EAAC,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAE,EAAE,EAAE,4BAAU,CAAC,MAAM,EAAC;SACjD,CAAC,CAAC,oBAAoB,EAAE,CAAC,EAAC,IAAI,EAAE,EAAE,EAAC,EAAE,EAAE;YACtC,MAAM,CAAC,IAAA,iCAAqB,EAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;QACrD,EAAE,CAAC,IAAI,CAAC;YACN,EAAC,IAAI,EAAE,4BAAU,CAAC,OAAO,EAAE,EAAE,EAAE,4BAAU,CAAC,IAAI,EAAC;YAC/C,EAAC,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAE,EAAE,EAAE,4BAAU,CAAC,OAAO,EAAC;YACjD,EAAC,IAAI,EAAE,4BAAU,CAAC,OAAO,EAAE,EAAE,EAAE,4BAAU,CAAC,MAAM,EAAC;YACjD,EAAC,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAE,EAAE,EAAE,4BAAU,CAAC,IAAI,EAAC;SAC/C,CAAC,CAAC,oBAAoB,EAAE,CAAC,EAAC,IAAI,EAAE,EAAE,EAAC,EAAE,EAAE;YACtC,MAAM,CAAC,IAAA,iCAAqB,EAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.axiosRequestHeaders = void 0;
|
|
4
|
+
const axiosRequestHeaders = (apikey) => {
|
|
5
|
+
const authorizationHeader = {
|
|
6
|
+
Authorization: `Bearer ${apikey}`,
|
|
7
|
+
};
|
|
8
|
+
const documentsRequestHeaders = Object.assign(Object.assign({}, authorizationHeader), { 'Content-Type': 'application/json', Accept: 'application/json' });
|
|
9
|
+
return { headers: documentsRequestHeaders };
|
|
10
|
+
};
|
|
11
|
+
exports.axiosRequestHeaders = axiosRequestHeaders;
|
|
12
|
+
//# sourceMappingURL=axiosUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"axiosUtils.js","sourceRoot":"","sources":["../../src/help/axiosUtils.ts"],"names":[],"mappings":";;;AAEO,MAAM,mBAAmB,GAAG,CAAC,MAAc,EAAsB,EAAE;IACxE,MAAM,mBAAmB,GAAG;QAC1B,aAAa,EAAE,UAAU,MAAM,EAAE;KAClC,CAAC;IAEF,MAAM,uBAAuB,mCACxB,mBAAmB,KACtB,cAAc,EAAE,kBAAkB,EAClC,MAAM,EAAE,kBAAkB,GAC3B,CAAC;IAEF,OAAO,EAAC,OAAO,EAAE,uBAAuB,EAAC,CAAC;AAC5C,CAAC,CAAC;AAZW,QAAA,mBAAmB,uBAY9B"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FileConsumer = void 0;
|
|
4
|
+
const parseFile_1 = require("../validation/parseFile");
|
|
5
|
+
const path_1 = require("path");
|
|
6
|
+
const generator_1 = require("./generator");
|
|
7
|
+
/**
|
|
8
|
+
* Util class to help injesting documents through a list of files and uploading generated document batches.
|
|
9
|
+
*/
|
|
10
|
+
class FileConsumer {
|
|
11
|
+
/**
|
|
12
|
+
* Creates an instance of FileConsumer.
|
|
13
|
+
* @param {(batch: BatchUpdateDocuments) => Promise<AxiosResponse>} upload the upload operation to apply to every document batch
|
|
14
|
+
* @param {Required<ConcurrentProcessing>} processingConfig
|
|
15
|
+
*/
|
|
16
|
+
constructor(upload, processingConfig) {
|
|
17
|
+
this.upload = upload;
|
|
18
|
+
this.processingConfig = processingConfig;
|
|
19
|
+
this.cbSuccess = () => { };
|
|
20
|
+
this.cbFail = () => { };
|
|
21
|
+
}
|
|
22
|
+
async consume(files) {
|
|
23
|
+
const fileNames = files.map((path) => (0, path_1.basename)(path));
|
|
24
|
+
const { chunksToUpload, close } = this.splitByChunkAndUpload(fileNames);
|
|
25
|
+
const docBuilderGenerator = function* (docBuilders) {
|
|
26
|
+
for (const upload of chunksToUpload(docBuilders)) {
|
|
27
|
+
yield upload();
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
// parallelize uploads across multiple files
|
|
31
|
+
const fileGenerator = function* () {
|
|
32
|
+
for (const filePath of files.values()) {
|
|
33
|
+
const docBuilders = (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath);
|
|
34
|
+
yield* docBuilderGenerator(docBuilders);
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
await (0, generator_1.consumeGenerator)(fileGenerator.bind(this), this.processingConfig.maxConcurrent);
|
|
38
|
+
await close();
|
|
39
|
+
}
|
|
40
|
+
onSuccess(callback) {
|
|
41
|
+
this.cbSuccess = callback;
|
|
42
|
+
}
|
|
43
|
+
onError(callback) {
|
|
44
|
+
this.cbFail = callback;
|
|
45
|
+
}
|
|
46
|
+
splitByChunkAndUpload(fileNames, accumulator = this.accumulator) {
|
|
47
|
+
const chunksToUpload = (documentBuilders) => {
|
|
48
|
+
const batchesToUpload = [];
|
|
49
|
+
for (const docBuilder of documentBuilders) {
|
|
50
|
+
const sizeOfDoc = Buffer.byteLength(JSON.stringify(docBuilder.marshal()));
|
|
51
|
+
if (accumulator.size + sizeOfDoc >= FileConsumer.maxContentLength) {
|
|
52
|
+
const chunks = accumulator.chunks;
|
|
53
|
+
if (chunks.length > 0) {
|
|
54
|
+
batchesToUpload.push(() => this.uploadBatch(chunks, fileNames));
|
|
55
|
+
}
|
|
56
|
+
accumulator.chunks = [docBuilder];
|
|
57
|
+
accumulator.size = sizeOfDoc;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
accumulator.size += sizeOfDoc;
|
|
61
|
+
accumulator.chunks.push(docBuilder);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return batchesToUpload;
|
|
65
|
+
};
|
|
66
|
+
const close = async () => {
|
|
67
|
+
await this.uploadBatch(accumulator.chunks, fileNames);
|
|
68
|
+
};
|
|
69
|
+
return { chunksToUpload, close };
|
|
70
|
+
}
|
|
71
|
+
async uploadBatch(batch, fileNames) {
|
|
72
|
+
let res;
|
|
73
|
+
try {
|
|
74
|
+
res = await this.upload({
|
|
75
|
+
addOrUpdate: batch,
|
|
76
|
+
delete: [],
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
this.cbFail(error, {
|
|
81
|
+
files: fileNames,
|
|
82
|
+
batch,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
this.cbSuccess({
|
|
86
|
+
files: fileNames,
|
|
87
|
+
batch,
|
|
88
|
+
res,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
get accumulator() {
|
|
92
|
+
return {
|
|
93
|
+
size: 0,
|
|
94
|
+
chunks: [],
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
exports.FileConsumer = FileConsumer;
|
|
99
|
+
FileConsumer.maxContentLength = 5 * 1024 * 1024;
|
|
100
|
+
//# sourceMappingURL=fileConsumer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fileConsumer.js","sourceRoot":"","sources":["../../src/help/fileConsumer.ts"],"names":[],"mappings":";;;AAMA,uDAAmF;AACnF,+BAA8B;AAC9B,2CAA6C;AAS7C;;GAEG;AACH,MAAa,YAAY;IAKvB;;;;OAIG;IACH,YACU,MAA+D,EAC/D,gBAAgD;QADhD,WAAM,GAAN,MAAM,CAAyD;QAC/D,qBAAgB,GAAhB,gBAAgB,CAAgC;QAVlD,cAAS,GAA6B,GAAG,EAAE,GAAE,CAAC,CAAC;QAC/C,WAAM,GAAyB,GAAG,EAAE,GAAE,CAAC,CAAC;IAU7C,CAAC;IAEG,KAAK,CAAC,OAAO,CAAC,KAAe;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAA,eAAQ,EAAC,IAAI,CAAC,CAAC,CAAC;QACtD,MAAM,EAAC,cAAc,EAAE,KAAK,EAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtE,MAAM,mBAAmB,GAAG,QAAQ,CAAC,EAAE,WAA8B;YACnE,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,WAAW,CAAC,EAAE;gBAChD,MAAM,MAAM,EAAE,CAAC;aAChB;QACH,CAAC,CAAC;QAEF,4CAA4C;QAC5C,MAAM,aAAa,GAAG,QAAQ,CAAC;YAC7B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;gBACrC,MAAM,WAAW,GACf,IAAA,sDAA0C,EAAC,QAAQ,CAAC,CAAC;gBACvD,KAAK,CAAC,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;aACzC;QACH,CAAC,CAAC;QAEF,MAAM,IAAA,4BAAgB,EACpB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EACxB,IAAI,CAAC,gBAAgB,CAAC,aAAa,CACpC,CAAC;QACF,MAAM,KAAK,EAAE,CAAC;IAChB,CAAC;IAEM,SAAS,CAAC,QAAkC;QACjD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAEM,OAAO,CAAC,QAA8B;QAC3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;IACzB,CAAC;IAEO,qBAAqB,CAC3B,SAAmB,EACnB,WAAW,GAAG,IAAI,CAAC,WAAW;QAE9B,MAAM,cAAc,GAAG,CAAC,gBAAmC,EAAE,EAAE;YAC7D,MAAM,eAAe,GAA+B,EAAE,CAAC;YAEvD,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE;gBACzC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CACjC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CACrC,CAAC;gBAEF,IAAI,WAAW,CAAC,IAAI,GAAG,SAAS,IAAI,YAAY,CAAC,gBAAgB,EAAE;oBACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;oBAClC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;wBACrB,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;qBACjE;oBACD,WAAW,CAAC,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC;oBAClC,WAAW,CAAC,IAAI,GAAG,SAAS,CAAC;iBAC9B;qBAAM;oBACL,WAAW,CAAC,IAAI,IAAI,SAAS,CAAC;oBAC9B,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;iBACrC;aACF;YACD,OAAO,eAAe,CAAC;QACzB,CAAC,CAAC;QACF,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACxD,CAAC,CAAC;QACF,OAAO,EAAC,cAAc,EAAE,KAAK,EAAC,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAwB,EAAE,SAAmB;QACrE,IAAI,GAA8B,CAAC;QACnC,IAAI;YACF,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBACtB,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;SACJ;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;gBACjB,KAAK,EAAE,SAAS;gBAChB,KAAK;aACN,CAAC,CAAC;SACJ;QAED,IAAI,CAAC,SAAS,CAAC;YACb,KAAK,EAAE,SAAS;YAChB,KAAK;YACL,GAAG;SACJ,CAAC,CAAC;IACL,CAAC;IAED,IAAY,WAAW;QACrB,OAAO;YACL,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;;AA3GH,oCA4GC;AA3GgB,6BAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
jest.mock('axios');
|
|
4
|
+
const path_1 = require("path");
|
|
5
|
+
const process_1 = require("process");
|
|
6
|
+
const fileConsumer_1 = require("./fileConsumer");
|
|
7
|
+
describe('FileConsumer', () => {
|
|
8
|
+
let fileConsumer;
|
|
9
|
+
const fakeUpload = jest.fn();
|
|
10
|
+
const pathToStub = (0, path_1.join)((0, process_1.cwd)(), 'src', '__stub__');
|
|
11
|
+
const entries = [
|
|
12
|
+
(0, path_1.join)(pathToStub, 'jsondocuments', 'batman.json'),
|
|
13
|
+
(0, path_1.join)(pathToStub, 'jsondocuments', 'fightclub.json'),
|
|
14
|
+
];
|
|
15
|
+
describe('when upload is successful', () => {
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
fileConsumer = new fileConsumer_1.FileConsumer(fakeUpload, { maxConcurrent: 10 });
|
|
18
|
+
fakeUpload.mockResolvedValueOnce({
|
|
19
|
+
status: 202,
|
|
20
|
+
statusText: 'All good',
|
|
21
|
+
});
|
|
22
|
+
});
|
|
23
|
+
it('should call the success callback', async () => {
|
|
24
|
+
const mockedHandleSuccess = jest.fn();
|
|
25
|
+
fileConsumer.onSuccess(mockedHandleSuccess);
|
|
26
|
+
await fileConsumer.consume(entries);
|
|
27
|
+
expect(mockedHandleSuccess).toHaveBeenCalled();
|
|
28
|
+
});
|
|
29
|
+
it('should not call the error callback', async () => {
|
|
30
|
+
const mockedHandleError = jest.fn();
|
|
31
|
+
fileConsumer.onError(mockedHandleError);
|
|
32
|
+
await fileConsumer.consume(entries);
|
|
33
|
+
expect(mockedHandleError).not.toHaveBeenCalled();
|
|
34
|
+
});
|
|
35
|
+
it('should only push JSON files', async () => {
|
|
36
|
+
const handleBatchUpload = (data) => {
|
|
37
|
+
const expected = [
|
|
38
|
+
'https://www.themoviedb.org/movie/268',
|
|
39
|
+
'https://www.themoviedb.org/movie/999',
|
|
40
|
+
'https://www.themoviedb.org/movie/550',
|
|
41
|
+
];
|
|
42
|
+
for (let i = 0; i < data.batch.length; i++) {
|
|
43
|
+
const documentBuilder = data.batch[i];
|
|
44
|
+
expect(documentBuilder.marshal().documentId).toEqual(expected[i]);
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
fileConsumer.onSuccess(handleBatchUpload);
|
|
48
|
+
await fileConsumer.consume(entries);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
describe('when upload is in error', () => {
|
|
52
|
+
let fileConsumer;
|
|
53
|
+
beforeEach(() => {
|
|
54
|
+
fileConsumer = new fileConsumer_1.FileConsumer(fakeUpload, { maxConcurrent: 10 });
|
|
55
|
+
fakeUpload.mockRejectedValueOnce({
|
|
56
|
+
status: 412,
|
|
57
|
+
statusText: 'BAD_REQUEST',
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
it('should call the error callback', async () => {
|
|
61
|
+
const mockedHandleError = jest.fn();
|
|
62
|
+
fileConsumer.onError(mockedHandleError);
|
|
63
|
+
await fileConsumer.consume(entries);
|
|
64
|
+
expect(mockedHandleError).toHaveBeenCalled();
|
|
65
|
+
});
|
|
66
|
+
it('should call the error callback', async () => {
|
|
67
|
+
const handleBatchError = (err) => {
|
|
68
|
+
expect(err).toEqual({ status: 412, statusText: 'BAD_REQUEST' });
|
|
69
|
+
};
|
|
70
|
+
fileConsumer.onError(handleBatchError);
|
|
71
|
+
await fileConsumer.consume(entries);
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
//# sourceMappingURL=fileConsumer.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fileConsumer.spec.js","sourceRoot":"","sources":["../../src/help/fileConsumer.spec.ts"],"names":[],"mappings":";;AAAA,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEnB,+BAA0B;AAC1B,qCAA4B;AAM5B,iDAA4C;AAE5C,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,IAAA,aAAG,GAAE,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG;QACd,IAAA,WAAI,EAAC,UAAU,EAAE,eAAe,EAAE,aAAa,CAAC;QAChD,IAAA,WAAI,EAAC,UAAU,EAAE,eAAe,EAAE,gBAAgB,CAAC;KACpD,CAAC;IAEF,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,UAAU,CAAC,GAAG,EAAE;YACd,YAAY,GAAG,IAAI,2BAAY,CAAC,UAAU,EAAE,EAAC,aAAa,EAAE,EAAE,EAAC,CAAC,CAAC;YACjE,UAAU,CAAC,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACtC,YAAY,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAC5C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,iBAAiB,GAA6B,CAClD,IAA6B,EAC7B,EAAE;gBACF,MAAM,QAAQ,GAAG;oBACf,sCAAsC;oBACtC,sCAAsC;oBACtC,sCAAsC;iBACvC,CAAC;gBACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBAC1C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtC,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;iBACnE;YACH,CAAC,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAC1C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,IAAI,YAA0B,CAAC;QAE/B,UAAU,CAAC,GAAG,EAAE;YACd,YAAY,GAAG,IAAI,2BAAY,CAAC,UAAU,EAAE,EAAC,aAAa,EAAE,EAAE,EAAC,CAAC,CAAC;YACjE,UAAU,CAAC,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAEpC,MAAM,CAAC,iBAAiB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,gBAAgB,GAAyB,CAAC,GAAY,EAAE,EAAE;gBAC9D,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAC,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAC,CAAC,CAAC;YAChE,CAAC,CAAC;YAEF,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getFileContainerAxiosConfig = exports.uploadContentToFileContainer = void 0;
|
|
4
|
+
const axios_1 = require("axios");
|
|
5
|
+
const url_1 = require("url");
|
|
6
|
+
const uploadContentToFileContainer = async (fileContainer, batch) => {
|
|
7
|
+
const uploadURL = new url_1.URL(fileContainer.uploadUri);
|
|
8
|
+
return axios_1.default.put(uploadURL.toString(), {
|
|
9
|
+
addOrUpdate: batch.addOrUpdate.map((docBuilder) => docBuilder.marshal()),
|
|
10
|
+
delete: batch.delete,
|
|
11
|
+
}, (0, exports.getFileContainerAxiosConfig)(fileContainer));
|
|
12
|
+
};
|
|
13
|
+
exports.uploadContentToFileContainer = uploadContentToFileContainer;
|
|
14
|
+
const getFileContainerAxiosConfig = (fileContainer) => {
|
|
15
|
+
return {
|
|
16
|
+
headers: fileContainer.requiredHeaders,
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
exports.getFileContainerAxiosConfig = getFileContainerAxiosConfig;
|
|
20
|
+
//# sourceMappingURL=fileContainer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fileContainer.js","sourceRoot":"","sources":["../../src/help/fileContainer.ts"],"names":[],"mappings":";;;AAAA,iCAAgD;AAChD,6BAAwB;AASjB,MAAM,4BAA4B,GAAG,KAAK,EAC/C,aAAoC,EACpC,KAA2B,EAC3B,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,SAAG,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;IACnD,OAAO,eAAK,CAAC,GAAG,CACd,SAAS,CAAC,QAAQ,EAAE,EACpB;QACE,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACxE,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,EACD,IAAA,mCAA2B,EAAC,aAAa,CAAC,CAC3C,CAAC;AACJ,CAAC,CAAC;AAbW,QAAA,4BAA4B,gCAavC;AAEK,MAAM,2BAA2B,GAAG,CACzC,aAAoC,EAChB,EAAE;IACtB,OAAO;QACL,OAAO,EAAE,aAAa,CAAC,eAAe;KACvC,CAAC;AACJ,CAAC,CAAC;AANW,QAAA,2BAA2B,+BAMtC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
jest.mock('axios');
|
|
4
|
+
const axios_1 = require("axios");
|
|
5
|
+
const __1 = require("..");
|
|
6
|
+
const fileContainer_1 = require("./fileContainer");
|
|
7
|
+
describe('FileContainer', () => {
|
|
8
|
+
const mockAxios = axios_1.default;
|
|
9
|
+
const fileContainerResponse = {
|
|
10
|
+
uploadUri: 'https://fake.upload.url',
|
|
11
|
+
fileId: 'file_id',
|
|
12
|
+
requiredHeaders: { foo: 'bar' },
|
|
13
|
+
};
|
|
14
|
+
const batch = {
|
|
15
|
+
addOrUpdate: [new __1.DocumentBuilder('http://some.url', 'Some document')],
|
|
16
|
+
delete: [],
|
|
17
|
+
};
|
|
18
|
+
it('sould perform an PUT request with the right params', async () => {
|
|
19
|
+
await (0, fileContainer_1.uploadContentToFileContainer)(fileContainerResponse, batch);
|
|
20
|
+
expect(mockAxios.put).toHaveBeenCalledWith('https://fake.upload.url/', expect.objectContaining({
|
|
21
|
+
addOrUpdate: expect.arrayContaining([
|
|
22
|
+
expect.objectContaining({
|
|
23
|
+
documentId: 'http://some.url',
|
|
24
|
+
}),
|
|
25
|
+
]),
|
|
26
|
+
delete: expect.arrayContaining([]),
|
|
27
|
+
}), {
|
|
28
|
+
headers: {
|
|
29
|
+
foo: 'bar',
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
//# sourceMappingURL=fileContainer.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fileContainer.spec.js","sourceRoot":"","sources":["../../src/help/fileContainer.spec.ts"],"names":[],"mappings":";;AAAA,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEnB,iCAA0B;AAC1B,0BAAyD;AACzD,mDAGyB;AAEzB,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,MAAM,SAAS,GAAG,eAAkC,CAAC;IACrD,MAAM,qBAAqB,GAA0B;QACnD,SAAS,EAAE,yBAAyB;QACpC,MAAM,EAAE,SAAS;QACjB,eAAe,EAAE,EAAC,GAAG,EAAE,KAAK,EAAC;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAyB;QAClC,WAAW,EAAE,CAAC,IAAI,mBAAe,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;QACtE,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,IAAA,4CAA4B,EAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAEjE,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,oBAAoB,CACxC,0BAA0B,EAC1B,MAAM,CAAC,gBAAgB,CAAC;YACtB,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;gBAClC,MAAM,CAAC,gBAAgB,CAAC;oBACtB,UAAU,EAAE,iBAAiB;iBAC9B,CAAC;aACH,CAAC;YACF,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;SACnC,CAAC,EACF;YACE,OAAO,EAAE;gBACP,GAAG,EAAE,KAAK;aACX;SACF,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.StreamUrlBuilder = exports.PushUrlBuilder = void 0;
|
|
4
|
+
const environment_1 = require("../environment");
|
|
5
|
+
class BaseUrlBuilder {
|
|
6
|
+
constructor(organizationId, options) {
|
|
7
|
+
this.organizationId = organizationId;
|
|
8
|
+
this.options = options;
|
|
9
|
+
}
|
|
10
|
+
get platformURL() {
|
|
11
|
+
return `${(0, environment_1.platformUrl)(this.options)}/${this.organizationId}`;
|
|
12
|
+
}
|
|
13
|
+
get fileContainerUrl() {
|
|
14
|
+
return new URL(`${this.platformURL}/files`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
class PushUrlBuilder extends BaseUrlBuilder {
|
|
18
|
+
constructor(sourceId, organizationId, options) {
|
|
19
|
+
super(organizationId, options);
|
|
20
|
+
this.sourceId = sourceId;
|
|
21
|
+
}
|
|
22
|
+
get baseURL() {
|
|
23
|
+
return new URL(`${this.platformURL}/sources/${this.sourceId}`);
|
|
24
|
+
}
|
|
25
|
+
get baseAPIURLForUpdate() {
|
|
26
|
+
return new URL(`${this.baseURL}/documents/batch`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
exports.PushUrlBuilder = PushUrlBuilder;
|
|
30
|
+
class StreamUrlBuilder extends BaseUrlBuilder {
|
|
31
|
+
constructor(sourceId, organizationId, options) {
|
|
32
|
+
super(organizationId, options);
|
|
33
|
+
this.sourceId = sourceId;
|
|
34
|
+
}
|
|
35
|
+
get baseStreamURL() {
|
|
36
|
+
return new URL(`${this.platformURL}/sources/${this.sourceId}/stream`);
|
|
37
|
+
}
|
|
38
|
+
get baseAPIURLForUpdate() {
|
|
39
|
+
return new URL(`${this.baseStreamURL}/update`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
exports.StreamUrlBuilder = StreamUrlBuilder;
|
|
43
|
+
//# sourceMappingURL=urlUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"urlUtils.js","sourceRoot":"","sources":["../../src/help/urlUtils.ts"],"names":[],"mappings":";;;AACA,gDAA2C;AAO3C,MAAe,cAAc;IAC3B,YACU,cAAsB,EACtB,OAAqC;QADrC,mBAAc,GAAd,cAAc,CAAQ;QACtB,YAAO,GAAP,OAAO,CAA8B;IAC5C,CAAC;IACJ,IAAc,WAAW;QACvB,OAAO,GAAG,IAAA,yBAAW,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;IAC/D,CAAC;IACD,IAAW,gBAAgB;QACzB,OAAO,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,QAAQ,CAAC,CAAC;IAC9C,CAAC;CAEF;AAED,MAAa,cAAe,SAAQ,cAAc;IAChD,YACU,QAAgB,EACxB,cAAsB,EACtB,OAAqC;QAErC,KAAK,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAJvB,aAAQ,GAAR,QAAQ,CAAQ;IAK1B,CAAC;IACD,IAAW,OAAO;QAChB,OAAO,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,IAAW,mBAAmB;QAC5B,OAAO,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,kBAAkB,CAAC,CAAC;IACpD,CAAC;CACF;AAdD,wCAcC;AAED,MAAa,gBAAiB,SAAQ,cAAc;IAClD,YACU,QAAgB,EACxB,cAAsB,EACtB,OAAqC;QAErC,KAAK,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAJvB,aAAQ,GAAR,QAAQ,CAAQ;IAK1B,CAAC;IACD,IAAW,aAAa;QACtB,OAAO,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,YAAY,IAAI,CAAC,QAAQ,SAAS,CAAC,CAAC;IACxE,CAAC;IACD,IAAW,mBAAmB;QAC5B,OAAO,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,aAAa,SAAS,CAAC,CAAC;IACjD,CAAC;CACF;AAdD,4CAcC"}
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Region = exports.PlatformEnvironment = exports.DocumentBuilder = exports.FieldAnalyser = exports.
|
|
3
|
+
exports.SourceVisibility = exports.Region = exports.PlatformEnvironment = exports.DocumentBuilder = exports.FieldAnalyser = exports.PushSource = exports.Source = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
tslib_1.__exportStar(require("./document"), exports);
|
|
6
|
-
var source_1 = require("./source");
|
|
6
|
+
var source_1 = require("./source/source");
|
|
7
7
|
Object.defineProperty(exports, "Source", { enumerable: true, get: function () { return source_1.Source; } });
|
|
8
|
-
|
|
8
|
+
var push_1 = require("./source/push");
|
|
9
|
+
Object.defineProperty(exports, "PushSource", { enumerable: true, get: function () { return push_1.PushSource; } });
|
|
9
10
|
var fieldAnalyser_1 = require("./fieldAnalyser/fieldAnalyser");
|
|
10
11
|
Object.defineProperty(exports, "FieldAnalyser", { enumerable: true, get: function () { return fieldAnalyser_1.FieldAnalyser; } });
|
|
11
12
|
var documentBuilder_1 = require("./documentBuilder");
|
|
@@ -14,4 +15,6 @@ tslib_1.__exportStar(require("./securityIdentityBuilder"), exports);
|
|
|
14
15
|
var environment_1 = require("./environment");
|
|
15
16
|
Object.defineProperty(exports, "PlatformEnvironment", { enumerable: true, get: function () { return environment_1.PlatformEnvironment; } });
|
|
16
17
|
Object.defineProperty(exports, "Region", { enumerable: true, get: function () { return environment_1.Region; } });
|
|
18
|
+
var platform_client_1 = require("@coveord/platform-client");
|
|
19
|
+
Object.defineProperty(exports, "SourceVisibility", { enumerable: true, get: function () { return platform_client_1.SourceVisibility; } });
|
|
17
20
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,qDAA2B;AAC3B,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,qDAA2B;AAC3B,0CAAuC;AAA/B,gGAAA,MAAM,OAAA;AACd,sCAAyC;AAAjC,kGAAA,UAAU,OAAA;AAUlB,+DAA4D;AAApD,8GAAA,aAAa,OAAA;AACrB,qDAAkD;AAA1C,kHAAA,eAAe,OAAA;AACvB,oEAA0C;AAC1C,6CAA8E;AAAtE,kHAAA,mBAAmB,OAAA;AAAE,qGAAA,MAAM,OAAA;AACnC,4DAA0D;AAAlD,mHAAA,gBAAgB,OAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../src/interfaces.ts"],"names":[],"mappings":""}
|
package/dist/localtest.js
CHANGED
|
@@ -4,12 +4,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
4
4
|
require("dotenv/config");
|
|
5
5
|
const documentBuilder_1 = require("./documentBuilder");
|
|
6
6
|
const securityIdentityBuilder_1 = require("./securityIdentityBuilder");
|
|
7
|
+
const push_1 = require("./source/push");
|
|
7
8
|
const API_KEY = process.env.API_KEY;
|
|
8
9
|
const ORG_ID = process.env.ORG_ID;
|
|
9
10
|
const SOURCE_ID = process.env.SOURCE_ID;
|
|
10
|
-
const source_1 = require("./source");
|
|
11
11
|
async function main() {
|
|
12
|
-
const source = new
|
|
12
|
+
const source = new push_1.PushSource(API_KEY, ORG_ID);
|
|
13
13
|
await source.setSourceStatus(SOURCE_ID, 'REFRESH');
|
|
14
14
|
const docBuilder = new documentBuilder_1.DocumentBuilder('https://perdu.com', 'hello world title')
|
|
15
15
|
.withAuthor('anonymous@coveo.com')
|
package/dist/localtest.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"localtest.js","sourceRoot":"","sources":["../src/localtest.ts"],"names":[],"mappings":";;AAAA,sDAAsD;AACtD,yBAAuB;AACvB,uDAAkD;AAClD,uEAAsE;
|
|
1
|
+
{"version":3,"file":"localtest.js","sourceRoot":"","sources":["../src/localtest.ts"],"names":[],"mappings":";;AAAA,sDAAsD;AACtD,yBAAuB;AACvB,uDAAkD;AAClD,uEAAsE;AAEtE,wCAAyC;AACzC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAiB,CAAC;AAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAgB,CAAC;AAC5C,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAmB,CAAC;AAElD,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,IAAI,iBAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,IAAI,iCAAe,CACpC,mBAAmB,EACnB,mBAAmB,CACpB;SACE,UAAU,CAAC,qBAAqB,CAAC;SACjC,gBAAgB,CAAC,yBAAyB,CAAC;SAC3C,QAAQ,CAAC,6BAA6B,CAAC;SACvC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC;SAC/B,QAAQ,CAAC,YAAY,CAAC;SACtB,iBAAiB,CAAC,OAAO,CAAC;SAC1B,sBAAsB,CACrB,IAAI,qDAA2B,CAAC,oBAAoB,CAAC,CACtD;SACA,qBAAqB,CACpB,IAAI,qDAA2B,CAAC;QAC9B,oBAAoB;QACpB,oBAAoB;KACrB,CAAC,CACH,CAAC;IACJ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAE5C,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,cAAc,CAC9C,SAAS,EACT,6BAA6B,CAC9B,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAElD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,SAAS,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,qBAAqB,GAAG,MAAM,MAAM,CAAC,wBAAwB,CACjE,SAAS,EACT,SAAS,CACV,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAEtE,MAAM,KAAK,GAAyB;QAClC,WAAW,EAAE,EAAE;QACf,MAAM,EAAE,EAAE;KACX,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE;QAC3B,MAAM,KAAK,GAAG,qBAAqB,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,CAAC,EAAE;YACT,KAAK,CAAC,WAAW,CAAC,IAAI,CACpB,IAAI,iCAAe,CAAC,KAAK,EAAE,cAAc,CAAC,EAAE,CAAC;iBAC1C,uBAAuB,CAAC,KAAK,CAAC;iBAC9B,sBAAsB,CACrB,IAAI,qDAA2B,CAAC,oBAAoB,CAAC,CACtD,CACJ,CAAC;SACH;aAAM;YACL,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAC,UAAU,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAC,CAAC,CAAC;SAC9D;KACF;IAED,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,IAAI,EAAE,CAAC"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BatchUploadDocumentsFromFilesReturn = void 0;
|
|
4
|
+
const fieldErrors_1 = require("../errors/fieldErrors");
|
|
5
|
+
const file_1 = require("../help/file");
|
|
6
|
+
const fileConsumer_1 = require("../help/fileConsumer");
|
|
7
|
+
const index_1 = require("../index");
|
|
8
|
+
const fieldUtils_1 = require("../fieldAnalyser/fieldUtils");
|
|
9
|
+
const parseFile_1 = require("../validation/parseFile");
|
|
10
|
+
class BatchUploadDocumentsFromFilesReturn {
|
|
11
|
+
constructor(platformClient, strategy, filesOrDirectories, options) {
|
|
12
|
+
const defaultOptions = {
|
|
13
|
+
maxConcurrent: 10,
|
|
14
|
+
createFields: true,
|
|
15
|
+
};
|
|
16
|
+
const { maxConcurrent, createFields } = Object.assign(Object.assign({}, defaultOptions), options);
|
|
17
|
+
this.consumer = new fileConsumer_1.FileConsumer((batch) => strategy.upload(batch), { maxConcurrent });
|
|
18
|
+
this.internalPromise = (async () => {
|
|
19
|
+
var _a;
|
|
20
|
+
const files = (0, file_1.getAllJsonFilesFromEntries)(filesOrDirectories);
|
|
21
|
+
if (createFields) {
|
|
22
|
+
const analyser = new index_1.FieldAnalyser(platformClient);
|
|
23
|
+
for (const filePath of files.values()) {
|
|
24
|
+
const docBuilders = (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath);
|
|
25
|
+
await analyser.add(docBuilders);
|
|
26
|
+
}
|
|
27
|
+
const { fields, inconsistencies } = analyser.report();
|
|
28
|
+
if (inconsistencies.size > 0) {
|
|
29
|
+
throw new fieldErrors_1.FieldTypeInconsistencyError(inconsistencies);
|
|
30
|
+
}
|
|
31
|
+
await (0, fieldUtils_1.createFields)(platformClient, fields);
|
|
32
|
+
}
|
|
33
|
+
await this.consumer.consume(files);
|
|
34
|
+
await ((_a = strategy.postUpload) === null || _a === void 0 ? void 0 : _a.call(strategy));
|
|
35
|
+
}).bind(this);
|
|
36
|
+
}
|
|
37
|
+
onBatchUpload(cb) {
|
|
38
|
+
this.consumer.onSuccess(cb);
|
|
39
|
+
return this;
|
|
40
|
+
}
|
|
41
|
+
onBatchError(cb) {
|
|
42
|
+
this.consumer.onError(cb);
|
|
43
|
+
return this;
|
|
44
|
+
}
|
|
45
|
+
batch() {
|
|
46
|
+
return this.internalPromise();
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
exports.BatchUploadDocumentsFromFilesReturn = BatchUploadDocumentsFromFilesReturn;
|
|
50
|
+
//# sourceMappingURL=batchUploadDocumentsFromFile.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batchUploadDocumentsFromFile.js","sourceRoot":"","sources":["../../src/source/batchUploadDocumentsFromFile.ts"],"names":[],"mappings":";;;AACA,uDAAkE;AAClE,uCAAwD;AACxD,uDAAkD;AAClD,oCAMkB;AAClB,4DAAmE;AAEnE,uDAAmF;AAEnF,MAAa,mCAAmC;IAI9C,YACE,cAA8B,EAC9B,QAAwB,EACxB,kBAA4B,EAC5B,OAAuC;QAEvC,MAAM,cAAc,GAAG;YACrB,aAAa,EAAE,EAAE;YACjB,YAAY,EAAE,IAAI;SACnB,CAAC;QACF,MAAM,EAAC,aAAa,EAAE,YAAY,EAAC,mCAC9B,cAAc,GACd,OAAO,CACX,CAAC;QAEF,IAAI,CAAC,QAAQ,GAAG,IAAI,2BAAY,CAC9B,CAAC,KAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EACvD,EAAC,aAAa,EAAC,CAChB,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,CAAC,KAAK,IAAI,EAAE;;YACjC,MAAM,KAAK,GAAG,IAAA,iCAA0B,EAAC,kBAAkB,CAAC,CAAC;YAC7D,IAAI,YAAY,EAAE;gBAChB,MAAM,QAAQ,GAAG,IAAI,qBAAa,CAAC,cAAc,CAAC,CAAC;gBACnD,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;oBACrC,MAAM,WAAW,GACf,IAAA,sDAA0C,EAAC,QAAQ,CAAC,CAAC;oBACvD,MAAM,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;iBACjC;gBACD,MAAM,EAAC,MAAM,EAAE,eAAe,EAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAEpD,IAAI,eAAe,CAAC,IAAI,GAAG,CAAC,EAAE;oBAC5B,MAAM,IAAI,yCAA2B,CAAC,eAAe,CAAC,CAAC;iBACxD;gBACD,MAAM,IAAA,yBAAM,EAAC,cAAc,EAAE,MAAM,CAAC,CAAC;aACtC;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAA,MAAA,QAAQ,CAAC,UAAU,wDAAI,CAAA,CAAC;QAChC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEM,aAAa,CAAC,EAA4B;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,YAAY,CAAC,EAAwB;QAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK;QACV,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;IAChC,CAAC;CACF;AA1DD,kFA0DC"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CatalogSource = void 0;
|
|
4
|
+
require('isomorphic-fetch');
|
|
5
|
+
require('abortcontroller-polyfill');
|
|
6
|
+
const platform_client_1 = require("@coveord/platform-client");
|
|
7
|
+
const environment_1 = require("../environment");
|
|
8
|
+
const securityIdenty_1 = require("./securityIdenty");
|
|
9
|
+
const uploadStrategy_1 = require("../uploadStrategy");
|
|
10
|
+
const axiosUtils_1 = require("../help/axiosUtils");
|
|
11
|
+
const documentUploader_1 = require("./documentUploader");
|
|
12
|
+
const urlUtils_1 = require("../help/urlUtils");
|
|
13
|
+
const batchUploadDocumentsFromFile_1 = require("./batchUploadDocumentsFromFile");
|
|
14
|
+
/**
|
|
15
|
+
* Manage a catalog source.
|
|
16
|
+
*
|
|
17
|
+
* Allows you to create a new catalog source, manage security identities and documents in a Coveo organization.
|
|
18
|
+
*/
|
|
19
|
+
class CatalogSource {
|
|
20
|
+
/**
|
|
21
|
+
* Creates an instance of CatalogSource.
|
|
22
|
+
* @param {string} apikey An apiKey capable of pushing documents and managing sources in a Coveo organization. See [Manage API Keys](https://docs.coveo.com/en/1718).
|
|
23
|
+
* @param {string} organizationid The Coveo Organization identifier.
|
|
24
|
+
* @param {PlatformUrlOptions} [opts=CatalogSource.defaultOptions]
|
|
25
|
+
*/
|
|
26
|
+
constructor(apikey, organizationid, opts = CatalogSource.defaultOptions) {
|
|
27
|
+
this.apikey = apikey;
|
|
28
|
+
this.organizationid = organizationid;
|
|
29
|
+
this.options = Object.assign(Object.assign({}, CatalogSource.defaultOptions), opts);
|
|
30
|
+
this.platformClient = new platform_client_1.PlatformClient({
|
|
31
|
+
accessToken: apikey,
|
|
32
|
+
environment: (0, environment_1.castEnvironmentToPlatformClient)(this.options.environment),
|
|
33
|
+
organizationId: organizationid,
|
|
34
|
+
region: this.options.region,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Create a new catalog source
|
|
39
|
+
* @param name The name of the source to create.
|
|
40
|
+
* @param sourceVisibility The security option that should be applied to the content of the source. See [Content Security](https://docs.coveo.com/en/1779).
|
|
41
|
+
* @returns
|
|
42
|
+
*/
|
|
43
|
+
create(name, sourceVisibility) {
|
|
44
|
+
return this.platformClient.source.create({
|
|
45
|
+
name,
|
|
46
|
+
streamEnabled: true,
|
|
47
|
+
sourceVisibility,
|
|
48
|
+
sourceType: platform_client_1.SourceType.CATALOG,
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
get identity() {
|
|
52
|
+
return new securityIdenty_1.SecurityIdentity(this.platformClient);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Manage batches of items in a catalog source.
|
|
56
|
+
* See [Full Document Update](https://docs.coveo.com/en/l62e0540)
|
|
57
|
+
* @param sourceId
|
|
58
|
+
* @param batch
|
|
59
|
+
* @param {BatchUpdateDocumentsOptions} [{createFields: createFields = true}={}]
|
|
60
|
+
* @returns
|
|
61
|
+
*/
|
|
62
|
+
batchUpdateDocuments(sourceId, batch, { createFields: createFields = true } = {}) {
|
|
63
|
+
const strategy = this.fileContainerStrategy(sourceId);
|
|
64
|
+
return (0, documentUploader_1.uploadBatch)(this.platformClient, strategy, batch, createFields);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Send your catalog data to your catalog source.
|
|
68
|
+
* See [How to Stream Your Catalog Data to Your Source](https://docs.coveo.com/en/lb4a0344)
|
|
69
|
+
* @param {string} sourceId
|
|
70
|
+
* @param {BatchUpdateDocuments} batch
|
|
71
|
+
* @param {BatchUpdateDocumentsOptions} [{createFields: createFields = true}={}]
|
|
72
|
+
*/
|
|
73
|
+
async batchStreamDocuments(sourceId, batch, { createFields: createFields = true } = {}) {
|
|
74
|
+
const strategy = this.streamChunkStrategy(sourceId);
|
|
75
|
+
await (0, documentUploader_1.uploadBatch)(this.platformClient, strategy, batch, createFields);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Manage batches of items in a catalog source from a list of JSON files. See [Full Document Update Source](https://docs.coveo.com/en/l62e0540)
|
|
79
|
+
* @param {string} sourceId The unique identifier of the target Push source
|
|
80
|
+
* @param {string[]} filesOrDirectories A list of JSON files or directories (containing JSON files) from which to extract documents.
|
|
81
|
+
* @param {BatchUpdateDocumentsFromFiles} options
|
|
82
|
+
*/
|
|
83
|
+
batchUpdateDocumentsFromFiles(sourceId, filesOrDirectories, options) {
|
|
84
|
+
return new batchUploadDocumentsFromFile_1.BatchUploadDocumentsFromFilesReturn(this.platformClient, this.fileContainerStrategy(sourceId), filesOrDirectories, options);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Send your catalog data to your catalog source from a list of JSON files.
|
|
88
|
+
* See [How to Stream Your Catalog Data to Your Source](https://docs.coveo.com/en/lb4a0344)
|
|
89
|
+
* @param {string} sourceId The unique identifier of the target Push source
|
|
90
|
+
* @param {string[]} filesOrDirectories A list of JSON files or directories (containing JSON files) from which to extract documents.
|
|
91
|
+
* @param {BatchUpdateDocumentsFromFiles} [options]
|
|
92
|
+
*/
|
|
93
|
+
batchStreamDocumentsFromFiles(sourceId, filesOrDirectories, options) {
|
|
94
|
+
return new batchUploadDocumentsFromFile_1.BatchUploadDocumentsFromFilesReturn(this.platformClient, this.streamChunkStrategy(sourceId), filesOrDirectories, options);
|
|
95
|
+
}
|
|
96
|
+
urlBuilder(sourceId) {
|
|
97
|
+
return new urlUtils_1.StreamUrlBuilder(sourceId, this.organizationid, this.options);
|
|
98
|
+
}
|
|
99
|
+
fileContainerStrategy(sourceId) {
|
|
100
|
+
const urlBuilder = this.urlBuilder(sourceId);
|
|
101
|
+
const documentsAxiosConfig = (0, axiosUtils_1.axiosRequestHeaders)(this.apikey);
|
|
102
|
+
return new uploadStrategy_1.FileContainerStrategy(urlBuilder, documentsAxiosConfig);
|
|
103
|
+
}
|
|
104
|
+
streamChunkStrategy(sourceId) {
|
|
105
|
+
const urlBuilder = this.urlBuilder(sourceId);
|
|
106
|
+
const documentsAxiosConfig = (0, axiosUtils_1.axiosRequestHeaders)(this.apikey);
|
|
107
|
+
return new uploadStrategy_1.StreamChunkStrategy(urlBuilder, documentsAxiosConfig);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
exports.CatalogSource = CatalogSource;
|
|
111
|
+
CatalogSource.defaultOptions = {
|
|
112
|
+
region: environment_1.DEFAULT_REGION,
|
|
113
|
+
environment: environment_1.DEFAULT_ENVIRONMENT,
|
|
114
|
+
};
|
|
115
|
+
//# sourceMappingURL=catalog.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"catalog.js","sourceRoot":"","sources":["../../src/source/catalog.ts"],"names":[],"mappings":";;;AAAA,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAC5B,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAEpC,8DAIkC;AAClC,gDAKwB;AACxB,qDAAkD;AAClD,sDAA6E;AAM7E,mDAAuD;AACvD,yDAA+C;AAC/C,+CAAkD;AAClD,iFAAmF;AAEnF;;;;GAIG;AACH,MAAa,aAAa;IAQxB;;;;;OAKG;IACH,YACU,MAAc,EACd,cAAsB,EAC9B,OAA2B,aAAa,CAAC,cAAc;QAF/C,WAAM,GAAN,MAAM,CAAQ;QACd,mBAAc,GAAd,cAAc,CAAQ;QAG9B,IAAI,CAAC,OAAO,mCAAO,aAAa,CAAC,cAAc,GAAK,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,cAAc,GAAG,IAAI,gCAAc,CAAC;YACvC,WAAW,EAAE,MAAM;YACnB,WAAW,EAAE,IAAA,6CAA+B,EAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YACtE,cAAc,EAAE,cAAc;YAC9B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,IAAY,EAAE,gBAAkC;QAC5D,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC;YACvC,IAAI;YACJ,aAAa,EAAE,IAAI;YACnB,gBAAgB;YAChB,UAAU,EAAE,4BAAU,CAAC,OAAO;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,IAAW,QAAQ;QACjB,OAAO,IAAI,iCAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACnD,CAAC;IAED;;;;;;;OAOG;IACI,oBAAoB,CACzB,QAAgB,EAChB,KAA2B,EAC3B,EAAC,YAAY,EAAE,YAAY,GAAG,IAAI,KAAiC,EAAE;QAErE,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACtD,OAAO,IAAA,8BAAW,EAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IACzE,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,oBAAoB,CAC/B,QAAgB,EAChB,KAA2B,EAC3B,EAAC,YAAY,EAAE,YAAY,GAAG,IAAI,KAAiC,EAAE;QAErE,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,IAAA,8BAAW,EAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IACxE,CAAC;IAED;;;;;OAKG;IACI,6BAA6B,CAClC,QAAgB,EAChB,kBAA4B,EAC5B,OAAuC;QAEvC,OAAO,IAAI,kEAAmC,CAC5C,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EACpC,kBAAkB,EAClB,OAAO,CACR,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACI,6BAA6B,CAClC,QAAgB,EAChB,kBAA4B,EAC5B,OAAuC;QAEvC,OAAO,IAAI,kEAAmC,CAC5C,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAClC,kBAAkB,EAClB,OAAO,CACR,CAAC;IACJ,CAAC;IAEO,UAAU,CAAC,QAAgB;QACjC,OAAO,IAAI,2BAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3E,CAAC;IAEO,qBAAqB,CAAC,QAAgB;QAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,oBAAoB,GAAG,IAAA,gCAAmB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,OAAO,IAAI,sCAAqB,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC;IACrE,CAAC;IAEO,mBAAmB,CAAC,QAAgB;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,oBAAoB,GAAG,IAAA,gCAAmB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,OAAO,IAAI,oCAAmB,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC;IACnE,CAAC;;AArIH,sCAsIC;AAnIgB,4BAAc,GAAiC;IAC5D,MAAM,EAAE,4BAAc;IACtB,WAAW,EAAE,iCAAmB;CACjC,CAAC"}
|