@coveo/push-api-client 1.6.8 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/definitions/environment.d.ts +3 -3
- package/dist/definitions/fieldAnalyser/fieldAnalyser.d.ts +2 -10
- package/dist/definitions/fieldAnalyser/{utils.d.ts → fieldUtils.d.ts} +0 -0
- package/dist/definitions/fieldAnalyser/{utils.spec.d.ts → fieldsUtils.spec.d.ts} +0 -0
- package/dist/definitions/fieldAnalyser/inconsistencies.d.ts +1 -1
- package/dist/definitions/fieldAnalyser/typeUtils.d.ts +3 -0
- package/dist/definitions/{source.spec.d.ts → fieldAnalyser/typeUtils.spec.d.ts} +0 -0
- package/dist/definitions/help/fileConsumer.d.ts +26 -0
- package/dist/definitions/help/fileConsumer.spec.d.ts +1 -0
- package/dist/definitions/index.d.ts +6 -2
- package/dist/definitions/interfaces.d.ts +36 -0
- package/dist/definitions/source/batchUpdateDocumentsFromFile.d.ts +10 -0
- package/dist/definitions/{source.d.ts → source/push.d.ts} +34 -69
- package/dist/definitions/source/push.spec.d.ts +1 -0
- package/dist/definitions/source/securityIdenty.d.ts +42 -0
- package/dist/definitions/source/source.d.ts +5 -0
- package/dist/docs/assets/search.js +1 -1
- package/dist/docs/classes/AnySecurityIdentityBuilder.html +3 -3
- package/dist/docs/classes/DocumentBuilder.html +17 -17
- package/dist/docs/classes/FieldAnalyser.html +3 -3
- package/dist/docs/classes/GroupSecurityIdentityBuilder.html +3 -3
- package/dist/docs/classes/PushSource.html +43 -0
- package/dist/docs/classes/UserSecurityIdentityBuilder.html +3 -3
- package/dist/docs/classes/VirtualGroupSecurityIdentityBuilder.html +3 -3
- package/dist/docs/enums/PlatformEnvironment.html +1 -1
- package/dist/docs/enums/Region.html +1 -1
- package/dist/docs/enums/SourceVisibility.html +1 -1
- package/dist/docs/index.html +1 -1
- package/dist/docs/interfaces/BatchUpdateDocuments.html +1 -1
- package/dist/docs/interfaces/Document.html +14 -14
- package/dist/docs/interfaces/SecurityIdentity.html +4 -4
- package/dist/docs/interfaces/SecurityIdentityBuilder.html +1 -1
- package/dist/docs/interfaces/UploadBatchCallbackData.html +1 -1
- package/dist/docs/modules.html +3 -2
- package/dist/environment.js.map +1 -1
- package/dist/fieldAnalyser/fieldAnalyser.js +21 -62
- package/dist/fieldAnalyser/fieldAnalyser.js.map +1 -1
- package/dist/fieldAnalyser/fieldAnalyser.spec.js +15 -22
- package/dist/fieldAnalyser/fieldAnalyser.spec.js.map +1 -1
- package/dist/fieldAnalyser/{utils.js → fieldUtils.js} +1 -1
- package/dist/fieldAnalyser/fieldUtils.js.map +1 -0
- package/dist/fieldAnalyser/{utils.spec.js → fieldsUtils.spec.js} +4 -4
- package/dist/fieldAnalyser/fieldsUtils.spec.js.map +1 -0
- package/dist/fieldAnalyser/inconsistencies.js.map +1 -1
- package/dist/fieldAnalyser/typeUtils.js +39 -0
- package/dist/fieldAnalyser/typeUtils.js.map +1 -0
- package/dist/fieldAnalyser/typeUtils.spec.js +48 -0
- package/dist/fieldAnalyser/typeUtils.spec.js.map +1 -0
- package/dist/help/fileConsumer.js +100 -0
- package/dist/help/fileConsumer.js.map +1 -0
- package/dist/help/fileConsumer.spec.js +75 -0
- package/dist/help/fileConsumer.spec.js.map +1 -0
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/dist/interfaces.js +3 -0
- package/dist/interfaces.js.map +1 -0
- package/dist/localtest.js +2 -2
- package/dist/localtest.js.map +1 -1
- package/dist/source/batchUpdateDocumentsFromFile.js +42 -0
- package/dist/source/batchUpdateDocumentsFromFile.js.map +1 -0
- package/dist/{source.js → source/push.js} +43 -128
- package/dist/source/push.js.map +1 -0
- package/dist/{source.spec.js → source/push.spec.js} +41 -27
- package/dist/source/push.spec.js.map +1 -0
- package/dist/source/securityIdenty.js +59 -0
- package/dist/source/securityIdenty.js.map +1 -0
- package/dist/source/source.js +9 -0
- package/dist/source/source.js.map +1 -0
- package/package.json +1 -1
- package/dist/docs/classes/Source.html +0 -45
- package/dist/docs/interfaces/BatchUpdateDocumentsFromFiles.html +0 -7
- package/dist/fieldAnalyser/utils.js.map +0 -1
- package/dist/fieldAnalyser/utils.spec.js.map +0 -1
- package/dist/source.js.map +0 -1
- package/dist/source.spec.js.map +0 -1
@@ -0,0 +1,100 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.FileConsumer = void 0;
|
4
|
+
const parseFile_1 = require("../validation/parseFile");
|
5
|
+
const path_1 = require("path");
|
6
|
+
const generator_1 = require("./generator");
|
7
|
+
/**
|
8
|
+
* Util class to help injesting documents through a list of files and uploading generated document batches.
|
9
|
+
*/
|
10
|
+
class FileConsumer {
|
11
|
+
/**
|
12
|
+
* Creates an instance of FileConsumer.
|
13
|
+
* @param {(batch: BatchUpdateDocuments) => Promise<AxiosResponse>} upload the upload operation to apply to every document batch
|
14
|
+
* @param {Required<ConcurrentProcessing>} processingConfig
|
15
|
+
*/
|
16
|
+
constructor(upload, processingConfig) {
|
17
|
+
this.upload = upload;
|
18
|
+
this.processingConfig = processingConfig;
|
19
|
+
this.cbSuccess = () => { };
|
20
|
+
this.cbFail = () => { };
|
21
|
+
}
|
22
|
+
async consume(files) {
|
23
|
+
const fileNames = files.map((path) => (0, path_1.basename)(path));
|
24
|
+
const { chunksToUpload, close } = this.splitByChunkAndUpload(fileNames);
|
25
|
+
const docBuilderGenerator = function* (docBuilders) {
|
26
|
+
for (const upload of chunksToUpload(docBuilders)) {
|
27
|
+
yield upload();
|
28
|
+
}
|
29
|
+
};
|
30
|
+
// parallelize uploads across multiple files
|
31
|
+
const fileGenerator = function* () {
|
32
|
+
for (const filePath of files.values()) {
|
33
|
+
const docBuilders = (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath);
|
34
|
+
yield* docBuilderGenerator(docBuilders);
|
35
|
+
}
|
36
|
+
};
|
37
|
+
await (0, generator_1.consumeGenerator)(fileGenerator.bind(this), this.processingConfig.maxConcurrent);
|
38
|
+
await close();
|
39
|
+
}
|
40
|
+
onSuccess(callback) {
|
41
|
+
this.cbSuccess = callback;
|
42
|
+
}
|
43
|
+
onError(callback) {
|
44
|
+
this.cbFail = callback;
|
45
|
+
}
|
46
|
+
splitByChunkAndUpload(fileNames, accumulator = this.accumulator) {
|
47
|
+
const chunksToUpload = (documentBuilders) => {
|
48
|
+
const batchesToUpload = [];
|
49
|
+
for (const docBuilder of documentBuilders) {
|
50
|
+
const sizeOfDoc = Buffer.byteLength(JSON.stringify(docBuilder.marshal()));
|
51
|
+
if (accumulator.size + sizeOfDoc >= FileConsumer.maxContentLength) {
|
52
|
+
const chunks = accumulator.chunks;
|
53
|
+
if (chunks.length > 0) {
|
54
|
+
batchesToUpload.push(() => this.uploadBatch(chunks, fileNames));
|
55
|
+
}
|
56
|
+
accumulator.chunks = [docBuilder];
|
57
|
+
accumulator.size = sizeOfDoc;
|
58
|
+
}
|
59
|
+
else {
|
60
|
+
accumulator.size += sizeOfDoc;
|
61
|
+
accumulator.chunks.push(docBuilder);
|
62
|
+
}
|
63
|
+
}
|
64
|
+
return batchesToUpload;
|
65
|
+
};
|
66
|
+
const close = async () => {
|
67
|
+
await this.uploadBatch(accumulator.chunks, fileNames);
|
68
|
+
};
|
69
|
+
return { chunksToUpload, close };
|
70
|
+
}
|
71
|
+
async uploadBatch(batch, fileNames) {
|
72
|
+
let res;
|
73
|
+
try {
|
74
|
+
res = await this.upload({
|
75
|
+
addOrUpdate: batch,
|
76
|
+
delete: [],
|
77
|
+
});
|
78
|
+
}
|
79
|
+
catch (error) {
|
80
|
+
this.cbFail(error, {
|
81
|
+
files: fileNames,
|
82
|
+
batch,
|
83
|
+
});
|
84
|
+
}
|
85
|
+
this.cbSuccess({
|
86
|
+
files: fileNames,
|
87
|
+
batch,
|
88
|
+
res,
|
89
|
+
});
|
90
|
+
}
|
91
|
+
get accumulator() {
|
92
|
+
return {
|
93
|
+
size: 0,
|
94
|
+
chunks: [],
|
95
|
+
};
|
96
|
+
}
|
97
|
+
}
|
98
|
+
exports.FileConsumer = FileConsumer;
|
99
|
+
FileConsumer.maxContentLength = 5 * 1024 * 1024;
|
100
|
+
//# sourceMappingURL=fileConsumer.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"fileConsumer.js","sourceRoot":"","sources":["../../src/help/fileConsumer.ts"],"names":[],"mappings":";;;AAMA,uDAAmF;AACnF,+BAA8B;AAC9B,2CAA6C;AAS7C;;GAEG;AACH,MAAa,YAAY;IAKvB;;;;OAIG;IACH,YACU,MAA+D,EAC/D,gBAAgD;QADhD,WAAM,GAAN,MAAM,CAAyD;QAC/D,qBAAgB,GAAhB,gBAAgB,CAAgC;QAVlD,cAAS,GAA6B,GAAG,EAAE,GAAE,CAAC,CAAC;QAC/C,WAAM,GAAyB,GAAG,EAAE,GAAE,CAAC,CAAC;IAU7C,CAAC;IAEG,KAAK,CAAC,OAAO,CAAC,KAAe;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAA,eAAQ,EAAC,IAAI,CAAC,CAAC,CAAC;QACtD,MAAM,EAAC,cAAc,EAAE,KAAK,EAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtE,MAAM,mBAAmB,GAAG,QAAQ,CAAC,EAAE,WAA8B;YACnE,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,WAAW,CAAC,EAAE;gBAChD,MAAM,MAAM,EAAE,CAAC;aAChB;QACH,CAAC,CAAC;QAEF,4CAA4C;QAC5C,MAAM,aAAa,GAAG,QAAQ,CAAC;YAC7B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;gBACrC,MAAM,WAAW,GACf,IAAA,sDAA0C,EAAC,QAAQ,CAAC,CAAC;gBACvD,KAAK,CAAC,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;aACzC;QACH,CAAC,CAAC;QAEF,MAAM,IAAA,4BAAgB,EACpB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EACxB,IAAI,CAAC,gBAAgB,CAAC,aAAa,CACpC,CAAC;QACF,MAAM,KAAK,EAAE,CAAC;IAChB,CAAC;IAEM,SAAS,CAAC,QAAkC;QACjD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAEM,OAAO,CAAC,QAA8B;QAC3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;IACzB,CAAC;IAEO,qBAAqB,CAC3B,SAAmB,EACnB,WAAW,GAAG,IAAI,CAAC,WAAW;QAE9B,MAAM,cAAc,GAAG,CAAC,gBAAmC,EAAE,EAAE;YAC7D,MAAM,eAAe,GAA+B,EAAE,CAAC;YAEvD,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE;gBACzC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CACjC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CACrC,CAAC;gBAEF,IAAI,WAAW,CAAC,IAAI,GAAG,SAAS,IAAI,YAAY,CAAC,gBAAgB,EAAE;oBACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;oBAClC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;wBACrB,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;qBACjE;oBACD,WAAW,CAAC,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC;oBAClC,WAAW,CAAC,IAAI,GAAG,SAAS,CAAC;iBAC9B;qBAAM;oBACL,WAAW,CAAC,IAAI,IAAI,SAAS,CAAC;oBAC9B,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;iBACrC;aACF;YACD,OAAO,eAAe,CAAC;QACzB,CAAC,CAAC;QACF,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACxD,CAAC,CAAC;QACF,OAAO,EAAC,cAAc,EAAE,KAAK,EAAC,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAwB,EAAE,SAAmB;QACrE,IAAI,GAA8B,CAAC;QACnC,IAAI;YACF,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBACtB,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;SACJ;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;gBACjB,KAAK,EAAE,SAAS;gBAChB,KAAK;aACN,CAAC,CAAC;SACJ;QAED,IAAI,CAAC,SAAS,CAAC;YACb,KAAK,EAAE,SAAS;YAChB,KAAK;YACL,GAAG;SACJ,CAAC,CAAC;IACL,CAAC;IAED,IAAY,WAAW;QACrB,OAAO;YACL,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;;AA3GH,oCA4GC;AA3GgB,6BAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC"}
|
@@ -0,0 +1,75 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
jest.mock('axios');
|
4
|
+
const path_1 = require("path");
|
5
|
+
const process_1 = require("process");
|
6
|
+
const fileConsumer_1 = require("./fileConsumer");
|
7
|
+
describe('FileConsumer', () => {
|
8
|
+
let fileConsumer;
|
9
|
+
const fakeUpload = jest.fn();
|
10
|
+
const pathToStub = (0, path_1.join)((0, process_1.cwd)(), 'src', '__stub__');
|
11
|
+
const entries = [
|
12
|
+
(0, path_1.join)(pathToStub, 'jsondocuments', 'batman.json'),
|
13
|
+
(0, path_1.join)(pathToStub, 'jsondocuments', 'fightclub.json'),
|
14
|
+
];
|
15
|
+
describe('when upload is successful', () => {
|
16
|
+
beforeEach(() => {
|
17
|
+
fileConsumer = new fileConsumer_1.FileConsumer(fakeUpload, { maxConcurrent: 10 });
|
18
|
+
fakeUpload.mockResolvedValueOnce({
|
19
|
+
status: 202,
|
20
|
+
statusText: 'All good',
|
21
|
+
});
|
22
|
+
});
|
23
|
+
it('should call the success callback', async () => {
|
24
|
+
const mockedHandleSuccess = jest.fn();
|
25
|
+
fileConsumer.onSuccess(mockedHandleSuccess);
|
26
|
+
await fileConsumer.consume(entries);
|
27
|
+
expect(mockedHandleSuccess).toHaveBeenCalled();
|
28
|
+
});
|
29
|
+
it('should not call the error callback', async () => {
|
30
|
+
const mockedHandleError = jest.fn();
|
31
|
+
fileConsumer.onError(mockedHandleError);
|
32
|
+
await fileConsumer.consume(entries);
|
33
|
+
expect(mockedHandleError).not.toHaveBeenCalled();
|
34
|
+
});
|
35
|
+
it('should only push JSON files', async () => {
|
36
|
+
const handleBatchUpload = (data) => {
|
37
|
+
const expected = [
|
38
|
+
'https://www.themoviedb.org/movie/268',
|
39
|
+
'https://www.themoviedb.org/movie/999',
|
40
|
+
'https://www.themoviedb.org/movie/550',
|
41
|
+
];
|
42
|
+
for (let i = 0; i < data.batch.length; i++) {
|
43
|
+
const documentBuilder = data.batch[i];
|
44
|
+
expect(documentBuilder.marshal().documentId).toEqual(expected[i]);
|
45
|
+
}
|
46
|
+
};
|
47
|
+
fileConsumer.onSuccess(handleBatchUpload);
|
48
|
+
await fileConsumer.consume(entries);
|
49
|
+
});
|
50
|
+
});
|
51
|
+
describe('when upload is in error', () => {
|
52
|
+
let fileConsumer;
|
53
|
+
beforeEach(() => {
|
54
|
+
fileConsumer = new fileConsumer_1.FileConsumer(fakeUpload, { maxConcurrent: 10 });
|
55
|
+
fakeUpload.mockRejectedValueOnce({
|
56
|
+
status: 412,
|
57
|
+
statusText: 'BAD_REQUEST',
|
58
|
+
});
|
59
|
+
});
|
60
|
+
it('should call the error callback', async () => {
|
61
|
+
const mockedHandleError = jest.fn();
|
62
|
+
fileConsumer.onError(mockedHandleError);
|
63
|
+
await fileConsumer.consume(entries);
|
64
|
+
expect(mockedHandleError).toHaveBeenCalled();
|
65
|
+
});
|
66
|
+
it('should call the error callback', async () => {
|
67
|
+
const handleBatchError = (err) => {
|
68
|
+
expect(err).toEqual({ status: 412, statusText: 'BAD_REQUEST' });
|
69
|
+
};
|
70
|
+
fileConsumer.onError(handleBatchError);
|
71
|
+
await fileConsumer.consume(entries);
|
72
|
+
});
|
73
|
+
});
|
74
|
+
});
|
75
|
+
//# sourceMappingURL=fileConsumer.spec.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"fileConsumer.spec.js","sourceRoot":"","sources":["../../src/help/fileConsumer.spec.ts"],"names":[],"mappings":";;AAAA,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEnB,+BAA0B;AAC1B,qCAA4B;AAM5B,iDAA4C;AAE5C,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,IAAA,aAAG,GAAE,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG;QACd,IAAA,WAAI,EAAC,UAAU,EAAE,eAAe,EAAE,aAAa,CAAC;QAChD,IAAA,WAAI,EAAC,UAAU,EAAE,eAAe,EAAE,gBAAgB,CAAC;KACpD,CAAC;IAEF,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,UAAU,CAAC,GAAG,EAAE;YACd,YAAY,GAAG,IAAI,2BAAY,CAAC,UAAU,EAAE,EAAC,aAAa,EAAE,EAAE,EAAC,CAAC,CAAC;YACjE,UAAU,CAAC,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACtC,YAAY,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAC5C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,iBAAiB,GAA6B,CAClD,IAA6B,EAC7B,EAAE;gBACF,MAAM,QAAQ,GAAG;oBACf,sCAAsC;oBACtC,sCAAsC;oBACtC,sCAAsC;iBACvC,CAAC;gBACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBAC1C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtC,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;iBACnE;YACH,CAAC,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAC1C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,IAAI,YAA0B,CAAC;QAE/B,UAAU,CAAC,GAAG,EAAE;YACd,YAAY,GAAG,IAAI,2BAAY,CAAC,UAAU,EAAE,EAAC,aAAa,EAAE,EAAE,EAAC,CAAC,CAAC;YACjE,UAAU,CAAC,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAEpC,MAAM,CAAC,iBAAiB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,gBAAgB,GAAyB,CAAC,GAAY,EAAE,EAAE;gBAC9D,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAC,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAC,CAAC,CAAC;YAChE,CAAC,CAAC;YAEF,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/index.js
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.Region = exports.PlatformEnvironment = exports.DocumentBuilder = exports.FieldAnalyser = exports.
|
3
|
+
exports.SourceVisibility = exports.Region = exports.PlatformEnvironment = exports.DocumentBuilder = exports.FieldAnalyser = exports.PushSource = exports.Source = void 0;
|
4
4
|
const tslib_1 = require("tslib");
|
5
5
|
tslib_1.__exportStar(require("./document"), exports);
|
6
|
-
var source_1 = require("./source");
|
6
|
+
var source_1 = require("./source/source");
|
7
7
|
Object.defineProperty(exports, "Source", { enumerable: true, get: function () { return source_1.Source; } });
|
8
|
-
|
8
|
+
var push_1 = require("./source/push");
|
9
|
+
Object.defineProperty(exports, "PushSource", { enumerable: true, get: function () { return push_1.PushSource; } });
|
9
10
|
var fieldAnalyser_1 = require("./fieldAnalyser/fieldAnalyser");
|
10
11
|
Object.defineProperty(exports, "FieldAnalyser", { enumerable: true, get: function () { return fieldAnalyser_1.FieldAnalyser; } });
|
11
12
|
var documentBuilder_1 = require("./documentBuilder");
|
@@ -14,4 +15,6 @@ tslib_1.__exportStar(require("./securityIdentityBuilder"), exports);
|
|
14
15
|
var environment_1 = require("./environment");
|
15
16
|
Object.defineProperty(exports, "PlatformEnvironment", { enumerable: true, get: function () { return environment_1.PlatformEnvironment; } });
|
16
17
|
Object.defineProperty(exports, "Region", { enumerable: true, get: function () { return environment_1.Region; } });
|
18
|
+
var platform_client_1 = require("@coveord/platform-client");
|
19
|
+
Object.defineProperty(exports, "SourceVisibility", { enumerable: true, get: function () { return platform_client_1.SourceVisibility; } });
|
17
20
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,qDAA2B;AAC3B,
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,qDAA2B;AAC3B,0CAAuC;AAA/B,gGAAA,MAAM,OAAA;AACd,sCAAyC;AAAjC,kGAAA,UAAU,OAAA;AAUlB,+DAA4D;AAApD,8GAAA,aAAa,OAAA;AACrB,qDAAkD;AAA1C,kHAAA,eAAe,OAAA;AACvB,oEAA0C;AAC1C,6CAA8E;AAAtE,kHAAA,mBAAmB,OAAA;AAAE,qGAAA,MAAM,OAAA;AACnC,4DAA0D;AAAlD,mHAAA,gBAAgB,OAAA"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../src/interfaces.ts"],"names":[],"mappings":""}
|
package/dist/localtest.js
CHANGED
@@ -4,12 +4,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
4
|
require("dotenv/config");
|
5
5
|
const documentBuilder_1 = require("./documentBuilder");
|
6
6
|
const securityIdentityBuilder_1 = require("./securityIdentityBuilder");
|
7
|
+
const push_1 = require("./source/push");
|
7
8
|
const API_KEY = process.env.API_KEY;
|
8
9
|
const ORG_ID = process.env.ORG_ID;
|
9
10
|
const SOURCE_ID = process.env.SOURCE_ID;
|
10
|
-
const source_1 = require("./source");
|
11
11
|
async function main() {
|
12
|
-
const source = new
|
12
|
+
const source = new push_1.PushSource(API_KEY, ORG_ID);
|
13
13
|
await source.setSourceStatus(SOURCE_ID, 'REFRESH');
|
14
14
|
const docBuilder = new documentBuilder_1.DocumentBuilder('https://perdu.com', 'hello world title')
|
15
15
|
.withAuthor('anonymous@coveo.com')
|
package/dist/localtest.js.map
CHANGED
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"localtest.js","sourceRoot":"","sources":["../src/localtest.ts"],"names":[],"mappings":";;AAAA,sDAAsD;AACtD,yBAAuB;AACvB,uDAAkD;AAClD,uEAAsE;
|
1
|
+
{"version":3,"file":"localtest.js","sourceRoot":"","sources":["../src/localtest.ts"],"names":[],"mappings":";;AAAA,sDAAsD;AACtD,yBAAuB;AACvB,uDAAkD;AAClD,uEAAsE;AAEtE,wCAAyC;AACzC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAiB,CAAC;AAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAgB,CAAC;AAC5C,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAmB,CAAC;AAElD,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,IAAI,iBAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,IAAI,iCAAe,CACpC,mBAAmB,EACnB,mBAAmB,CACpB;SACE,UAAU,CAAC,qBAAqB,CAAC;SACjC,gBAAgB,CAAC,yBAAyB,CAAC;SAC3C,QAAQ,CAAC,6BAA6B,CAAC;SACvC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC;SAC/B,QAAQ,CAAC,YAAY,CAAC;SACtB,iBAAiB,CAAC,OAAO,CAAC;SAC1B,sBAAsB,CACrB,IAAI,qDAA2B,CAAC,oBAAoB,CAAC,CACtD;SACA,qBAAqB,CACpB,IAAI,qDAA2B,CAAC;QAC9B,oBAAoB;QACpB,oBAAoB;KACrB,CAAC,CACH,CAAC;IACJ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAE5C,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,cAAc,CAC9C,SAAS,EACT,6BAA6B,CAC9B,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAElD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,SAAS,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,qBAAqB,GAAG,MAAM,MAAM,CAAC,wBAAwB,CACjE,SAAS,EACT,SAAS,CACV,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAEtE,MAAM,KAAK,GAAyB;QAClC,WAAW,EAAE,EAAE;QACf,MAAM,EAAE,EAAE;KACX,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE;QAC3B,MAAM,KAAK,GAAG,qBAAqB,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,CAAC,EAAE;YACT,KAAK,CAAC,WAAW,CAAC,IAAI,CACpB,IAAI,iCAAe,CAAC,KAAK,EAAE,cAAc,CAAC,EAAE,CAAC;iBAC1C,uBAAuB,CAAC,KAAK,CAAC;iBAC9B,sBAAsB,CACrB,IAAI,qDAA2B,CAAC,oBAAoB,CAAC,CACtD,CACJ,CAAC;SACH;aAAM;YACL,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAC,UAAU,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAC,CAAC,CAAC;SAC9D;KACF;IAED,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,IAAI,EAAE,CAAC"}
|
@@ -0,0 +1,42 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.BatchUpdateDocumentsFromFilesReturn = void 0;
|
4
|
+
const file_1 = require("../help/file");
|
5
|
+
const fileConsumer_1 = require("../help/fileConsumer");
|
6
|
+
const index_1 = require("../index");
|
7
|
+
const parseFile_1 = require("../validation/parseFile");
|
8
|
+
class BatchUpdateDocumentsFromFilesReturn {
|
9
|
+
constructor(pushSource, sourceId, filesOrDirectories, options) {
|
10
|
+
const defaultOptions = {
|
11
|
+
maxConcurrent: 10,
|
12
|
+
createFields: true,
|
13
|
+
};
|
14
|
+
const { maxConcurrent, createFields } = Object.assign(Object.assign({}, defaultOptions), options);
|
15
|
+
this.batchConsumer = new fileConsumer_1.FileConsumer((batch) => pushSource.uploadBatch(sourceId, batch), { maxConcurrent });
|
16
|
+
this.internalPromise = (async () => {
|
17
|
+
const files = (0, file_1.getAllJsonFilesFromEntries)(filesOrDirectories);
|
18
|
+
if (createFields) {
|
19
|
+
const analyser = new index_1.FieldAnalyser(pushSource.platformClient);
|
20
|
+
for (const filePath of files.values()) {
|
21
|
+
const docBuilders = (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath);
|
22
|
+
await analyser.add(docBuilders);
|
23
|
+
}
|
24
|
+
await pushSource.createFields(analyser);
|
25
|
+
}
|
26
|
+
await this.batchConsumer.consume(files);
|
27
|
+
}).bind(this);
|
28
|
+
}
|
29
|
+
onBatchUpload(cb) {
|
30
|
+
this.batchConsumer.onSuccess(cb);
|
31
|
+
return this;
|
32
|
+
}
|
33
|
+
onBatchError(cb) {
|
34
|
+
this.batchConsumer.onError(cb);
|
35
|
+
return this;
|
36
|
+
}
|
37
|
+
batch() {
|
38
|
+
return this.internalPromise();
|
39
|
+
}
|
40
|
+
}
|
41
|
+
exports.BatchUpdateDocumentsFromFilesReturn = BatchUpdateDocumentsFromFilesReturn;
|
42
|
+
//# sourceMappingURL=batchUpdateDocumentsFromFile.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"batchUpdateDocumentsFromFile.js","sourceRoot":"","sources":["../../src/source/batchUpdateDocumentsFromFile.ts"],"names":[],"mappings":";;;AAAA,uCAAwD;AACxD,uDAAkD;AAClD,oCAMkB;AAClB,uDAAmF;AAGnF,MAAa,mCAAmC;IAI9C,YACE,UAAsB,EACtB,QAAgB,EAChB,kBAA4B,EAC5B,OAAuC;QAEvC,MAAM,cAAc,GAAG;YACrB,aAAa,EAAE,EAAE;YACjB,YAAY,EAAE,IAAI;SACnB,CAAC;QACF,MAAM,EAAC,aAAa,EAAE,YAAY,EAAC,mCAC9B,cAAc,GACd,OAAO,CACX,CAAC;QAEF,IAAI,CAAC,aAAa,GAAG,IAAI,2BAAY,CACnC,CAAC,KAA2B,EAAE,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,EACxE,EAAC,aAAa,EAAC,CAChB,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,CAAC,KAAK,IAAI,EAAE;YACjC,MAAM,KAAK,GAAG,IAAA,iCAA0B,EAAC,kBAAkB,CAAC,CAAC;YAC7D,IAAI,YAAY,EAAE;gBAChB,MAAM,QAAQ,GAAG,IAAI,qBAAa,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAC9D,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;oBACrC,MAAM,WAAW,GACf,IAAA,sDAA0C,EAAC,QAAQ,CAAC,CAAC;oBACvD,MAAM,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;iBACjC;gBACD,MAAM,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;aACzC;YACD,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEM,aAAa,CAAC,EAA4B;QAC/C,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,YAAY,CAAC,EAAwB;QAC1C,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK;QACV,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;IAChC,CAAC;CACF;AApDD,kFAoDC"}
|
@@ -1,6 +1,6 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.
|
3
|
+
exports.PushSource = exports.SourceVisibility = void 0;
|
4
4
|
require('isomorphic-fetch');
|
5
5
|
require('abortcontroller-polyfill');
|
6
6
|
const platform_client_1 = require("@coveord/platform-client");
|
@@ -9,20 +9,18 @@ Object.defineProperty(exports, "SourceVisibility", { enumerable: true, get: func
|
|
9
9
|
const axios_1 = require("axios");
|
10
10
|
const dayjs = require("dayjs");
|
11
11
|
const url_1 = require("url");
|
12
|
-
const
|
13
|
-
const
|
14
|
-
const
|
15
|
-
const
|
16
|
-
const
|
17
|
-
const
|
18
|
-
const fieldErrors_1 = require("./errors/fieldErrors");
|
19
|
-
const utils_1 = require("./fieldAnalyser/utils");
|
12
|
+
const environment_1 = require("../environment");
|
13
|
+
const fieldAnalyser_1 = require("../fieldAnalyser/fieldAnalyser");
|
14
|
+
const fieldErrors_1 = require("../errors/fieldErrors");
|
15
|
+
const fieldUtils_1 = require("../fieldAnalyser/fieldUtils");
|
16
|
+
const securityIdenty_1 = require("./securityIdenty");
|
17
|
+
const batchUpdateDocumentsFromFile_1 = require("./batchUpdateDocumentsFromFile");
|
20
18
|
/**
|
21
19
|
* Manage a push source.
|
22
20
|
*
|
23
21
|
* Allows you to create a new push source, manage security identities and documents in a Coveo organization.
|
24
22
|
*/
|
25
|
-
class
|
23
|
+
class PushSource {
|
26
24
|
/**
|
27
25
|
*
|
28
26
|
* @param apikey An apiKey capable of pushing documents and managing sources in a Coveo organization. See [Manage API Keys](https://docs.coveo.com/en/1718).
|
@@ -31,7 +29,7 @@ class Source {
|
|
31
29
|
constructor(apikey, organizationid, options) {
|
32
30
|
this.apikey = apikey;
|
33
31
|
this.organizationid = organizationid;
|
34
|
-
this.options = Object.assign(Object.assign({},
|
32
|
+
this.options = Object.assign(Object.assign({}, PushSource.defaultOptions), options);
|
35
33
|
this.platformClient = new platform_client_1.PlatformClient({
|
36
34
|
accessToken: this.apikey,
|
37
35
|
environment: (0, environment_1.castEnvironmentToPlatformClient)(this.options.environment),
|
@@ -54,46 +52,47 @@ class Source {
|
|
54
52
|
});
|
55
53
|
}
|
56
54
|
/**
|
57
|
-
*
|
58
|
-
*
|
59
|
-
* @
|
60
|
-
* @returns
|
55
|
+
* @deprecated use `identity.createSecurityIdentity`
|
56
|
+
*
|
57
|
+
* See {@link Source.identity}
|
61
58
|
*/
|
62
59
|
createSecurityIdentity(securityProviderId, securityIdentity) {
|
63
|
-
return this.
|
60
|
+
return this.identity.createSecurityIdentity(securityProviderId, securityIdentity);
|
64
61
|
}
|
65
62
|
/**
|
66
|
-
*
|
67
|
-
*
|
68
|
-
* @
|
69
|
-
* @returns
|
63
|
+
* @deprecated use `identity.createOrUpdateSecurityIdentityAlias`
|
64
|
+
*
|
65
|
+
* See {@link Source.identity}
|
70
66
|
*/
|
71
67
|
createOrUpdateSecurityIdentityAlias(securityProviderId, securityIdentityAlias) {
|
72
|
-
return this.
|
68
|
+
return this.identity.createOrUpdateSecurityIdentityAlias(securityProviderId, securityIdentityAlias);
|
73
69
|
}
|
74
70
|
/**
|
75
|
-
*
|
76
|
-
*
|
77
|
-
* @
|
78
|
-
* @returns
|
71
|
+
* @deprecated use `identity.deleteSecurityIdentity`
|
72
|
+
*
|
73
|
+
* See {@link Source.identity}
|
79
74
|
*/
|
80
75
|
deleteSecurityIdentity(securityProviderId, securityIdentityToDelete) {
|
81
|
-
return this.
|
76
|
+
return this.identity.deleteSecurityIdentity(securityProviderId, securityIdentityToDelete);
|
82
77
|
}
|
83
78
|
/**
|
84
|
-
*
|
85
|
-
*
|
86
|
-
* @
|
87
|
-
* @returns
|
79
|
+
* @deprecated use `identity.deleteOldSecurityIdentities`
|
80
|
+
*
|
81
|
+
* See {@link Source.identity}
|
88
82
|
*/
|
89
83
|
deleteOldSecurityIdentities(securityProviderId, batchDelete) {
|
90
|
-
return this.
|
84
|
+
return this.identity.deleteOldSecurityIdentities(securityProviderId, batchDelete);
|
91
85
|
}
|
92
86
|
/**
|
93
|
-
*
|
87
|
+
* @deprecated use `identity.manageSecurityIdentities`
|
88
|
+
*
|
89
|
+
* See {@link Source.identity}
|
94
90
|
*/
|
95
91
|
manageSecurityIdentities(securityProviderId, batchConfig) {
|
96
|
-
return this.
|
92
|
+
return this.identity.manageSecurityIdentities(securityProviderId, batchConfig);
|
93
|
+
}
|
94
|
+
get identity() {
|
95
|
+
return new securityIdenty_1.SecurityIdentity(this.platformClient);
|
97
96
|
}
|
98
97
|
/**
|
99
98
|
* Adds or updates an individual item in a push source. See [Adding a Single Item in a Push Source](https://docs.coveo.com/en/133).
|
@@ -137,46 +136,8 @@ class Source {
|
|
137
136
|
* @param {UploadBatchCallback} callback Callback executed when a batch of documents is either successfully uploaded or when an error occurs during the upload
|
138
137
|
* @param {BatchUpdateDocumentsFromFiles} options
|
139
138
|
*/
|
140
|
-
|
141
|
-
|
142
|
-
maxConcurrent: 10,
|
143
|
-
createFields: true,
|
144
|
-
};
|
145
|
-
const { maxConcurrent, createFields: createFields } = Object.assign(Object.assign({}, defaultOptions), options);
|
146
|
-
const files = (0, file_1.getAllJsonFilesFromEntries)(filesOrDirectories);
|
147
|
-
if (createFields) {
|
148
|
-
const analyser = new fieldAnalyser_1.FieldAnalyser(this.platformClient);
|
149
|
-
for (const filePath of files.values()) {
|
150
|
-
const docBuilders = (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath);
|
151
|
-
await analyser.add(docBuilders);
|
152
|
-
}
|
153
|
-
await this.createFields(analyser);
|
154
|
-
}
|
155
|
-
const fileNames = files.map((path) => (0, path_1.basename)(path));
|
156
|
-
const { chunksToUpload, close } = this.splitByChunkAndUpload(sourceID, fileNames, callback);
|
157
|
-
if (createFields) {
|
158
|
-
const analyser = new fieldAnalyser_1.FieldAnalyser(this.platformClient);
|
159
|
-
for (const filePath of files.values()) {
|
160
|
-
const docBuilders = (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath);
|
161
|
-
await analyser.add(docBuilders);
|
162
|
-
}
|
163
|
-
await this.createFields(analyser);
|
164
|
-
}
|
165
|
-
// parallelize uploads within the same file
|
166
|
-
const docBuilderGenerator = function* (docBuilders) {
|
167
|
-
for (const upload of chunksToUpload(docBuilders)) {
|
168
|
-
yield upload();
|
169
|
-
}
|
170
|
-
};
|
171
|
-
// parallelize uploads across multiple files
|
172
|
-
const fileGenerator = function* () {
|
173
|
-
for (const filePath of files.values()) {
|
174
|
-
const docBuilders = (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath);
|
175
|
-
yield* docBuilderGenerator(docBuilders);
|
176
|
-
}
|
177
|
-
};
|
178
|
-
await (0, generator_1.consumeGenerator)(fileGenerator.bind(this), maxConcurrent);
|
179
|
-
await close();
|
139
|
+
batchUpdateDocumentsFromFiles(sourceID, filesOrDirectories, options) {
|
140
|
+
return new batchUpdateDocumentsFromFile_1.BatchUpdateDocumentsFromFilesReturn(this, sourceID, filesOrDirectories, options);
|
180
141
|
}
|
181
142
|
/**
|
182
143
|
* Deletes a specific item from a Push source. Optionally, the child items of that item can also be deleted. See [Deleting an Item in a Push Source](https://docs.coveo.com/en/171).
|
@@ -214,6 +175,11 @@ class Source {
|
|
214
175
|
urlStatus.searchParams.append('statusType', status);
|
215
176
|
return axios_1.default.post(urlStatus.toString(), {}, this.documentsAxiosConfig);
|
216
177
|
}
|
178
|
+
async uploadBatch(sourceID, batch) {
|
179
|
+
const fileContainer = await this.createFileContainer();
|
180
|
+
await this.uploadContentToFileContainer(fileContainer, batch);
|
181
|
+
return this.pushFileContainerContent(sourceID, fileContainer);
|
182
|
+
}
|
217
183
|
get baseAPIURL() {
|
218
184
|
return `${(0, environment_1.platformUrl)(this.options)}/${this.organizationid}`;
|
219
185
|
}
|
@@ -230,7 +196,7 @@ class Source {
|
|
230
196
|
if (inconsistencies.size > 0) {
|
231
197
|
throw new fieldErrors_1.FieldTypeInconsistencyError(inconsistencies);
|
232
198
|
}
|
233
|
-
await (0,
|
199
|
+
await (0, fieldUtils_1.createFields)(this.platformClient, fields);
|
234
200
|
}
|
235
201
|
getFileContainerAxiosConfig(fileContainer) {
|
236
202
|
return {
|
@@ -262,61 +228,10 @@ class Source {
|
|
262
228
|
pushURL.searchParams.append('fileId', fileContainer.fileId);
|
263
229
|
return axios_1.default.put(pushURL.toString(), {}, this.documentsAxiosConfig);
|
264
230
|
}
|
265
|
-
splitByChunkAndUpload(sourceID, fileNames, callback, accumulator = this.accumulator) {
|
266
|
-
const chunksToUpload = (documentBuilders) => {
|
267
|
-
const batchesToUpload = [];
|
268
|
-
for (const docBuilder of documentBuilders) {
|
269
|
-
const sizeOfDoc = Buffer.byteLength(JSON.stringify(docBuilder.marshal()));
|
270
|
-
if (accumulator.size + sizeOfDoc >= Source.maxContentLength) {
|
271
|
-
const chunks = accumulator.chunks;
|
272
|
-
if (chunks.length > 0) {
|
273
|
-
batchesToUpload.push(() => this.uploadBatch(sourceID, chunks, fileNames, callback));
|
274
|
-
}
|
275
|
-
accumulator.chunks = [docBuilder];
|
276
|
-
accumulator.size = sizeOfDoc;
|
277
|
-
}
|
278
|
-
else {
|
279
|
-
accumulator.size += sizeOfDoc;
|
280
|
-
accumulator.chunks.push(docBuilder);
|
281
|
-
}
|
282
|
-
}
|
283
|
-
return batchesToUpload;
|
284
|
-
};
|
285
|
-
const close = async () => {
|
286
|
-
await this.uploadBatch(sourceID, accumulator.chunks, fileNames, callback);
|
287
|
-
};
|
288
|
-
return { chunksToUpload, close };
|
289
|
-
}
|
290
|
-
async uploadBatch(sourceID, batch, fileNames, callback) {
|
291
|
-
try {
|
292
|
-
const res = await this.batchUpdateDocuments(sourceID, {
|
293
|
-
addOrUpdate: batch,
|
294
|
-
delete: [],
|
295
|
-
}, { createFields: false });
|
296
|
-
callback(null, {
|
297
|
-
files: fileNames,
|
298
|
-
batch,
|
299
|
-
res,
|
300
|
-
});
|
301
|
-
}
|
302
|
-
catch (e) {
|
303
|
-
callback(e, {
|
304
|
-
files: fileNames,
|
305
|
-
batch,
|
306
|
-
});
|
307
|
-
}
|
308
|
-
}
|
309
|
-
get accumulator() {
|
310
|
-
return {
|
311
|
-
size: 0,
|
312
|
-
chunks: [],
|
313
|
-
};
|
314
|
-
}
|
315
231
|
}
|
316
|
-
exports.
|
317
|
-
|
232
|
+
exports.PushSource = PushSource;
|
233
|
+
PushSource.defaultOptions = {
|
318
234
|
region: environment_1.DEFAULT_REGION,
|
319
235
|
environment: environment_1.DEFAULT_ENVIRONMENT,
|
320
236
|
};
|
321
|
-
|
322
|
-
//# sourceMappingURL=source.js.map
|
237
|
+
//# sourceMappingURL=push.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"push.js","sourceRoot":"","sources":["../../src/source/push.ts"],"names":[],"mappings":";;;AAAA,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAC5B,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAEpC,8DASkC;AAClC,4DAA0D;AAAlD,mHAAA,gBAAgB,OAAA;AACxB,iCAAgD;AAEhD,+BAAgC;AAChC,6BAAwB;AACxB,gDAMwB;AACxB,kEAA6D;AAC7D,uDAAkE;AAClE,4DAAyD;AACzD,qDAAkD;AAMlD,iFAAmF;AAUnF;;;;GAIG;AACH,MAAa,UAAU;IAOrB;;;;OAIG;IACH,YACU,MAAc,EACd,cAAsB,EAC9B,OAA4B;QAFpB,WAAM,GAAN,MAAM,CAAQ;QACd,mBAAc,GAAd,cAAc,CAAQ;QAG9B,IAAI,CAAC,OAAO,mCAAO,UAAU,CAAC,cAAc,GAAK,OAAO,CAAC,CAAC;QAC1D,IAAI,CAAC,cAAc,GAAG,IAAI,gCAAc,CAAC;YACvC,WAAW,EAAE,IAAI,CAAC,MAAM;YACxB,WAAW,EAAE,IAAA,6CAA+B,EAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YACtE,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,IAAY,EAAE,gBAAkC;QAC5D,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC;YACvC,UAAU,EAAE,4BAAU,CAAC,IAAI;YAC3B,WAAW,EAAE,IAAI;YACjB,IAAI;YACJ,gBAAgB;SACjB,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACI,sBAAsB,CAC3B,kBAA0B,EAC1B,gBAAuC;QAEvC,OAAO,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CACzC,kBAAkB,EAClB,gBAAgB,CACjB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACI,mCAAmC,CACxC,kBAA0B,EAC1B,qBAAiD;QAEjD,OAAO,IAAI,CAAC,QAAQ,CAAC,mCAAmC,CACtD,kBAAkB,EAClB,qBAAqB,CACtB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACI,sBAAsB,CAC3B,kBAA0B,EAC1B,wBAAgD;QAEhD,OAAO,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CACzC,kBAAkB,EAClB,wBAAwB,CACzB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACI,2BAA2B,CAChC,kBAA0B,EAC1B,WAA0C;QAE1C,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAC9C,kBAAkB,EAClB,WAAW,CACZ,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACI,wBAAwB,CAC7B,kBAA0B,EAC1B,WAAwC;QAExC,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAC3C,kBAAkB,EAClB,WAAW,CACZ,CAAC;IACJ,CAAC;IAED,IAAW,QAAQ;QACjB,OAAO,IAAI,iCAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACnD,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,mBAAmB,CAC9B,QAAgB,EAChB,UAA2B,EAC3B,EAAC,YAAY,EAAE,YAAY,GAAG,IAAI,KAAiC,EAAE;QAErE,IAAI,YAAY,EAAE;YAChB,MAAM,QAAQ,GAAG,IAAI,6BAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;YACjC,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;SACnC;QAED,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAI,SAAG,CAAC,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjE,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAClD,OAAO,eAAK,CAAC,GAAG,CACd,MAAM,CAAC,QAAQ,EAAE,EACjB,UAAU,CAAC,OAAO,EAAE,EACpB,IAAI,CAAC,oBAAoB,CAC1B,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,oBAAoB,CAC/B,QAAgB,EAChB,KAA2B,EAC3B,EAAC,YAAY,EAAE,YAAY,GAAG,IAAI,KAAiC,EAAE;QAErE,IAAI,YAAY,EAAE;YAChB,MAAM,QAAQ,GAAG,IAAI,6BAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACtC,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;SACnC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACvD,MAAM,IAAI,CAAC,4BAA4B,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,wBAAwB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAChE,CAAC;IAED;;;;;;;OAOG;IACI,6BAA6B,CAClC,QAAgB,EAChB,kBAA4B,EAC5B,OAAuC;QAEvC,OAAO,IAAI,kEAAmC,CAC5C,IAAI,EACJ,QAAQ,EACR,kBAAkB,EAClB,OAAO,CACR,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACI,cAAc,CACnB,QAAgB,EAChB,UAAkB,EAClB,cAAc,GAAG,KAAK;QAEtB,MAAM,SAAS,GAAG,IAAI,SAAG,CAAC,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpE,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QACxD,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,gBAAgB,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;QACrE,OAAO,eAAK,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;;;OAKG;IACI,wBAAwB,CAC7B,QAAgB,EAChB,SAAiC;QAEjC,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,SAAG,CACvB,GAAG,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,YAAY,CACxD,CAAC;QACF,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACjE,OAAO,eAAK,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;;;OAKG;IACI,eAAe,CAAC,QAAgB,EAAE,MAAoB;QAC3D,MAAM,SAAS,GAAG,IAAI,SAAG,CAAC,GAAG,IAAI,CAAC,UAAU,YAAY,QAAQ,SAAS,CAAC,CAAC;QAC3E,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACpD,OAAO,eAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACzE,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,KAA2B;QACpE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACvD,MAAM,IAAI,CAAC,4BAA4B,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,wBAAwB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAChE,CAAC;IAED,IAAY,UAAU;QACpB,OAAO,GAAG,IAAA,yBAAW,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;IAC/D,CAAC;IAEO,yBAAyB,CAAC,QAAgB;QAChD,OAAO,GAAG,IAAI,CAAC,UAAU,YAAY,QAAQ,YAAY,CAAC;IAC5D,CAAC;IAED,IAAY,oBAAoB;QAC9B,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,uBAAuB;SACtC,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,QAAuB;QAC/C,MAAM,EAAC,MAAM,EAAE,eAAe,EAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;QAEpD,IAAI,eAAe,CAAC,IAAI,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,yCAA2B,CAAC,eAAe,CAAC,CAAC;SACxD;QACD,MAAM,IAAA,yBAAY,EAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;IAEO,2BAA2B,CACjC,aAAoC;QAEpC,OAAO;YACL,OAAO,EAAE,aAAa,CAAC,eAAe;SACvC,CAAC;IACJ,CAAC;IAED,IAAY,uBAAuB;QACjC,uCACK,IAAI,CAAC,mBAAmB,KAC3B,cAAc,EAAE,kBAAkB,EAClC,MAAM,EAAE,kBAAkB,IAC1B;IACJ,CAAC;IAED,IAAY,mBAAmB;QAC7B,OAAO;YACL,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;SACvC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,gBAAgB,GAAG,IAAI,SAAG,CAAC,GAAG,IAAI,CAAC,UAAU,QAAQ,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,MAAM,eAAK,CAAC,IAAI,CAC1B,gBAAgB,CAAC,QAAQ,EAAE,EAC3B,EAAE,EACF,IAAI,CAAC,oBAAoB,CAC1B,CAAC;QACF,OAAO,GAAG,CAAC,IAA6B,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,4BAA4B,CACxC,aAAoC,EACpC,KAA2B;QAE3B,MAAM,SAAS,GAAG,IAAI,SAAG,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACnD,OAAO,MAAM,eAAK,CAAC,GAAG,CACpB,SAAS,CAAC,QAAQ,EAAE,EACpB;YACE,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAChD,UAAU,CAAC,OAAO,EAAE,CACrB;YACD,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,EACD,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAChD,CAAC;IACJ,CAAC;IAEO,wBAAwB,CAC9B,QAAgB,EAChB,aAAoC;QAEpC,MAAM,OAAO,GAAG,IAAI,SAAG,CACrB,GAAG,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,QAAQ,CACpD,CAAC;QACF,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QAC5D,OAAO,eAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACtE,CAAC;;AAtUH,gCAuUC;AApUgB,yBAAc,GAAiC;IAC5D,MAAM,EAAE,4BAAc;IACtB,WAAW,EAAE,iCAAmB;CACjC,CAAC"}
|