@coveo/push-api-client 2.0.0 → 2.1.0

package/dist/source/batchUploadDocumentsFromFile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchUploadDocumentsFromFile.js","sourceRoot":"","sources":["../../src/source/batchUploadDocumentsFromFile.ts"],"names":[],"mappings":";;;AACA,uDAAkE;AAClE,uCAAwD;AACxD,uDAAkD;AAClD,oCAMkB;AAClB,4DAAmE;AAEnE,uDAAmF;AAEnF,MAAa,mCAAmC;IAI9C,YACE,cAA8B,EAC9B,QAAwB,EACxB,kBAA4B,EAC5B,OAAuC;QAEvC,MAAM,cAAc,GAAG;YACrB,aAAa,EAAE,EAAE;YACjB,YAAY,EAAE,IAAI;SACnB,CAAC;QACF,MAAM,EAAC,aAAa,EAAE,YAAY,EAAC,mCAC9B,cAAc,GACd,OAAO,CACX,CAAC;QAEF,IAAI,CAAC,QAAQ,GAAG,IAAI,2BAAY,CAC9B,CAAC,KAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EACvD,EAAC,aAAa,EAAC,CAChB,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,CAAC,KAAK,IAAI,EAAE;;YACjC,MAAM,KAAK,GAAG,IAAA,iCAA0B,EAAC,kBAAkB,CAAC,CAAC;YAC7D,IAAI,YAAY,EAAE;gBAChB,MAAM,QAAQ,GAAG,IAAI,qBAAa,CAAC,cAAc,CAAC,CAAC;gBACnD,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;oBACrC,MAAM,WAAW,GACf,IAAA,sDAA0C,EAAC,QAAQ,CAAC,CAAC;oBACvD,MAAM,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;iBACjC;gBACD,MAAM,EAAC,MAAM,EAAE,eAAe,EAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAEpD,IAAI,eAAe,CAAC,IAAI,GAAG,CAAC,EAAE;oBAC5B,MAAM,IAAI,yCAA2B,CAAC,eAAe,CAAC,CAAC;iBACxD;gBACD,MAAM,IAAA,yBAAM,EAAC,cAAc,EAAE,MAAM,CAAC,CAAC;aACtC;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAA,MAAA,QAAQ,CAAC,UAAU,wDAAI,CAAA,CAAC;QAChC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEM,aAAa,CAAC,EAA4B;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,YAAY,CAAC,EAAwB;QAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK;QACV,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;IAChC,CAAC;CACF;AA1DD,kFA0DC"}

package/dist/source/catalog.js

@@ -0,0 +1,115 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CatalogSource = void 0;
+require('isomorphic-fetch');
+require('abortcontroller-polyfill');
+const platform_client_1 = require("@coveord/platform-client");
+const environment_1 = require("../environment");
+const securityIdenty_1 = require("./securityIdenty");
+const uploadStrategy_1 = require("../uploadStrategy");
+const axiosUtils_1 = require("../help/axiosUtils");
+const documentUploader_1 = require("./documentUploader");
+const urlUtils_1 = require("../help/urlUtils");
+const batchUploadDocumentsFromFile_1 = require("./batchUploadDocumentsFromFile");
+/**
+ * Manage a catalog source.
+ *
+ * Allows you to create a new catalog source, manage security identities and documents in a Coveo organization.
+ */
+class CatalogSource {
+    /**
+     * Creates an instance of CatalogSource.
+     * @param {string} apikey An apiKey capable of pushing documents and managing sources in a Coveo organization. See [Manage API Keys](https://docs.coveo.com/en/1718).
+     * @param {string} organizationid The Coveo Organization identifier.
+     * @param {PlatformUrlOptions} [opts=CatalogSource.defaultOptions]
+     */
+    constructor(apikey, organizationid, opts = CatalogSource.defaultOptions) {
+        this.apikey = apikey;
+        this.organizationid = organizationid;
+        this.options = Object.assign(Object.assign({}, CatalogSource.defaultOptions), opts);
+        this.platformClient = new platform_client_1.PlatformClient({
+            accessToken: apikey,
+            environment: (0, environment_1.castEnvironmentToPlatformClient)(this.options.environment),
+            organizationId: organizationid,
+            region: this.options.region,
+        });
+    }
+    /**
+     * Create a new catalog source
+     * @param name The name of the source to create.
+     * @param sourceVisibility The security option that should be applied to the content of the source. See [Content Security](https://docs.coveo.com/en/1779).
+     * @returns
+     */
+    create(name, sourceVisibility) {
+        return this.platformClient.source.create({
+            name,
+            streamEnabled: true,
+            sourceVisibility,
+            sourceType: platform_client_1.SourceType.CATALOG,
+        });
+    }
+    get identity() {
+        return new securityIdenty_1.SecurityIdentity(this.platformClient);
+    }
+    /**
+     * Manage batches of items in a catalog source.
+     * See [Full Document Update](https://docs.coveo.com/en/l62e0540)
+     * @param sourceId
+     * @param batch
+     * @param {BatchUpdateDocumentsOptions} [{createFields: createFields = true}={}]
+     * @returns
+     */
+    batchUpdateDocuments(sourceId, batch, { createFields: createFields = true } = {}) {
+        const strategy = this.fileContainerStrategy(sourceId);
+        return (0, documentUploader_1.uploadBatch)(this.platformClient, strategy, batch, createFields);
+    }
+    /**
+     * Send your catalog data to your catalog source.
+     * See [How to Stream Your Catalog Data to Your Source](https://docs.coveo.com/en/lb4a0344)
+     * @param {string} sourceId
+     * @param {BatchUpdateDocuments} batch
+     * @param {BatchUpdateDocumentsOptions} [{createFields: createFields = true}={}]
+     */
+    async batchStreamDocuments(sourceId, batch, { createFields: createFields = true } = {}) {
+        const strategy = this.streamChunkStrategy(sourceId);
+        await (0, documentUploader_1.uploadBatch)(this.platformClient, strategy, batch, createFields);
+    }
+    /**
+     * Manage batches of items in a catalog source from a list of JSON files. See [Full Document Update Source](https://docs.coveo.com/en/l62e0540)
+     * @param {string} sourceId The unique identifier of the target Push source
+     * @param {string[]} filesOrDirectories A list of JSON files or directories (containing JSON files) from which to extract documents.
+     * @param {BatchUpdateDocumentsFromFiles} options
+     */
+    batchUpdateDocumentsFromFiles(sourceId, filesOrDirectories, options) {
+        return new batchUploadDocumentsFromFile_1.BatchUploadDocumentsFromFilesReturn(this.platformClient, this.fileContainerStrategy(sourceId), filesOrDirectories, options);
+    }
+    /**
+     * Send your catalog data to your catalog source from a list of JSON files.
+     * See [How to Stream Your Catalog Data to Your Source](https://docs.coveo.com/en/lb4a0344)
+     * @param {string} sourceId The unique identifier of the target Push source
+     * @param {string[]} filesOrDirectories A list of JSON files or directories (containing JSON files) from which to extract documents.
+     * @param {BatchUpdateDocumentsFromFiles} [options]
+     */
+    batchStreamDocumentsFromFiles(sourceId, filesOrDirectories, options) {
+        return new batchUploadDocumentsFromFile_1.BatchUploadDocumentsFromFilesReturn(this.platformClient, this.streamChunkStrategy(sourceId), filesOrDirectories, options);
+    }
+    urlBuilder(sourceId) {
+        return new urlUtils_1.StreamUrlBuilder(sourceId, this.organizationid, this.options);
+    }
+    fileContainerStrategy(sourceId) {
+        const urlBuilder = this.urlBuilder(sourceId);
+        const documentsAxiosConfig = (0, axiosUtils_1.axiosRequestHeaders)(this.apikey);
+        return new uploadStrategy_1.FileContainerStrategy(urlBuilder, documentsAxiosConfig);
+    }
+    streamChunkStrategy(sourceId) {
+        const urlBuilder = this.urlBuilder(sourceId);
+        const documentsAxiosConfig = (0, axiosUtils_1.axiosRequestHeaders)(this.apikey);
+        return new uploadStrategy_1.StreamChunkStrategy(urlBuilder, documentsAxiosConfig);
+    }
+}
+exports.CatalogSource = CatalogSource;
+CatalogSource.defaultOptions = {
+    region: environment_1.DEFAULT_REGION,
+    environment: environment_1.DEFAULT_ENVIRONMENT,
+};
+//# sourceMappingURL=catalog.js.map

package/dist/source/catalog.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"catalog.js","sourceRoot":"","sources":["../../src/source/catalog.ts"],"names":[],"mappings":";;;AAAA,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAC5B,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAEpC,8DAIkC;AAClC,gDAKwB;AACxB,qDAAkD;AAClD,sDAA6E;AAM7E,mDAAuD;AACvD,yDAA+C;AAC/C,+CAAkD;AAClD,iFAAmF;AAEnF;;;;GAIG;AACH,MAAa,aAAa;IAQxB;;;;;OAKG;IACH,YACU,MAAc,EACd,cAAsB,EAC9B,OAA2B,aAAa,CAAC,cAAc;QAF/C,WAAM,GAAN,MAAM,CAAQ;QACd,mBAAc,GAAd,cAAc,CAAQ;QAG9B,IAAI,CAAC,OAAO,mCAAO,aAAa,CAAC,cAAc,GAAK,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,cAAc,GAAG,IAAI,gCAAc,CAAC;YACvC,WAAW,EAAE,MAAM;YACnB,WAAW,EAAE,IAAA,6CAA+B,EAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YACtE,cAAc,EAAE,cAAc;YAC9B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,IAAY,EAAE,gBAAkC;QAC5D,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC;YACvC,IAAI;YACJ,aAAa,EAAE,IAAI;YACnB,gBAAgB;YAChB,UAAU,EAAE,4BAAU,CAAC,OAAO;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,IAAW,QAAQ;QACjB,OAAO,IAAI,iCAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACnD,CAAC;IAED;;;;;;;OAOG;IACI,oBAAoB,CACzB,QAAgB,EAChB,KAA2B,EAC3B,EAAC,YAAY,EAAE,YAAY,GAAG,IAAI,KAAiC,EAAE;QAErE,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACtD,OAAO,IAAA,8BAAW,EAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IACzE,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,oBAAoB,CAC/B,QAAgB,EAChB,KAA2B,EAC3B,EAAC,YAAY,EAAE,YAAY,GAAG,IAAI,KAAiC,EAAE;QAErE,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,IAAA,8BAAW,EAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IACxE,CAAC;IAED;;;;;OAKG;IACI,6BAA6B,CAClC,QAAgB,EAChB,kBAA4B,EAC5B,OAAuC;QAEvC,OAAO,IAAI,kEAAmC,CAC5C,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EACpC,kBAAkB,EAClB,OAAO,CACR,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACI,6BAA6B,CAClC,QAAgB,EAChB,kBAA4B,EAC5B,OAAuC;QAEvC,OAAO,IAAI,kEAAmC,CAC5C,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAClC,kBAAkB,EAClB,OAAO,CACR,CAAC;IACJ,CAAC;IAEO,UAAU,CAAC,QAAgB;QACjC,OAAO,IAAI,2BAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3E,CAAC;IAEO,qBAAqB,CAAC,QAAgB;QAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,oBAAoB,GAAG,IAAA,gCAAmB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,OAAO,IAAI,sCAAqB,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC;IACrE,CAAC;IAEO,mBAAmB,CAAC,QAAgB;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,oBAAoB,GAAG,IAAA,gCAAmB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,OAAO,IAAI,oCAAmB,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC;IACnE,CAAC;;AArIH,sCAsIC;AAnIgB,4BAAc,GAAiC;IAC5D,MAAM,EAAE,4BAAc;IACtB,WAAW,EAAE,iCAAmB;CACjC,CAAC"}

package/dist/source/catalog.spec.js

@@ -0,0 +1,239 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable node/no-unpublished-import */
+jest.mock('@coveord/platform-client');
+jest.mock('axios');
+jest.mock('../fieldAnalyser/fieldAnalyser');
+const platform_client_1 = require("@coveord/platform-client");
+const documentBuilder_1 = require("../documentBuilder");
+const axios_1 = require("axios");
+const path_1 = require("path");
+const process_1 = require("process");
+const inconsistencies_1 = require("../fieldAnalyser/inconsistencies");
+const fieldErrors_1 = require("../errors/fieldErrors");
+const catalog_1 = require("./catalog");
+const __1 = require("..");
+const mockAxios = axios_1.default;
+const mockedFieldAnalyser = jest.mocked(__1.FieldAnalyser);
+const mockedPlatformClient = jest.mocked(platform_client_1.default);
+const mockCreateSource = jest.fn();
+const mockEvaluate = jest.fn();
+const mockCreateField = jest.fn();
+const mockAnalyserAdd = jest.fn();
+const mockAnalyserReport = jest.fn();
+const mockedSuccessCallback = jest.fn();
+const mockedErrorCallback = jest.fn();
+const pathToStub = (0, path_1.join)((0, process_1.cwd)(), 'src', '__stub__');
+const expectedDocumentsHeaders = {
+    headers: {
+        Accept: 'application/json',
+        Authorization: 'Bearer the_key',
+        'Content-Type': 'application/json',
+    },
+};
+const doAxiosMockFileContainerResponse = () => ({
+    data: {
+        uploadUri: 'https://fake.upload.url',
+        fileId: 'file_id',
+        requiredHeaders: { foo: 'bar' },
+    },
+});
+const doAxiosMockPost = () => {
+    mockAxios.post.mockImplementationOnce((url) => {
+        if (url.match(/files/)) {
+            return Promise.resolve(doAxiosMockFileContainerResponse());
+        }
+        return Promise.resolve();
+    });
+};
+const doMockPlatformClient = () => {
+    mockedPlatformClient.mockImplementation(() => ({
+        privilegeEvaluator: { evaluate: mockEvaluate },
+        source: {
+            create: mockCreateSource,
+        },
+        field: {
+            createFields: mockCreateField,
+        },
+    }));
+};
+const doMockFieldAnalyser = () => {
+    mockedFieldAnalyser.mockImplementation(() => ({
+        add: mockAnalyserAdd,
+        report: mockAnalyserReport,
+    }));
+};
+describe('CatalogSource - Push', () => {
+    let source;
+    let batch;
+    beforeAll(() => {
+        doMockPlatformClient();
+        doMockFieldAnalyser();
+    });
+    beforeEach(() => {
+        mockEvaluate.mockResolvedValue({ approved: true });
+        source = new catalog_1.CatalogSource('the_key', 'the_org');
+        batch = {
+            addOrUpdate: [
+                new documentBuilder_1.DocumentBuilder('the_uri_1', 'the_title_1'),
+                new documentBuilder_1.DocumentBuilder('the_uri_2', 'the_title_2'),
+            ],
+            delete: [{ documentId: 'the_uri_3', deleteChildren: true }],
+        };
+    });
+    it('should call platform client on creation', () => {
+        source.create('the_name', platform_client_1.SourceVisibility.SHARED);
+        expect(mockCreateSource).toHaveBeenCalledWith({
+            name: 'the_name',
+            streamEnabled: true,
+            sourceType: 'CATALOG',
+            sourceVisibility: 'SHARED',
+        });
+    });
+    describe('when doing batch update', () => {
+        beforeEach(() => {
+            doAxiosMockPost();
+        });
+        it('should create a file container', async () => {
+            await source.batchUpdateDocuments('the_id', batch, {
+                createFields: false,
+            });
+            expect(mockAxios.post).toHaveBeenCalledWith('https://api.cloud.coveo.com/push/v1/organizations/the_org/files', expect.objectContaining({}), expectedDocumentsHeaders);
+        });
+        it('should upload files to container with returned upload uri and required headers ', async () => {
+            await source.batchUpdateDocuments('the_id', batch, {
+                createFields: false,
+            });
+            expect(mockAxios.put).toHaveBeenCalledWith('https://fake.upload.url/', expect.objectContaining({
+                addOrUpdate: expect.arrayContaining([
+                    expect.objectContaining({ documentId: 'the_uri_1' }),
+                    expect.objectContaining({ documentId: 'the_uri_2' }),
+                ]),
+                delete: expect.arrayContaining([
+                    expect.objectContaining({ documentId: 'the_uri_3' }),
+                ]),
+            }), { headers: { foo: 'bar' } });
+        });
+        it('should push content to source with returned fileId', async () => {
+            await source.batchUpdateDocuments('the_id', batch, {
+                createFields: false,
+            });
+            expect(mockAxios.put).toHaveBeenCalledWith('https://api.cloud.coveo.com/push/v1/organizations/the_org/sources/the_id/stream/update?fileId=file_id', {}, expectedDocumentsHeaders);
+        });
+    });
+    describe('when doing batch update from local files', () => {
+        afterAll(() => {
+            mockAxios.post.mockReset();
+        });
+        beforeEach(() => {
+            doAxiosMockPost();
+        });
+        it('should upload documents from local file', async () => {
+            await source
+                .batchUpdateDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                .batch();
+            expect(mockAxios.put).toHaveBeenCalledWith('https://fake.upload.url/', expect.objectContaining({
+                addOrUpdate: expect.arrayContaining([
+                    expect.objectContaining({
+                        documentId: 'https://www.themoviedb.org/movie/268',
+                    }),
+                    expect.objectContaining({
+                        documentId: 'https://www.themoviedb.org/movie/999',
+                    }),
+                ]),
+                delete: expect.arrayContaining([]),
+            }), {
+                headers: {
+                    foo: 'bar',
+                },
+            });
+        });
+        it('should throw an error if the path is invalid', () => {
+            expect(() => source
+                .batchUpdateDocumentsFromFiles('the_id', ['path/to/invalid/document'], { createFields: false })
+                .batch()).rejects.toThrow("no such file or directory, lstat 'path/to/invalid/document'");
+        });
+        it('should call the callback without error when uploading documents', async () => {
+            await source
+                .batchUpdateDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                .onBatchError(mockedErrorCallback)
+                .batch();
+            expect(mockedErrorCallback).not.toHaveBeenCalled();
+        });
+        it('should only push JSON files', async () => {
+            await source
+                .batchUpdateDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                .onBatchUpload(mockedSuccessCallback)
+                .batch();
+            expect(mockedSuccessCallback).toHaveBeenCalledWith(expect.objectContaining({ files: ['valid.json'] }));
+        });
+        it('should call the errorCallback on a failure from the API', async () => {
+            mockAxios.post.mockReset();
+            mockAxios.post.mockRejectedValue({ message: 'Error Message' });
+            await source
+                .batchUpdateDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                .onBatchError(mockedErrorCallback)
+                .batch();
+            expect(mockedErrorCallback).toHaveBeenCalledWith({
+                message: 'Error Message',
+            }, expect.anything());
+        });
+    });
+    describe('when enabling auto field creation', () => {
+        beforeEach(() => {
+            doAxiosMockPost();
+        });
+        describe('when there are no inconsistencies', () => {
+            beforeEach(() => {
+                const inconsistencies = new inconsistencies_1.Inconsistencies();
+                mockAnalyserReport.mockReturnValueOnce({ fields: [], inconsistencies });
+            });
+            it('should analyse document builder batch', async () => {
+                await source.batchUpdateDocuments('the_id', batch);
+                expect(mockAnalyserAdd).toHaveBeenCalledWith(batch.addOrUpdate);
+            });
+        });
+        describe('when document batches contain type inconsistencies', () => {
+            beforeEach(() => {
+                const inconsistencies = new inconsistencies_1.Inconsistencies().add('foo', [
+                    platform_client_1.FieldTypes.STRING,
+                    platform_client_1.FieldTypes.DOUBLE,
+                ]);
+                mockAnalyserReport.mockReturnValueOnce({ fields: [], inconsistencies });
+            });
+            it('should throw', () => {
+                expect(() => source.batchUpdateDocuments('the_id', batch)).rejects.toThrow(fieldErrors_1.FieldTypeInconsistencyError);
+            });
+        });
+        describe('when document batches contain missing fields', () => {
+            beforeEach(() => {
+                const inconsistencies = new inconsistencies_1.Inconsistencies();
+                mockAnalyserReport.mockReturnValueOnce({
+                    fields: [
+                        { name: 'stringfield', type: platform_client_1.FieldTypes.STRING },
+                        { name: 'numericalfield', type: platform_client_1.FieldTypes.DOUBLE },
+                    ],
+                    inconsistencies,
+                });
+            });
+            it('should create fields', async () => {
+                await source.batchUpdateDocuments('the_id', batch);
+                expect(mockCreateField).toHaveBeenCalledWith([
+                    { name: 'stringfield', type: platform_client_1.FieldTypes.STRING },
+                    { name: 'numericalfield', type: platform_client_1.FieldTypes.DOUBLE },
+                ]);
+            });
+        });
+        describe('when document batches do not contain missing fields', () => {
+            beforeEach(() => {
+                const inconsistencies = new inconsistencies_1.Inconsistencies();
+                mockAnalyserReport.mockReturnValueOnce({ fields: [], inconsistencies });
+            });
+            it('should not create fields', async () => {
+                source.batchUpdateDocuments('the_id', batch);
+                expect(mockCreateField).not.toHaveBeenCalled();
+            });
+        });
+    });
+});
+//# sourceMappingURL=catalog.spec.js.map

package/dist/source/catalog.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"catalog.spec.js","sourceRoot":"","sources":["../../src/source/catalog.spec.ts"],"names":[],"mappings":";;AAAA,+CAA+C;AAC/C,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;AACtC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACnB,IAAI,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;AAC5C,8DAGkC;AAClC,wDAAmD;AACnD,iCAA0B;AAC1B,+BAA0B;AAC1B,qCAA4B;AAC5B,sEAAiE;AACjE,uDAAkE;AAElE,uCAAwC;AACxC,0BAAiC;AACjC,MAAM,SAAS,GAAG,eAAkC,CAAC;AAErD,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAa,CAAC,CAAC;AACvD,MAAM,oBAAoB,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAc,CAAC,CAAC;AACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACnC,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAC/B,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAClC,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAClC,MAAM,kBAAkB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACrC,MAAM,qBAAqB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACxC,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACtC,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,IAAA,aAAG,GAAE,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;AAElD,MAAM,wBAAwB,GAAG;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,gBAAgB;QAC/B,cAAc,EAAE,kBAAkB;KACnC;CACF,CAAC;AAEF,MAAM,gCAAgC,GAAG,GAAG,EAAE,CAAC,CAAC;IAC9C,IAAI,EAAE;QACJ,SAAS,EAAE,yBAAyB;QACpC,MAAM,EAAE,SAAS;QACjB,eAAe,EAAE,EAAC,GAAG,EAAE,KAAK,EAAC;KAC9B;CACF,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,GAAG,EAAE;IAC3B,SAAS,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,GAAW,EAAE,EAAE;QACpD,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE;YACtB,OAAO,OAAO,CAAC,OAAO,CAAC,gCAAgC,EAAE,CAAC,CAAC;SAC5D;QACD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,GAAG,EAAE;IAChC,oBAAoB,CAAC,kBAAkB,CACrC,GAAG,EAAE,CACH,CAAC;QACC,kBAAkB,EAAE,EAAC,QAAQ,EAAE,YAAY,EAAC;QAC5C,MAAM,EAAE;YACN,MAAM,EAAE,gBAAgB;SACzB;QACD,KAAK,EAAE;YACL,YAAY,EAAE,eAAe;SAC9B;KAC4B,CAAA,CAClC,CAAC;AACJ,CAAC,CAAC;AACF,MAAM,mBAAmB,GAAG,GAAG,EAAE;IAC/B,mBAAmB,CAAC,kBAAkB,CACpC,GAAG,EAAE,CACH,CAAC;QACC,GAAG,EAAE,eAAe;QACpB,MAAM,EAAE,kBAAkB;KACE,CAAA,CACjC,CAAC;AACJ,CAAC,CAAC;AAEF,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,IAAI,MAAqB,CAAC;IAC1B,IAAI,KAA2B,CAAC;IAChC,SAAS,CAAC,GAAG,EAAE;QACb,oBAAoB,EAAE,CAAC;QACvB,mBAAmB,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,GAAG,EAAE;QACd,YAAY,CAAC,iBAAiB,CAAC,EAAC,QAAQ,EAAE,IAAI,EAAC,CAAC,CAAC;QACjD,MAAM,GAAG,IAAI,uBAAa,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACjD,KAAK,GAAG;YACN,WAAW,EAAE;gBACX,IAAI,iCAAe,CAAC,WAAW,EAAE,aAAa,CAAC;gBAC/C,IAAI,iCAAe,CAAC,WAAW,EAAE,aAAa,CAAC;aAChD;YACD,MAAM,EAAE,CAAC,EAAC,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,IAAI,EAAC,CAAC;SAC1D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,kCAAgB,CAAC,MAAM,CAAC,CAAC;QAEnD,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC;YAC5C,IAAI,EAAE,UAAU;YAChB,aAAa,EAAE,IAAI;YACnB,UAAU,EAAE,SAAS;YACrB,gBAAgB,EAAE,QAAQ;SAC3B,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,UAAU,CAAC,GAAG,EAAE;YACd,eAAe,EAAE,CAAC;QACpB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,CAAC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE;gBACjD,YAAY,EAAE,KAAK;aACpB,CAAC,CAAC;YACH,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,oBAAoB,CACzC,iEAAiE,EACjE,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAC3B,wBAAwB,CACzB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iFAAiF,EAAE,KAAK,IAAI,EAAE;YAC/F,MAAM,MAAM,CAAC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE;gBACjD,YAAY,EAAE,KAAK;aACpB,CAAC,CAAC;YACH,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,oBAAoB,CACxC,0BAA0B,EAC1B,MAAM,CAAC,gBAAgB,CAAC;gBACtB,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;oBAClC,MAAM,CAAC,gBAAgB,CAAC,EAAC,UAAU,EAAE,WAAW,EAAC,CAAC;oBAClD,MAAM,CAAC,gBAAgB,CAAC,EAAC,UAAU,EAAE,WAAW,EAAC,CAAC;iBACnD,CAAC;gBACF,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC;oBAC7B,MAAM,CAAC,gBAAgB,CAAC,EAAC,UAAU,EAAE,WAAW,EAAC,CAAC;iBACnD,CAAC;aACH,CAAC,EACF,EAAC,OAAO,EAAE,EAAC,GAAG,EAAE,KAAK,EAAC,EAAC,CACxB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,MAAM,CAAC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE;gBACjD,YAAY,EAAE,KAAK;aACpB,CAAC,CAAC;YACH,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,oBAAoB,CACxC,uGAAuG,EACvG,EAAE,EACF,wBAAwB,CACzB,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,0CAA0C,EAAE,GAAG,EAAE;QACxD,QAAQ,CAAC,GAAG,EAAE;YACZ,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,GAAG,EAAE;YACd,eAAe,EAAE,CAAC;QACpB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,MAAM;iBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,KAAK,EAAE,CAAC;YAEX,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,oBAAoB,CACxC,0BAA0B,EAC1B,MAAM,CAAC,gBAAgB,CAAC;gBACtB,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;oBAClC,MAAM,CAAC,gBAAgB,CAAC;wBACtB,UAAU,EAAE,sCAAsC;qBACnD,CAAC;oBACF,MAAM,CAAC,gBAAgB,CAAC;wBACtB,UAAU,EAAE,sCAAsC;qBACnD,CAAC;iBACH,CAAC;gBACF,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;aACnC,CAAC,EACF;gBACE,OAAO,EAAE;oBACP,GAAG,EAAE,KAAK;iBACX;aACF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,CAAC,GAAG,EAAE,CACV,MAAM;iBACH,6BAA6B,CAC5B,QAAQ,EACR,CAAC,0BAA0B,CAAC,EAC5B,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,KAAK,EAAE,CACX,CAAC,OAAO,CAAC,OAAO,CACf,6DAA6D,CAC9D,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YAC/E,MAAM,MAAM;iBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,YAAY,CAAC,mBAAmB,CAAC;iBACjC,KAAK,EAAE,CAAC;YACX,MAAM,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,MAAM;iBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,aAAa,CAAC,qBAAqB,CAAC;iBACpC,KAAK,EAAE,CAAC;YAEX,MAAM,CAAC,qBAAqB,CAAC,CAAC,oBAAoB,CAChD,MAAM,CAAC,gBAAgB,CAAC,EAAC,KAAK,EAAE,CAAC,YAAY,CAAC,EAAC,CAAC,CACjD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3B,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAC,OAAO,EAAE,eAAe,EAAC,CAAC,CAAC;YAC7D,MAAM,MAAM;iBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,YAAY,CAAC,mBAAmB,CAAC;iBACjC,KAAK,EAAE,CAAC;YAEX,MAAM,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CAC9C;gBACE,OAAO,EAAE,eAAe;aACzB,EACD,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;QACjD,UAAU,CAAC,GAAG,EAAE;YACd,eAAe,EAAE,CAAC;QACpB,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;YACjD,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,eAAe,GAAG,IAAI,iCAAe,EAAE,CAAC;gBAC9C,kBAAkB,CAAC,mBAAmB,CAAC,EAAC,MAAM,EAAE,EAAE,EAAE,eAAe,EAAC,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;gBACrD,MAAM,MAAM,CAAC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACnD,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAClE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,oDAAoD,EAAE,GAAG,EAAE;YAClE,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,eAAe,GAAG,IAAI,iCAAe,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE;oBACvD,4BAAU,CAAC,MAAM;oBACjB,4BAAU,CAAC,MAAM;iBAClB,CAAC,CAAC;gBACH,kBAAkB,CAAC,mBAAmB,CAAC,EAAC,MAAM,EAAE,EAAE,EAAE,eAAe,EAAC,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;YACH,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;gBACtB,MAAM,CAAC,GAAG,EAAE,CACV,MAAM,CAAC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAC7C,CAAC,OAAO,CAAC,OAAO,CAAC,yCAA2B,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,8CAA8C,EAAE,GAAG,EAAE;YAC5D,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,eAAe,GAAG,IAAI,iCAAe,EAAE,CAAC;gBAC9C,kBAAkB,CAAC,mBAAmB,CAAC;oBACrC,MAAM,EAAE;wBACN,EAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;wBAC9C,EAAC,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;qBAClD;oBACD,eAAe;iBAChB,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YACH,EAAE,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;gBACpC,MAAM,MAAM,CAAC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACnD,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC;oBAC3C,EAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;oBAC9C,EAAC,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,4BAAU,CAAC,MAAM,EAAC;iBAClD,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,qDAAqD,EAAE,GAAG,EAAE;YACnE,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,eAAe,GAAG,IAAI,iCAAe,EAAE,CAAC;gBAC9C,kBAAkB,CAAC,mBAAmB,CAAC,EAAC,MAAM,EAAE,EAAE,EAAE,eAAe,EAAC,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;gBACxC,MAAM,CAAC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAC7C,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACjD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/source/catalog.stream.spec.js

@@ -0,0 +1,189 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable node/no-unpublished-import */
+jest.mock('@coveord/platform-client');
+jest.mock('axios');
+jest.mock('../fieldAnalyser/fieldAnalyser');
+const platform_client_1 = require("@coveord/platform-client");
+const axios_1 = require("axios");
+const path_1 = require("path");
+const process_1 = require("process");
+const catalog_1 = require("./catalog");
+const __1 = require("..");
+const mockAxios = axios_1.default;
+const mockedFieldAnalyser = jest.mocked(__1.FieldAnalyser);
+const mockedPlatformClient = jest.mocked(platform_client_1.default);
+const mockCreateSource = jest.fn();
+const mockCreateField = jest.fn();
+const mockEvaluate = jest.fn();
+const mockAnalyserAdd = jest.fn();
+const mockAnalyserReport = jest.fn();
+const mockedSuccessCallback = jest.fn();
+const mockedErrorCallback = jest.fn();
+const pathToStub = (0, path_1.join)((0, process_1.cwd)(), 'src', '__stub__');
+const expectedDocumentsHeaders = {
+    headers: {
+        Accept: 'application/json',
+        Authorization: 'Bearer the_key',
+        'Content-Type': 'application/json',
+    },
+};
+const doAxiosMockFileContainerResponse = () => ({
+    data: {
+        uploadUri: 'https://fake.upload.url',
+        fileId: 'file_id',
+        requiredHeaders: { foo: 'bar' },
+    },
+});
+const doAxiosMockOpenStream = () => ({
+    data: {
+        streamId: 'the_stream_id',
+    },
+});
+const doMockPlatformClient = () => {
+    mockedPlatformClient.mockImplementation(() => ({
+        privilegeEvaluator: { evaluate: mockEvaluate },
+        source: {
+            create: mockCreateSource,
+        },
+        field: {
+            createFields: mockCreateField,
+        },
+    }));
+};
+const doMockFieldAnalyser = () => {
+    mockedFieldAnalyser.mockImplementation(() => ({
+        add: mockAnalyserAdd,
+        report: mockAnalyserReport,
+    }));
+};
+const mockAxiosForStreamCalls = () => {
+    mockAxios.post.mockImplementation((url) => {
+        if (url.match(/chunk/)) {
+            return Promise.resolve(doAxiosMockFileContainerResponse());
+        }
+        if (url.match(/stream\/open/)) {
+            return Promise.resolve(doAxiosMockOpenStream());
+        }
+        return Promise.resolve({ data: {} });
+    });
+};
+const basicStreamExpectations = () => {
+    it('should open a stream', async () => {
+        expect(mockAxios.post).toHaveReturnedTimes(3);
+    });
+    it('should open a stream', async () => {
+        expect(mockAxios.post).toHaveBeenNthCalledWith(1, 'https://api.cloud.coveo.com/push/v1/organizations/the_org/sources/the_id/stream/open', {}, expectedDocumentsHeaders);
+    });
+    it('should request a stream chunk', async () => {
+        expect(mockAxios.post).toHaveBeenNthCalledWith(2, 'https://api.cloud.coveo.com/push/v1/organizations/the_org/sources/the_id/stream/the_stream_id/chunk', {}, expectedDocumentsHeaders);
+    });
+    it('should close a stream', async () => {
+        expect(mockAxios.post).toHaveBeenNthCalledWith(3, 'https://api.cloud.coveo.com/push/v1/organizations/the_org/sources/the_id/stream/the_stream_id/close', {}, expectedDocumentsHeaders);
+    });
+};
+describe('CatalogSource - Stream', () => {
+    let source;
+    beforeAll(() => {
+        doMockPlatformClient();
+        doMockFieldAnalyser();
+    });
+    beforeEach(() => {
+        mockEvaluate.mockResolvedValue({ approved: true });
+        source = new catalog_1.CatalogSource('the_key', 'the_org');
+    });
+    describe('when streaming data from a batch', () => {
+        beforeEach(async () => {
+            mockAxiosForStreamCalls();
+            await source
+                .batchStreamDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                .batch();
+        });
+        afterAll(() => {
+            mockAxios.post.mockReset();
+        });
+        basicStreamExpectations();
+        it('should upload content to stream chunk', async () => {
+            expect(mockAxios.put).toHaveBeenCalledWith('https://fake.upload.url/', expect.objectContaining({
+                addOrUpdate: expect.arrayContaining([
+                    expect.objectContaining({
+                        documentId: 'https://www.themoviedb.org/movie/268',
+                    }),
+                    expect.objectContaining({
+                        documentId: 'https://www.themoviedb.org/movie/999',
+                    }),
+                ]),
+                delete: expect.arrayContaining([]),
+            }), {
+                headers: {
+                    foo: 'bar',
+                },
+            });
+        });
+    });
+    describe('when streaming data from local files', () => {
+        beforeEach(async () => {
+            mockAxiosForStreamCalls();
+        });
+        afterAll(() => {
+            mockAxios.post.mockReset();
+        });
+        describe('API call expectations', () => {
+            beforeEach(async () => {
+                mockAxiosForStreamCalls();
+                await source
+                    .batchStreamDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                    .batch();
+            });
+            basicStreamExpectations();
+            it('should upload documents from local file', async () => {
+                expect(mockAxios.put).toHaveBeenCalledWith('https://fake.upload.url/', expect.objectContaining({
+                    addOrUpdate: expect.arrayContaining([
+                        expect.objectContaining({
+                            documentId: 'https://www.themoviedb.org/movie/268',
+                        }),
+                        expect.objectContaining({
+                            documentId: 'https://www.themoviedb.org/movie/999',
+                        }),
+                    ]),
+                    delete: expect.arrayContaining([]),
+                }), {
+                    headers: {
+                        foo: 'bar',
+                    },
+                });
+            });
+        });
+        it('should call the callback without error when uploading documents', async () => {
+            await source
+                .batchStreamDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                .onBatchError(mockedErrorCallback)
+                .batch();
+            expect(mockedErrorCallback).not.toHaveBeenCalled();
+        });
+        it('should only push JSON files', async () => {
+            await source
+                .batchStreamDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                .onBatchUpload(mockedSuccessCallback)
+                .batch();
+            expect(mockedSuccessCallback).toHaveBeenCalledWith(expect.objectContaining({ files: ['valid.json'] }));
+        });
+        it('should call the errorCallback on a failure from the API', async () => {
+            mockAxios.post.mockReset();
+            mockAxios.post.mockRejectedValue({ message: 'Error Message' });
+            await source
+                .batchStreamDocumentsFromFiles('the_id', [(0, path_1.join)(pathToStub, 'mixdocuments')], { createFields: false })
+                .onBatchError(mockedErrorCallback)
+                .batch();
+            expect(mockedErrorCallback).toHaveBeenCalledWith({
+                message: 'Error Message',
+            }, expect.anything());
+        });
+        it('should throw an error if the path is invalid', () => {
+            expect(() => source
+                .batchStreamDocumentsFromFiles('the_id', ['path/to/invalid/document'], { createFields: false })
+                .batch()).rejects.toThrow("no such file or directory, lstat 'path/to/invalid/document'");
+        });
+    });
+});
+//# sourceMappingURL=catalog.stream.spec.js.map

package/dist/source/catalog.stream.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"catalog.stream.spec.js","sourceRoot":"","sources":["../../src/source/catalog.stream.spec.ts"],"names":[],"mappings":";;AAAA,+CAA+C;AAC/C,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;AACtC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACnB,IAAI,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;AAC5C,8DAAsD;AACtD,iCAA0B;AAC1B,+BAA0B;AAC1B,qCAA4B;AAC5B,uCAAwC;AACxC,0BAAiC;AACjC,MAAM,SAAS,GAAG,eAAkC,CAAC;AAErD,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAa,CAAC,CAAC;AACvD,MAAM,oBAAoB,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAc,CAAC,CAAC;AACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACnC,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAClC,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAC/B,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAClC,MAAM,kBAAkB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACrC,MAAM,qBAAqB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACxC,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AACtC,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,IAAA,aAAG,GAAE,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;AAElD,MAAM,wBAAwB,GAAG;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,kBAAkB;QAC1B,aAAa,EAAE,gBAAgB;QAC/B,cAAc,EAAE,kBAAkB;KACnC;CACF,CAAC;AAEF,MAAM,gCAAgC,GAAG,GAAG,EAAE,CAAC,CAAC;IAC9C,IAAI,EAAE;QACJ,SAAS,EAAE,yBAAyB;QACpC,MAAM,EAAE,SAAS;QACjB,eAAe,EAAE,EAAC,GAAG,EAAE,KAAK,EAAC;KAC9B;CACF,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,GAAG,EAAE,CAAC,CAAC;IACnC,IAAI,EAAE;QACJ,QAAQ,EAAE,eAAe;KAC1B;CACF,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,GAAG,EAAE;IAChC,oBAAoB,CAAC,kBAAkB,CACrC,GAAG,EAAE,CACH,CAAC;QACC,kBAAkB,EAAE,EAAC,QAAQ,EAAE,YAAY,EAAC;QAC5C,MAAM,EAAE;YACN,MAAM,EAAE,gBAAgB;SACzB;QACD,KAAK,EAAE;YACL,YAAY,EAAE,eAAe;SAC9B;KAC4B,CAAA,CAClC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,GAAG,EAAE;IAC/B,mBAAmB,CAAC,kBAAkB,CACpC,GAAG,EAAE,CACH,CAAC;QACC,GAAG,EAAE,eAAe;QACpB,MAAM,EAAE,kBAAkB;KACE,CAAA,CACjC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,GAAG,EAAE;IACnC,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,GAAW,EAAE,EAAE;QAChD,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE;YACtB,OAAO,OAAO,CAAC,OAAO,CAAC,gCAAgC,EAAE,CAAC,CAAC;SAC5D;QACD,IAAI,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE;YAC7B,OAAO,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC;SACjD;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,EAAC,IAAI,EAAE,EAAE,EAAC,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,GAAG,EAAE;IACnC,EAAE,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QACpC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QACpC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,uBAAuB,CAC5C,CAAC,EACD,sFAAsF,EACtF,EAAE,EACF,wBAAwB,CACzB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,uBAAuB,CAC5C,CAAC,EACD,qGAAqG,EACrG,EAAE,EACF,wBAAwB,CACzB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACrC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,uBAAuB,CAC5C,CAAC,EACD,qGAAqG,EACrG,EAAE,EACF,wBAAwB,CACzB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,IAAI,MAAqB,CAAC;IAC1B,SAAS,CAAC,GAAG,EAAE;QACb,oBAAoB,EAAE,CAAC;QACvB,mBAAmB,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,GAAG,EAAE;QACd,YAAY,CAAC,iBAAiB,CAAC,EAAC,QAAQ,EAAE,IAAI,EAAC,CAAC,CAAC;QACjD,MAAM,GAAG,IAAI,uBAAa,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAChD,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,uBAAuB,EAAE,CAAC;YAC1B,MAAM,MAAM;iBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,KAAK,EAAE,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,GAAG,EAAE;YACZ,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,uBAAuB,EAAE,CAAC;QAE1B,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,oBAAoB,CACxC,0BAA0B,EAC1B,MAAM,CAAC,gBAAgB,CAAC;gBACtB,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;oBAClC,MAAM,CAAC,gBAAgB,CAAC;wBACtB,UAAU,EAAE,sCAAsC;qBACnD,CAAC;oBACF,MAAM,CAAC,gBAAgB,CAAC;wBACtB,UAAU,EAAE,sCAAsC;qBACnD,CAAC;iBACH,CAAC;gBACF,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;aACnC,CAAC,EACF;gBACE,OAAO,EAAE;oBACP,GAAG,EAAE,KAAK;iBACX;aACF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sCAAsC,EAAE,GAAG,EAAE;QACpD,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,uBAAuB,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,GAAG,EAAE;YACZ,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;YACrC,UAAU,CAAC,KAAK,IAAI,EAAE;gBACpB,uBAAuB,EAAE,CAAC;gBAC1B,MAAM,MAAM;qBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;qBACA,KAAK,EAAE,CAAC;YACb,CAAC,CAAC,CAAC;YACH,uBAAuB,EAAE,CAAC;YAE1B,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;gBACvD,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,oBAAoB,CACxC,0BAA0B,EAC1B,MAAM,CAAC,gBAAgB,CAAC;oBACtB,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;wBAClC,MAAM,CAAC,gBAAgB,CAAC;4BACtB,UAAU,EAAE,sCAAsC;yBACnD,CAAC;wBACF,MAAM,CAAC,gBAAgB,CAAC;4BACtB,UAAU,EAAE,sCAAsC;yBACnD,CAAC;qBACH,CAAC;oBACF,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;iBACnC,CAAC,EACF;oBACE,OAAO,EAAE;wBACP,GAAG,EAAE,KAAK;qBACX;iBACF,CACF,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YAC/E,MAAM,MAAM;iBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,YAAY,CAAC,mBAAmB,CAAC;iBACjC,KAAK,EAAE,CAAC;YACX,MAAM,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,MAAM;iBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,aAAa,CAAC,qBAAqB,CAAC;iBACpC,KAAK,EAAE,CAAC;YAEX,MAAM,CAAC,qBAAqB,CAAC,CAAC,oBAAoB,CAChD,MAAM,CAAC,gBAAgB,CAAC,EAAC,KAAK,EAAE,CAAC,YAAY,CAAC,EAAC,CAAC,CACjD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3B,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAC,OAAO,EAAE,eAAe,EAAC,CAAC,CAAC;YAE7D,MAAM,MAAM;iBACT,6BAA6B,CAC5B,QAAQ,EACR,CAAC,IAAA,WAAI,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC,EAClC,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,YAAY,CAAC,mBAAmB,CAAC;iBACjC,KAAK,EAAE,CAAC;YACX,MAAM,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CAC9C;gBACE,OAAO,EAAE,eAAe;aACzB,EACD,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,CAAC,GAAG,EAAE,CACV,MAAM;iBACH,6BAA6B,CAC5B,QAAQ,EACR,CAAC,0BAA0B,CAAC,EAC5B,EAAC,YAAY,EAAE,KAAK,EAAC,CACtB;iBACA,KAAK,EAAE,CACX,CAAC,OAAO,CAAC,OAAO,CACf,6DAA6D,CAC9D,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/source/documentUploader.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadBatch = void 0;
+const __1 = require("..");
+const fieldErrors_1 = require("../errors/fieldErrors");
+const fieldUtils_1 = require("../fieldAnalyser/fieldUtils");
+async function uploadBatch(platformClient, strategy, batch, createFields = true) {
+    var _a;
+    if (createFields) {
+        const analyser = new __1.FieldAnalyser(platformClient);
+        await analyser.add(batch.addOrUpdate);
+        const { fields, inconsistencies } = analyser.report();
+        if (inconsistencies.size > 0) {
+            throw new fieldErrors_1.FieldTypeInconsistencyError(inconsistencies);
+        }
+        await (0, fieldUtils_1.createFields)(platformClient, fields);
+    }
+    const res = await strategy.upload(batch);
+    await ((_a = strategy.postUpload) === null || _a === void 0 ? void 0 : _a.call(strategy));
+    return res;
+}
+exports.uploadBatch = uploadBatch;
+//# sourceMappingURL=documentUploader.js.map

package/dist/source/documentUploader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"documentUploader.js","sourceRoot":"","sources":["../../src/source/documentUploader.ts"],"names":[],"mappings":";;;AAEA,0BAAiC;AACjC,uDAAkE;AAClE,4DAAmE;AAI5D,KAAK,UAAU,WAAW,CAC/B,cAA8B,EAC9B,QAAwB,EACxB,KAA2B,EAC3B,YAAY,GAAG,IAAI;;IAEnB,IAAI,YAAY,EAAE;QAChB,MAAM,QAAQ,GAAG,IAAI,iBAAa,CAAC,cAAc,CAAC,CAAC;QACnD,MAAM,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,EAAC,MAAM,EAAE,eAAe,EAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;QAEpD,IAAI,eAAe,CAAC,IAAI,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,yCAA2B,CAAC,eAAe,CAAC,CAAC;SACxD;QAED,MAAM,IAAA,yBAAM,EAAC,cAAc,EAAE,MAAM,CAAC,CAAC;KACtC;IACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,CAAA,MAAA,QAAQ,CAAC,UAAU,wDAAI,CAAA,CAAC;IAE9B,OAAO,GAAG,CAAC;AACb,CAAC;AArBD,kCAqBC"}