@terascope/elasticsearch-api 4.12.2 → 4.13.0

package/test/bulk-send-dlq-spec.js

@@ -1,96 +0,0 @@
-import { jest } from '@jest/globals';
-import { debugLogger, cloneDeep, DataEntity } from '@terascope/utils';
-import { ElasticsearchTestHelpers } from '@terascope/opensearch-client';
-import elasticsearchAPI from '../index.js';
-
-const {
-    makeClient, cleanupIndex, EvenDateData,
-    TEST_INDEX_PREFIX, createMappingFromDatatype,
-    formatUploadData
-} = ElasticsearchTestHelpers;
-
-const THREE_MINUTES = 3 * 60 * 1000;
-
-jest.setTimeout(THREE_MINUTES + 60000);
-
-describe('bulkSend', () => {
-    let client;
-    let api;
-
-    beforeAll(async () => {
-        client = await makeClient();
-    });
-
-    describe('can return non-retryable records', () => {
-        const logger = debugLogger('congested_test');
-        const index = `${TEST_INDEX_PREFIX}_non-retryable-records`;
-
-        beforeAll(async () => {
-            await cleanupIndex(client, index);
-            api = elasticsearchAPI(client, logger, { _dead_letter_action: 'kafka_dead_letter' });
-
-            const overrides = {
-                settings: {
-                    'index.number_of_shards': 1,
-                    'index.number_of_replicas': 0,
-                },
-            };
-
-            const mapping = await createMappingFromDatatype(
-                client, EvenDateData.EvenDataType, '_doc', overrides
-            );
-
-            mapping.index = index;
-
-            await client.indices.create(mapping);
-        });
-
-        afterAll(async () => {
-            await cleanupIndex(client, index);
-        });
-
-        it('should send records if no issues and dlq not set', async () => {
-            const diffApi = elasticsearchAPI(client, logger);
-
-            const docs = cloneDeep(EvenDateData.data.slice(0, 2));
-
-            const result = await diffApi.bulkSend(formatUploadData(index, docs, true));
-
-            expect(result).toBe(2);
-        });
-
-        it('should throw an error if dlq is not set', async () => {
-            const diffApi = elasticsearchAPI(client, logger);
-
-            const docs = cloneDeep(EvenDateData.data.slice(0, 2));
-
-            docs[0].bytes = 'this is a bad value';
-
-            await expect(diffApi.bulkSend(formatUploadData(index, docs, true)))
-                .rejects.toThrow();
-        });
-
-        it('should update metadata for records that are not retryable and return results for successful updates if dlq is set', async () => {
-            const docs = cloneDeep(EvenDateData.data.slice(0, 2))
-                .map((doc, i) => DataEntity.make(doc, { _key: i + 1 }));
-
-            docs[0].bytes = 'this is a bad value';
-
-            const result = await api.bulkSend(formatUploadData(index, docs, true));
-
-            // 1 good doc  - so only 1 row affected
-            expect(result).toBe(1);
-
-            expect(docs[0].getMetadata('_bulk_sender_rejection')).toInclude('mapper_parsing_exception--failed to parse field [bytes]');
-            expect(docs[1].getMetadata('_bulk_sender_rejection')).toBeUndefined();
-        });
-
-        it('should return a count if not un-retryable records if dlq is set', async () => {
-            const docs = cloneDeep(EvenDateData.data.slice(0, 2));
-
-            const result = await api.bulkSend(formatUploadData(index, docs, true));
-
-            expect(result).toBe(2);
-        });
-    });
-});

package/test/bulk-send-limit-spec.js

@@ -1,49 +0,0 @@
-import { jest } from '@jest/globals';
-import { debugLogger, chunk, pMap } from '@terascope/utils';
-import { ElasticsearchTestHelpers } from '@terascope/opensearch-client';
-import elasticsearchAPI from '../index.js';
-
-const {
-    makeClient, cleanupIndex, waitForData,
-    EvenDateData, TEST_INDEX_PREFIX, formatUploadData
-} = ElasticsearchTestHelpers;
-
-const THREE_MINUTES = 3 * 60 * 1000;
-
-jest.setTimeout(THREE_MINUTES + 60000);
-
-describe('bulkSend', () => {
-    let client;
-    let api;
-
-    beforeAll(async () => {
-        client = await makeClient();
-    });
-
-    describe('can work with congested queues', () => {
-        const logger = debugLogger('congested_test');
-        const index = `${TEST_INDEX_PREFIX}_congested_queues_`;
-
-        beforeAll(async () => {
-            await cleanupIndex(client, index);
-            api = elasticsearchAPI(client, logger);
-        });
-
-        afterAll(async () => {
-            await cleanupIndex(client, index);
-        });
-
-        it('can get correct data even with congested queues', async () => {
-            const chunkedData = chunk(EvenDateData.data, 50);
-
-            await pMap(chunkedData, async (cData) => {
-                const formattedData = formatUploadData(index, cData, true);
-                return api.bulkSend(formattedData);
-            }, { concurrency: 9 });
-
-            await expect(
-                waitForData(client, index, EvenDateData.data.length, logger, THREE_MINUTES)
-            ).resolves.not.toThrow();
-        });
-    });
-});

package/test/retry-spec.js

@@ -1,36 +0,0 @@
-import { ElasticsearchTestHelpers } from '@terascope/opensearch-client';
-import { debugLogger } from '@terascope/utils';
-import opensearch from '@opensearch-project/opensearch';
-import API from '../index.js';
-
-const logger = debugLogger('retry spec');
-const port = '1111';
-
-const { data } = ElasticsearchTestHelpers.EvenDateData;
-
-const body = ElasticsearchTestHelpers.formatUploadData('retry-error-test', data);
-
-describe('retry behavior', () => {
-    it('will work with opensearch when connection cannot be established', async () => {
-        expect.assertions(4);
-        const client = new opensearch.Client({ node: `http://127.0.0.1:${port}` });
-        const { isErrorRetryable } = API(client, logger);
-
-        try {
-            await client.bulk({ body });
-            throw Error('should have thrown');
-        } catch (err) {
-            expect(err).toBeDefined();
-            expect(isErrorRetryable(err)).toBeTrue();
-        }
-
-        // we try again as we have seen the second call be marked as unretryable
-        try {
-            await client.bulk({ body });
-            throw Error('should have thrown');
-        } catch (err) {
-            expect(err).toBeDefined();
-            expect(isErrorRetryable(err)).toBeTrue();
-        }
-    });
-});

package/types/index.d.ts

@@ -1,81 +0,0 @@
-// Type definitions for elasticsearch-api
-// Project: @terascope/elasticsearch-api
-
-import { ClientParams, ClientResponse } from '@terascope/types';
-import { Logger } from '@terascope/utils';
-import { ClientMetadata } from '@terascope/types'
-
-export default elasticsearchAPI;
-
-declare function elasticsearchAPI(client: Client, logger: Logger, config?: elasticsearchAPI.Config): elasticsearchAPI.Client;
-
-declare namespace elasticsearchAPI {
-    export interface Config {
-        index?: string;
-        full_response?: boolean;
-        connection?: string;
-    }
-
-    export interface Client {
-        search: (query: ClientParams.SearchParams) => Promise<ClientResponse.SearchResponse | any[]>;
-        count: (query: ClientParams.CountParams) => Promise<number>;
-        get: (query: ClientParams.GetParams, fullResponse?: boolean) => Promise<any>;
-        mget: (query: ClientParams.MGetParams) => Promise<any>;
-        index: (query: ClientParams.IndexParams) => Promise<any>;
-        indexWithId: (query: ClientParams.IndexParams) => Promise<any>;
-        isAvailable: (index: string, recordType?: string) => Promise<any>;
-        create: (query: ClientParams.CreateParams) => Promise<any>;
-        update: (query: ClientParams.UpdateParams) => Promise<any>;
-        remove: (query: ClientParams.DeleteParams) => Promise<es.DeleteDocumentResponse>;
-        version: () => Promise<boolean>;
-        putTemplate: (template: any, name: string) => Promise<any>;
-        /**
-         * The new and improved bulk send with proper retry support
-         *
-         * @returns the number of affected rows
-        */
-        bulkSend: (data: BulkRecord[]) => Promise<number>;
-        nodeInfo: (query: any) => Promise<any>;
-        nodeStats: (query: any) => Promise<any>;
-        buildQuery: (opConfig: Config, msg: any) => ClientParams.SearchParams;
-        indexExists: (query: ClientParams.ExistsParams) => Promise<boolean>;
-        indexCreate: (query: ClientParams.IndicesCreateParams) => Promise<any>;
-        indexRefresh: (query: ClientParams.IndicesRefreshParams) => Promise<any>;
-        indexRecovery: (query: ClientParams.IndicesRecoveryParams) => Promise<any>;
-        indexSetup: (clusterName, newIndex, migrantIndexName, mapping, recordType, clientName) => Promise<boolean>;
-        verifyClient: () => boolean;
-        validateGeoParameters: (opConfig: any) => any;
-        /** This api is deprecated, please use getClientMetadata */
-        getESVersion: () => number;
-        getClientMetadata: () => ClientMetadata;
-        isElasticsearch6: () => boolean
-    }
-
-    /**
-     * This is used for improved bulk sending function
-    */
-    export interface BulkRecord {
-        action: AnyBulkAction;
-        data?: UpdateConfig | DataEntity;
-    }
-
-    /**
-     * This is used for improved bulk sending function
-    */
-    export interface AnyBulkAction {
-        update?: Partial<BulkActionMetadata>;
-        index?: Partial<BulkActionMetadata>;
-        create?: Partial<BulkActionMetadata>;
-        delete?: Partial<BulkActionMetadata>;
-    }
-
-    /**
-     * This is used for improved bulk sending function
-    */
-    export interface BulkActionMetadata {
-        _index: string;
-        _type: string;
-        _id: string | number;
-        retry_on_conflict?: number;
-    }
-}