@webiny/api-elasticsearch-tasks 0.0.0-unstable.06b2ede40f

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["ElasticsearchToDynamoDbSynchronization","constructor","params","manager","indexManager","synchronize","fetcher","run","input","lastIndex","elasticsearchToDynamoDb","index","cursor","indexes","fetchAllIndexes","next","findIndex","currentIndex","isAborted","response","aborted","isCloseToTimeout","continue","result","fetch","limit","syncResult","execute","done","items","undefined","finished","list","length","Error","exports"],"sources":["ElasticsearchToDynamoDbSynchronization.ts"],"sourcesContent":["import type {\n    IDataSynchronizationInput,\n    IDataSynchronizationManager,\n    IElasticsearchSyncParams,\n    ISynchronization,\n    ISynchronizationRunResult\n} from \"../types\";\nimport type { IIndexManager } from \"~/settings/types\";\nimport type { NonEmptyArray } from \"@webiny/api/types\";\nimport type { IElasticsearchSynchronize } from \"./abstractions/ElasticsearchSynchronize\";\nimport type { IElasticsearchFetcher } from \"./abstractions/ElasticsearchFetcher\";\n\nexport class ElasticsearchToDynamoDbSynchronization implements ISynchronization {\n    private readonly manager: IDataSynchronizationManager;\n    private readonly indexManager: IIndexManager;\n    private readonly synchronize: IElasticsearchSynchronize;\n    private readonly fetcher: IElasticsearchFetcher;\n\n    public constructor(params: IElasticsearchSyncParams) {\n        this.manager = params.manager;\n        this.indexManager = params.indexManager;\n        this.synchronize = params.synchronize;\n        this.fetcher = params.fetcher;\n    }\n\n    public async run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult> {\n        const lastIndex = input.elasticsearchToDynamoDb?.index;\n        let cursor = input.elasticsearchToDynamoDb?.cursor;\n        const indexes = await this.fetchAllIndexes();\n\n        let next = 0;\n        if (lastIndex) {\n            next = indexes.findIndex(index => index === lastIndex);\n        }\n\n        let currentIndex = indexes[next];\n\n        while (currentIndex) {\n            if (this.manager.isAborted()) {\n                return this.manager.response.aborted();\n            }\n            /**\n             * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly.\n             * We want to leave enough time for possible retries.\n             */\n            //\n            else if (this.manager.isCloseToTimeout(180)) {\n                return this.manager.response.continue({\n                    ...input,\n                    elasticsearchToDynamoDb: {\n                        ...input.elasticsearchToDynamoDb,\n                        index: currentIndex,\n                        cursor\n                    }\n                });\n            }\n\n            const result = await this.fetcher.fetch({\n                index: currentIndex,\n                cursor,\n                limit: 100\n            });\n\n            const syncResult = await this.synchronize.execute({\n                done: result.done,\n                index: currentIndex,\n                items: result.items\n            });\n\n            if (!syncResult.done && result.cursor) {\n                cursor = result.cursor;\n                continue;\n            }\n            cursor = undefined;\n\n            const next = indexes.findIndex(index => index === currentIndex) + 1;\n            currentIndex = indexes[next];\n        }\n\n        return this.manager.response.continue({\n            ...input,\n            elasticsearchToDynamoDb: {\n                finished: true\n            }\n        });\n    }\n\n    private async fetchAllIndexes(): Promise<NonEmptyArray<string>> {\n        const result = await this.indexManager.list();\n        if (result.length > 0) {\n            return result as NonEmptyArray<string>;\n        }\n        throw new Error(\"No Elasticsearch / OpenSearch indexes found.\");\n    }\n}\n"],"mappings":";;;;;;AAYO,MAAMA,sCAAsC,CAA6B;EAMrEC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO;IAC7B,IAAI,CAACC,YAAY,GAAGF,MAAM,CAACE,YAAY;IACvC,IAAI,CAACC,WAAW,GAAGH,MAAM,CAACG,WAAW;IACrC,IAAI,CAACC,OAAO,GAAGJ,MAAM,CAACI,OAAO;EACjC;EAEA,MAAaC,GAAGA,CAACC,KAAgC,EAAsC;IACnF,MAAMC,SAAS,GAAGD,KAAK,CAACE,uBAAuB,EAAEC,KAAK;IACtD,IAAIC,MAAM,GAAGJ,KAAK,CAACE,uBAAuB,EAAEE,MAAM;IAClD,MAAMC,OAAO,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAE5C,IAAIC,IAAI,GAAG,CAAC;IACZ,IAAIN,SAAS,EAAE;MACXM,IAAI,GAAGF,OAAO,CAACG,SAAS,CAACL,KAAK,IAAIA,KAAK,KAAKF,SAAS,CAAC;IAC1D;IAEA,IAAIQ,YAAY,GAAGJ,OAAO,CAACE,IAAI,CAAC;IAEhC,OAAOE,YAAY,EAAE;MACjB,IAAI,IAAI,CAACd,OAAO,CAACe,SAAS,CAAC,CAAC,EAAE;QAC1B,OAAO,IAAI,CAACf,OAAO,CAACgB,QAAQ,CAACC,OAAO,CAAC,CAAC;MAC1C;MACA;AACZ;AACA;AACA;MACY;MAAA,KACK,IAAI,IAAI,CAACjB,OAAO,CAACkB,gBAAgB,CAAC,GAAG,CAAC,EAAE;QACzC,OAAO,IAAI,CAAClB,OAAO,CAACgB,QAAQ,CAACG,QAAQ,CAAC;UAClC,GAAGd,KAAK;UACRE,uBAAuB,EAAE;YACrB,GAAGF,KAAK,CAACE,uBAAuB;YAChCC,KAAK,EAAEM,YAAY;YACnBL;UACJ;QACJ,CAAC,CAAC;MACN;MAEA,MAAMW,MAAM,GAAG,MAAM,IAAI,CAACjB,OAAO,CAACkB,KAAK,CAAC;QACpCb,KAAK,EAAEM,YAAY;QACnBL,MAAM;QACNa,KAAK,EAAE;MACX,CAAC,CAAC;MAEF,MAAMC,UAAU,GAAG,MAAM,IAAI,CAACrB,WAAW,CAACsB,OAAO,CAAC;QAC9CC,IAAI,EAAEL,MAAM,CAACK,IAAI;QACjBjB,KAAK,EAAEM,YAAY;QACnBY,KAAK,EAAEN,MAAM,CAACM;MAClB,CAAC,CAAC;MAEF,IAAI,CAACH,UAAU,CAACE,IAAI,IAAIL,MAAM,CAACX,MAAM,EAAE;QACnCA,MAAM,GAAGW,MAAM,CAACX,MAAM;QACtB;MACJ;MACAA,MAAM,GAAGkB,SAAS;MAElB,MAAMf,IAAI,GAAGF,OAAO,CAACG,SAAS,CAACL,KAAK,IAAIA,KAAK,KAAKM,YAAY,CAAC,GAAG,CAAC;MACnEA,YAAY,GAAGJ,OAAO,CAACE,IAAI,CAAC;IAChC;IAEA,OAAO,IAAI,CAACZ,OAAO,CAACgB,QAAQ,CAACG,QAAQ,CAAC;MAClC,GAAGd,KAAK;MACRE,uBAAuB,EAAE;QACrBqB,QAAQ,EAAE;MACd;IACJ,CAAC,CAAC;EACN;EAEA,MAAcjB,eAAeA,CAAA,EAAmC;IAC5D,MAAMS,MAAM,GAAG,MAAM,IAAI,CAACnB,YAAY,CAAC4B,IAAI,CAAC,CAAC;IAC7C,IAAIT,MAAM,CAACU,MAAM,GAAG,CAAC,EAAE;MACnB,OAAOV,MAAM;IACjB;IACA,MAAM,IAAIW,KAAK,CAAC,8CAA8C,CAAC;EACnE;AACJ;AAACC,OAAA,CAAAnC,sCAAA,GAAAA,sCAAA","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.d.ts

@@ -0,0 +1,21 @@
+import type { PrimitiveValue } from "@webiny/api-elasticsearch/types";
+export interface IElasticsearchFetcherFetchResponseItem {
+    PK: string;
+    SK: string;
+    _id: string;
+    index: string;
+}
+export interface IElasticsearchFetcherFetchParams {
+    index: string;
+    cursor?: PrimitiveValue[];
+    limit: number;
+}
+export interface IElasticsearchFetcherFetchResponse {
+    done: boolean;
+    totalCount: number;
+    cursor?: PrimitiveValue[];
+    items: IElasticsearchFetcherFetchResponseItem[];
+}
+export interface IElasticsearchFetcher {
+    fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;
+}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+//# sourceMappingURL=ElasticsearchFetcher.js.map

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { PrimitiveValue } from \"@webiny/api-elasticsearch/types\";\n\nexport interface IElasticsearchFetcherFetchResponseItem {\n    PK: string;\n    SK: string;\n    _id: string;\n    index: string;\n}\n\nexport interface IElasticsearchFetcherFetchParams {\n    index: string;\n    cursor?: PrimitiveValue[];\n    limit: number;\n}\n\nexport interface IElasticsearchFetcherFetchResponse {\n    done: boolean;\n    totalCount: number;\n    cursor?: PrimitiveValue[];\n    items: IElasticsearchFetcherFetchResponseItem[];\n}\n\nexport interface IElasticsearchFetcher {\n    fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;\n}\n"],"mappings":"","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.d.ts

@@ -0,0 +1,17 @@
+export interface IElasticsearchSynchronizeExecuteParamsItem {
+    PK: string;
+    SK: string;
+    _id: string;
+    index: string;
+}
+export interface IElasticsearchSynchronizeExecuteParams {
+    done: boolean;
+    index: string;
+    items: IElasticsearchSynchronizeExecuteParamsItem[];
+}
+export interface IElasticsearchSynchronizeExecuteResponse {
+    done: boolean;
+}
+export interface IElasticsearchSynchronize {
+    execute(params: IElasticsearchSynchronizeExecuteParams): Promise<IElasticsearchSynchronizeExecuteResponse>;
+}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+//# sourceMappingURL=ElasticsearchSynchronize.js.map

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["ElasticsearchSynchronize.ts"],"sourcesContent":["export interface IElasticsearchSynchronizeExecuteParamsItem {\n    PK: string;\n    SK: string;\n    _id: string;\n    index: string;\n}\n\nexport interface IElasticsearchSynchronizeExecuteParams {\n    done: boolean;\n    index: string;\n    items: IElasticsearchSynchronizeExecuteParamsItem[];\n}\n\nexport interface IElasticsearchSynchronizeExecuteResponse {\n    done: boolean;\n}\nexport interface IElasticsearchSynchronize {\n    execute(\n        params: IElasticsearchSynchronizeExecuteParams\n    ): Promise<IElasticsearchSynchronizeExecuteResponse>;\n}\n"],"mappings":"","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.d.ts

@@ -0,0 +1,2 @@
+import type WebinyError from "@webiny/error";
+export declare const shouldIgnoreEsResponseError: (error: WebinyError) => boolean;

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js

@@ -0,0 +1,13 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.shouldIgnoreEsResponseError = void 0;
+const IGNORED_ES_SEARCH_EXCEPTIONS = ["index_not_found_exception", "search_phase_execution_exception", "illegal_argument_exception"];
+const shouldIgnoreEsResponseError = error => {
+  return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message);
+};
+exports.shouldIgnoreEsResponseError = shouldIgnoreEsResponseError;
+
+//# sourceMappingURL=shouldIgnoreEsResponseError.js.map

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["IGNORED_ES_SEARCH_EXCEPTIONS","shouldIgnoreEsResponseError","error","includes","message","exports"],"sources":["shouldIgnoreEsResponseError.ts"],"sourcesContent":["import type WebinyError from \"@webiny/error\";\n\nconst IGNORED_ES_SEARCH_EXCEPTIONS = [\n    \"index_not_found_exception\",\n    \"search_phase_execution_exception\",\n    \"illegal_argument_exception\"\n];\n\nexport const shouldIgnoreEsResponseError = (error: WebinyError) => {\n    return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message);\n};\n"],"mappings":";;;;;;AAEA,MAAMA,4BAA4B,GAAG,CACjC,2BAA2B,EAC3B,kCAAkC,EAClC,4BAA4B,CAC/B;AAEM,MAAMC,2BAA2B,GAAIC,KAAkB,IAAK;EAC/D,OAAOF,4BAA4B,CAACG,QAAQ,CAACD,KAAK,CAACE,OAAO,CAAC;AAC/D,CAAC;AAACC,OAAA,CAAAJ,2BAAA,GAAAA,2BAAA","ignoreList":[]}