@webiny/api-elasticsearch-tasks 5.41.1 → 5.41.2-beta.0

package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js

@@ -0,0 +1,98 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ElasticsearchFetcher = void 0;
+var _shouldIgnoreEsResponseError = require("./shouldIgnoreEsResponseError");
+var _nodeUtil = require("node:util");
+class ElasticsearchFetcher {
+  constructor(params) {
+    this.client = params.client;
+  }
+  async fetch({
+    index,
+    cursor,
+    limit
+  }) {
+    let response;
+    try {
+      response = await this.client.search({
+        index,
+        body: {
+          query: {
+            match_all: {}
+          },
+          sort: {
+            "id.keyword": {
+              order: "asc"
+            }
+          },
+          size: limit + 1,
+          track_total_hits: true,
+          search_after: cursor,
+          _source: false
+        }
+      });
+    } catch (ex) {
+      /**
+       * If we ignore the error, we can continue with the next index.
+       */
+      if ((0, _shouldIgnoreEsResponseError.shouldIgnoreEsResponseError)(ex)) {
+        if (process.env.DEBUG === "true") {
+          console.error((0, _nodeUtil.inspect)(ex, {
+            depth: 5,
+            showHidden: true
+          }));
+        }
+        return {
+          done: true,
+          totalCount: 0,
+          items: []
+        };
+      }
+      console.error("Failed to fetch data from Elasticsearch.", ex);
+      throw ex;
+    }
+    const {
+      hits,
+      total
+    } = response.body.hits;
+    if (hits.length === 0) {
+      return {
+        done: true,
+        cursor: undefined,
+        totalCount: total.value,
+        items: []
+      };
+    }
+    const hasMoreItems = hits.length > limit;
+    let nextCursor;
+    if (hasMoreItems) {
+      hits.pop();
+      nextCursor = hits.at(-1)?.sort;
+    }
+    const items = hits.reduce((collection, hit) => {
+      const [PK, SK] = hit._id.split(":");
+      if (!PK || !SK) {
+        return collection;
+      }
+      collection.push({
+        PK,
+        SK,
+        _id: hit._id,
+        index: hit._index
+      });
+      return collection;
+    }, []);
+    return {
+      totalCount: total.value,
+      cursor: nextCursor,
+      done: !nextCursor,
+      items
+    };
+  }
+}
+exports.ElasticsearchFetcher = ElasticsearchFetcher;
+
+//# sourceMappingURL=ElasticsearchFetcher.js.map

package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_shouldIgnoreEsResponseError","require","_nodeUtil","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","shouldIgnoreEsResponseError","process","env","DEBUG","console","error","inspect","depth","showHidden","done","totalCount","items","hits","total","length","undefined","value","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index","exports"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import { Client } from \"@webiny/api-elasticsearch\";\nimport {\n    IElasticsearchFetcher,\n    IElasticsearchFetcherFetchParams,\n    IElasticsearchFetcherFetchResponse,\n    IElasticsearchFetcherFetchResponseItem\n} from \"./abstractions/ElasticsearchFetcher\";\nimport { ElasticsearchSearchResponse, PrimitiveValue } from \"@webiny/api-elasticsearch/types\";\nimport { shouldIgnoreEsResponseError } from \"./shouldIgnoreEsResponseError\";\nimport { inspect } from \"node:util\";\n\nexport interface IElasticsearchFetcherParams {\n    client: Client;\n}\n\nexport class ElasticsearchFetcher implements IElasticsearchFetcher {\n    private readonly client: Client;\n\n    public constructor(params: IElasticsearchFetcherParams) {\n        this.client = params.client;\n    }\n    public async fetch({\n        index,\n        cursor,\n        limit\n    }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse> {\n        let response: ElasticsearchSearchResponse<undefined>;\n        try {\n            response = await this.client.search({\n                index,\n                body: {\n                    query: {\n                        match_all: {}\n                    },\n                    sort: {\n                        \"id.keyword\": {\n                            order: \"asc\"\n                        }\n                    },\n                    size: limit + 1,\n                    track_total_hits: true,\n                    search_after: cursor,\n                    _source: false\n                }\n            });\n        } catch (ex) {\n            /**\n             * If we ignore the error, we can continue with the next index.\n             */\n            if (shouldIgnoreEsResponseError(ex)) {\n                if (process.env.DEBUG === \"true\") {\n                    console.error(\n                        inspect(ex, {\n                            depth: 5,\n                            showHidden: true\n                        })\n                    );\n                }\n                return {\n                    done: true,\n                    totalCount: 0,\n                    items: []\n                };\n            }\n            console.error(\"Failed to fetch data from Elasticsearch.\", ex);\n            throw ex;\n        }\n\n        const { hits, total } = response.body.hits;\n        if (hits.length === 0) {\n            return {\n                done: true,\n                cursor: undefined,\n                totalCount: total.value,\n                items: []\n            };\n        }\n\n        const hasMoreItems = hits.length > limit;\n        let nextCursor: PrimitiveValue[] | undefined;\n        if (hasMoreItems) {\n            hits.pop();\n            nextCursor = hits.at(-1)?.sort;\n        }\n        const items = hits.reduce<IElasticsearchFetcherFetchResponseItem[]>((collection, hit) => {\n            const [PK, SK] = hit._id.split(\":\");\n            if (!PK || !SK) {\n                return collection;\n            }\n            collection.push({\n                PK,\n                SK,\n                _id: hit._id,\n                index: hit._index\n            });\n\n            return collection;\n        }, []);\n\n        return {\n            totalCount: total.value,\n            cursor: nextCursor,\n            done: !nextCursor,\n            items\n        };\n    }\n}\n"],"mappings":";;;;;;AAQA,IAAAA,4BAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AAMO,MAAME,oBAAoB,CAAkC;EAGxDC,WAAWA,CAACC,MAAmC,EAAE;IACpD,IAAI,CAACC,MAAM,GAAGD,MAAM,CAACC,MAAM;EAC/B;EACA,MAAaC,KAAKA,CAAC;IACfC,KAAK;IACLC,MAAM;IACNC;EAC8B,CAAC,EAA+C;IAC9E,IAAIC,QAAgD;IACpD,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACL,MAAM,CAACM,MAAM,CAAC;QAChCJ,KAAK;QACLK,IAAI,EAAE;UACFC,KAAK,EAAE;YACHC,SAAS,EAAE,CAAC;UAChB,CAAC;UACDC,IAAI,EAAE;YACF,YAAY,EAAE;cACVC,KAAK,EAAE;YACX;UACJ,CAAC;UACDC,IAAI,EAAER,KAAK,GAAG,CAAC;UACfS,gBAAgB,EAAE,IAAI;UACtBC,YAAY,EAAEX,MAAM;UACpBY,OAAO,EAAE;QACb;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAI,IAAAC,wDAA2B,EAACD,EAAE,CAAC,EAAE;QACjC,IAAIE,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9BC,OAAO,CAACC,KAAK,CACT,IAAAC,iBAAO,EAACP,EAAE,EAAE;YACRQ,KAAK,EAAE,CAAC;YACRC,UAAU,EAAE;UAChB,CAAC,CACL,CAAC;QACL;QACA,OAAO;UACHC,IAAI,EAAE,IAAI;UACVC,UAAU,EAAE,CAAC;UACbC,KAAK,EAAE;QACX,CAAC;MACL;MACAP,OAAO,CAACC,KAAK,CAAC,0CAA0C,EAAEN,EAAE,CAAC;MAC7D,MAAMA,EAAE;IACZ;IAEA,MAAM;MAAEa,IAAI;MAAEC;IAAM,CAAC,GAAGzB,QAAQ,CAACE,IAAI,CAACsB,IAAI;IAC1C,IAAIA,IAAI,CAACE,MAAM,KAAK,CAAC,EAAE;MACnB,OAAO;QACHL,IAAI,EAAE,IAAI;QACVvB,MAAM,EAAE6B,SAAS;QACjBL,UAAU,EAAEG,KAAK,CAACG,KAAK;QACvBL,KAAK,EAAE;MACX,CAAC;IACL;IAEA,MAAMM,YAAY,GAAGL,IAAI,CAACE,MAAM,GAAG3B,KAAK;IACxC,IAAI+B,UAAwC;IAC5C,IAAID,YAAY,EAAE;MACdL,IAAI,CAACO,GAAG,CAAC,CAAC;MACVD,UAAU,GAAGN,IAAI,CAACQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE3B,IAAI;IAClC;IACA,MAAMkB,KAAK,GAAGC,IAAI,CAACS,MAAM,CAA2C,CAACC,UAAU,EAAEC,GAAG,KAAK;MACrF,MAAM,CAACC,EAAE,EAAEC,EAAE,CAAC,GAAGF,GAAG,CAACG,GAAG,CAACC,KAAK,CAAC,GAAG,CAAC;MACnC,IAAI,CAACH,EAAE,IAAI,CAACC,EAAE,EAAE;QACZ,OAAOH,UAAU;MACrB;MACAA,UAAU,CAACM,IAAI,CAAC;QACZJ,EAAE;QACFC,EAAE;QACFC,GAAG,EAAEH,GAAG,CAACG,GAAG;QACZzC,KAAK,EAAEsC,GAAG,CAACM;MACf,CAAC,CAAC;MAEF,OAAOP,UAAU;IACrB,CAAC,EAAE,EAAE,CAAC;IAEN,OAAO;MACHZ,UAAU,EAAEG,KAAK,CAACG,KAAK;MACvB9B,MAAM,EAAEgC,UAAU;MAClBT,IAAI,EAAE,CAACS,UAAU;MACjBP;IACJ,CAAC;EACL;AACJ;AAACmB,OAAA,CAAAlD,oBAAA,GAAAA,oBAAA","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.d.ts

@@ -0,0 +1,14 @@
+import { ITimer } from "@webiny/handler-aws";
+import { Context } from "../../../types";
+import { IElasticsearchSynchronize, IElasticsearchSynchronizeExecuteParams, IElasticsearchSynchronizeExecuteResponse } from "./abstractions/ElasticsearchSynchronize";
+export interface IElasticsearchSynchronizeParams {
+    timer: ITimer;
+    context: Context;
+}
+export declare class ElasticsearchSynchronize implements IElasticsearchSynchronize {
+    private readonly timer;
+    private readonly context;
+    constructor(params: IElasticsearchSynchronizeParams);
+    execute(params: IElasticsearchSynchronizeExecuteParams): Promise<IElasticsearchSynchronizeExecuteResponse>;
+    private getEntity;
+}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js

@@ -0,0 +1,76 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ElasticsearchSynchronize = void 0;
+var _dbDynamodb = require("@webiny/db-dynamodb");
+var _apiDynamodbToElasticsearch = require("@webiny/api-dynamodb-to-elasticsearch");
+var _entities = require("../entities");
+class ElasticsearchSynchronize {
+  constructor(params) {
+    this.timer = params.timer;
+    this.context = params.context;
+  }
+  async execute(params) {
+    const {
+      items,
+      done,
+      index
+    } = params;
+    if (items.length === 0) {
+      return {
+        done: true
+      };
+    }
+    const table = (0, _entities.getTable)({
+      type: "es",
+      context: this.context
+    });
+    const readableItems = items.map(item => {
+      const entity = this.getEntity(item);
+      return entity.item.getBatch({
+        PK: item.PK,
+        SK: item.SK
+      });
+    });
+    const tableItems = await (0, _dbDynamodb.batchReadAll)({
+      items: readableItems,
+      table
+    });
+    const elasticsearchSyncBuilder = (0, _apiDynamodbToElasticsearch.createSynchronizationBuilder)({
+      timer: this.timer,
+      context: this.context
+    });
+    /**
+     * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table.
+     */
+    for (const item of items) {
+      const exists = tableItems.some(ddbItem => {
+        return ddbItem.PK === item.PK && ddbItem.SK === item.SK;
+      });
+      if (exists) {
+        continue;
+      }
+      elasticsearchSyncBuilder.delete({
+        index,
+        id: item._id
+      });
+    }
+    const executeWithRetry = elasticsearchSyncBuilder.build();
+    await executeWithRetry();
+    return {
+      done
+    };
+  }
+  getEntity(params) {
+    const type = (0, _entities.getElasticsearchEntityType)(params);
+    return (0, _entities.getElasticsearchEntity)({
+      type,
+      context: this.context
+    });
+  }
+}
+exports.ElasticsearchSynchronize = ElasticsearchSynchronize;
+
+//# sourceMappingURL=ElasticsearchSynchronize.js.map

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_dbDynamodb","require","_apiDynamodbToElasticsearch","_entities","ElasticsearchSynchronize","constructor","params","timer","context","execute","items","done","index","length","table","getTable","type","readableItems","map","item","entity","getEntity","getBatch","PK","SK","tableItems","batchReadAll","elasticsearchSyncBuilder","createSynchronizationBuilder","exists","some","ddbItem","delete","id","_id","executeWithRetry","build","getElasticsearchEntityType","getElasticsearchEntity","exports"],"sources":["ElasticsearchSynchronize.ts"],"sourcesContent":["import { batchReadAll } from \"@webiny/db-dynamodb\";\nimport { createSynchronizationBuilder } from \"@webiny/api-dynamodb-to-elasticsearch\";\nimport {\n    getElasticsearchEntity,\n    getElasticsearchEntityType,\n    getTable,\n    IGetElasticsearchEntityTypeParams\n} from \"~/tasks/dataSynchronization/entities\";\nimport { ITimer } from \"@webiny/handler-aws\";\nimport { Context } from \"~/types\";\nimport {\n    IElasticsearchSynchronize,\n    IElasticsearchSynchronizeExecuteParams,\n    IElasticsearchSynchronizeExecuteResponse\n} from \"./abstractions/ElasticsearchSynchronize\";\n\nexport interface IElasticsearchSynchronizeParams {\n    timer: ITimer;\n    context: Context;\n}\n\ninterface IDynamoDbItem {\n    PK: string;\n    SK: string;\n}\n\nexport class ElasticsearchSynchronize implements IElasticsearchSynchronize {\n    private readonly timer: ITimer;\n    private readonly context: Context;\n\n    public constructor(params: IElasticsearchSynchronizeParams) {\n        this.timer = params.timer;\n        this.context = params.context;\n    }\n\n    public async execute(\n        params: IElasticsearchSynchronizeExecuteParams\n    ): Promise<IElasticsearchSynchronizeExecuteResponse> {\n        const { items, done, index } = params;\n        if (items.length === 0) {\n            return {\n                done: true\n            };\n        }\n\n        const table = getTable({\n            type: \"es\",\n            context: this.context\n        });\n\n        const readableItems = items.map(item => {\n            const entity = this.getEntity(item);\n            return entity.item.getBatch({\n                PK: item.PK,\n                SK: item.SK\n            });\n        });\n\n        const tableItems = await batchReadAll<IDynamoDbItem>({\n            items: readableItems,\n            table\n        });\n\n        const elasticsearchSyncBuilder = createSynchronizationBuilder({\n            timer: this.timer,\n            context: this.context\n        });\n        /**\n         * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table.\n         */\n        for (const item of items) {\n            const exists = tableItems.some(ddbItem => {\n                return ddbItem.PK === item.PK && ddbItem.SK === item.SK;\n            });\n            if (exists) {\n                continue;\n            }\n            elasticsearchSyncBuilder.delete({\n                index,\n                id: item._id\n            });\n        }\n\n        const executeWithRetry = elasticsearchSyncBuilder.build();\n        await executeWithRetry();\n\n        return {\n            done\n        };\n    }\n\n    private getEntity(\n        params: IGetElasticsearchEntityTypeParams\n    ): ReturnType<typeof getElasticsearchEntity> {\n        const type = getElasticsearchEntityType(params);\n        return getElasticsearchEntity({\n            type,\n            context: this.context\n        });\n    }\n}\n"],"mappings":";;;;;;AAAA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,2BAAA,GAAAD,OAAA;AACA,IAAAE,SAAA,GAAAF,OAAA;AAwBO,MAAMG,wBAAwB,CAAsC;EAIhEC,WAAWA,CAACC,MAAuC,EAAE;IACxD,IAAI,CAACC,KAAK,GAAGD,MAAM,CAACC,KAAK;IACzB,IAAI,CAACC,OAAO,GAAGF,MAAM,CAACE,OAAO;EACjC;EAEA,MAAaC,OAAOA,CAChBH,MAA8C,EACG;IACjD,MAAM;MAAEI,KAAK;MAAEC,IAAI;MAAEC;IAAM,CAAC,GAAGN,MAAM;IACrC,IAAII,KAAK,CAACG,MAAM,KAAK,CAAC,EAAE;MACpB,OAAO;QACHF,IAAI,EAAE;MACV,CAAC;IACL;IAEA,MAAMG,KAAK,GAAG,IAAAC,kBAAQ,EAAC;MACnBC,IAAI,EAAE,IAAI;MACVR,OAAO,EAAE,IAAI,CAACA;IAClB,CAAC,CAAC;IAEF,MAAMS,aAAa,GAAGP,KAAK,CAACQ,GAAG,CAACC,IAAI,IAAI;MACpC,MAAMC,MAAM,GAAG,IAAI,CAACC,SAAS,CAACF,IAAI,CAAC;MACnC,OAAOC,MAAM,CAACD,IAAI,CAACG,QAAQ,CAAC;QACxBC,EAAE,EAAEJ,IAAI,CAACI,EAAE;QACXC,EAAE,EAAEL,IAAI,CAACK;MACb,CAAC,CAAC;IACN,CAAC,CAAC;IAEF,MAAMC,UAAU,GAAG,MAAM,IAAAC,wBAAY,EAAgB;MACjDhB,KAAK,EAAEO,aAAa;MACpBH;IACJ,CAAC,CAAC;IAEF,MAAMa,wBAAwB,GAAG,IAAAC,wDAA4B,EAAC;MAC1DrB,KAAK,EAAE,IAAI,CAACA,KAAK;MACjBC,OAAO,EAAE,IAAI,CAACA;IAClB,CAAC,CAAC;IACF;AACR;AACA;IACQ,KAAK,MAAMW,IAAI,IAAIT,KAAK,EAAE;MACtB,MAAMmB,MAAM,GAAGJ,UAAU,CAACK,IAAI,CAACC,OAAO,IAAI;QACtC,OAAOA,OAAO,CAACR,EAAE,KAAKJ,IAAI,CAACI,EAAE,IAAIQ,OAAO,CAACP,EAAE,KAAKL,IAAI,CAACK,EAAE;MAC3D,CAAC,CAAC;MACF,IAAIK,MAAM,EAAE;QACR;MACJ;MACAF,wBAAwB,CAACK,MAAM,CAAC;QAC5BpB,KAAK;QACLqB,EAAE,EAAEd,IAAI,CAACe;MACb,CAAC,CAAC;IACN;IAEA,MAAMC,gBAAgB,GAAGR,wBAAwB,CAACS,KAAK,CAAC,CAAC;IACzD,MAAMD,gBAAgB,CAAC,CAAC;IAExB,OAAO;MACHxB;IACJ,CAAC;EACL;EAEQU,SAASA,CACbf,MAAyC,EACA;IACzC,MAAMU,IAAI,GAAG,IAAAqB,oCAA0B,EAAC/B,MAAM,CAAC;IAC/C,OAAO,IAAAgC,gCAAsB,EAAC;MAC1BtB,IAAI;MACJR,OAAO,EAAE,IAAI,CAACA;IAClB,CAAC,CAAC;EACN;AACJ;AAAC+B,OAAA,CAAAnC,wBAAA,GAAAA,wBAAA","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.d.ts

@@ -0,0 +1,10 @@
+import { IDataSynchronizationInput, IElasticsearchSyncParams, ISynchronization, ISynchronizationRunResult } from "../types";
+export declare class ElasticsearchToDynamoDbSynchronization implements ISynchronization {
+    private readonly manager;
+    private readonly indexManager;
+    private readonly synchronize;
+    private readonly fetcher;
+    constructor(params: IElasticsearchSyncParams);
+    run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult>;
+    private fetchAllIndexes;
+}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js

@@ -0,0 +1,77 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ElasticsearchToDynamoDbSynchronization = void 0;
+class ElasticsearchToDynamoDbSynchronization {
+  constructor(params) {
+    this.manager = params.manager;
+    this.indexManager = params.indexManager;
+    this.synchronize = params.synchronize;
+    this.fetcher = params.fetcher;
+  }
+  async run(input) {
+    const lastIndex = input.elasticsearchToDynamoDb?.index;
+    let cursor = input.elasticsearchToDynamoDb?.cursor;
+    const indexes = await this.fetchAllIndexes();
+    let next = 0;
+    if (lastIndex) {
+      next = indexes.findIndex(index => index === lastIndex);
+    }
+    let currentIndex = indexes[next];
+    while (currentIndex) {
+      if (this.manager.isAborted()) {
+        return this.manager.response.aborted();
+      }
+      /**
+       * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly.
+       * We want to leave enough time for possible retries.
+       */
+      //
+      else if (this.manager.isCloseToTimeout(180)) {
+        return this.manager.response.continue({
+          ...input,
+          elasticsearchToDynamoDb: {
+            ...input.elasticsearchToDynamoDb,
+            index: currentIndex,
+            cursor
+          }
+        });
+      }
+      const result = await this.fetcher.fetch({
+        index: currentIndex,
+        cursor,
+        limit: 100
+      });
+      const syncResult = await this.synchronize.execute({
+        done: result.done,
+        index: currentIndex,
+        items: result.items
+      });
+      if (!syncResult.done && result.cursor) {
+        cursor = result.cursor;
+        continue;
+      }
+      cursor = undefined;
+      const next = indexes.findIndex(index => index === currentIndex) + 1;
+      currentIndex = indexes[next];
+    }
+    return this.manager.response.continue({
+      ...input,
+      elasticsearchToDynamoDb: {
+        finished: true
+      }
+    });
+  }
+  async fetchAllIndexes() {
+    const result = await this.indexManager.list();
+    if (result.length > 0) {
+      return result;
+    }
+    throw new Error("No Elasticsearch / OpenSearch indexes found.");
+  }
+}
+exports.ElasticsearchToDynamoDbSynchronization = ElasticsearchToDynamoDbSynchronization;
+
+//# sourceMappingURL=ElasticsearchToDynamoDbSynchronization.js.map

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["ElasticsearchToDynamoDbSynchronization","constructor","params","manager","indexManager","synchronize","fetcher","run","input","lastIndex","elasticsearchToDynamoDb","index","cursor","indexes","fetchAllIndexes","next","findIndex","currentIndex","isAborted","response","aborted","isCloseToTimeout","continue","result","fetch","limit","syncResult","execute","done","items","undefined","finished","list","length","Error","exports"],"sources":["ElasticsearchToDynamoDbSynchronization.ts"],"sourcesContent":["import {\n    IDataSynchronizationInput,\n    IDataSynchronizationManager,\n    IElasticsearchSyncParams,\n    ISynchronization,\n    ISynchronizationRunResult\n} from \"../types\";\nimport { IIndexManager } from \"~/settings/types\";\nimport { NonEmptyArray } from \"@webiny/api/types\";\nimport { IElasticsearchSynchronize } from \"./abstractions/ElasticsearchSynchronize\";\nimport { IElasticsearchFetcher } from \"./abstractions/ElasticsearchFetcher\";\n\nexport class ElasticsearchToDynamoDbSynchronization implements ISynchronization {\n    private readonly manager: IDataSynchronizationManager;\n    private readonly indexManager: IIndexManager;\n    private readonly synchronize: IElasticsearchSynchronize;\n    private readonly fetcher: IElasticsearchFetcher;\n\n    public constructor(params: IElasticsearchSyncParams) {\n        this.manager = params.manager;\n        this.indexManager = params.indexManager;\n        this.synchronize = params.synchronize;\n        this.fetcher = params.fetcher;\n    }\n\n    public async run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult> {\n        const lastIndex = input.elasticsearchToDynamoDb?.index;\n        let cursor = input.elasticsearchToDynamoDb?.cursor;\n        const indexes = await this.fetchAllIndexes();\n\n        let next = 0;\n        if (lastIndex) {\n            next = indexes.findIndex(index => index === lastIndex);\n        }\n\n        let currentIndex = indexes[next];\n\n        while (currentIndex) {\n            if (this.manager.isAborted()) {\n                return this.manager.response.aborted();\n            }\n            /**\n             * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly.\n             * We want to leave enough time for possible retries.\n             */\n            //\n            else if (this.manager.isCloseToTimeout(180)) {\n                return this.manager.response.continue({\n                    ...input,\n                    elasticsearchToDynamoDb: {\n                        ...input.elasticsearchToDynamoDb,\n                        index: currentIndex,\n                        cursor\n                    }\n                });\n            }\n\n            const result = await this.fetcher.fetch({\n                index: currentIndex,\n                cursor,\n                limit: 100\n            });\n\n            const syncResult = await this.synchronize.execute({\n                done: result.done,\n                index: currentIndex,\n                items: result.items\n            });\n\n            if (!syncResult.done && result.cursor) {\n                cursor = result.cursor;\n                continue;\n            }\n            cursor = undefined;\n\n            const next = indexes.findIndex(index => index === currentIndex) + 1;\n            currentIndex = indexes[next];\n        }\n\n        return this.manager.response.continue({\n            ...input,\n            elasticsearchToDynamoDb: {\n                finished: true\n            }\n        });\n    }\n\n    private async fetchAllIndexes(): Promise<NonEmptyArray<string>> {\n        const result = await this.indexManager.list();\n        if (result.length > 0) {\n            return result as NonEmptyArray<string>;\n        }\n        throw new Error(\"No Elasticsearch / OpenSearch indexes found.\");\n    }\n}\n"],"mappings":";;;;;;AAYO,MAAMA,sCAAsC,CAA6B;EAMrEC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO;IAC7B,IAAI,CAACC,YAAY,GAAGF,MAAM,CAACE,YAAY;IACvC,IAAI,CAACC,WAAW,GAAGH,MAAM,CAACG,WAAW;IACrC,IAAI,CAACC,OAAO,GAAGJ,MAAM,CAACI,OAAO;EACjC;EAEA,MAAaC,GAAGA,CAACC,KAAgC,EAAsC;IACnF,MAAMC,SAAS,GAAGD,KAAK,CAACE,uBAAuB,EAAEC,KAAK;IACtD,IAAIC,MAAM,GAAGJ,KAAK,CAACE,uBAAuB,EAAEE,MAAM;IAClD,MAAMC,OAAO,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAE5C,IAAIC,IAAI,GAAG,CAAC;IACZ,IAAIN,SAAS,EAAE;MACXM,IAAI,GAAGF,OAAO,CAACG,SAAS,CAACL,KAAK,IAAIA,KAAK,KAAKF,SAAS,CAAC;IAC1D;IAEA,IAAIQ,YAAY,GAAGJ,OAAO,CAACE,IAAI,CAAC;IAEhC,OAAOE,YAAY,EAAE;MACjB,IAAI,IAAI,CAACd,OAAO,CAACe,SAAS,CAAC,CAAC,EAAE;QAC1B,OAAO,IAAI,CAACf,OAAO,CAACgB,QAAQ,CAACC,OAAO,CAAC,CAAC;MAC1C;MACA;AACZ;AACA;AACA;MACY;MAAA,KACK,IAAI,IAAI,CAACjB,OAAO,CAACkB,gBAAgB,CAAC,GAAG,CAAC,EAAE;QACzC,OAAO,IAAI,CAAClB,OAAO,CAACgB,QAAQ,CAACG,QAAQ,CAAC;UAClC,GAAGd,KAAK;UACRE,uBAAuB,EAAE;YACrB,GAAGF,KAAK,CAACE,uBAAuB;YAChCC,KAAK,EAAEM,YAAY;YACnBL;UACJ;QACJ,CAAC,CAAC;MACN;MAEA,MAAMW,MAAM,GAAG,MAAM,IAAI,CAACjB,OAAO,CAACkB,KAAK,CAAC;QACpCb,KAAK,EAAEM,YAAY;QACnBL,MAAM;QACNa,KAAK,EAAE;MACX,CAAC,CAAC;MAEF,MAAMC,UAAU,GAAG,MAAM,IAAI,CAACrB,WAAW,CAACsB,OAAO,CAAC;QAC9CC,IAAI,EAAEL,MAAM,CAACK,IAAI;QACjBjB,KAAK,EAAEM,YAAY;QACnBY,KAAK,EAAEN,MAAM,CAACM;MAClB,CAAC,CAAC;MAEF,IAAI,CAACH,UAAU,CAACE,IAAI,IAAIL,MAAM,CAACX,MAAM,EAAE;QACnCA,MAAM,GAAGW,MAAM,CAACX,MAAM;QACtB;MACJ;MACAA,MAAM,GAAGkB,SAAS;MAElB,MAAMf,IAAI,GAAGF,OAAO,CAACG,SAAS,CAACL,KAAK,IAAIA,KAAK,KAAKM,YAAY,CAAC,GAAG,CAAC;MACnEA,YAAY,GAAGJ,OAAO,CAACE,IAAI,CAAC;IAChC;IAEA,OAAO,IAAI,CAACZ,OAAO,CAACgB,QAAQ,CAACG,QAAQ,CAAC;MAClC,GAAGd,KAAK;MACRE,uBAAuB,EAAE;QACrBqB,QAAQ,EAAE;MACd;IACJ,CAAC,CAAC;EACN;EAEA,MAAcjB,eAAeA,CAAA,EAAmC;IAC5D,MAAMS,MAAM,GAAG,MAAM,IAAI,CAACnB,YAAY,CAAC4B,IAAI,CAAC,CAAC;IAC7C,IAAIT,MAAM,CAACU,MAAM,GAAG,CAAC,EAAE;MACnB,OAAOV,MAAM;IACjB;IACA,MAAM,IAAIW,KAAK,CAAC,8CAA8C,CAAC;EACnE;AACJ;AAACC,OAAA,CAAAnC,sCAAA,GAAAA,sCAAA","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.d.ts

@@ -0,0 +1,21 @@
+import { PrimitiveValue } from "@webiny/api-elasticsearch/types";
+export interface IElasticsearchFetcherFetchResponseItem {
+    PK: string;
+    SK: string;
+    _id: string;
+    index: string;
+}
+export interface IElasticsearchFetcherFetchParams {
+    index: string;
+    cursor?: PrimitiveValue[];
+    limit: number;
+}
+export interface IElasticsearchFetcherFetchResponse {
+    done: boolean;
+    totalCount: number;
+    cursor?: PrimitiveValue[];
+    items: IElasticsearchFetcherFetchResponseItem[];
+}
+export interface IElasticsearchFetcher {
+    fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;
+}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+//# sourceMappingURL=ElasticsearchFetcher.js.map

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import { PrimitiveValue } from \"@webiny/api-elasticsearch/types\";\n\nexport interface IElasticsearchFetcherFetchResponseItem {\n    PK: string;\n    SK: string;\n    _id: string;\n    index: string;\n}\n\nexport interface IElasticsearchFetcherFetchParams {\n    index: string;\n    cursor?: PrimitiveValue[];\n    limit: number;\n}\n\nexport interface IElasticsearchFetcherFetchResponse {\n    done: boolean;\n    totalCount: number;\n    cursor?: PrimitiveValue[];\n    items: IElasticsearchFetcherFetchResponseItem[];\n}\n\nexport interface IElasticsearchFetcher {\n    fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;\n}\n"],"mappings":"","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.d.ts

@@ -0,0 +1,17 @@
+export interface IElasticsearchSynchronizeExecuteParamsItem {
+    PK: string;
+    SK: string;
+    _id: string;
+    index: string;
+}
+export interface IElasticsearchSynchronizeExecuteParams {
+    done: boolean;
+    index: string;
+    items: IElasticsearchSynchronizeExecuteParamsItem[];
+}
+export interface IElasticsearchSynchronizeExecuteResponse {
+    done: boolean;
+}
+export interface IElasticsearchSynchronize {
+    execute(params: IElasticsearchSynchronizeExecuteParams): Promise<IElasticsearchSynchronizeExecuteResponse>;
+}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+//# sourceMappingURL=ElasticsearchSynchronize.js.map

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["ElasticsearchSynchronize.ts"],"sourcesContent":["export interface IElasticsearchSynchronizeExecuteParamsItem {\n    PK: string;\n    SK: string;\n    _id: string;\n    index: string;\n}\n\nexport interface IElasticsearchSynchronizeExecuteParams {\n    done: boolean;\n    index: string;\n    items: IElasticsearchSynchronizeExecuteParamsItem[];\n}\n\nexport interface IElasticsearchSynchronizeExecuteResponse {\n    done: boolean;\n}\nexport interface IElasticsearchSynchronize {\n    execute(\n        params: IElasticsearchSynchronizeExecuteParams\n    ): Promise<IElasticsearchSynchronizeExecuteResponse>;\n}\n"],"mappings":"","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.d.ts

@@ -0,0 +1,2 @@
+import WebinyError from "@webiny/error";
+export declare const shouldIgnoreEsResponseError: (error: WebinyError) => boolean;

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js

@@ -0,0 +1,13 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.shouldIgnoreEsResponseError = void 0;
+const IGNORED_ES_SEARCH_EXCEPTIONS = ["index_not_found_exception", "search_phase_execution_exception", "illegal_argument_exception"];
+const shouldIgnoreEsResponseError = error => {
+  return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message);
+};
+exports.shouldIgnoreEsResponseError = shouldIgnoreEsResponseError;
+
+//# sourceMappingURL=shouldIgnoreEsResponseError.js.map

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["IGNORED_ES_SEARCH_EXCEPTIONS","shouldIgnoreEsResponseError","error","includes","message","exports"],"sources":["shouldIgnoreEsResponseError.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\n\nconst IGNORED_ES_SEARCH_EXCEPTIONS = [\n    \"index_not_found_exception\",\n    \"search_phase_execution_exception\",\n    \"illegal_argument_exception\"\n];\n\nexport const shouldIgnoreEsResponseError = (error: WebinyError) => {\n    return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message);\n};\n"],"mappings":";;;;;;AAEA,MAAMA,4BAA4B,GAAG,CACjC,2BAA2B,EAC3B,kCAAkC,EAClC,4BAA4B,CAC/B;AAEM,MAAMC,2BAA2B,GAAIC,KAAkB,IAAK;EAC/D,OAAOF,4BAA4B,CAACG,QAAQ,CAACD,KAAK,CAACE,OAAO,CAAC;AAC/D,CAAC;AAACC,OAAA,CAAAJ,2BAAA,GAAAA,2BAAA","ignoreList":[]}