@webiny/api-elasticsearch-tasks 0.0.0-unstable.06b2ede40f

package/tasks/createIndexes/index.js

@@ -0,0 +1,68 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createIndexesTaskDefinition = void 0;
+var _tasks = require("@webiny/tasks");
+var _CreateIndexesTaskRunner = require("./CreateIndexesTaskRunner");
+const createIndexesTaskDefinition = params => {
+  return (0, _tasks.createTaskDefinition)({
+    id: "elasticsearchCreateIndexes",
+    title: "Create Missing Elasticsearch Indexes",
+    /**
+     * Maximum number of iterations before the task goes into the error state.
+     * No point in having more than 2 runs, as the create index operations should not even take 1 full run, no matter how much indeexs is there to create.
+     */
+    maxIterations: 2,
+    run: async ({
+      response,
+      context,
+      isCloseToTimeout,
+      isAborted,
+      store,
+      input,
+      timer
+    }) => {
+      const {
+        Manager
+      } = await import(/* webpackChunkName: "Manager" */"../Manager");
+      const {
+        IndexManager
+      } = await import(/* webpackChunkName: "IndexManager" */"../../settings");
+      const manager = new Manager({
+        elasticsearchClient: params?.elasticsearchClient,
+        documentClient: params?.documentClient,
+        response,
+        context,
+        isAborted,
+        isCloseToTimeout,
+        store,
+        timer
+      });
+      const indexManager = new IndexManager(manager.elasticsearch, {});
+      const createIndexesTaskRunner = new _CreateIndexesTaskRunner.CreateIndexesTaskRunner(manager, indexManager);
+      return createIndexesTaskRunner.execute(input.matching, Array.from(input.done || []));
+    },
+    async onBeforeTrigger({
+      context
+    }) {
+      // Let's create a new index for the tasks first.
+      const {
+        IndexManager
+      } = await import(/* webpackChunkName: "IndexManager" */"../../settings");
+      const indexManager = new IndexManager(context.elasticsearch, {});
+      const {
+        OnBeforeTrigger
+      } = await import(/* webpackChunkName: "OnBeforeTrigger" */"./OnBeforeTrigger");
+      const onBeforeTrigger = new OnBeforeTrigger({
+        indexManager,
+        context
+      });
+      await onBeforeTrigger.run(["webinytask"]);
+    }
+  });
+};
+exports.createIndexesTaskDefinition = createIndexesTaskDefinition;
+
+//# sourceMappingURL=index.js.map

package/tasks/createIndexes/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_tasks","require","_CreateIndexesTaskRunner","createIndexesTaskDefinition","params","createTaskDefinition","id","title","maxIterations","run","response","context","isCloseToTimeout","isAborted","store","input","timer","Manager","IndexManager","manager","elasticsearchClient","documentClient","indexManager","elasticsearch","createIndexesTaskRunner","CreateIndexesTaskRunner","execute","matching","Array","from","done","onBeforeTrigger","OnBeforeTrigger","exports"],"sources":["index.ts"],"sourcesContent":["import { createTaskDefinition } from \"@webiny/tasks\";\nimport type { Context, IElasticsearchTaskConfig } from \"~/types\";\nimport type { IElasticsearchCreateIndexesTaskInput } from \"~/tasks/createIndexes/types\";\nimport { CreateIndexesTaskRunner } from \"./CreateIndexesTaskRunner\";\n\nexport const createIndexesTaskDefinition = (params?: IElasticsearchTaskConfig) => {\n    return createTaskDefinition<Context, IElasticsearchCreateIndexesTaskInput>({\n        id: \"elasticsearchCreateIndexes\",\n        title: \"Create Missing Elasticsearch Indexes\",\n        /**\n         * Maximum number of iterations before the task goes into the error state.\n         * No point in having more than 2 runs, as the create index operations should not even take 1 full run, no matter how much indeexs is there to create.\n         */\n        maxIterations: 2,\n        run: async ({ response, context, isCloseToTimeout, isAborted, store, input, timer }) => {\n            const { Manager } = await import(\n                /* webpackChunkName: \"Manager\" */\n                \"../Manager\"\n            );\n            const { IndexManager } = await import(\n                /* webpackChunkName: \"IndexManager\" */ \"~/settings\"\n            );\n\n            const manager = new Manager<IElasticsearchCreateIndexesTaskInput>({\n                elasticsearchClient: params?.elasticsearchClient,\n                documentClient: params?.documentClient,\n                response,\n                context,\n                isAborted,\n                isCloseToTimeout,\n                store,\n                timer\n            });\n\n            const indexManager = new IndexManager(manager.elasticsearch, {});\n\n            const createIndexesTaskRunner = new CreateIndexesTaskRunner(manager, indexManager);\n\n            return createIndexesTaskRunner.execute(input.matching, Array.from(input.done || []));\n        },\n        async onBeforeTrigger({ context }) {\n            // Let's create a new index for the tasks first.\n            const { IndexManager } = await import(\n                /* webpackChunkName: \"IndexManager\" */ \"~/settings\"\n            );\n            const indexManager = new IndexManager(context.elasticsearch, {});\n            const { OnBeforeTrigger } = await import(\n                /* webpackChunkName: \"OnBeforeTrigger\" */\n                \"./OnBeforeTrigger\"\n            );\n\n            const onBeforeTrigger = new OnBeforeTrigger({\n                indexManager,\n                context\n            });\n            await onBeforeTrigger.run([\"webinytask\"]);\n        }\n    });\n};\n"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAGA,IAAAC,wBAAA,GAAAD,OAAA;AAEO,MAAME,2BAA2B,GAAIC,MAAiC,IAAK;EAC9E,OAAO,IAAAC,2BAAoB,EAAgD;IACvEC,EAAE,EAAE,4BAA4B;IAChCC,KAAK,EAAE,sCAAsC;IAC7C;AACR;AACA;AACA;IACQC,aAAa,EAAE,CAAC;IAChBC,GAAG,EAAE,MAAAA,CAAO;MAAEC,QAAQ;MAAEC,OAAO;MAAEC,gBAAgB;MAAEC,SAAS;MAAEC,KAAK;MAAEC,KAAK;MAAEC;IAAM,CAAC,KAAK;MACpF,MAAM;QAAEC;MAAQ,CAAC,GAAG,MAAM,MAAM,CAC5B,6CAEJ,CAAC;MACD,MAAM;QAAEC;MAAa,CAAC,GAAG,MAAM,MAAM,CACjC,sDACJ,CAAC;MAED,MAAMC,OAAO,GAAG,IAAIF,OAAO,CAAuC;QAC9DG,mBAAmB,EAAEhB,MAAM,EAAEgB,mBAAmB;QAChDC,cAAc,EAAEjB,MAAM,EAAEiB,cAAc;QACtCX,QAAQ;QACRC,OAAO;QACPE,SAAS;QACTD,gBAAgB;QAChBE,KAAK;QACLE;MACJ,CAAC,CAAC;MAEF,MAAMM,YAAY,GAAG,IAAIJ,YAAY,CAACC,OAAO,CAACI,aAAa,EAAE,CAAC,CAAC,CAAC;MAEhE,MAAMC,uBAAuB,GAAG,IAAIC,gDAAuB,CAACN,OAAO,EAAEG,YAAY,CAAC;MAElF,OAAOE,uBAAuB,CAACE,OAAO,CAACX,KAAK,CAACY,QAAQ,EAAEC,KAAK,CAACC,IAAI,CAACd,KAAK,CAACe,IAAI,IAAI,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,MAAMC,eAAeA,CAAC;MAAEpB;IAAQ,CAAC,EAAE;MAC/B;MACA,MAAM;QAAEO;MAAa,CAAC,GAAG,MAAM,MAAM,CACjC,sDACJ,CAAC;MACD,MAAMI,YAAY,GAAG,IAAIJ,YAAY,CAACP,OAAO,CAACY,aAAa,EAAE,CAAC,CAAC,CAAC;MAChE,MAAM;QAAES;MAAgB,CAAC,GAAG,MAAM,MAAM,CACpC,4DAEJ,CAAC;MAED,MAAMD,eAAe,GAAG,IAAIC,eAAe,CAAC;QACxCV,YAAY;QACZX;MACJ,CAAC,CAAC;MACF,MAAMoB,eAAe,CAACtB,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;IAC7C;EACJ,CAAC,CAAC;AACN,CAAC;AAACwB,OAAA,CAAA9B,2BAAA,GAAAA,2BAAA","ignoreList":[]}

package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.d.ts

@@ -0,0 +1,4 @@
+import { CreateElasticsearchIndexTaskPlugin } from "./CreateElasticsearchIndexTaskPlugin";
+import type { Context } from "../../types";
+import type { PluginsContainer } from "@webiny/plugins";
+export declare const listCreateElasticsearchIndexTaskPlugin: <C extends Context = Context>(plugins: PluginsContainer) => CreateElasticsearchIndexTaskPlugin<C>[];

package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.js

@@ -0,0 +1,13 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.listCreateElasticsearchIndexTaskPlugin = void 0;
+var _CreateElasticsearchIndexTaskPlugin = require("./CreateElasticsearchIndexTaskPlugin");
+const listCreateElasticsearchIndexTaskPlugin = plugins => {
+  return plugins.byType(_CreateElasticsearchIndexTaskPlugin.CreateElasticsearchIndexTaskPlugin.type);
+};
+exports.listCreateElasticsearchIndexTaskPlugin = listCreateElasticsearchIndexTaskPlugin;
+
+//# sourceMappingURL=listCreateElasticsearchIndexTaskPlugin.js.map

package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_CreateElasticsearchIndexTaskPlugin","require","listCreateElasticsearchIndexTaskPlugin","plugins","byType","CreateElasticsearchIndexTaskPlugin","type","exports"],"sources":["listCreateElasticsearchIndexTaskPlugin.ts"],"sourcesContent":["import { CreateElasticsearchIndexTaskPlugin } from \"~/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin\";\nimport type { Context } from \"~/types\";\nimport type { PluginsContainer } from \"@webiny/plugins\";\n\nexport const listCreateElasticsearchIndexTaskPlugin = <C extends Context = Context>(\n    plugins: PluginsContainer\n): CreateElasticsearchIndexTaskPlugin<C>[] => {\n    return plugins.byType<CreateElasticsearchIndexTaskPlugin<Context>>(\n        CreateElasticsearchIndexTaskPlugin.type\n    );\n};\n"],"mappings":";;;;;;AAAA,IAAAA,mCAAA,GAAAC,OAAA;AAIO,MAAMC,sCAAsC,GAC/CC,OAAyB,IACiB;EAC1C,OAAOA,OAAO,CAACC,MAAM,CACjBC,sEAAkC,CAACC,IACvC,CAAC;AACL,CAAC;AAACC,OAAA,CAAAL,sCAAA,GAAAA,sCAAA","ignoreList":[]}

package/tasks/createIndexes/listIndexes.d.ts

@@ -0,0 +1,12 @@
+import type { CreateElasticsearchIndexTaskPluginIndex } from "./CreateElasticsearchIndexTaskPlugin";
+import type { CreateElasticsearchIndexTaskPlugin } from "./CreateElasticsearchIndexTaskPlugin";
+import type { Context } from "../../types";
+import type { Tenant } from "@webiny/api-tenancy/types";
+import type { I18NLocale } from "@webiny/api-i18n/types";
+export interface IListIndexesParams {
+    context: Context;
+    plugins: CreateElasticsearchIndexTaskPlugin<Context>[];
+    tenants?: Tenant[];
+    locales?: I18NLocale[];
+}
+export declare const listIndexes: (params: IListIndexesParams) => Promise<CreateElasticsearchIndexTaskPluginIndex[]>;

package/tasks/createIndexes/listIndexes.js

@@ -0,0 +1,53 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.listIndexes = void 0;
+const listIndexes = async params => {
+  const {
+    context,
+    plugins,
+    tenants: inputTenants,
+    locales: inputLocales
+  } = params;
+  if (plugins.length === 0) {
+    return [];
+  }
+  const indexes = [];
+  const tenants = inputTenants || (await context.tenancy.listTenants());
+  const initialTenant = context.tenancy.getCurrentTenant();
+  try {
+    for (const tenant of tenants) {
+      context.tenancy.setCurrentTenant(tenant);
+      let locales = inputLocales ? [...inputLocales] : [];
+      if (locales.length === 0) {
+        const [localesResult] = await context.i18n.locales.listLocales({
+          limit: 10000
+        });
+        locales = localesResult;
+      }
+      for (const locale of locales) {
+        for (const plugin of plugins) {
+          const results = await plugin.getIndexList({
+            context,
+            tenant: tenant.id,
+            locale: locale.code
+          });
+          for (const result of results) {
+            if (indexes.some(i => i.index === result.index)) {
+              continue;
+            }
+            indexes.push(result);
+          }
+        }
+      }
+    }
+  } finally {
+    context.tenancy.setCurrentTenant(initialTenant);
+  }
+  return indexes;
+};
+exports.listIndexes = listIndexes;
+
+//# sourceMappingURL=listIndexes.js.map

package/tasks/createIndexes/listIndexes.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["listIndexes","params","context","plugins","tenants","inputTenants","locales","inputLocales","length","indexes","tenancy","listTenants","initialTenant","getCurrentTenant","tenant","setCurrentTenant","localesResult","i18n","listLocales","limit","locale","plugin","results","getIndexList","id","code","result","some","i","index","push","exports"],"sources":["listIndexes.ts"],"sourcesContent":["import type { CreateElasticsearchIndexTaskPluginIndex } from \"~/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin\";\nimport type { CreateElasticsearchIndexTaskPlugin } from \"~/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin\";\nimport type { Context } from \"~/types\";\nimport type { Tenant } from \"@webiny/api-tenancy/types\";\nimport type { I18NLocale } from \"@webiny/api-i18n/types\";\n\nexport interface IListIndexesParams {\n    context: Context;\n    plugins: CreateElasticsearchIndexTaskPlugin<Context>[];\n    tenants?: Tenant[];\n    locales?: I18NLocale[];\n}\n\nexport const listIndexes = async (\n    params: IListIndexesParams\n): Promise<CreateElasticsearchIndexTaskPluginIndex[]> => {\n    const { context, plugins, tenants: inputTenants, locales: inputLocales } = params;\n    if (plugins.length === 0) {\n        return [];\n    }\n\n    const indexes: CreateElasticsearchIndexTaskPluginIndex[] = [];\n    const tenants = inputTenants || (await context.tenancy.listTenants());\n    const initialTenant = context.tenancy.getCurrentTenant();\n    try {\n        for (const tenant of tenants) {\n            context.tenancy.setCurrentTenant(tenant);\n\n            let locales = inputLocales ? [...inputLocales] : [];\n            if (locales.length === 0) {\n                const [localesResult] = await context.i18n.locales.listLocales({\n                    limit: 10000\n                });\n                locales = localesResult;\n            }\n\n            for (const locale of locales) {\n                for (const plugin of plugins) {\n                    const results = await plugin.getIndexList({\n                        context,\n                        tenant: tenant.id,\n                        locale: locale.code\n                    });\n                    for (const result of results) {\n                        if (indexes.some(i => i.index === result.index)) {\n                            continue;\n                        }\n                        indexes.push(result);\n                    }\n                }\n            }\n        }\n    } finally {\n        context.tenancy.setCurrentTenant(initialTenant);\n    }\n\n    return indexes;\n};\n"],"mappings":";;;;;;AAaO,MAAMA,WAAW,GAAG,MACvBC,MAA0B,IAC2B;EACrD,MAAM;IAAEC,OAAO;IAAEC,OAAO;IAAEC,OAAO,EAAEC,YAAY;IAAEC,OAAO,EAAEC;EAAa,CAAC,GAAGN,MAAM;EACjF,IAAIE,OAAO,CAACK,MAAM,KAAK,CAAC,EAAE;IACtB,OAAO,EAAE;EACb;EAEA,MAAMC,OAAkD,GAAG,EAAE;EAC7D,MAAML,OAAO,GAAGC,YAAY,KAAK,MAAMH,OAAO,CAACQ,OAAO,CAACC,WAAW,CAAC,CAAC,CAAC;EACrE,MAAMC,aAAa,GAAGV,OAAO,CAACQ,OAAO,CAACG,gBAAgB,CAAC,CAAC;EACxD,IAAI;IACA,KAAK,MAAMC,MAAM,IAAIV,OAAO,EAAE;MAC1BF,OAAO,CAACQ,OAAO,CAACK,gBAAgB,CAACD,MAAM,CAAC;MAExC,IAAIR,OAAO,GAAGC,YAAY,GAAG,CAAC,GAAGA,YAAY,CAAC,GAAG,EAAE;MACnD,IAAID,OAAO,CAACE,MAAM,KAAK,CAAC,EAAE;QACtB,MAAM,CAACQ,aAAa,CAAC,GAAG,MAAMd,OAAO,CAACe,IAAI,CAACX,OAAO,CAACY,WAAW,CAAC;UAC3DC,KAAK,EAAE;QACX,CAAC,CAAC;QACFb,OAAO,GAAGU,aAAa;MAC3B;MAEA,KAAK,MAAMI,MAAM,IAAId,OAAO,EAAE;QAC1B,KAAK,MAAMe,MAAM,IAAIlB,OAAO,EAAE;UAC1B,MAAMmB,OAAO,GAAG,MAAMD,MAAM,CAACE,YAAY,CAAC;YACtCrB,OAAO;YACPY,MAAM,EAAEA,MAAM,CAACU,EAAE;YACjBJ,MAAM,EAAEA,MAAM,CAACK;UACnB,CAAC,CAAC;UACF,KAAK,MAAMC,MAAM,IAAIJ,OAAO,EAAE;YAC1B,IAAIb,OAAO,CAACkB,IAAI,CAACC,CAAC,IAAIA,CAAC,CAACC,KAAK,KAAKH,MAAM,CAACG,KAAK,CAAC,EAAE;cAC7C;YACJ;YACApB,OAAO,CAACqB,IAAI,CAACJ,MAAM,CAAC;UACxB;QACJ;MACJ;IACJ;EACJ,CAAC,SAAS;IACNxB,OAAO,CAACQ,OAAO,CAACK,gBAAgB,CAACH,aAAa,CAAC;EACnD;EAEA,OAAOH,OAAO;AAClB,CAAC;AAACsB,OAAA,CAAA/B,WAAA,GAAAA,WAAA","ignoreList":[]}

package/tasks/createIndexes/types.d.ts

@@ -0,0 +1,4 @@
+export interface IElasticsearchCreateIndexesTaskInput {
+    matching?: string;
+    done?: string[];
+}

package/tasks/createIndexes/types.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+//# sourceMappingURL=types.js.map

package/tasks/createIndexes/types.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["export interface IElasticsearchCreateIndexesTaskInput {\n    matching?: string;\n    done?: string[];\n}\n"],"mappings":"","ignoreList":[]}

package/tasks/dataSynchronization/DataSynchronizationTaskRunner.d.ts

@@ -0,0 +1,15 @@
+import type { IDataSynchronizationInput, IDataSynchronizationManager, IFactories } from "./types";
+import type { IIndexManager } from "../../settings/types";
+export interface IDataSynchronizationTaskRunnerParams {
+    manager: IDataSynchronizationManager;
+    indexManager: IIndexManager;
+    factories: IFactories;
+}
+export declare class DataSynchronizationTaskRunner {
+    private readonly manager;
+    private readonly indexManager;
+    private readonly factories;
+    constructor(params: IDataSynchronizationTaskRunnerParams);
+    run(input: IDataSynchronizationInput): Promise<import("@webiny/tasks").ITaskResponseErrorResult | import("@webiny/tasks").ITaskResponseAbortedResult | import("@webiny/tasks").ITaskResponseDoneResult<import("@webiny/tasks").ITaskResponseDoneResultOutput> | import("@webiny/tasks").ITaskResponseContinueResult<IDataSynchronizationInput>>;
+    private validateFlow;
+}

package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js

@@ -0,0 +1,55 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.DataSynchronizationTaskRunner = void 0;
+var _ElasticsearchSynchronize = require("./elasticsearch/ElasticsearchSynchronize");
+var _ElasticsearchFetcher = require("./elasticsearch/ElasticsearchFetcher");
+class DataSynchronizationTaskRunner {
+  constructor(params) {
+    this.manager = params.manager;
+    this.indexManager = params.indexManager;
+    this.factories = params.factories;
+  }
+  async run(input) {
+    this.validateFlow(input);
+    /**
+     * Go through the Elasticsearch and delete records which do not exist in the Elasticsearch table.
+     */
+    //
+    if (input.flow === "elasticsearchToDynamoDb" && !input.elasticsearchToDynamoDb?.finished) {
+      const sync = this.factories.elasticsearchToDynamoDb({
+        manager: this.manager,
+        indexManager: this.indexManager,
+        synchronize: new _ElasticsearchSynchronize.ElasticsearchSynchronize({
+          context: this.manager.context,
+          timer: this.manager.timer
+        }),
+        fetcher: new _ElasticsearchFetcher.ElasticsearchFetcher({
+          client: this.manager.elasticsearch
+        })
+      });
+      try {
+        return await sync.run(input);
+      } catch (ex) {
+        return this.manager.response.error(ex);
+      }
+    }
+    /**
+     * We are done.
+     */
+    return this.manager.response.done();
+  }
+  validateFlow(input) {
+    if (!input.flow) {
+      throw new Error(`Missing "flow" in the input.`);
+    } else if (this.factories[input.flow]) {
+      return;
+    }
+    throw new Error(`Invalid flow "${input.flow}". Allowed flows: ${Object.keys(this.factories).join(", ")}.`);
+  }
+}
+exports.DataSynchronizationTaskRunner = DataSynchronizationTaskRunner;
+
+//# sourceMappingURL=DataSynchronizationTaskRunner.js.map

package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_ElasticsearchSynchronize","require","_ElasticsearchFetcher","DataSynchronizationTaskRunner","constructor","params","manager","indexManager","factories","run","input","validateFlow","flow","elasticsearchToDynamoDb","finished","sync","synchronize","ElasticsearchSynchronize","context","timer","fetcher","ElasticsearchFetcher","client","elasticsearch","ex","response","error","done","Error","Object","keys","join","exports"],"sources":["DataSynchronizationTaskRunner.ts"],"sourcesContent":["import type {\n    IDataSynchronizationInput,\n    IDataSynchronizationManager,\n    IFactories\n} from \"~/tasks/dataSynchronization/types\";\nimport type { IIndexManager } from \"~/settings/types\";\nimport { ElasticsearchSynchronize } from \"~/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize\";\nimport { ElasticsearchFetcher } from \"~/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher\";\n\nexport interface IDataSynchronizationTaskRunnerParams {\n    manager: IDataSynchronizationManager;\n    indexManager: IIndexManager;\n    factories: IFactories;\n}\n\nexport class DataSynchronizationTaskRunner {\n    private readonly manager: IDataSynchronizationManager;\n    private readonly indexManager: IIndexManager;\n    private readonly factories: IFactories;\n\n    public constructor(params: IDataSynchronizationTaskRunnerParams) {\n        this.manager = params.manager;\n        this.indexManager = params.indexManager;\n        this.factories = params.factories;\n    }\n\n    public async run(input: IDataSynchronizationInput) {\n        this.validateFlow(input);\n        /**\n         * Go through the Elasticsearch and delete records which do not exist in the Elasticsearch table.\n         */\n        //\n        if (input.flow === \"elasticsearchToDynamoDb\" && !input.elasticsearchToDynamoDb?.finished) {\n            const sync = this.factories.elasticsearchToDynamoDb({\n                manager: this.manager,\n                indexManager: this.indexManager,\n                synchronize: new ElasticsearchSynchronize({\n                    context: this.manager.context,\n                    timer: this.manager.timer\n                }),\n                fetcher: new ElasticsearchFetcher({\n                    client: this.manager.elasticsearch\n                })\n            });\n            try {\n                return await sync.run(input);\n            } catch (ex) {\n                return this.manager.response.error(ex);\n            }\n        }\n        /**\n         * We are done.\n         */\n        return this.manager.response.done();\n    }\n\n    private validateFlow(input: IDataSynchronizationInput): void {\n        if (!input.flow) {\n            throw new Error(`Missing \"flow\" in the input.`);\n        } else if (this.factories[input.flow]) {\n            return;\n        }\n        throw new Error(\n            `Invalid flow \"${input.flow}\". Allowed flows: ${Object.keys(this.factories).join(\n                \", \"\n            )}.`\n        );\n    }\n}\n"],"mappings":";;;;;;AAMA,IAAAA,yBAAA,GAAAC,OAAA;AACA,IAAAC,qBAAA,GAAAD,OAAA;AAQO,MAAME,6BAA6B,CAAC;EAKhCC,WAAWA,CAACC,MAA4C,EAAE;IAC7D,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO;IAC7B,IAAI,CAACC,YAAY,GAAGF,MAAM,CAACE,YAAY;IACvC,IAAI,CAACC,SAAS,GAAGH,MAAM,CAACG,SAAS;EACrC;EAEA,MAAaC,GAAGA,CAACC,KAAgC,EAAE;IAC/C,IAAI,CAACC,YAAY,CAACD,KAAK,CAAC;IACxB;AACR;AACA;IACQ;IACA,IAAIA,KAAK,CAACE,IAAI,KAAK,yBAAyB,IAAI,CAACF,KAAK,CAACG,uBAAuB,EAAEC,QAAQ,EAAE;MACtF,MAAMC,IAAI,GAAG,IAAI,CAACP,SAAS,CAACK,uBAAuB,CAAC;QAChDP,OAAO,EAAE,IAAI,CAACA,OAAO;QACrBC,YAAY,EAAE,IAAI,CAACA,YAAY;QAC/BS,WAAW,EAAE,IAAIC,kDAAwB,CAAC;UACtCC,OAAO,EAAE,IAAI,CAACZ,OAAO,CAACY,OAAO;UAC7BC,KAAK,EAAE,IAAI,CAACb,OAAO,CAACa;QACxB,CAAC,CAAC;QACFC,OAAO,EAAE,IAAIC,0CAAoB,CAAC;UAC9BC,MAAM,EAAE,IAAI,CAAChB,OAAO,CAACiB;QACzB,CAAC;MACL,CAAC,CAAC;MACF,IAAI;QACA,OAAO,MAAMR,IAAI,CAACN,GAAG,CAACC,KAAK,CAAC;MAChC,CAAC,CAAC,OAAOc,EAAE,EAAE;QACT,OAAO,IAAI,CAAClB,OAAO,CAACmB,QAAQ,CAACC,KAAK,CAACF,EAAE,CAAC;MAC1C;IACJ;IACA;AACR;AACA;IACQ,OAAO,IAAI,CAAClB,OAAO,CAACmB,QAAQ,CAACE,IAAI,CAAC,CAAC;EACvC;EAEQhB,YAAYA,CAACD,KAAgC,EAAQ;IACzD,IAAI,CAACA,KAAK,CAACE,IAAI,EAAE;MACb,MAAM,IAAIgB,KAAK,CAAC,8BAA8B,CAAC;IACnD,CAAC,MAAM,IAAI,IAAI,CAACpB,SAAS,CAACE,KAAK,CAACE,IAAI,CAAC,EAAE;MACnC;IACJ;IACA,MAAM,IAAIgB,KAAK,CACX,iBAAiBlB,KAAK,CAACE,IAAI,qBAAqBiB,MAAM,CAACC,IAAI,CAAC,IAAI,CAACtB,SAAS,CAAC,CAACuB,IAAI,CAC5E,IACJ,CAAC,GACL,CAAC;EACL;AACJ;AAACC,OAAA,CAAA7B,6BAAA,GAAAA,6BAAA","ignoreList":[]}

package/tasks/dataSynchronization/createFactories.d.ts

@@ -0,0 +1,2 @@
+import type { IFactories } from "./types";
+export declare const createFactories: () => IFactories;

package/tasks/dataSynchronization/createFactories.js

@@ -0,0 +1,17 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createFactories = void 0;
+var _ElasticsearchToDynamoDbSynchronization = require("./elasticsearch/ElasticsearchToDynamoDbSynchronization");
+const createFactories = () => {
+  return {
+    elasticsearchToDynamoDb: params => {
+      return new _ElasticsearchToDynamoDbSynchronization.ElasticsearchToDynamoDbSynchronization(params);
+    }
+  };
+};
+exports.createFactories = createFactories;
+
+//# sourceMappingURL=createFactories.js.map

package/tasks/dataSynchronization/createFactories.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_ElasticsearchToDynamoDbSynchronization","require","createFactories","elasticsearchToDynamoDb","params","ElasticsearchToDynamoDbSynchronization","exports"],"sources":["createFactories.ts"],"sourcesContent":["import type { IFactories } from \"./types\";\nimport { ElasticsearchToDynamoDbSynchronization } from \"./elasticsearch/ElasticsearchToDynamoDbSynchronization\";\n\nexport const createFactories = (): IFactories => {\n    return {\n        elasticsearchToDynamoDb: params => {\n            return new ElasticsearchToDynamoDbSynchronization(params);\n        }\n    };\n};\n"],"mappings":";;;;;;AACA,IAAAA,uCAAA,GAAAC,OAAA;AAEO,MAAMC,eAAe,GAAGA,CAAA,KAAkB;EAC7C,OAAO;IACHC,uBAAuB,EAAEC,MAAM,IAAI;MAC/B,OAAO,IAAIC,8EAAsC,CAACD,MAAM,CAAC;IAC7D;EACJ,CAAC;AACL,CAAC;AAACE,OAAA,CAAAJ,eAAA,GAAAA,eAAA","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.d.ts

@@ -0,0 +1,10 @@
+import type { Client } from "@webiny/api-elasticsearch";
+import type { IElasticsearchFetcher, IElasticsearchFetcherFetchParams, IElasticsearchFetcherFetchResponse } from "./abstractions/ElasticsearchFetcher";
+export interface IElasticsearchFetcherParams {
+    client: Client;
+}
+export declare class ElasticsearchFetcher implements IElasticsearchFetcher {
+    private readonly client;
+    constructor(params: IElasticsearchFetcherParams);
+    fetch({ index, cursor, limit }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;
+}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js

@@ -0,0 +1,98 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ElasticsearchFetcher = void 0;
+var _shouldIgnoreEsResponseError = require("./shouldIgnoreEsResponseError");
+var _nodeUtil = require("node:util");
+class ElasticsearchFetcher {
+  constructor(params) {
+    this.client = params.client;
+  }
+  async fetch({
+    index,
+    cursor,
+    limit
+  }) {
+    let response;
+    try {
+      response = await this.client.search({
+        index,
+        body: {
+          query: {
+            match_all: {}
+          },
+          sort: {
+            "id.keyword": {
+              order: "asc"
+            }
+          },
+          size: limit + 1,
+          track_total_hits: true,
+          search_after: cursor,
+          _source: false
+        }
+      });
+    } catch (ex) {
+      /**
+       * If we ignore the error, we can continue with the next index.
+       */
+      if ((0, _shouldIgnoreEsResponseError.shouldIgnoreEsResponseError)(ex)) {
+        if (process.env.DEBUG === "true") {
+          console.error((0, _nodeUtil.inspect)(ex, {
+            depth: 5,
+            showHidden: true
+          }));
+        }
+        return {
+          done: true,
+          totalCount: 0,
+          items: []
+        };
+      }
+      console.error("Failed to fetch data from Elasticsearch.", ex);
+      throw ex;
+    }
+    const {
+      hits,
+      total
+    } = response.body.hits;
+    if (hits.length === 0) {
+      return {
+        done: true,
+        cursor: undefined,
+        totalCount: total.value,
+        items: []
+      };
+    }
+    const hasMoreItems = hits.length > limit;
+    let nextCursor;
+    if (hasMoreItems) {
+      hits.pop();
+      nextCursor = hits.at(-1)?.sort;
+    }
+    const items = hits.reduce((collection, hit) => {
+      const [PK, SK] = hit._id.split(":");
+      if (!PK || !SK) {
+        return collection;
+      }
+      collection.push({
+        PK,
+        SK,
+        _id: hit._id,
+        index: hit._index
+      });
+      return collection;
+    }, []);
+    return {
+      totalCount: total.value,
+      cursor: nextCursor,
+      done: !nextCursor,
+      items
+    };
+  }
+}
+exports.ElasticsearchFetcher = ElasticsearchFetcher;
+
+//# sourceMappingURL=ElasticsearchFetcher.js.map

package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_shouldIgnoreEsResponseError","require","_nodeUtil","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","shouldIgnoreEsResponseError","process","env","DEBUG","console","error","inspect","depth","showHidden","done","totalCount","items","hits","total","length","undefined","value","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index","exports"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-elasticsearch\";\nimport type {\n    IElasticsearchFetcher,\n    IElasticsearchFetcherFetchParams,\n    IElasticsearchFetcherFetchResponse,\n    IElasticsearchFetcherFetchResponseItem\n} from \"./abstractions/ElasticsearchFetcher\";\nimport type { ElasticsearchSearchResponse, PrimitiveValue } from \"@webiny/api-elasticsearch/types\";\nimport { shouldIgnoreEsResponseError } from \"./shouldIgnoreEsResponseError\";\nimport { inspect } from \"node:util\";\n\nexport interface IElasticsearchFetcherParams {\n    client: Client;\n}\n\nexport class ElasticsearchFetcher implements IElasticsearchFetcher {\n    private readonly client: Client;\n\n    public constructor(params: IElasticsearchFetcherParams) {\n        this.client = params.client;\n    }\n    public async fetch({\n        index,\n        cursor,\n        limit\n    }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse> {\n        let response: ElasticsearchSearchResponse<undefined>;\n        try {\n            response = await this.client.search({\n                index,\n                body: {\n                    query: {\n                        match_all: {}\n                    },\n                    sort: {\n                        \"id.keyword\": {\n                            order: \"asc\"\n                        }\n                    },\n                    size: limit + 1,\n                    track_total_hits: true,\n                    search_after: cursor,\n                    _source: false\n                }\n            });\n        } catch (ex) {\n            /**\n             * If we ignore the error, we can continue with the next index.\n             */\n            if (shouldIgnoreEsResponseError(ex)) {\n                if (process.env.DEBUG === \"true\") {\n                    console.error(\n                        inspect(ex, {\n                            depth: 5,\n                            showHidden: true\n                        })\n                    );\n                }\n                return {\n                    done: true,\n                    totalCount: 0,\n                    items: []\n                };\n            }\n            console.error(\"Failed to fetch data from Elasticsearch.\", ex);\n            throw ex;\n        }\n\n        const { hits, total } = response.body.hits;\n        if (hits.length === 0) {\n            return {\n                done: true,\n                cursor: undefined,\n                totalCount: total.value,\n                items: []\n            };\n        }\n\n        const hasMoreItems = hits.length > limit;\n        let nextCursor: PrimitiveValue[] | undefined;\n        if (hasMoreItems) {\n            hits.pop();\n            nextCursor = hits.at(-1)?.sort;\n        }\n        const items = hits.reduce<IElasticsearchFetcherFetchResponseItem[]>((collection, hit) => {\n            const [PK, SK] = hit._id.split(\":\");\n            if (!PK || !SK) {\n                return collection;\n            }\n            collection.push({\n                PK,\n                SK,\n                _id: hit._id,\n                index: hit._index\n            });\n\n            return collection;\n        }, []);\n\n        return {\n            totalCount: total.value,\n            cursor: nextCursor,\n            done: !nextCursor,\n            items\n        };\n    }\n}\n"],"mappings":";;;;;;AAQA,IAAAA,4BAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AAMO,MAAME,oBAAoB,CAAkC;EAGxDC,WAAWA,CAACC,MAAmC,EAAE;IACpD,IAAI,CAACC,MAAM,GAAGD,MAAM,CAACC,MAAM;EAC/B;EACA,MAAaC,KAAKA,CAAC;IACfC,KAAK;IACLC,MAAM;IACNC;EAC8B,CAAC,EAA+C;IAC9E,IAAIC,QAAgD;IACpD,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACL,MAAM,CAACM,MAAM,CAAC;QAChCJ,KAAK;QACLK,IAAI,EAAE;UACFC,KAAK,EAAE;YACHC,SAAS,EAAE,CAAC;UAChB,CAAC;UACDC,IAAI,EAAE;YACF,YAAY,EAAE;cACVC,KAAK,EAAE;YACX;UACJ,CAAC;UACDC,IAAI,EAAER,KAAK,GAAG,CAAC;UACfS,gBAAgB,EAAE,IAAI;UACtBC,YAAY,EAAEX,MAAM;UACpBY,OAAO,EAAE;QACb;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAI,IAAAC,wDAA2B,EAACD,EAAE,CAAC,EAAE;QACjC,IAAIE,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9BC,OAAO,CAACC,KAAK,CACT,IAAAC,iBAAO,EAACP,EAAE,EAAE;YACRQ,KAAK,EAAE,CAAC;YACRC,UAAU,EAAE;UAChB,CAAC,CACL,CAAC;QACL;QACA,OAAO;UACHC,IAAI,EAAE,IAAI;UACVC,UAAU,EAAE,CAAC;UACbC,KAAK,EAAE;QACX,CAAC;MACL;MACAP,OAAO,CAACC,KAAK,CAAC,0CAA0C,EAAEN,EAAE,CAAC;MAC7D,MAAMA,EAAE;IACZ;IAEA,MAAM;MAAEa,IAAI;MAAEC;IAAM,CAAC,GAAGzB,QAAQ,CAACE,IAAI,CAACsB,IAAI;IAC1C,IAAIA,IAAI,CAACE,MAAM,KAAK,CAAC,EAAE;MACnB,OAAO;QACHL,IAAI,EAAE,IAAI;QACVvB,MAAM,EAAE6B,SAAS;QACjBL,UAAU,EAAEG,KAAK,CAACG,KAAK;QACvBL,KAAK,EAAE;MACX,CAAC;IACL;IAEA,MAAMM,YAAY,GAAGL,IAAI,CAACE,MAAM,GAAG3B,KAAK;IACxC,IAAI+B,UAAwC;IAC5C,IAAID,YAAY,EAAE;MACdL,IAAI,CAACO,GAAG,CAAC,CAAC;MACVD,UAAU,GAAGN,IAAI,CAACQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE3B,IAAI;IAClC;IACA,MAAMkB,KAAK,GAAGC,IAAI,CAACS,MAAM,CAA2C,CAACC,UAAU,EAAEC,GAAG,KAAK;MACrF,MAAM,CAACC,EAAE,EAAEC,EAAE,CAAC,GAAGF,GAAG,CAACG,GAAG,CAACC,KAAK,CAAC,GAAG,CAAC;MACnC,IAAI,CAACH,EAAE,IAAI,CAACC,EAAE,EAAE;QACZ,OAAOH,UAAU;MACrB;MACAA,UAAU,CAACM,IAAI,CAAC;QACZJ,EAAE;QACFC,EAAE;QACFC,GAAG,EAAEH,GAAG,CAACG,GAAG;QACZzC,KAAK,EAAEsC,GAAG,CAACM;MACf,CAAC,CAAC;MAEF,OAAOP,UAAU;IACrB,CAAC,EAAE,EAAE,CAAC;IAEN,OAAO;MACHZ,UAAU,EAAEG,KAAK,CAACG,KAAK;MACvB9B,MAAM,EAAEgC,UAAU;MAClBT,IAAI,EAAE,CAACS,UAAU;MACjBP;IACJ,CAAC;EACL;AACJ;AAACmB,OAAA,CAAAlD,oBAAA,GAAAA,oBAAA","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.d.ts

@@ -0,0 +1,14 @@
+import type { ITimer } from "@webiny/handler-aws";
+import type { Context } from "../../../types";
+import type { IElasticsearchSynchronize, IElasticsearchSynchronizeExecuteParams, IElasticsearchSynchronizeExecuteResponse } from "./abstractions/ElasticsearchSynchronize";
+export interface IElasticsearchSynchronizeParams {
+    timer: ITimer;
+    context: Context;
+}
+export declare class ElasticsearchSynchronize implements IElasticsearchSynchronize {
+    private readonly timer;
+    private readonly context;
+    constructor(params: IElasticsearchSynchronizeParams);
+    execute(params: IElasticsearchSynchronizeExecuteParams): Promise<IElasticsearchSynchronizeExecuteResponse>;
+    private getEntity;
+}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js

@@ -0,0 +1,76 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ElasticsearchSynchronize = void 0;
+var _dbDynamodb = require("@webiny/db-dynamodb");
+var _apiDynamodbToElasticsearch = require("@webiny/api-dynamodb-to-elasticsearch");
+var _entities = require("../entities");
+class ElasticsearchSynchronize {
+  constructor(params) {
+    this.timer = params.timer;
+    this.context = params.context;
+  }
+  async execute(params) {
+    const {
+      items,
+      done,
+      index
+    } = params;
+    if (items.length === 0) {
+      return {
+        done: true
+      };
+    }
+    const table = (0, _entities.getTable)({
+      type: "es",
+      context: this.context
+    });
+    const readableItems = items.map(item => {
+      const entity = this.getEntity(item);
+      return entity.item.getBatch({
+        PK: item.PK,
+        SK: item.SK
+      });
+    });
+    const tableItems = await (0, _dbDynamodb.batchReadAll)({
+      items: readableItems,
+      table
+    });
+    const elasticsearchSyncBuilder = (0, _apiDynamodbToElasticsearch.createSynchronizationBuilder)({
+      timer: this.timer,
+      context: this.context
+    });
+    /**
+     * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table.
+     */
+    for (const item of items) {
+      const exists = tableItems.some(ddbItem => {
+        return ddbItem.PK === item.PK && ddbItem.SK === item.SK;
+      });
+      if (exists) {
+        continue;
+      }
+      elasticsearchSyncBuilder.delete({
+        index,
+        id: item._id
+      });
+    }
+    const executeWithRetry = elasticsearchSyncBuilder.build();
+    await executeWithRetry();
+    return {
+      done
+    };
+  }
+  getEntity(params) {
+    const type = (0, _entities.getElasticsearchEntityType)(params);
+    return (0, _entities.getElasticsearchEntity)({
+      type,
+      context: this.context
+    });
+  }
+}
+exports.ElasticsearchSynchronize = ElasticsearchSynchronize;
+
+//# sourceMappingURL=ElasticsearchSynchronize.js.map

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_dbDynamodb","require","_apiDynamodbToElasticsearch","_entities","ElasticsearchSynchronize","constructor","params","timer","context","execute","items","done","index","length","table","getTable","type","readableItems","map","item","entity","getEntity","getBatch","PK","SK","tableItems","batchReadAll","elasticsearchSyncBuilder","createSynchronizationBuilder","exists","some","ddbItem","delete","id","_id","executeWithRetry","build","getElasticsearchEntityType","getElasticsearchEntity","exports"],"sources":["ElasticsearchSynchronize.ts"],"sourcesContent":["import { batchReadAll } from \"@webiny/db-dynamodb\";\nimport { createSynchronizationBuilder } from \"@webiny/api-dynamodb-to-elasticsearch\";\nimport type { IGetElasticsearchEntityTypeParams } from \"~/tasks/dataSynchronization/entities\";\nimport {\n    getElasticsearchEntity,\n    getElasticsearchEntityType,\n    getTable\n} from \"~/tasks/dataSynchronization/entities\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport type { Context } from \"~/types\";\nimport type {\n    IElasticsearchSynchronize,\n    IElasticsearchSynchronizeExecuteParams,\n    IElasticsearchSynchronizeExecuteResponse\n} from \"./abstractions/ElasticsearchSynchronize\";\n\nexport interface IElasticsearchSynchronizeParams {\n    timer: ITimer;\n    context: Context;\n}\n\ninterface IDynamoDbItem {\n    PK: string;\n    SK: string;\n}\n\nexport class ElasticsearchSynchronize implements IElasticsearchSynchronize {\n    private readonly timer: ITimer;\n    private readonly context: Context;\n\n    public constructor(params: IElasticsearchSynchronizeParams) {\n        this.timer = params.timer;\n        this.context = params.context;\n    }\n\n    public async execute(\n        params: IElasticsearchSynchronizeExecuteParams\n    ): Promise<IElasticsearchSynchronizeExecuteResponse> {\n        const { items, done, index } = params;\n        if (items.length === 0) {\n            return {\n                done: true\n            };\n        }\n\n        const table = getTable({\n            type: \"es\",\n            context: this.context\n        });\n\n        const readableItems = items.map(item => {\n            const entity = this.getEntity(item);\n            return entity.item.getBatch({\n                PK: item.PK,\n                SK: item.SK\n            });\n        });\n\n        const tableItems = await batchReadAll<IDynamoDbItem>({\n            items: readableItems,\n            table\n        });\n\n        const elasticsearchSyncBuilder = createSynchronizationBuilder({\n            timer: this.timer,\n            context: this.context\n        });\n        /**\n         * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table.\n         */\n        for (const item of items) {\n            const exists = tableItems.some(ddbItem => {\n                return ddbItem.PK === item.PK && ddbItem.SK === item.SK;\n            });\n            if (exists) {\n                continue;\n            }\n            elasticsearchSyncBuilder.delete({\n                index,\n                id: item._id\n            });\n        }\n\n        const executeWithRetry = elasticsearchSyncBuilder.build();\n        await executeWithRetry();\n\n        return {\n            done\n        };\n    }\n\n    private getEntity(\n        params: IGetElasticsearchEntityTypeParams\n    ): ReturnType<typeof getElasticsearchEntity> {\n        const type = getElasticsearchEntityType(params);\n        return getElasticsearchEntity({\n            type,\n            context: this.context\n        });\n    }\n}\n"],"mappings":";;;;;;AAAA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,2BAAA,GAAAD,OAAA;AAEA,IAAAE,SAAA,GAAAF,OAAA;AAuBO,MAAMG,wBAAwB,CAAsC;EAIhEC,WAAWA,CAACC,MAAuC,EAAE;IACxD,IAAI,CAACC,KAAK,GAAGD,MAAM,CAACC,KAAK;IACzB,IAAI,CAACC,OAAO,GAAGF,MAAM,CAACE,OAAO;EACjC;EAEA,MAAaC,OAAOA,CAChBH,MAA8C,EACG;IACjD,MAAM;MAAEI,KAAK;MAAEC,IAAI;MAAEC;IAAM,CAAC,GAAGN,MAAM;IACrC,IAAII,KAAK,CAACG,MAAM,KAAK,CAAC,EAAE;MACpB,OAAO;QACHF,IAAI,EAAE;MACV,CAAC;IACL;IAEA,MAAMG,KAAK,GAAG,IAAAC,kBAAQ,EAAC;MACnBC,IAAI,EAAE,IAAI;MACVR,OAAO,EAAE,IAAI,CAACA;IAClB,CAAC,CAAC;IAEF,MAAMS,aAAa,GAAGP,KAAK,CAACQ,GAAG,CAACC,IAAI,IAAI;MACpC,MAAMC,MAAM,GAAG,IAAI,CAACC,SAAS,CAACF,IAAI,CAAC;MACnC,OAAOC,MAAM,CAACD,IAAI,CAACG,QAAQ,CAAC;QACxBC,EAAE,EAAEJ,IAAI,CAACI,EAAE;QACXC,EAAE,EAAEL,IAAI,CAACK;MACb,CAAC,CAAC;IACN,CAAC,CAAC;IAEF,MAAMC,UAAU,GAAG,MAAM,IAAAC,wBAAY,EAAgB;MACjDhB,KAAK,EAAEO,aAAa;MACpBH;IACJ,CAAC,CAAC;IAEF,MAAMa,wBAAwB,GAAG,IAAAC,wDAA4B,EAAC;MAC1DrB,KAAK,EAAE,IAAI,CAACA,KAAK;MACjBC,OAAO,EAAE,IAAI,CAACA;IAClB,CAAC,CAAC;IACF;AACR;AACA;IACQ,KAAK,MAAMW,IAAI,IAAIT,KAAK,EAAE;MACtB,MAAMmB,MAAM,GAAGJ,UAAU,CAACK,IAAI,CAACC,OAAO,IAAI;QACtC,OAAOA,OAAO,CAACR,EAAE,KAAKJ,IAAI,CAACI,EAAE,IAAIQ,OAAO,CAACP,EAAE,KAAKL,IAAI,CAACK,EAAE;MAC3D,CAAC,CAAC;MACF,IAAIK,MAAM,EAAE;QACR;MACJ;MACAF,wBAAwB,CAACK,MAAM,CAAC;QAC5BpB,KAAK;QACLqB,EAAE,EAAEd,IAAI,CAACe;MACb,CAAC,CAAC;IACN;IAEA,MAAMC,gBAAgB,GAAGR,wBAAwB,CAACS,KAAK,CAAC,CAAC;IACzD,MAAMD,gBAAgB,CAAC,CAAC;IAExB,OAAO;MACHxB;IACJ,CAAC;EACL;EAEQU,SAASA,CACbf,MAAyC,EACA;IACzC,MAAMU,IAAI,GAAG,IAAAqB,oCAA0B,EAAC/B,MAAM,CAAC;IAC/C,OAAO,IAAAgC,gCAAsB,EAAC;MAC1BtB,IAAI;MACJR,OAAO,EAAE,IAAI,CAACA;IAClB,CAAC,CAAC;EACN;AACJ;AAAC+B,OAAA,CAAAnC,wBAAA,GAAAA,wBAAA","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.d.ts

@@ -0,0 +1,10 @@
+import type { IDataSynchronizationInput, IElasticsearchSyncParams, ISynchronization, ISynchronizationRunResult } from "../types";
+export declare class ElasticsearchToDynamoDbSynchronization implements ISynchronization {
+    private readonly manager;
+    private readonly indexManager;
+    private readonly synchronize;
+    private readonly fetcher;
+    constructor(params: IElasticsearchSyncParams);
+    run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult>;
+    private fetchAllIndexes;
+}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js

@@ -0,0 +1,77 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ElasticsearchToDynamoDbSynchronization = void 0;
+class ElasticsearchToDynamoDbSynchronization {
+  constructor(params) {
+    this.manager = params.manager;
+    this.indexManager = params.indexManager;
+    this.synchronize = params.synchronize;
+    this.fetcher = params.fetcher;
+  }
+  async run(input) {
+    const lastIndex = input.elasticsearchToDynamoDb?.index;
+    let cursor = input.elasticsearchToDynamoDb?.cursor;
+    const indexes = await this.fetchAllIndexes();
+    let next = 0;
+    if (lastIndex) {
+      next = indexes.findIndex(index => index === lastIndex);
+    }
+    let currentIndex = indexes[next];
+    while (currentIndex) {
+      if (this.manager.isAborted()) {
+        return this.manager.response.aborted();
+      }
+      /**
+       * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly.
+       * We want to leave enough time for possible retries.
+       */
+      //
+      else if (this.manager.isCloseToTimeout(180)) {
+        return this.manager.response.continue({
+          ...input,
+          elasticsearchToDynamoDb: {
+            ...input.elasticsearchToDynamoDb,
+            index: currentIndex,
+            cursor
+          }
+        });
+      }
+      const result = await this.fetcher.fetch({
+        index: currentIndex,
+        cursor,
+        limit: 100
+      });
+      const syncResult = await this.synchronize.execute({
+        done: result.done,
+        index: currentIndex,
+        items: result.items
+      });
+      if (!syncResult.done && result.cursor) {
+        cursor = result.cursor;
+        continue;
+      }
+      cursor = undefined;
+      const next = indexes.findIndex(index => index === currentIndex) + 1;
+      currentIndex = indexes[next];
+    }
+    return this.manager.response.continue({
+      ...input,
+      elasticsearchToDynamoDb: {
+        finished: true
+      }
+    });
+  }
+  async fetchAllIndexes() {
+    const result = await this.indexManager.list();
+    if (result.length > 0) {
+      return result;
+    }
+    throw new Error("No Elasticsearch / OpenSearch indexes found.");
+  }
+}
+exports.ElasticsearchToDynamoDbSynchronization = ElasticsearchToDynamoDbSynchronization;
+
+//# sourceMappingURL=ElasticsearchToDynamoDbSynchronization.js.map