@webiny/api-elasticsearch-tasks 6.0.0-beta.0 → 6.0.0-rc.0

package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.d.ts

@@ -0,0 +1,10 @@
+import type { Client } from "@webiny/api-elasticsearch";
+import type { IElasticsearchFetcher, IElasticsearchFetcherFetchParams, IElasticsearchFetcherFetchResponse } from "./abstractions/ElasticsearchFetcher.js";
+export interface IElasticsearchFetcherParams {
+    client: Client;
+}
+export declare class ElasticsearchFetcher implements IElasticsearchFetcher {
+    private readonly client;
+    constructor(params: IElasticsearchFetcherParams);
+    fetch({ index, cursor, limit }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;
+}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js

@@ -0,0 +1,91 @@
+import { shouldIgnoreEsResponseError } from "./shouldIgnoreEsResponseError.js";
+import { inspect } from "node:util";
+export class ElasticsearchFetcher {
+  constructor(params) {
+    this.client = params.client;
+  }
+  async fetch({
+    index,
+    cursor,
+    limit
+  }) {
+    let response;
+    try {
+      response = await this.client.search({
+        index,
+        body: {
+          query: {
+            match_all: {}
+          },
+          sort: {
+            "id.keyword": {
+              order: "asc"
+            }
+          },
+          size: limit + 1,
+          track_total_hits: true,
+          search_after: cursor,
+          _source: false
+        }
+      });
+    } catch (ex) {
+      /**
+       * If we ignore the error, we can continue with the next index.
+       */
+      if (shouldIgnoreEsResponseError(ex)) {
+        if (process.env.DEBUG === "true") {
+          console.error(inspect(ex, {
+            depth: 5,
+            showHidden: true
+          }));
+        }
+        return {
+          done: true,
+          totalCount: 0,
+          items: []
+        };
+      }
+      console.error("Failed to fetch data from Elasticsearch.", ex);
+      throw ex;
+    }
+    const {
+      hits,
+      total
+    } = response.body.hits;
+    if (hits.length === 0) {
+      return {
+        done: true,
+        cursor: undefined,
+        totalCount: total.value,
+        items: []
+      };
+    }
+    const hasMoreItems = hits.length > limit;
+    let nextCursor;
+    if (hasMoreItems) {
+      hits.pop();
+      nextCursor = hits.at(-1)?.sort;
+    }
+    const items = hits.reduce((collection, hit) => {
+      const [PK, SK] = hit._id.split(":");
+      if (!PK || !SK) {
+        return collection;
+      }
+      collection.push({
+        PK,
+        SK,
+        _id: hit._id,
+        index: hit._index
+      });
+      return collection;
+    }, []);
+    return {
+      totalCount: total.value,
+      cursor: nextCursor,
+      done: !nextCursor,
+      items
+    };
+  }
+}
+
+//# sourceMappingURL=ElasticsearchFetcher.js.map

package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["shouldIgnoreEsResponseError","inspect","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","process","env","DEBUG","console","error","depth","showHidden","done","totalCount","items","hits","total","length","undefined","value","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-elasticsearch\";\nimport type {\n    IElasticsearchFetcher,\n    IElasticsearchFetcherFetchParams,\n    IElasticsearchFetcherFetchResponse,\n    IElasticsearchFetcherFetchResponseItem\n} from \"./abstractions/ElasticsearchFetcher.js\";\nimport type {\n    ElasticsearchSearchResponse,\n    PrimitiveValue\n} from \"@webiny/api-elasticsearch/types.js\";\nimport { shouldIgnoreEsResponseError } from \"./shouldIgnoreEsResponseError.js\";\nimport { inspect } from \"node:util\";\n\nexport interface IElasticsearchFetcherParams {\n    client: Client;\n}\n\nexport class ElasticsearchFetcher implements IElasticsearchFetcher {\n    private readonly client: Client;\n\n    public constructor(params: IElasticsearchFetcherParams) {\n        this.client = params.client;\n    }\n    public async fetch({\n        index,\n        cursor,\n        limit\n    }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse> {\n        let response: ElasticsearchSearchResponse<undefined>;\n        try {\n            response = await this.client.search({\n                index,\n                body: {\n                    query: {\n                        match_all: {}\n                    },\n                    sort: {\n                        \"id.keyword\": {\n                            order: \"asc\"\n                        }\n                    },\n                    size: limit + 1,\n                    track_total_hits: true,\n                    search_after: cursor,\n                    _source: false\n                }\n            });\n        } catch (ex) {\n            /**\n             * If we ignore the error, we can continue with the next index.\n             */\n            if (shouldIgnoreEsResponseError(ex)) {\n                if (process.env.DEBUG === \"true\") {\n                    console.error(\n                        inspect(ex, {\n                            depth: 5,\n                            showHidden: true\n                        })\n                    );\n                }\n                return {\n                    done: true,\n                    totalCount: 0,\n                    items: []\n                };\n            }\n            console.error(\"Failed to fetch data from Elasticsearch.\", ex);\n            throw ex;\n        }\n\n        const { hits, total } = response.body.hits;\n        if (hits.length === 0) {\n            return {\n                done: true,\n                cursor: undefined,\n                totalCount: total.value,\n                items: []\n            };\n        }\n\n        const hasMoreItems = hits.length > limit;\n        let nextCursor: PrimitiveValue[] | undefined;\n        if (hasMoreItems) {\n            hits.pop();\n            nextCursor = hits.at(-1)?.sort;\n        }\n        const items = hits.reduce<IElasticsearchFetcherFetchResponseItem[]>((collection, hit) => {\n            const [PK, SK] = hit._id.split(\":\");\n            if (!PK || !SK) {\n                return collection;\n            }\n            collection.push({\n                PK,\n                SK,\n                _id: hit._id,\n                index: hit._index\n            });\n\n            return collection;\n        }, []);\n\n        return {\n            totalCount: total.value,\n            cursor: nextCursor,\n            done: !nextCursor,\n            items\n        };\n    }\n}\n"],"mappings":"AAWA,SAASA,2BAA2B;AACpC,SAASC,OAAO,QAAQ,WAAW;AAMnC,OAAO,MAAMC,oBAAoB,CAAkC;EAGxDC,WAAWA,CAACC,MAAmC,EAAE;IACpD,IAAI,CAACC,MAAM,GAAGD,MAAM,CAACC,MAAM;EAC/B;EACA,MAAaC,KAAKA,CAAC;IACfC,KAAK;IACLC,MAAM;IACNC;EAC8B,CAAC,EAA+C;IAC9E,IAAIC,QAAgD;IACpD,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACL,MAAM,CAACM,MAAM,CAAC;QAChCJ,KAAK;QACLK,IAAI,EAAE;UACFC,KAAK,EAAE;YACHC,SAAS,EAAE,CAAC;UAChB,CAAC;UACDC,IAAI,EAAE;YACF,YAAY,EAAE;cACVC,KAAK,EAAE;YACX;UACJ,CAAC;UACDC,IAAI,EAAER,KAAK,GAAG,CAAC;UACfS,gBAAgB,EAAE,IAAI;UACtBC,YAAY,EAAEX,MAAM;UACpBY,OAAO,EAAE;QACb;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAIrB,2BAA2B,CAACqB,EAAE,CAAC,EAAE;QACjC,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9BC,OAAO,CAACC,KAAK,CACTzB,OAAO,CAACoB,EAAE,EAAE;YACRM,KAAK,EAAE,CAAC;YACRC,UAAU,EAAE;UAChB,CAAC,CACL,CAAC;QACL;QACA,OAAO;UACHC,IAAI,EAAE,IAAI;UACVC,UAAU,EAAE,CAAC;UACbC,KAAK,EAAE;QACX,CAAC;MACL;MACAN,OAAO,CAACC,KAAK,CAAC,0CAA0C,EAAEL,EAAE,CAAC;MAC7D,MAAMA,EAAE;IACZ;IAEA,MAAM;MAAEW,IAAI;MAAEC;IAAM,CAAC,GAAGvB,QAAQ,CAACE,IAAI,CAACoB,IAAI;IAC1C,IAAIA,IAAI,CAACE,MAAM,KAAK,CAAC,EAAE;MACnB,OAAO;QACHL,IAAI,EAAE,IAAI;QACVrB,MAAM,EAAE2B,SAAS;QACjBL,UAAU,EAAEG,KAAK,CAACG,KAAK;QACvBL,KAAK,EAAE;MACX,CAAC;IACL;IAEA,MAAMM,YAAY,GAAGL,IAAI,CAACE,MAAM,GAAGzB,KAAK;IACxC,IAAI6B,UAAwC;IAC5C,IAAID,YAAY,EAAE;MACdL,IAAI,CAACO,GAAG,CAAC,CAAC;MACVD,UAAU,GAAGN,IAAI,CAACQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAEzB,IAAI;IAClC;IACA,MAAMgB,KAAK,GAAGC,IAAI,CAACS,MAAM,CAA2C,CAACC,UAAU,EAAEC,GAAG,KAAK;MACrF,MAAM,CAACC,EAAE,EAAEC,EAAE,CAAC,GAAGF,GAAG,CAACG,GAAG,CAACC,KAAK,CAAC,GAAG,CAAC;MACnC,IAAI,CAACH,EAAE,IAAI,CAACC,EAAE,EAAE;QACZ,OAAOH,UAAU;MACrB;MACAA,UAAU,CAACM,IAAI,CAAC;QACZJ,EAAE;QACFC,EAAE;QACFC,GAAG,EAAEH,GAAG,CAACG,GAAG;QACZvC,KAAK,EAAEoC,GAAG,CAACM;MACf,CAAC,CAAC;MAEF,OAAOP,UAAU;IACrB,CAAC,EAAE,EAAE,CAAC;IAEN,OAAO;MACHZ,UAAU,EAAEG,KAAK,CAACG,KAAK;MACvB5B,MAAM,EAAE8B,UAAU;MAClBT,IAAI,EAAE,CAACS,UAAU;MACjBP;IACJ,CAAC;EACL;AACJ","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.d.ts

@@ -0,0 +1,12 @@
+import type { TaskController } from "@webiny/api-core/features/task/TaskController/index.js";
+import type { IElasticsearchSynchronize, IElasticsearchSynchronizeExecuteParams, IElasticsearchSynchronizeExecuteResponse } from "./abstractions/ElasticsearchSynchronize.js";
+import { SynchronizationContext } from "../../../abstractions/SynchronizationContext.js";
+import { DbRegistry } from "../../../abstractions/DbRegistry.js";
+export declare class ElasticsearchSynchronize implements IElasticsearchSynchronize {
+    private controller;
+    private dbRegistry;
+    private context;
+    constructor(controller: TaskController.Interface, dbRegistry: DbRegistry.Interface, context: SynchronizationContext.Interface);
+    execute(params: IElasticsearchSynchronizeExecuteParams): Promise<IElasticsearchSynchronizeExecuteResponse>;
+    private getEntity;
+}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js

@@ -0,0 +1,70 @@
+import { batchReadAll } from "@webiny/db-dynamodb/utils/batch/batchRead.js";
+import { createSynchronizationBuilder } from "@webiny/api-dynamodb-to-elasticsearch";
+import { getElasticsearchEntity, getElasticsearchEntityType, getTable } from "../entities/index.js";
+export class ElasticsearchSynchronize {
+  constructor(controller, dbRegistry, context) {
+    this.controller = controller;
+    this.dbRegistry = dbRegistry;
+    this.context = context;
+  }
+  async execute(params) {
+    const {
+      items,
+      done,
+      index
+    } = params;
+    if (items.length === 0) {
+      return {
+        done: true
+      };
+    }
+    const table = getTable({
+      type: "es",
+      dbRegistry: this.dbRegistry
+    });
+    const readableItems = items.map(item => {
+      const entity = this.getEntity(item);
+      return entity.item.entity.getBatch({
+        PK: item.PK,
+        SK: item.SK
+      });
+    });
+    const tableItems = await batchReadAll({
+      items: readableItems,
+      table
+    });
+    const elasticsearchSyncBuilder = createSynchronizationBuilder({
+      context: this.context,
+      timer: this.controller.runtime
+    });
+    /**
+     * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table.
+     */
+    for (const item of items) {
+      const exists = tableItems.some(ddbItem => {
+        return ddbItem.PK === item.PK && ddbItem.SK === item.SK;
+      });
+      if (exists) {
+        continue;
+      }
+      elasticsearchSyncBuilder.delete({
+        index,
+        id: item._id
+      });
+    }
+    const executeWithRetry = elasticsearchSyncBuilder.build();
+    await executeWithRetry();
+    return {
+      done
+    };
+  }
+  getEntity(params) {
+    const type = getElasticsearchEntityType(params);
+    return getElasticsearchEntity({
+      type,
+      dbRegistry: this.dbRegistry
+    });
+  }
+}
+
+//# sourceMappingURL=ElasticsearchSynchronize.js.map

package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["batchReadAll","createSynchronizationBuilder","getElasticsearchEntity","getElasticsearchEntityType","getTable","ElasticsearchSynchronize","constructor","controller","dbRegistry","context","execute","params","items","done","index","length","table","type","readableItems","map","item","entity","getEntity","getBatch","PK","SK","tableItems","elasticsearchSyncBuilder","timer","runtime","exists","some","ddbItem","delete","id","_id","executeWithRetry","build"],"sources":["ElasticsearchSynchronize.ts"],"sourcesContent":["import { batchReadAll } from \"@webiny/db-dynamodb/utils/batch/batchRead.js\";\nimport { createSynchronizationBuilder } from \"@webiny/api-dynamodb-to-elasticsearch\";\nimport type { IGetElasticsearchEntityTypeParams } from \"~/tasks/dataSynchronization/entities/index.js\";\nimport {\n    getElasticsearchEntity,\n    getElasticsearchEntityType,\n    getTable\n} from \"~/tasks/dataSynchronization/entities/index.js\";\nimport type { TaskController } from \"@webiny/api-core/features/task/TaskController/index.js\";\nimport type {\n    IElasticsearchSynchronize,\n    IElasticsearchSynchronizeExecuteParams,\n    IElasticsearchSynchronizeExecuteResponse\n} from \"./abstractions/ElasticsearchSynchronize.js\";\nimport { SynchronizationContext } from \"~/abstractions/SynchronizationContext.js\";\nimport { DbRegistry } from \"~/abstractions/DbRegistry.js\";\n\ninterface IDynamoDbItem {\n    PK: string;\n    SK: string;\n}\n\nexport class ElasticsearchSynchronize implements IElasticsearchSynchronize {\n    public constructor(\n        private controller: TaskController.Interface,\n        private dbRegistry: DbRegistry.Interface,\n        private context: SynchronizationContext.Interface\n    ) {}\n\n    public async execute(\n        params: IElasticsearchSynchronizeExecuteParams\n    ): Promise<IElasticsearchSynchronizeExecuteResponse> {\n        const { items, done, index } = params;\n        if (items.length === 0) {\n            return {\n                done: true\n            };\n        }\n\n        const table = getTable({\n            type: \"es\",\n            dbRegistry: this.dbRegistry\n        });\n\n        const readableItems = items.map(item => {\n            const entity = this.getEntity(item);\n            return entity.item.entity.getBatch({\n                PK: item.PK,\n                SK: item.SK\n            });\n        });\n\n        const tableItems = await batchReadAll<IDynamoDbItem>({\n            items: readableItems,\n            table\n        });\n\n        const elasticsearchSyncBuilder = createSynchronizationBuilder({\n            context: this.context,\n            timer: this.controller.runtime\n        });\n        /**\n         * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table.\n         */\n        for (const item of items) {\n            const exists = tableItems.some(ddbItem => {\n                return ddbItem.PK === item.PK && ddbItem.SK === item.SK;\n            });\n            if (exists) {\n                continue;\n            }\n            elasticsearchSyncBuilder.delete({\n                index,\n                id: item._id\n            });\n        }\n\n        const executeWithRetry = elasticsearchSyncBuilder.build();\n        await executeWithRetry();\n\n        return {\n            done\n        };\n    }\n\n    private getEntity(\n        params: IGetElasticsearchEntityTypeParams\n    ): ReturnType<typeof getElasticsearchEntity> {\n        const type = getElasticsearchEntityType(params);\n        return getElasticsearchEntity({\n            type,\n            dbRegistry: this.dbRegistry\n        });\n    }\n}\n"],"mappings":"AAAA,SAASA,YAAY,QAAQ,8CAA8C;AAC3E,SAASC,4BAA4B,QAAQ,uCAAuC;AAEpF,SACIC,sBAAsB,EACtBC,0BAA0B,EAC1BC,QAAQ;AAgBZ,OAAO,MAAMC,wBAAwB,CAAsC;EAChEC,WAAWA,CACNC,UAAoC,EACpCC,UAAgC,EAChCC,OAAyC,EACnD;IAAA,KAHUF,UAAoC,GAApCA,UAAoC;IAAA,KACpCC,UAAgC,GAAhCA,UAAgC;IAAA,KAChCC,OAAyC,GAAzCA,OAAyC;EAClD;EAEH,MAAaC,OAAOA,CAChBC,MAA8C,EACG;IACjD,MAAM;MAAEC,KAAK;MAAEC,IAAI;MAAEC;IAAM,CAAC,GAAGH,MAAM;IACrC,IAAIC,KAAK,CAACG,MAAM,KAAK,CAAC,EAAE;MACpB,OAAO;QACHF,IAAI,EAAE;MACV,CAAC;IACL;IAEA,MAAMG,KAAK,GAAGZ,QAAQ,CAAC;MACnBa,IAAI,EAAE,IAAI;MACVT,UAAU,EAAE,IAAI,CAACA;IACrB,CAAC,CAAC;IAEF,MAAMU,aAAa,GAAGN,KAAK,CAACO,GAAG,CAACC,IAAI,IAAI;MACpC,MAAMC,MAAM,GAAG,IAAI,CAACC,SAAS,CAACF,IAAI,CAAC;MACnC,OAAOC,MAAM,CAACD,IAAI,CAACC,MAAM,CAACE,QAAQ,CAAC;QAC/BC,EAAE,EAAEJ,IAAI,CAACI,EAAE;QACXC,EAAE,EAAEL,IAAI,CAACK;MACb,CAAC,CAAC;IACN,CAAC,CAAC;IAEF,MAAMC,UAAU,GAAG,MAAM1B,YAAY,CAAgB;MACjDY,KAAK,EAAEM,aAAa;MACpBF;IACJ,CAAC,CAAC;IAEF,MAAMW,wBAAwB,GAAG1B,4BAA4B,CAAC;MAC1DQ,OAAO,EAAE,IAAI,CAACA,OAAO;MACrBmB,KAAK,EAAE,IAAI,CAACrB,UAAU,CAACsB;IAC3B,CAAC,CAAC;IACF;AACR;AACA;IACQ,KAAK,MAAMT,IAAI,IAAIR,KAAK,EAAE;MACtB,MAAMkB,MAAM,GAAGJ,UAAU,CAACK,IAAI,CAACC,OAAO,IAAI;QACtC,OAAOA,OAAO,CAACR,EAAE,KAAKJ,IAAI,CAACI,EAAE,IAAIQ,OAAO,CAACP,EAAE,KAAKL,IAAI,CAACK,EAAE;MAC3D,CAAC,CAAC;MACF,IAAIK,MAAM,EAAE;QACR;MACJ;MACAH,wBAAwB,CAACM,MAAM,CAAC;QAC5BnB,KAAK;QACLoB,EAAE,EAAEd,IAAI,CAACe;MACb,CAAC,CAAC;IACN;IAEA,MAAMC,gBAAgB,GAAGT,wBAAwB,CAACU,KAAK,CAAC,CAAC;IACzD,MAAMD,gBAAgB,CAAC,CAAC;IAExB,OAAO;MACHvB;IACJ,CAAC;EACL;EAEQS,SAASA,CACbX,MAAyC,EACA;IACzC,MAAMM,IAAI,GAAGd,0BAA0B,CAACQ,MAAM,CAAC;IAC/C,OAAOT,sBAAsB,CAAC;MAC1Be,IAAI;MACJT,UAAU,EAAE,IAAI,CAACA;IACrB,CAAC,CAAC;EACN;AACJ","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.d.ts

@@ -0,0 +1,10 @@
+import type { IDataSynchronizationInput, IElasticsearchSyncParams, ISynchronization, ISynchronizationRunResult } from "../types.js";
+export declare class ElasticsearchToDynamoDbSynchronization implements ISynchronization {
+    private readonly manager;
+    private readonly indexManager;
+    private readonly synchronize;
+    private readonly fetcher;
+    constructor(params: IElasticsearchSyncParams);
+    run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult>;
+    private fetchAllIndexes;
+}

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js

@@ -0,0 +1,70 @@
+export class ElasticsearchToDynamoDbSynchronization {
+  constructor(params) {
+    this.manager = params.manager;
+    this.indexManager = params.indexManager;
+    this.synchronize = params.synchronize;
+    this.fetcher = params.fetcher;
+  }
+  async run(input) {
+    const lastIndex = input.elasticsearchToDynamoDb?.index;
+    let cursor = input.elasticsearchToDynamoDb?.cursor;
+    const indexes = await this.fetchAllIndexes();
+    let next = 0;
+    if (lastIndex) {
+      next = indexes.findIndex(index => index === lastIndex);
+    }
+    let currentIndex = indexes[next];
+    while (currentIndex) {
+      if (this.manager.controller.runtime.isAborted()) {
+        return this.manager.controller.response.aborted();
+      }
+      /**
+       * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly.
+       * We want to leave enough time for possible retries.
+       */
+      //
+      else if (this.manager.controller.runtime.isCloseToTimeout(180)) {
+        return this.manager.controller.response.continue({
+          ...input,
+          elasticsearchToDynamoDb: {
+            ...input.elasticsearchToDynamoDb,
+            index: currentIndex,
+            cursor
+          }
+        });
+      }
+      const result = await this.fetcher.fetch({
+        index: currentIndex,
+        cursor,
+        limit: 100
+      });
+      const syncResult = await this.synchronize.execute({
+        done: result.done,
+        index: currentIndex,
+        items: result.items
+      });
+      if (!syncResult.done && result.cursor) {
+        cursor = result.cursor;
+        continue;
+      }
+      cursor = undefined;
+      const next = indexes.findIndex(index => index === currentIndex) + 1;
+      currentIndex = indexes[next];
+    }
+    return this.manager.controller.response.continue({
+      ...input,
+      elasticsearchToDynamoDb: {
+        finished: true
+      }
+    });
+  }
+  async fetchAllIndexes() {
+    const result = await this.indexManager.list();
+    if (result.length > 0) {
+      return result;
+    }
+    throw new Error("No Elasticsearch / OpenSearch indexes found.");
+  }
+}
+
+//# sourceMappingURL=ElasticsearchToDynamoDbSynchronization.js.map

package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["ElasticsearchToDynamoDbSynchronization","constructor","params","manager","indexManager","synchronize","fetcher","run","input","lastIndex","elasticsearchToDynamoDb","index","cursor","indexes","fetchAllIndexes","next","findIndex","currentIndex","controller","runtime","isAborted","response","aborted","isCloseToTimeout","continue","result","fetch","limit","syncResult","execute","done","items","undefined","finished","list","length","Error"],"sources":["ElasticsearchToDynamoDbSynchronization.ts"],"sourcesContent":["import type {\n    IDataSynchronizationInput,\n    IDataSynchronizationManager,\n    IElasticsearchSyncParams,\n    ISynchronization,\n    ISynchronizationRunResult\n} from \"../types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { NonEmptyArray } from \"@webiny/api/types.js\";\nimport type { IElasticsearchSynchronize } from \"./abstractions/ElasticsearchSynchronize.js\";\nimport type { IElasticsearchFetcher } from \"./abstractions/ElasticsearchFetcher.js\";\n\nexport class ElasticsearchToDynamoDbSynchronization implements ISynchronization {\n    private readonly manager: IDataSynchronizationManager;\n    private readonly indexManager: IIndexManager;\n    private readonly synchronize: IElasticsearchSynchronize;\n    private readonly fetcher: IElasticsearchFetcher;\n\n    public constructor(params: IElasticsearchSyncParams) {\n        this.manager = params.manager;\n        this.indexManager = params.indexManager;\n        this.synchronize = params.synchronize;\n        this.fetcher = params.fetcher;\n    }\n\n    public async run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult> {\n        const lastIndex = input.elasticsearchToDynamoDb?.index;\n        let cursor = input.elasticsearchToDynamoDb?.cursor;\n        const indexes = await this.fetchAllIndexes();\n\n        let next = 0;\n        if (lastIndex) {\n            next = indexes.findIndex(index => index === lastIndex);\n        }\n\n        let currentIndex = indexes[next];\n\n        while (currentIndex) {\n            if (this.manager.controller.runtime.isAborted()) {\n                return this.manager.controller.response.aborted();\n            }\n            /**\n             * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly.\n             * We want to leave enough time for possible retries.\n             */\n            //\n            else if (this.manager.controller.runtime.isCloseToTimeout(180)) {\n                return this.manager.controller.response.continue({\n                    ...input,\n                    elasticsearchToDynamoDb: {\n                        ...input.elasticsearchToDynamoDb,\n                        index: currentIndex,\n                        cursor\n                    }\n                });\n            }\n\n            const result = await this.fetcher.fetch({\n                index: currentIndex,\n                cursor,\n                limit: 100\n            });\n\n            const syncResult = await this.synchronize.execute({\n                done: result.done,\n                index: currentIndex,\n                items: result.items\n            });\n\n            if (!syncResult.done && result.cursor) {\n                cursor = result.cursor;\n                continue;\n            }\n            cursor = undefined;\n\n            const next = indexes.findIndex(index => index === currentIndex) + 1;\n            currentIndex = indexes[next];\n        }\n\n        return this.manager.controller.response.continue({\n            ...input,\n            elasticsearchToDynamoDb: {\n                finished: true\n            }\n        });\n    }\n\n    private async fetchAllIndexes(): Promise<NonEmptyArray<string>> {\n        const result = await this.indexManager.list();\n        if (result.length > 0) {\n            return result as NonEmptyArray<string>;\n        }\n        throw new Error(\"No Elasticsearch / OpenSearch indexes found.\");\n    }\n}\n"],"mappings":"AAYA,OAAO,MAAMA,sCAAsC,CAA6B;EAMrEC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO;IAC7B,IAAI,CAACC,YAAY,GAAGF,MAAM,CAACE,YAAY;IACvC,IAAI,CAACC,WAAW,GAAGH,MAAM,CAACG,WAAW;IACrC,IAAI,CAACC,OAAO,GAAGJ,MAAM,CAACI,OAAO;EACjC;EAEA,MAAaC,GAAGA,CAACC,KAAgC,EAAsC;IACnF,MAAMC,SAAS,GAAGD,KAAK,CAACE,uBAAuB,EAAEC,KAAK;IACtD,IAAIC,MAAM,GAAGJ,KAAK,CAACE,uBAAuB,EAAEE,MAAM;IAClD,MAAMC,OAAO,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAE5C,IAAIC,IAAI,GAAG,CAAC;IACZ,IAAIN,SAAS,EAAE;MACXM,IAAI,GAAGF,OAAO,CAACG,SAAS,CAACL,KAAK,IAAIA,KAAK,KAAKF,SAAS,CAAC;IAC1D;IAEA,IAAIQ,YAAY,GAAGJ,OAAO,CAACE,IAAI,CAAC;IAEhC,OAAOE,YAAY,EAAE;MACjB,IAAI,IAAI,CAACd,OAAO,CAACe,UAAU,CAACC,OAAO,CAACC,SAAS,CAAC,CAAC,EAAE;QAC7C,OAAO,IAAI,CAACjB,OAAO,CAACe,UAAU,CAACG,QAAQ,CAACC,OAAO,CAAC,CAAC;MACrD;MACA;AACZ;AACA;AACA;MACY;MAAA,KACK,IAAI,IAAI,CAACnB,OAAO,CAACe,UAAU,CAACC,OAAO,CAACI,gBAAgB,CAAC,GAAG,CAAC,EAAE;QAC5D,OAAO,IAAI,CAACpB,OAAO,CAACe,UAAU,CAACG,QAAQ,CAACG,QAAQ,CAAC;UAC7C,GAAGhB,KAAK;UACRE,uBAAuB,EAAE;YACrB,GAAGF,KAAK,CAACE,uBAAuB;YAChCC,KAAK,EAAEM,YAAY;YACnBL;UACJ;QACJ,CAAC,CAAC;MACN;MAEA,MAAMa,MAAM,GAAG,MAAM,IAAI,CAACnB,OAAO,CAACoB,KAAK,CAAC;QACpCf,KAAK,EAAEM,YAAY;QACnBL,MAAM;QACNe,KAAK,EAAE;MACX,CAAC,CAAC;MAEF,MAAMC,UAAU,GAAG,MAAM,IAAI,CAACvB,WAAW,CAACwB,OAAO,CAAC;QAC9CC,IAAI,EAAEL,MAAM,CAACK,IAAI;QACjBnB,KAAK,EAAEM,YAAY;QACnBc,KAAK,EAAEN,MAAM,CAACM;MAClB,CAAC,CAAC;MAEF,IAAI,CAACH,UAAU,CAACE,IAAI,IAAIL,MAAM,CAACb,MAAM,EAAE;QACnCA,MAAM,GAAGa,MAAM,CAACb,MAAM;QACtB;MACJ;MACAA,MAAM,GAAGoB,SAAS;MAElB,MAAMjB,IAAI,GAAGF,OAAO,CAACG,SAAS,CAACL,KAAK,IAAIA,KAAK,KAAKM,YAAY,CAAC,GAAG,CAAC;MACnEA,YAAY,GAAGJ,OAAO,CAACE,IAAI,CAAC;IAChC;IAEA,OAAO,IAAI,CAACZ,OAAO,CAACe,UAAU,CAACG,QAAQ,CAACG,QAAQ,CAAC;MAC7C,GAAGhB,KAAK;MACRE,uBAAuB,EAAE;QACrBuB,QAAQ,EAAE;MACd;IACJ,CAAC,CAAC;EACN;EAEA,MAAcnB,eAAeA,CAAA,EAAmC;IAC5D,MAAMW,MAAM,GAAG,MAAM,IAAI,CAACrB,YAAY,CAAC8B,IAAI,CAAC,CAAC;IAC7C,IAAIT,MAAM,CAACU,MAAM,GAAG,CAAC,EAAE;MACnB,OAAOV,MAAM;IACjB;IACA,MAAM,IAAIW,KAAK,CAAC,8CAA8C,CAAC;EACnE;AACJ","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.d.ts

@@ -0,0 +1,21 @@
+import type { PrimitiveValue } from "@webiny/api-elasticsearch/types.js";
+export interface IElasticsearchFetcherFetchResponseItem {
+    PK: string;
+    SK: string;
+    _id: string;
+    index: string;
+}
+export interface IElasticsearchFetcherFetchParams {
+    index: string;
+    cursor?: PrimitiveValue[];
+    limit: number;
+}
+export interface IElasticsearchFetcherFetchResponse {
+    done: boolean;
+    totalCount: number;
+    cursor?: PrimitiveValue[];
+    items: IElasticsearchFetcherFetchResponseItem[];
+}
+export interface IElasticsearchFetcher {
+    fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;
+}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js

@@ -0,0 +1,3 @@
+export {};
+
+//# sourceMappingURL=ElasticsearchFetcher.js.map

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { PrimitiveValue } from \"@webiny/api-elasticsearch/types.js\";\n\nexport interface IElasticsearchFetcherFetchResponseItem {\n    PK: string;\n    SK: string;\n    _id: string;\n    index: string;\n}\n\nexport interface IElasticsearchFetcherFetchParams {\n    index: string;\n    cursor?: PrimitiveValue[];\n    limit: number;\n}\n\nexport interface IElasticsearchFetcherFetchResponse {\n    done: boolean;\n    totalCount: number;\n    cursor?: PrimitiveValue[];\n    items: IElasticsearchFetcherFetchResponseItem[];\n}\n\nexport interface IElasticsearchFetcher {\n    fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;\n}\n"],"mappings":"","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.d.ts

@@ -0,0 +1,17 @@
+export interface IElasticsearchSynchronizeExecuteParamsItem {
+    PK: string;
+    SK: string;
+    _id: string;
+    index: string;
+}
+export interface IElasticsearchSynchronizeExecuteParams {
+    done: boolean;
+    index: string;
+    items: IElasticsearchSynchronizeExecuteParamsItem[];
+}
+export interface IElasticsearchSynchronizeExecuteResponse {
+    done: boolean;
+}
+export interface IElasticsearchSynchronize {
+    execute(params: IElasticsearchSynchronizeExecuteParams): Promise<IElasticsearchSynchronizeExecuteResponse>;
+}

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js

@@ -0,0 +1,3 @@
+export {};
+
+//# sourceMappingURL=ElasticsearchSynchronize.js.map

package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["ElasticsearchSynchronize.ts"],"sourcesContent":["export interface IElasticsearchSynchronizeExecuteParamsItem {\n    PK: string;\n    SK: string;\n    _id: string;\n    index: string;\n}\n\nexport interface IElasticsearchSynchronizeExecuteParams {\n    done: boolean;\n    index: string;\n    items: IElasticsearchSynchronizeExecuteParamsItem[];\n}\n\nexport interface IElasticsearchSynchronizeExecuteResponse {\n    done: boolean;\n}\nexport interface IElasticsearchSynchronize {\n    execute(\n        params: IElasticsearchSynchronizeExecuteParams\n    ): Promise<IElasticsearchSynchronizeExecuteResponse>;\n}\n"],"mappings":"","ignoreList":[]}

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.d.ts

@@ -0,0 +1,2 @@
+import type WebinyError from "@webiny/error";
+export declare const shouldIgnoreEsResponseError: (error: WebinyError) => boolean;

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js

@@ -0,0 +1,6 @@
+const IGNORED_ES_SEARCH_EXCEPTIONS = ["index_not_found_exception", "search_phase_execution_exception", "illegal_argument_exception"];
+export const shouldIgnoreEsResponseError = error => {
+  return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message);
+};
+
+//# sourceMappingURL=shouldIgnoreEsResponseError.js.map

package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["IGNORED_ES_SEARCH_EXCEPTIONS","shouldIgnoreEsResponseError","error","includes","message"],"sources":["shouldIgnoreEsResponseError.ts"],"sourcesContent":["import type WebinyError from \"@webiny/error\";\n\nconst IGNORED_ES_SEARCH_EXCEPTIONS = [\n    \"index_not_found_exception\",\n    \"search_phase_execution_exception\",\n    \"illegal_argument_exception\"\n];\n\nexport const shouldIgnoreEsResponseError = (error: WebinyError) => {\n    return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message);\n};\n"],"mappings":"AAEA,MAAMA,4BAA4B,GAAG,CACjC,2BAA2B,EAC3B,kCAAkC,EAClC,4BAA4B,CAC/B;AAED,OAAO,MAAMC,2BAA2B,GAAIC,KAAkB,IAAK;EAC/D,OAAOF,4BAA4B,CAACG,QAAQ,CAACD,KAAK,CAACE,OAAO,CAAC;AAC/D,CAAC","ignoreList":[]}

package/tasks/dataSynchronization/entities/getElasticsearchEntity.d.ts

@@ -0,0 +1,10 @@
+import { EntityType } from "./getElasticsearchEntityType.js";
+import { DbRegistry } from "../../../abstractions/DbRegistry.js";
+import type { IEntity, IStandardEntityAttributes } from "@webiny/db-dynamodb";
+export interface IGetElasticsearchEntityParams {
+    type: EntityType | unknown;
+    dbRegistry: DbRegistry.Interface;
+}
+export declare const getElasticsearchEntity: (params: IGetElasticsearchEntityParams) => {
+    item: IEntity<IStandardEntityAttributes>;
+};

package/tasks/dataSynchronization/entities/getElasticsearchEntity.js

@@ -0,0 +1,24 @@
+import { EntityType } from "./getElasticsearchEntityType.js";
+const createPredicate = (app, tags) => {
+  return item => {
+    return item.app === app && tags.every(tag => item.tags.includes(tag));
+  };
+};
+export const getElasticsearchEntity = params => {
+  const {
+    type,
+    dbRegistry
+  } = params;
+  const getByPredicate = predicate => {
+    return dbRegistry.getOneItem(predicate);
+  };
+  try {
+    switch (type) {
+      case EntityType.CMS:
+        return getByPredicate(createPredicate("cms", ["es"]));
+    }
+  } catch {}
+  throw new Error(`Unknown entity type "${type}".`);
+};
+
+//# sourceMappingURL=getElasticsearchEntity.js.map

package/tasks/dataSynchronization/entities/getElasticsearchEntity.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["EntityType","createPredicate","app","tags","item","every","tag","includes","getElasticsearchEntity","params","type","dbRegistry","getByPredicate","predicate","getOneItem","CMS","Error"],"sources":["getElasticsearchEntity.ts"],"sourcesContent":["import type { NonEmptyArray } from \"@webiny/api/types.js\";\nimport type { IRegistryItem } from \"@webiny/db\";\nimport { EntityType } from \"./getElasticsearchEntityType.js\";\nimport { DbRegistry } from \"~/abstractions/DbRegistry.js\";\nimport type { IEntity, IStandardEntityAttributes } from \"@webiny/db-dynamodb\";\n\nexport interface IGetElasticsearchEntityParams {\n    type: EntityType | unknown;\n    dbRegistry: DbRegistry.Interface;\n}\n\nconst createPredicate = (app: string, tags: NonEmptyArray<string>) => {\n    return (item: IRegistryItem) => {\n        return item.app === app && tags.every(tag => item.tags.includes(tag));\n    };\n};\n\nexport const getElasticsearchEntity = (params: IGetElasticsearchEntityParams) => {\n    const { type, dbRegistry } = params;\n\n    const getByPredicate = (predicate: (item: IRegistryItem) => boolean) => {\n        return dbRegistry.getOneItem<IEntity<IStandardEntityAttributes>>(predicate);\n    };\n\n    try {\n        switch (type) {\n            case EntityType.CMS:\n                return getByPredicate(createPredicate(\"cms\", [\"es\"]));\n        }\n    } catch {}\n    throw new Error(`Unknown entity type \"${type}\".`);\n};\n"],"mappings":"AAEA,SAASA,UAAU;AASnB,MAAMC,eAAe,GAAGA,CAACC,GAAW,EAAEC,IAA2B,KAAK;EAClE,OAAQC,IAAmB,IAAK;IAC5B,OAAOA,IAAI,CAACF,GAAG,KAAKA,GAAG,IAAIC,IAAI,CAACE,KAAK,CAACC,GAAG,IAAIF,IAAI,CAACD,IAAI,CAACI,QAAQ,CAACD,GAAG,CAAC,CAAC;EACzE,CAAC;AACL,CAAC;AAED,OAAO,MAAME,sBAAsB,GAAIC,MAAqC,IAAK;EAC7E,MAAM;IAAEC,IAAI;IAAEC;EAAW,CAAC,GAAGF,MAAM;EAEnC,MAAMG,cAAc,GAAIC,SAA2C,IAAK;IACpE,OAAOF,UAAU,CAACG,UAAU,CAAqCD,SAAS,CAAC;EAC/E,CAAC;EAED,IAAI;IACA,QAAQH,IAAI;MACR,KAAKV,UAAU,CAACe,GAAG;QACf,OAAOH,cAAc,CAACX,eAAe,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D;EACJ,CAAC,CAAC,MAAM,CAAC;EACT,MAAM,IAAIe,KAAK,CAAC,wBAAwBN,IAAI,IAAI,CAAC;AACrD,CAAC","ignoreList":[]}

package/tasks/dataSynchronization/entities/getElasticsearchEntityType.d.ts

@@ -0,0 +1,8 @@
+export declare enum EntityType {
+    CMS = "headless-cms"
+}
+export interface IGetElasticsearchEntityTypeParams {
+    SK: string;
+    index: string;
+}
+export declare const getElasticsearchEntityType: (params: IGetElasticsearchEntityTypeParams) => EntityType;

package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js

@@ -0,0 +1,12 @@
+export let EntityType = /*#__PURE__*/function (EntityType) {
+  EntityType["CMS"] = "headless-cms";
+  return EntityType;
+}({});
+export const getElasticsearchEntityType = params => {
+  if (params.index.includes("-headless-cms-")) {
+    return EntityType.CMS;
+  }
+  throw new Error(`Unknown entity type for item "${JSON.stringify(params)}".`);
+};
+
+//# sourceMappingURL=getElasticsearchEntityType.js.map

package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["EntityType","getElasticsearchEntityType","params","index","includes","CMS","Error","JSON","stringify"],"sources":["getElasticsearchEntityType.ts"],"sourcesContent":["export enum EntityType {\n    CMS = \"headless-cms\"\n}\n\nexport interface IGetElasticsearchEntityTypeParams {\n    SK: string;\n    index: string;\n}\n\nexport const getElasticsearchEntityType = (\n    params: IGetElasticsearchEntityTypeParams\n): EntityType => {\n    if (params.index.includes(\"-headless-cms-\")) {\n        return EntityType.CMS;\n    }\n\n    throw new Error(`Unknown entity type for item \"${JSON.stringify(params)}\".`);\n};\n"],"mappings":"AAAA,WAAYA,UAAU,0BAAVA,UAAU;EAAVA,UAAU;EAAA,OAAVA,UAAU;AAAA;AAStB,OAAO,MAAMC,0BAA0B,GACnCC,MAAyC,IAC5B;EACb,IAAIA,MAAM,CAACC,KAAK,CAACC,QAAQ,CAAC,gBAAgB,CAAC,EAAE;IACzC,OAAOJ,UAAU,CAACK,GAAG;EACzB;EAEA,MAAM,IAAIC,KAAK,CAAC,iCAAiCC,IAAI,CAACC,SAAS,CAACN,MAAM,CAAC,IAAI,CAAC;AAChF,CAAC","ignoreList":[]}

package/tasks/dataSynchronization/entities/getTable.d.ts

@@ -0,0 +1,7 @@
+import type { TableDef } from "@webiny/db-dynamodb/toolbox.js";
+import type { DbRegistry } from "../../../abstractions/DbRegistry.js";
+export interface IGetTableParams {
+    dbRegistry: DbRegistry.Interface;
+    type: "regular" | "es";
+}
+export declare const getTable: (params: IGetTableParams) => TableDef;

package/tasks/dataSynchronization/entities/getTable.js

@@ -0,0 +1,22 @@
+const createPredicate = (app, tags) => {
+  return item => {
+    return item.app === app && tags.every(tag => item.tags.includes(tag));
+  };
+};
+export const getTable = params => {
+  const {
+    dbRegistry,
+    type
+  } = params;
+  const getByPredicate = predicate => {
+    const item = dbRegistry.getOneItem(predicate);
+    return item.item;
+  };
+  const entity = getByPredicate(createPredicate("cms", [type]));
+  if (!entity) {
+    throw new Error(`Unknown entity type "${type}".`);
+  }
+  return entity.table;
+};
+
+//# sourceMappingURL=getTable.js.map

package/tasks/dataSynchronization/entities/getTable.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["createPredicate","app","tags","item","every","tag","includes","getTable","params","dbRegistry","type","getByPredicate","predicate","getOneItem","entity","Error","table"],"sources":["getTable.ts"],"sourcesContent":["import type { Entity, TableDef } from \"@webiny/db-dynamodb/toolbox.js\";\nimport type { NonEmptyArray } from \"@webiny/api/types.js\";\nimport type { IRegistryItem } from \"@webiny/db\";\nimport type { DbRegistry } from \"~/abstractions/DbRegistry.js\";\n\nexport interface IGetTableParams {\n    dbRegistry: DbRegistry.Interface;\n    type: \"regular\" | \"es\";\n}\n\nconst createPredicate = (app: string, tags: NonEmptyArray<string>) => {\n    return (item: IRegistryItem) => {\n        return item.app === app && tags.every(tag => item.tags.includes(tag));\n    };\n};\n\nexport const getTable = (params: IGetTableParams): TableDef => {\n    const { dbRegistry, type } = params;\n\n    const getByPredicate = (predicate: (item: IRegistryItem) => boolean) => {\n        const item = dbRegistry.getOneItem<Entity>(predicate);\n        return item.item;\n    };\n\n    const entity = getByPredicate(createPredicate(\"cms\", [type]));\n    if (!entity) {\n        throw new Error(`Unknown entity type \"${type}\".`);\n    }\n    return entity.table as TableDef;\n};\n"],"mappings":"AAUA,MAAMA,eAAe,GAAGA,CAACC,GAAW,EAAEC,IAA2B,KAAK;EAClE,OAAQC,IAAmB,IAAK;IAC5B,OAAOA,IAAI,CAACF,GAAG,KAAKA,GAAG,IAAIC,IAAI,CAACE,KAAK,CAACC,GAAG,IAAIF,IAAI,CAACD,IAAI,CAACI,QAAQ,CAACD,GAAG,CAAC,CAAC;EACzE,CAAC;AACL,CAAC;AAED,OAAO,MAAME,QAAQ,GAAIC,MAAuB,IAAe;EAC3D,MAAM;IAAEC,UAAU;IAAEC;EAAK,CAAC,GAAGF,MAAM;EAEnC,MAAMG,cAAc,GAAIC,SAA2C,IAAK;IACpE,MAAMT,IAAI,GAAGM,UAAU,CAACI,UAAU,CAASD,SAAS,CAAC;IACrD,OAAOT,IAAI,CAACA,IAAI;EACpB,CAAC;EAED,MAAMW,MAAM,GAAGH,cAAc,CAACX,eAAe,CAAC,KAAK,EAAE,CAACU,IAAI,CAAC,CAAC,CAAC;EAC7D,IAAI,CAACI,MAAM,EAAE;IACT,MAAM,IAAIC,KAAK,CAAC,wBAAwBL,IAAI,IAAI,CAAC;EACrD;EACA,OAAOI,MAAM,CAACE,KAAK;AACvB,CAAC","ignoreList":[]}

package/tasks/dataSynchronization/entities/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./getElasticsearchEntity.js";
+export * from "./getElasticsearchEntityType.js";
+export * from "./getTable.js";

package/tasks/dataSynchronization/entities/index.js

@@ -0,0 +1,5 @@
+export * from "./getElasticsearchEntity.js";
+export * from "./getElasticsearchEntityType.js";
+export * from "./getTable.js";
+
+//# sourceMappingURL=index.js.map

package/tasks/dataSynchronization/entities/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./getElasticsearchEntity.js\";\nexport * from \"./getElasticsearchEntityType.js\";\nexport * from \"./getTable.js\";\n"],"mappings":"AAAA;AACA;AACA","ignoreList":[]}

package/tasks/dataSynchronization/index.d.ts

@@ -0,0 +1,3 @@
+import type { Context, IElasticsearchTaskConfig } from "../../types.js";
+export { DATA_SYNCHRONIZATION_TASK } from "./DataSynchronizationTask.js";
+export declare const createDataSynchronization: (params?: Partial<IElasticsearchTaskConfig>) => import("@webiny/api").ContextPlugin<Context>;

package/tasks/dataSynchronization/index.js

@@ -0,0 +1,28 @@
+import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js";
+import { createContextPlugin } from "@webiny/api";
+import { DataSynchronizationTask } from "./DataSynchronizationTask.js";
+import { getClients } from "../../helpers/getClients.js";
+import { ElasticsearchSynchronize } from "./elasticsearch/ElasticsearchSynchronize.js";
+import { TaskController } from "@webiny/api-core/features/task/TaskController/index.js";
+import { DbRegistry } from "../../abstractions/DbRegistry.js";
+import { SynchronizationContext } from "../../abstractions/SynchronizationContext.js";
+export { DATA_SYNCHRONIZATION_TASK } from "./DataSynchronizationTask.js";
+export const createDataSynchronization = params => {
+  return createContextPlugin(async context => {
+    const {
+      documentClient,
+      elasticsearchClient
+    } = getClients(context, params);
+
+    // Register the task definition with DbRegistry injected
+    context.container.registerFactory(TaskDefinition, () => {
+      const elasticsearchSynchronize = context.container.resolveWithDependencies({
+        implementation: ElasticsearchSynchronize,
+        dependencies: [TaskController, DbRegistry, SynchronizationContext]
+      });
+      return new DataSynchronizationTask(elasticsearchClient, documentClient, elasticsearchSynchronize);
+    });
+  });
+};
+
+//# sourceMappingURL=index.js.map

package/tasks/dataSynchronization/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["TaskDefinition","createContextPlugin","DataSynchronizationTask","getClients","ElasticsearchSynchronize","TaskController","DbRegistry","SynchronizationContext","DATA_SYNCHRONIZATION_TASK","createDataSynchronization","params","context","documentClient","elasticsearchClient","container","registerFactory","elasticsearchSynchronize","resolveWithDependencies","implementation","dependencies"],"sources":["index.ts"],"sourcesContent":["import { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport type { Context, IElasticsearchTaskConfig } from \"~/types.js\";\nimport { createContextPlugin } from \"@webiny/api\";\nimport { DataSynchronizationTask } from \"./DataSynchronizationTask.js\";\nimport { getClients } from \"~/helpers/getClients.js\";\nimport { ElasticsearchSynchronize } from \"~/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js\";\nimport { TaskController } from \"@webiny/api-core/features/task/TaskController/index.js\";\nimport { DbRegistry } from \"~/abstractions/DbRegistry.js\";\nimport { SynchronizationContext } from \"~/abstractions/SynchronizationContext.js\";\n\nexport { DATA_SYNCHRONIZATION_TASK } from \"./DataSynchronizationTask.js\";\n\nexport const createDataSynchronization = (params?: Partial<IElasticsearchTaskConfig>) => {\n    return createContextPlugin<Context>(async context => {\n        const { documentClient, elasticsearchClient } = getClients(context, params);\n\n        // Register the task definition with DbRegistry injected\n        context.container.registerFactory(TaskDefinition, () => {\n            const elasticsearchSynchronize = context.container.resolveWithDependencies({\n                implementation: ElasticsearchSynchronize,\n                dependencies: [TaskController, DbRegistry, SynchronizationContext]\n            });\n\n            return new DataSynchronizationTask(\n                elasticsearchClient,\n                documentClient,\n                elasticsearchSynchronize\n            );\n        });\n    });\n};\n"],"mappings":"AAAA,SAASA,cAAc,QAAQ,wDAAwD;AAEvF,SAASC,mBAAmB,QAAQ,aAAa;AACjD,SAASC,uBAAuB;AAChC,SAASC,UAAU;AACnB,SAASC,wBAAwB;AACjC,SAASC,cAAc,QAAQ,wDAAwD;AACvF,SAASC,UAAU;AACnB,SAASC,sBAAsB;AAE/B,SAASC,yBAAyB;AAElC,OAAO,MAAMC,yBAAyB,GAAIC,MAA0C,IAAK;EACrF,OAAOT,mBAAmB,CAAU,MAAMU,OAAO,IAAI;IACjD,MAAM;MAAEC,cAAc;MAAEC;IAAoB,CAAC,GAAGV,UAAU,CAACQ,OAAO,EAAED,MAAM,CAAC;;IAE3E;IACAC,OAAO,CAACG,SAAS,CAACC,eAAe,CAACf,cAAc,EAAE,MAAM;MACpD,MAAMgB,wBAAwB,GAAGL,OAAO,CAACG,SAAS,CAACG,uBAAuB,CAAC;QACvEC,cAAc,EAAEd,wBAAwB;QACxCe,YAAY,EAAE,CAACd,cAAc,EAAEC,UAAU,EAAEC,sBAAsB;MACrE,CAAC,CAAC;MAEF,OAAO,IAAIL,uBAAuB,CAC9BW,mBAAmB,EACnBD,cAAc,EACdI,wBACJ,CAAC;IACL,CAAC,CAAC;EACN,CAAC,CAAC;AACN,CAAC","ignoreList":[]}

package/tasks/dataSynchronization/types.d.ts

@@ -0,0 +1,39 @@
+import type { IManager } from "../../types.js";
+import type { PrimitiveValue } from "@webiny/api-elasticsearch/types.js";
+import type { IIndexManager } from "../../settings/types.js";
+import type { IElasticsearchSynchronize } from "../../tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js";
+import type { IElasticsearchFetcher } from "../../tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js";
+import type { IGenericOutput } from "@webiny/api-core/features/task/TaskService/index.js";
+import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js";
+export interface IDataSynchronizationInputValue {
+    finished?: boolean;
+}
+export interface IDataSynchronizationInputElasticsearchToDynamoDbValue extends IDataSynchronizationInputValue {
+    index?: string;
+    cursor?: PrimitiveValue[];
+}
+export interface IDataSynchronizationInput {
+    flow: "elasticsearchToDynamoDb";
+    elasticsearchToDynamoDb?: IDataSynchronizationInputElasticsearchToDynamoDbValue;
+}
+export type IDataSynchronizationOutput = IGenericOutput;
+export type ISynchronizationRunResult = TaskDefinition.ResultContinue<IDataSynchronizationInput> | TaskDefinition.ResultDone<IDataSynchronizationOutput> | TaskDefinition.ResultError | TaskDefinition.ResultAborted;
+export interface ISynchronization {
+    run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult>;
+}
+export interface IElasticsearchSyncParams {
+    manager: IDataSynchronizationManager;
+    indexManager: IIndexManager;
+    synchronize: IElasticsearchSynchronize;
+    fetcher: IElasticsearchFetcher;
+}
+export interface IElasticsearchSyncFactory {
+    (params: IElasticsearchSyncParams): ISynchronization;
+}
+export interface IFactories {
+    /**
+     * Delete all the records which are in the Elasticsearch but not in the Elasticsearch DynamoDB table.
+     */
+    elasticsearchToDynamoDb: IElasticsearchSyncFactory;
+}
+export type IDataSynchronizationManager = IManager<IDataSynchronizationInput, IDataSynchronizationOutput>;

package/tasks/dataSynchronization/types.js

@@ -0,0 +1,3 @@
+export {};
+
+//# sourceMappingURL=types.js.map

package/tasks/dataSynchronization/types.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { IManager } from \"~/types.js\";\nimport type { PrimitiveValue } from \"@webiny/api-elasticsearch/types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { IElasticsearchSynchronize } from \"~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js\";\nimport type { IElasticsearchFetcher } from \"~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js\";\nimport type { IGenericOutput } from \"@webiny/api-core/features/task/TaskService/index.js\";\nimport { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\n\nexport interface IDataSynchronizationInputValue {\n    finished?: boolean;\n}\n\nexport interface IDataSynchronizationInputElasticsearchToDynamoDbValue\n    extends IDataSynchronizationInputValue {\n    index?: string;\n    cursor?: PrimitiveValue[];\n}\n\nexport interface IDataSynchronizationInput {\n    flow: \"elasticsearchToDynamoDb\";\n    elasticsearchToDynamoDb?: IDataSynchronizationInputElasticsearchToDynamoDbValue;\n}\n\nexport type IDataSynchronizationOutput = IGenericOutput;\n\nexport type ISynchronizationRunResult =\n    | TaskDefinition.ResultContinue<IDataSynchronizationInput>\n    | TaskDefinition.ResultDone<IDataSynchronizationOutput>\n    | TaskDefinition.ResultError\n    | TaskDefinition.ResultAborted;\n\nexport interface ISynchronization {\n    run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult>;\n}\n\nexport interface IElasticsearchSyncParams {\n    manager: IDataSynchronizationManager;\n    indexManager: IIndexManager;\n    synchronize: IElasticsearchSynchronize;\n    fetcher: IElasticsearchFetcher;\n}\n\nexport interface IElasticsearchSyncFactory {\n    (params: IElasticsearchSyncParams): ISynchronization;\n}\n\nexport interface IFactories {\n    /**\n     * Delete all the records which are in the Elasticsearch but not in the Elasticsearch DynamoDB table.\n     */\n    elasticsearchToDynamoDb: IElasticsearchSyncFactory;\n}\n\nexport type IDataSynchronizationManager = IManager<\n    IDataSynchronizationInput,\n    IDataSynchronizationOutput\n>;\n"],"mappings":"","ignoreList":[]}