@webiny/api-elasticsearch-tasks 0.0.0-unstable.2696f9d9e8

package/settings/IndexSettingsManager.js

@@ -0,0 +1,44 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.IndexSettingsManager = void 0;
+var _errors = require("../errors");
+class IndexSettingsManager {
+  constructor(elasticsearch) {
+    this.elasticsearch = elasticsearch;
+  }
+  async getSettings(index) {
+    try {
+      const response = await this.elasticsearch.indices.getSettings({
+        index
+      });
+      const setting = response.body[index].settings.index;
+      return {
+        numberOfReplicas: setting.number_of_replicas,
+        refreshInterval: setting.refresh_interval
+      };
+    } catch (ex) {
+      throw new _errors.IndexSettingsGetError(ex, index);
+    }
+  }
+  async setSettings(index, settings) {
+    try {
+      await this.elasticsearch.indices.putSettings({
+        index,
+        body: {
+          index: {
+            number_of_replicas: settings.numberOfReplicas,
+            refresh_interval: settings.refreshInterval
+          }
+        }
+      });
+    } catch (ex) {
+      throw new _errors.IndexSettingsSetError(ex, index);
+    }
+  }
+}
+exports.IndexSettingsManager = IndexSettingsManager;
+
+//# sourceMappingURL=IndexSettingsManager.js.map

package/settings/IndexSettingsManager.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_errors","require","IndexSettingsManager","constructor","elasticsearch","getSettings","index","response","indices","setting","body","settings","numberOfReplicas","number_of_replicas","refreshInterval","refresh_interval","ex","IndexSettingsGetError","setSettings","putSettings","IndexSettingsSetError","exports"],"sources":["IndexSettingsManager.ts"],"sourcesContent":["import { Client } from \"@webiny/api-elasticsearch\";\nimport { IndexSettingsGetError, IndexSettingsSetError } from \"~/errors\";\nimport { IIndexSettingsValues } from \"~/types\";\n\nexport class IndexSettingsManager {\n    private readonly elasticsearch: Client;\n\n    public constructor(elasticsearch: Client) {\n        this.elasticsearch = elasticsearch;\n    }\n\n    public async getSettings(index: string): Promise<IIndexSettingsValues> {\n        try {\n            const response = await this.elasticsearch.indices.getSettings({\n                index\n            });\n\n            const setting = response.body[index].settings.index;\n\n            return {\n                numberOfReplicas: setting.number_of_replicas,\n                refreshInterval: setting.refresh_interval\n            };\n        } catch (ex) {\n            throw new IndexSettingsGetError(ex, index);\n        }\n    }\n\n    public async setSettings(index: string, settings: IIndexSettingsValues): Promise<void> {\n        try {\n            await this.elasticsearch.indices.putSettings({\n                index,\n                body: {\n                    index: {\n                        number_of_replicas: settings.numberOfReplicas,\n                        refresh_interval: settings.refreshInterval\n                    }\n                }\n            });\n        } catch (ex) {\n            throw new IndexSettingsSetError(ex, index);\n        }\n    }\n}\n"],"mappings":";;;;;;AACA,IAAAA,OAAA,GAAAC,OAAA;AAGO,MAAMC,oBAAoB,CAAC;EAGvBC,WAAWA,CAACC,aAAqB,EAAE;IACtC,IAAI,CAACA,aAAa,GAAGA,aAAa;EACtC;EAEA,MAAaC,WAAWA,CAACC,KAAa,EAAiC;IACnE,IAAI;MACA,MAAMC,QAAQ,GAAG,MAAM,IAAI,CAACH,aAAa,CAACI,OAAO,CAACH,WAAW,CAAC;QAC1DC;MACJ,CAAC,CAAC;MAEF,MAAMG,OAAO,GAAGF,QAAQ,CAACG,IAAI,CAACJ,KAAK,CAAC,CAACK,QAAQ,CAACL,KAAK;MAEnD,OAAO;QACHM,gBAAgB,EAAEH,OAAO,CAACI,kBAAkB;QAC5CC,eAAe,EAAEL,OAAO,CAACM;MAC7B,CAAC;IACL,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT,MAAM,IAAIC,6BAAqB,CAACD,EAAE,EAAEV,KAAK,CAAC;IAC9C;EACJ;EAEA,MAAaY,WAAWA,CAACZ,KAAa,EAAEK,QAA8B,EAAiB;IACnF,IAAI;MACA,MAAM,IAAI,CAACP,aAAa,CAACI,OAAO,CAACW,WAAW,CAAC;QACzCb,KAAK;QACLI,IAAI,EAAE;UACFJ,KAAK,EAAE;YACHO,kBAAkB,EAAEF,QAAQ,CAACC,gBAAgB;YAC7CG,gBAAgB,EAAEJ,QAAQ,CAACG;UAC/B;QACJ;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,MAAM,IAAII,6BAAqB,CAACJ,EAAE,EAAEV,KAAK,CAAC;IAC9C;EACJ;AACJ;AAACe,OAAA,CAAAnB,oBAAA,GAAAA,oBAAA"}

package/settings/index.d.ts

@@ -0,0 +1 @@
+export * from "./IndexManager";

package/settings/index.js

@@ -0,0 +1,18 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+var _IndexManager = require("./IndexManager");
+Object.keys(_IndexManager).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _IndexManager[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _IndexManager[key];
+    }
+  });
+});
+
+//# sourceMappingURL=index.js.map

package/settings/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_IndexManager","require","Object","keys","forEach","key","exports","defineProperty","enumerable","get"],"sources":["index.ts"],"sourcesContent":["export * from \"./IndexManager\";\n"],"mappings":";;;;;AAAA,IAAAA,aAAA,GAAAC,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAH,aAAA,EAAAI,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAL,aAAA,CAAAK,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAT,aAAA,CAAAK,GAAA;IAAA;EAAA;AAAA"}

package/settings/types.d.ts

@@ -0,0 +1,6 @@
+import { IElasticsearchIndexingTaskValuesSettings, IIndexSettingsValues } from "../types";
+export interface IIndexManager {
+    settings: IElasticsearchIndexingTaskValuesSettings;
+    disableIndexing(index: string): Promise<IIndexSettingsValues>;
+    enableIndexing(index?: string): Promise<void>;
+}

package/settings/types.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+//# sourceMappingURL=types.js.map

package/settings/types.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import { IElasticsearchIndexingTaskValuesSettings, IIndexSettingsValues } from \"~/types\";\n\nexport interface IIndexManager {\n    settings: IElasticsearchIndexingTaskValuesSettings;\n\n    disableIndexing(index: string): Promise<IIndexSettingsValues>;\n    enableIndexing(index?: string): Promise<void>;\n}\n"],"mappings":""}

package/tasks/Manager.d.ts

@@ -0,0 +1,32 @@
+import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
+import { Client } from "@webiny/api-elasticsearch";
+import { createTable } from "../definitions";
+import { Context, IElasticsearchIndexingTaskValues, IManager } from "../types";
+import { Entity } from "@webiny/db-dynamodb/toolbox";
+import { ITaskResponse } from "@webiny/tasks/response/abstractions";
+import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions";
+import { BatchReadItem, BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb";
+export interface ManagerParams {
+    context: Context;
+    documentClient?: DynamoDBDocument;
+    elasticsearchClient?: Client;
+    isCloseToTimeout: () => boolean;
+    isAborted: () => boolean;
+    response: ITaskResponse;
+    store: ITaskManagerStore<IElasticsearchIndexingTaskValues>;
+}
+export declare class Manager implements IManager {
+    readonly documentClient: DynamoDBDocument;
+    readonly elasticsearch: Client;
+    readonly context: Context;
+    readonly table: ReturnType<typeof createTable>;
+    readonly isCloseToTimeout: () => boolean;
+    readonly isAborted: () => boolean;
+    readonly response: ITaskResponse;
+    readonly store: ITaskManagerStore<IElasticsearchIndexingTaskValues>;
+    private readonly entities;
+    constructor(params: ManagerParams);
+    getEntity(name: string): Entity<any>;
+    read<T>(items: BatchReadItem[]): Promise<T[]>;
+    write(items: BatchWriteItem[]): Promise<BatchWriteResult>;
+}

package/tasks/Manager.js

@@ -0,0 +1,52 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.Manager = void 0;
+var _clientDynamodb = require("@webiny/aws-sdk/client-dynamodb");
+var _apiElasticsearch = require("@webiny/api-elasticsearch");
+var _definitions = require("../definitions");
+var _entry = require("../definitions/entry");
+var _dbDynamodb = require("@webiny/db-dynamodb");
+class Manager {
+  entities = {};
+  constructor(params) {
+    this.context = params.context;
+    this.documentClient = params?.documentClient || (0, _clientDynamodb.getDocumentClient)();
+    this.elasticsearch = params?.elasticsearchClient || params.context.elasticsearch || (0, _apiElasticsearch.createElasticsearchClient)({
+      endpoint: `https://${process.env.ELASTIC_SEARCH_ENDPOINT}`
+    });
+    this.table = (0, _definitions.createTable)({
+      documentClient: this.documentClient
+    });
+    this.isCloseToTimeout = params.isCloseToTimeout;
+    this.isAborted = params.isAborted;
+    this.response = params.response;
+    this.store = params.store;
+  }
+  getEntity(name) {
+    if (this.entities[name]) {
+      return this.entities[name];
+    }
+    return this.entities[name] = (0, _entry.createEntry)({
+      table: this.table,
+      entityName: name
+    });
+  }
+  async read(items) {
+    return await (0, _dbDynamodb.batchReadAll)({
+      table: this.table,
+      items
+    });
+  }
+  async write(items) {
+    return await (0, _dbDynamodb.batchWriteAll)({
+      table: this.table,
+      items
+    });
+  }
+}
+exports.Manager = Manager;
+
+//# sourceMappingURL=Manager.js.map

package/tasks/Manager.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_clientDynamodb","require","_apiElasticsearch","_definitions","_entry","_dbDynamodb","Manager","entities","constructor","params","context","documentClient","getDocumentClient","elasticsearch","elasticsearchClient","createElasticsearchClient","endpoint","process","env","ELASTIC_SEARCH_ENDPOINT","table","createTable","isCloseToTimeout","isAborted","response","store","getEntity","name","createEntry","entityName","read","items","batchReadAll","write","batchWriteAll","exports"],"sources":["Manager.ts"],"sourcesContent":["import { DynamoDBDocument, getDocumentClient } from \"@webiny/aws-sdk/client-dynamodb\";\nimport { Client, createElasticsearchClient } from \"@webiny/api-elasticsearch\";\nimport { createTable } from \"~/definitions\";\nimport { Context, IElasticsearchIndexingTaskValues, IManager } from \"~/types\";\nimport { createEntry } from \"~/definitions/entry\";\nimport { Entity } from \"@webiny/db-dynamodb/toolbox\";\nimport { ITaskResponse } from \"@webiny/tasks/response/abstractions\";\nimport { ITaskManagerStore } from \"@webiny/tasks/runner/abstractions\";\nimport {\n    batchReadAll,\n    BatchReadItem,\n    batchWriteAll,\n    BatchWriteItem,\n    BatchWriteResult\n} from \"@webiny/db-dynamodb\";\n\nexport interface ManagerParams {\n    context: Context;\n    documentClient?: DynamoDBDocument;\n    elasticsearchClient?: Client;\n    isCloseToTimeout: () => boolean;\n    isAborted: () => boolean;\n    response: ITaskResponse;\n    store: ITaskManagerStore<IElasticsearchIndexingTaskValues>;\n}\n\nexport class Manager implements IManager {\n    public readonly documentClient: DynamoDBDocument;\n    public readonly elasticsearch: Client;\n    public readonly context: Context;\n    public readonly table: ReturnType<typeof createTable>;\n    public readonly isCloseToTimeout: () => boolean;\n    public readonly isAborted: () => boolean;\n    public readonly response: ITaskResponse;\n    public readonly store: ITaskManagerStore<IElasticsearchIndexingTaskValues>;\n\n    private readonly entities: Record<string, Entity<any>> = {};\n\n    public constructor(params: ManagerParams) {\n        this.context = params.context;\n        this.documentClient = params?.documentClient || getDocumentClient();\n\n        this.elasticsearch =\n            params?.elasticsearchClient ||\n            params.context.elasticsearch ||\n            createElasticsearchClient({\n                endpoint: `https://${process.env.ELASTIC_SEARCH_ENDPOINT}`\n            });\n\n        this.table = createTable({\n            documentClient: this.documentClient\n        });\n        this.isCloseToTimeout = params.isCloseToTimeout;\n        this.isAborted = params.isAborted;\n        this.response = params.response;\n        this.store = params.store;\n    }\n\n    public getEntity(name: string): Entity<any> {\n        if (this.entities[name]) {\n            return this.entities[name];\n        }\n\n        return (this.entities[name] = createEntry({\n            table: this.table,\n            entityName: name\n        }));\n    }\n\n    public async read<T>(items: BatchReadItem[]) {\n        return await batchReadAll<T>({\n            table: this.table,\n            items\n        });\n    }\n\n    public async write(items: BatchWriteItem[]): Promise<BatchWriteResult> {\n        return await batchWriteAll({\n            table: this.table,\n            items\n        });\n    }\n}\n"],"mappings":";;;;;;AAAA,IAAAA,eAAA,GAAAC,OAAA;AACA,IAAAC,iBAAA,GAAAD,OAAA;AACA,IAAAE,YAAA,GAAAF,OAAA;AAEA,IAAAG,MAAA,GAAAH,OAAA;AAIA,IAAAI,WAAA,GAAAJ,OAAA;AAkBO,MAAMK,OAAO,CAAqB;EAUpBC,QAAQ,GAAgC,CAAC,CAAC;EAEpDC,WAAWA,CAACC,MAAqB,EAAE;IACtC,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO;IAC7B,IAAI,CAACC,cAAc,GAAGF,MAAM,EAAEE,cAAc,IAAI,IAAAC,iCAAiB,EAAC,CAAC;IAEnE,IAAI,CAACC,aAAa,GACdJ,MAAM,EAAEK,mBAAmB,IAC3BL,MAAM,CAACC,OAAO,CAACG,aAAa,IAC5B,IAAAE,2CAAyB,EAAC;MACtBC,QAAQ,EAAG,WAAUC,OAAO,CAACC,GAAG,CAACC,uBAAwB;IAC7D,CAAC,CAAC;IAEN,IAAI,CAACC,KAAK,GAAG,IAAAC,wBAAW,EAAC;MACrBV,cAAc,EAAE,IAAI,CAACA;IACzB,CAAC,CAAC;IACF,IAAI,CAACW,gBAAgB,GAAGb,MAAM,CAACa,gBAAgB;IAC/C,IAAI,CAACC,SAAS,GAAGd,MAAM,CAACc,SAAS;IACjC,IAAI,CAACC,QAAQ,GAAGf,MAAM,CAACe,QAAQ;IAC/B,IAAI,CAACC,KAAK,GAAGhB,MAAM,CAACgB,KAAK;EAC7B;EAEOC,SAASA,CAACC,IAAY,EAAe;IACxC,IAAI,IAAI,CAACpB,QAAQ,CAACoB,IAAI,CAAC,EAAE;MACrB,OAAO,IAAI,CAACpB,QAAQ,CAACoB,IAAI,CAAC;IAC9B;IAEA,OAAQ,IAAI,CAACpB,QAAQ,CAACoB,IAAI,CAAC,GAAG,IAAAC,kBAAW,EAAC;MACtCR,KAAK,EAAE,IAAI,CAACA,KAAK;MACjBS,UAAU,EAAEF;IAChB,CAAC,CAAC;EACN;EAEA,MAAaG,IAAIA,CAAIC,KAAsB,EAAE;IACzC,OAAO,MAAM,IAAAC,wBAAY,EAAI;MACzBZ,KAAK,EAAE,IAAI,CAACA,KAAK;MACjBW;IACJ,CAAC,CAAC;EACN;EAEA,MAAaE,KAAKA,CAACF,KAAuB,EAA6B;IACnE,OAAO,MAAM,IAAAG,yBAAa,EAAC;MACvBd,KAAK,EAAE,IAAI,CAACA,KAAK;MACjBW;IACJ,CAAC,CAAC;EACN;AACJ;AAACI,OAAA,CAAA7B,OAAA,GAAAA,OAAA"}

package/tasks/index.d.ts

@@ -0,0 +1 @@
+export * from "./reindexing";

package/tasks/index.js

@@ -0,0 +1,18 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+var _reindexing = require("./reindexing");
+Object.keys(_reindexing).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _reindexing[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _reindexing[key];
+    }
+  });
+});
+
+//# sourceMappingURL=index.js.map

package/tasks/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_reindexing","require","Object","keys","forEach","key","exports","defineProperty","enumerable","get"],"sources":["index.ts"],"sourcesContent":["export * from \"./reindexing\";\n"],"mappings":";;;;;AAAA,IAAAA,WAAA,GAAAC,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAH,WAAA,EAAAI,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAL,WAAA,CAAAK,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAT,WAAA,CAAAK,GAAA;IAAA;EAAA;AAAA"}

package/tasks/reindexing/ReindexingTaskRunner.d.ts

@@ -0,0 +1,16 @@
+import { IElasticsearchIndexingTaskValuesKeys, IManager } from "../../types";
+import { ITaskResponseResult } from "@webiny/tasks/response/abstractions";
+import { IndexManager } from "../../settings";
+export declare class ReindexingTaskRunner {
+    private readonly manager;
+    private keys?;
+    private readonly indexManager;
+    private readonly response;
+    constructor(manager: IManager, indexManager: IndexManager);
+    /**
+     * When running the task, we always must check:
+     * * if task is close to timeout
+     * * if task was aborted
+     */
+    exec(keys?: IElasticsearchIndexingTaskValuesKeys | undefined, limit?: number): Promise<ITaskResponseResult>;
+}

package/tasks/reindexing/ReindexingTaskRunner.js

@@ -0,0 +1,126 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ReindexingTaskRunner = void 0;
+var _scan = require("../../helpers/scan");
+const getKeys = results => {
+  if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {
+    return {
+      PK: results.lastEvaluatedKey.PK,
+      SK: results.lastEvaluatedKey.SK
+    };
+  }
+  return undefined;
+};
+class ReindexingTaskRunner {
+  constructor(manager, indexManager) {
+    this.manager = manager;
+    this.response = manager.response;
+    this.indexManager = indexManager;
+  }
+
+  /**
+   * When running the task, we always must check:
+   * * if task is close to timeout
+   * * if task was aborted
+   */
+  async exec(keys = undefined, limit = 200) {
+    this.keys = keys;
+    const isIndexAllowed = index => {
+      const input = this.manager.store.getInput();
+      if (typeof input.matching !== "string" || !input.matching) {
+        return true;
+      }
+      return index.includes(input.matching);
+    };
+    try {
+      while (this.manager.isCloseToTimeout() === false) {
+        if (this.manager.isAborted()) {
+          return this.response.aborted();
+        }
+        const results = await (0, _scan.scan)({
+          table: this.manager.table,
+          keys: this.keys,
+          options: {
+            limit
+          }
+        });
+        if (results.items.length === 0) {
+          await this.indexManager.enableIndexing();
+          return this.response.done("No more items to process.");
+        }
+        const batch = [];
+        for (const item of results.items) {
+          /**
+           * No index defined? Impossible but let's skip if really happens.
+           */
+          if (!item.index) {
+            continue;
+          }
+          if (isIndexAllowed(item.index) === false) {
+            continue;
+          }
+          /**
+           * Is there a possibility that entityName does not exist? What do we do at that point?
+           */
+          const entityName = item._et || item.entity;
+          /**
+           * Let's skip for now.
+           */
+          if (!entityName) {
+            continue;
+          }
+          const entity = this.manager.getEntity(entityName);
+          /**
+           * Disable the indexing for the current index.
+           * Method does nothing if indexing is already disabled.
+           */
+          await this.indexManager.disableIndexing(item.index);
+          /**
+           * Reindexing will be triggered by the `putBatch` method.
+           */
+          batch.push(entity.putBatch({
+            ...item,
+            modified: new Date().toISOString()
+          }));
+        }
+        await this.manager.write(batch);
+        /**
+         * We always store the index settings, so we can restore them later.
+         * Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.
+         */
+        this.keys = getKeys(results);
+        await this.manager.store.updateInput({
+          settings: this.indexManager.settings,
+          keys: this.keys
+        });
+        /**
+         * We want to make sure that, if there are no last evaluated keys, we enable indexing.
+         */
+        if (!this.keys) {
+          await this.indexManager.enableIndexing();
+          return this.response.done("No more items to process - no last evaluated keys.");
+        }
+      }
+      return this.response.continue({
+        keys: this.keys
+      });
+    } catch (ex) {
+      /**
+       * We want to enable indexing if there was an error.
+       */
+      try {
+        await this.indexManager.enableIndexing();
+      } catch (er) {
+        er.data = ex;
+        return this.response.error(er);
+      }
+      return this.response.error(ex);
+    }
+  }
+}
+exports.ReindexingTaskRunner = ReindexingTaskRunner;
+
+//# sourceMappingURL=ReindexingTaskRunner.js.map

package/tasks/reindexing/ReindexingTaskRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_scan","require","getKeys","results","lastEvaluatedKey","PK","SK","undefined","ReindexingTaskRunner","constructor","manager","indexManager","response","exec","keys","limit","isIndexAllowed","index","input","store","getInput","matching","includes","isCloseToTimeout","isAborted","aborted","scan","table","options","items","length","enableIndexing","done","batch","item","entityName","_et","entity","getEntity","disableIndexing","push","putBatch","modified","Date","toISOString","write","updateInput","settings","continue","ex","er","data","error","exports"],"sources":["ReindexingTaskRunner.ts"],"sourcesContent":["import {\n    IDynamoDbElasticsearchRecord,\n    IElasticsearchIndexingTaskValuesKeys,\n    IManager\n} from \"~/types\";\nimport { ITaskResponse, ITaskResponseResult } from \"@webiny/tasks/response/abstractions\";\nimport { scan } from \"~/helpers/scan\";\nimport { BatchWriteItem, ScanResponse } from \"@webiny/db-dynamodb\";\nimport { IndexManager } from \"~/settings\";\nimport { IIndexManager } from \"~/settings/types\";\n\nconst getKeys = (results: ScanResponse): IElasticsearchIndexingTaskValuesKeys | undefined => {\n    if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {\n        return {\n            PK: results.lastEvaluatedKey.PK as unknown as string,\n            SK: results.lastEvaluatedKey.SK as unknown as string\n        };\n    }\n    return undefined;\n};\n\nexport class ReindexingTaskRunner {\n    private readonly manager: IManager;\n    private keys?: IElasticsearchIndexingTaskValuesKeys;\n\n    private readonly indexManager: IIndexManager;\n    private readonly response: ITaskResponse;\n\n    public constructor(manager: IManager, indexManager: IndexManager) {\n        this.manager = manager;\n        this.response = manager.response;\n        this.indexManager = indexManager;\n    }\n\n    /**\n     * When running the task, we always must check:\n     * * if task is close to timeout\n     * * if task was aborted\n     */\n    public async exec(\n        keys: IElasticsearchIndexingTaskValuesKeys | undefined = undefined,\n        limit = 200\n    ): Promise<ITaskResponseResult> {\n        this.keys = keys;\n\n        const isIndexAllowed = (index: string): boolean => {\n            const input = this.manager.store.getInput();\n            if (typeof input.matching !== \"string\" || !input.matching) {\n                return true;\n            }\n            return index.includes(input.matching);\n        };\n\n        try {\n            while (this.manager.isCloseToTimeout() === false) {\n                if (this.manager.isAborted()) {\n                    return this.response.aborted();\n                }\n\n                const results = await scan<IDynamoDbElasticsearchRecord>({\n                    table: this.manager.table,\n                    keys: this.keys,\n                    options: {\n                        limit\n                    }\n                });\n                if (results.items.length === 0) {\n                    await this.indexManager.enableIndexing();\n                    return this.response.done(\"No more items to process.\");\n                }\n\n                const batch: BatchWriteItem[] = [];\n                for (const item of results.items) {\n                    /**\n                     * No index defined? Impossible but let's skip if really happens.\n                     */\n                    if (!item.index) {\n                        continue;\n                    }\n                    if (isIndexAllowed(item.index) === false) {\n                        continue;\n                    }\n                    /**\n                     * Is there a possibility that entityName does not exist? What do we do at that point?\n                     */\n                    const entityName = item._et || item.entity;\n                    /**\n                     * Let's skip for now.\n                     */\n                    if (!entityName) {\n                        continue;\n                    }\n                    const entity = this.manager.getEntity(entityName);\n                    /**\n                     * Disable the indexing for the current index.\n                     * Method does nothing if indexing is already disabled.\n                     */\n                    await this.indexManager.disableIndexing(item.index);\n                    /**\n                     * Reindexing will be triggered by the `putBatch` method.\n                     */\n                    batch.push(\n                        entity.putBatch({\n                            ...item,\n                            modified: new Date().toISOString()\n                        })\n                    );\n                }\n                await this.manager.write(batch);\n                /**\n                 * We always store the index settings, so we can restore them later.\n                 * Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.\n                 */\n                this.keys = getKeys(results);\n                await this.manager.store.updateInput({\n                    settings: this.indexManager.settings,\n                    keys: this.keys\n                });\n                /**\n                 * We want to make sure that, if there are no last evaluated keys, we enable indexing.\n                 */\n                if (!this.keys) {\n                    await this.indexManager.enableIndexing();\n                    return this.response.done(\"No more items to process - no last evaluated keys.\");\n                }\n            }\n            return this.response.continue({\n                keys: this.keys\n            });\n        } catch (ex) {\n            /**\n             * We want to enable indexing if there was an error.\n             */\n            try {\n                await this.indexManager.enableIndexing();\n            } catch (er) {\n                er.data = ex;\n                return this.response.error(er);\n            }\n            return this.response.error(ex);\n        }\n    }\n}\n"],"mappings":";;;;;;AAMA,IAAAA,KAAA,GAAAC,OAAA;AAKA,MAAMC,OAAO,GAAIC,OAAqB,IAAuD;EACzF,IAAIA,OAAO,CAACC,gBAAgB,EAAEC,EAAE,IAAIF,OAAO,CAACC,gBAAgB,EAAEE,EAAE,EAAE;IAC9D,OAAO;MACHD,EAAE,EAAEF,OAAO,CAACC,gBAAgB,CAACC,EAAuB;MACpDC,EAAE,EAAEH,OAAO,CAACC,gBAAgB,CAACE;IACjC,CAAC;EACL;EACA,OAAOC,SAAS;AACpB,CAAC;AAEM,MAAMC,oBAAoB,CAAC;EAOvBC,WAAWA,CAACC,OAAiB,EAAEC,YAA0B,EAAE;IAC9D,IAAI,CAACD,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACE,QAAQ,GAAGF,OAAO,CAACE,QAAQ;IAChC,IAAI,CAACD,YAAY,GAAGA,YAAY;EACpC;;EAEA;AACJ;AACA;AACA;AACA;EACI,MAAaE,IAAIA,CACbC,IAAsD,GAAGP,SAAS,EAClEQ,KAAK,GAAG,GAAG,EACiB;IAC5B,IAAI,CAACD,IAAI,GAAGA,IAAI;IAEhB,MAAME,cAAc,GAAIC,KAAa,IAAc;MAC/C,MAAMC,KAAK,GAAG,IAAI,CAACR,OAAO,CAACS,KAAK,CAACC,QAAQ,CAAC,CAAC;MAC3C,IAAI,OAAOF,KAAK,CAACG,QAAQ,KAAK,QAAQ,IAAI,CAACH,KAAK,CAACG,QAAQ,EAAE;QACvD,OAAO,IAAI;MACf;MACA,OAAOJ,KAAK,CAACK,QAAQ,CAACJ,KAAK,CAACG,QAAQ,CAAC;IACzC,CAAC;IAED,IAAI;MACA,OAAO,IAAI,CAACX,OAAO,CAACa,gBAAgB,CAAC,CAAC,KAAK,KAAK,EAAE;QAC9C,IAAI,IAAI,CAACb,OAAO,CAACc,SAAS,CAAC,CAAC,EAAE;UAC1B,OAAO,IAAI,CAACZ,QAAQ,CAACa,OAAO,CAAC,CAAC;QAClC;QAEA,MAAMtB,OAAO,GAAG,MAAM,IAAAuB,UAAI,EAA+B;UACrDC,KAAK,EAAE,IAAI,CAACjB,OAAO,CAACiB,KAAK;UACzBb,IAAI,EAAE,IAAI,CAACA,IAAI;UACfc,OAAO,EAAE;YACLb;UACJ;QACJ,CAAC,CAAC;QACF,IAAIZ,OAAO,CAAC0B,KAAK,CAACC,MAAM,KAAK,CAAC,EAAE;UAC5B,MAAM,IAAI,CAACnB,YAAY,CAACoB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACnB,QAAQ,CAACoB,IAAI,CAAC,2BAA2B,CAAC;QAC1D;QAEA,MAAMC,KAAuB,GAAG,EAAE;QAClC,KAAK,MAAMC,IAAI,IAAI/B,OAAO,CAAC0B,KAAK,EAAE;UAC9B;AACpB;AACA;UACoB,IAAI,CAACK,IAAI,CAACjB,KAAK,EAAE;YACb;UACJ;UACA,IAAID,cAAc,CAACkB,IAAI,CAACjB,KAAK,CAAC,KAAK,KAAK,EAAE;YACtC;UACJ;UACA;AACpB;AACA;UACoB,MAAMkB,UAAU,GAAGD,IAAI,CAACE,GAAG,IAAIF,IAAI,CAACG,MAAM;UAC1C;AACpB;AACA;UACoB,IAAI,CAACF,UAAU,EAAE;YACb;UACJ;UACA,MAAME,MAAM,GAAG,IAAI,CAAC3B,OAAO,CAAC4B,SAAS,CAACH,UAAU,CAAC;UACjD;AACpB;AACA;AACA;UACoB,MAAM,IAAI,CAACxB,YAAY,CAAC4B,eAAe,CAACL,IAAI,CAACjB,KAAK,CAAC;UACnD;AACpB;AACA;UACoBgB,KAAK,CAACO,IAAI,CACNH,MAAM,CAACI,QAAQ,CAAC;YACZ,GAAGP,IAAI;YACPQ,QAAQ,EAAE,IAAIC,IAAI,CAAC,CAAC,CAACC,WAAW,CAAC;UACrC,CAAC,CACL,CAAC;QACL;QACA,MAAM,IAAI,CAAClC,OAAO,CAACmC,KAAK,CAACZ,KAAK,CAAC;QAC/B;AAChB;AACA;AACA;QACgB,IAAI,CAACnB,IAAI,GAAGZ,OAAO,CAACC,OAAO,CAAC;QAC5B,MAAM,IAAI,CAACO,OAAO,CAACS,KAAK,CAAC2B,WAAW,CAAC;UACjCC,QAAQ,EAAE,IAAI,CAACpC,YAAY,CAACoC,QAAQ;UACpCjC,IAAI,EAAE,IAAI,CAACA;QACf,CAAC,CAAC;QACF;AAChB;AACA;QACgB,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;UACZ,MAAM,IAAI,CAACH,YAAY,CAACoB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACnB,QAAQ,CAACoB,IAAI,CAAC,oDAAoD,CAAC;QACnF;MACJ;MACA,OAAO,IAAI,CAACpB,QAAQ,CAACoC,QAAQ,CAAC;QAC1BlC,IAAI,EAAE,IAAI,CAACA;MACf,CAAC,CAAC;IACN,CAAC,CAAC,OAAOmC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAI;QACA,MAAM,IAAI,CAACtC,YAAY,CAACoB,cAAc,CAAC,CAAC;MAC5C,CAAC,CAAC,OAAOmB,EAAE,EAAE;QACTA,EAAE,CAACC,IAAI,GAAGF,EAAE;QACZ,OAAO,IAAI,CAACrC,QAAQ,CAACwC,KAAK,CAACF,EAAE,CAAC;MAClC;MACA,OAAO,IAAI,CAACtC,QAAQ,CAACwC,KAAK,CAACH,EAAE,CAAC;IAClC;EACJ;AACJ;AAACI,OAAA,CAAA7C,oBAAA,GAAAA,oBAAA"}

package/tasks/reindexing/index.d.ts

@@ -0,0 +1 @@
+export * from "./reindexingTaskDefinition";

package/tasks/reindexing/index.js

@@ -0,0 +1,18 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+var _reindexingTaskDefinition = require("./reindexingTaskDefinition");
+Object.keys(_reindexingTaskDefinition).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _reindexingTaskDefinition[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _reindexingTaskDefinition[key];
+    }
+  });
+});
+
+//# sourceMappingURL=index.js.map

package/tasks/reindexing/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_reindexingTaskDefinition","require","Object","keys","forEach","key","exports","defineProperty","enumerable","get"],"sources":["index.ts"],"sourcesContent":["export * from \"./reindexingTaskDefinition\";\n"],"mappings":";;;;;AAAA,IAAAA,yBAAA,GAAAC,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAH,yBAAA,EAAAI,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAL,yBAAA,CAAAK,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAT,yBAAA,CAAAK,GAAA;IAAA;EAAA;AAAA"}

package/tasks/reindexing/reindexingTaskDefinition.d.ts

@@ -0,0 +1,8 @@
+import { Context, IElasticsearchIndexingTaskValues } from "../../types";
+import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
+import { Client } from "@webiny/api-elasticsearch";
+export interface CreateElasticsearchIndexTaskConfig {
+    documentClient?: DynamoDBDocument;
+    elasticsearchClient?: Client;
+}
+export declare const createElasticsearchReindexingTask: (params?: CreateElasticsearchIndexTaskConfig) => import("@webiny/tasks").TaskDefinitionPlugin<Context, IElasticsearchIndexingTaskValues>;

package/tasks/reindexing/reindexingTaskDefinition.js

@@ -0,0 +1,47 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createElasticsearchReindexingTask = void 0;
+var _tasks = require("@webiny/tasks");
+const createElasticsearchReindexingTask = params => {
+  return (0, _tasks.createTaskDefinition)({
+    id: "elasticsearchReindexing",
+    title: "Elasticsearch reindexing",
+    run: async ({
+      context,
+      isCloseToTimeout,
+      response,
+      input,
+      isAborted,
+      store
+    }) => {
+      const {
+        Manager
+      } = await import("../Manager");
+      const {
+        IndexManager
+      } = await import("../../settings");
+      const {
+        ReindexingTaskRunner
+      } = await import("./ReindexingTaskRunner");
+      const manager = new Manager({
+        elasticsearchClient: params?.elasticsearchClient,
+        documentClient: params?.documentClient,
+        response,
+        context,
+        isAborted,
+        isCloseToTimeout,
+        store
+      });
+      const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});
+      const reindexing = new ReindexingTaskRunner(manager, indexManager);
+      const keys = input.keys || undefined;
+      return reindexing.exec(keys, 200);
+    }
+  });
+};
+exports.createElasticsearchReindexingTask = createElasticsearchReindexingTask;
+
+//# sourceMappingURL=reindexingTaskDefinition.js.map

package/tasks/reindexing/reindexingTaskDefinition.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_tasks","require","createElasticsearchReindexingTask","params","createTaskDefinition","id","title","run","context","isCloseToTimeout","response","input","isAborted","store","Manager","IndexManager","ReindexingTaskRunner","manager","elasticsearchClient","documentClient","indexManager","elasticsearch","settings","reindexing","keys","undefined","exec","exports"],"sources":["reindexingTaskDefinition.ts"],"sourcesContent":["import { createTaskDefinition } from \"@webiny/tasks\";\nimport { Context, IElasticsearchIndexingTaskValues } from \"~/types\";\nimport { DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb\";\nimport { Client } from \"@webiny/api-elasticsearch\";\n\nexport interface CreateElasticsearchIndexTaskConfig {\n    documentClient?: DynamoDBDocument;\n    elasticsearchClient?: Client;\n}\n\nexport const createElasticsearchReindexingTask = (params?: CreateElasticsearchIndexTaskConfig) => {\n    return createTaskDefinition<Context, IElasticsearchIndexingTaskValues>({\n        id: \"elasticsearchReindexing\",\n        title: \"Elasticsearch reindexing\",\n        run: async ({ context, isCloseToTimeout, response, input, isAborted, store }) => {\n            const { Manager } = await import(\"../Manager\");\n            const { IndexManager } = await import(\"~/settings\");\n            const { ReindexingTaskRunner } = await import(\"./ReindexingTaskRunner\");\n\n            const manager = new Manager({\n                elasticsearchClient: params?.elasticsearchClient,\n                documentClient: params?.documentClient,\n                response,\n                context,\n                isAborted,\n                isCloseToTimeout,\n                store\n            });\n\n            const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});\n            const reindexing = new ReindexingTaskRunner(manager, indexManager);\n\n            const keys = input.keys || undefined;\n            return reindexing.exec(keys, 200);\n        }\n    });\n};\n"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAUO,MAAMC,iCAAiC,GAAIC,MAA2C,IAAK;EAC9F,OAAO,IAAAC,2BAAoB,EAA4C;IACnEC,EAAE,EAAE,yBAAyB;IAC7BC,KAAK,EAAE,0BAA0B;IACjCC,GAAG,EAAE,MAAAA,CAAO;MAAEC,OAAO;MAAEC,gBAAgB;MAAEC,QAAQ;MAAEC,KAAK;MAAEC,SAAS;MAAEC;IAAM,CAAC,KAAK;MAC7E,MAAM;QAAEC;MAAQ,CAAC,GAAG,MAAM,MAAM,aAAa,CAAC;MAC9C,MAAM;QAAEC;MAAa,CAAC,GAAG,MAAM,MAAM,iBAAa,CAAC;MACnD,MAAM;QAAEC;MAAqB,CAAC,GAAG,MAAM,MAAM,yBAAyB,CAAC;MAEvE,MAAMC,OAAO,GAAG,IAAIH,OAAO,CAAC;QACxBI,mBAAmB,EAAEf,MAAM,EAAEe,mBAAmB;QAChDC,cAAc,EAAEhB,MAAM,EAAEgB,cAAc;QACtCT,QAAQ;QACRF,OAAO;QACPI,SAAS;QACTH,gBAAgB;QAChBI;MACJ,CAAC,CAAC;MAEF,MAAMO,YAAY,GAAG,IAAIL,YAAY,CAACE,OAAO,CAACI,aAAa,EAAEV,KAAK,CAACW,QAAQ,IAAI,CAAC,CAAC,CAAC;MAClF,MAAMC,UAAU,GAAG,IAAIP,oBAAoB,CAACC,OAAO,EAAEG,YAAY,CAAC;MAElE,MAAMI,IAAI,GAAGb,KAAK,CAACa,IAAI,IAAIC,SAAS;MACpC,OAAOF,UAAU,CAACG,IAAI,CAACF,IAAI,EAAE,GAAG,CAAC;IACrC;EACJ,CAAC,CAAC;AACN,CAAC;AAACG,OAAA,CAAAzB,iCAAA,GAAAA,iCAAA"}

package/types.d.ts

@@ -0,0 +1,52 @@
+import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
+import { Entity } from "@webiny/db-dynamodb/toolbox";
+import { Context as TasksContext } from "@webiny/tasks/types";
+import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
+import { Client } from "@webiny/api-elasticsearch";
+import { createTable } from "./definitions";
+import { ITaskResponse } from "@webiny/tasks/response/abstractions";
+import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions";
+import { BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb";
+export interface Context extends ElasticsearchContext, TasksContext {
+}
+export interface IElasticsearchIndexingTaskValuesKeys {
+    PK: string;
+    SK: string;
+}
+export interface IIndexSettingsValues {
+    numberOfReplicas: number;
+    refreshInterval: string;
+}
+export interface IElasticsearchIndexingTaskValuesSettings {
+    [key: string]: IIndexSettingsValues;
+}
+export interface IElasticsearchIndexingTaskValues {
+    matching?: string;
+    keys?: IElasticsearchIndexingTaskValuesKeys;
+    settings?: IElasticsearchIndexingTaskValuesSettings;
+}
+export interface AugmentedError extends Error {
+    data?: Record<string, any>;
+    [key: string]: any;
+}
+export interface IDynamoDbElasticsearchRecord {
+    PK: string;
+    SK: string;
+    index: string;
+    _et?: string;
+    entity: string;
+    data: Record<string, any>;
+    modified: string;
+}
+export interface IManager {
+    readonly documentClient: DynamoDBDocument;
+    readonly elasticsearch: Client;
+    readonly context: Context;
+    readonly table: ReturnType<typeof createTable>;
+    readonly isCloseToTimeout: () => boolean;
+    readonly isAborted: () => boolean;
+    readonly response: ITaskResponse;
+    readonly store: ITaskManagerStore<IElasticsearchIndexingTaskValues>;
+    getEntity: (name: string) => Entity<any>;
+    write: (items: BatchWriteItem[]) => Promise<BatchWriteResult>;
+}

package/types.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+//# sourceMappingURL=types.js.map

package/types.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import { ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { Entity } from \"@webiny/db-dynamodb/toolbox\";\nimport { Context as TasksContext } from \"@webiny/tasks/types\";\nimport { DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb\";\nimport { Client } from \"@webiny/api-elasticsearch\";\nimport { createTable } from \"~/definitions\";\nimport { ITaskResponse } from \"@webiny/tasks/response/abstractions\";\nimport { ITaskManagerStore } from \"@webiny/tasks/runner/abstractions\";\nimport { BatchWriteItem, BatchWriteResult } from \"@webiny/db-dynamodb\";\n\nexport interface Context extends ElasticsearchContext, TasksContext {}\n\nexport interface IElasticsearchIndexingTaskValuesKeys {\n    PK: string;\n    SK: string;\n}\n\nexport interface IIndexSettingsValues {\n    numberOfReplicas: number;\n    refreshInterval: string;\n}\n\nexport interface IElasticsearchIndexingTaskValuesSettings {\n    [key: string]: IIndexSettingsValues;\n}\n\nexport interface IElasticsearchIndexingTaskValues {\n    matching?: string;\n    keys?: IElasticsearchIndexingTaskValuesKeys;\n    settings?: IElasticsearchIndexingTaskValuesSettings;\n}\n\nexport interface AugmentedError extends Error {\n    data?: Record<string, any>;\n    [key: string]: any;\n}\n\nexport interface IDynamoDbElasticsearchRecord {\n    PK: string;\n    SK: string;\n    index: string;\n    _et?: string;\n    entity: string;\n    data: Record<string, any>;\n    modified: string;\n}\n\nexport interface IManager {\n    readonly documentClient: DynamoDBDocument;\n    readonly elasticsearch: Client;\n    readonly context: Context;\n    readonly table: ReturnType<typeof createTable>;\n    readonly isCloseToTimeout: () => boolean;\n    readonly isAborted: () => boolean;\n    readonly response: ITaskResponse;\n    readonly store: ITaskManagerStore<IElasticsearchIndexingTaskValues>;\n\n    getEntity: (name: string) => Entity<any>;\n\n    write: (items: BatchWriteItem[]) => Promise<BatchWriteResult>;\n}\n"],"mappings":""}