@webiny/api-dynamodb-to-elasticsearch 5.33.4 → 5.33.5

package/index.js

@@ -15,6 +15,8 @@ var _apiElasticsearch = require("@webiny/api-elasticsearch");
 
 var _handlerAws = require("@webiny/handler-aws");
 
+var _pRetry = _interopRequireDefault(require("p-retry"));
+
 var Operations;
 
 (function (Operations) {
@@ -37,6 +39,17 @@ const getError = item => {
   return reason;
 };
 
+const getNumberEnvVariable = (name, def) => {
+  const input = process.env[name];
+  const value = Number(input);
+
+  if (value > 0) {
+    return value;
+  }
+
+  return def;
+};
+
 const checkErrors = result => {
   if (!result || !result.body || !result.body.items) {
     return;
@@ -71,100 +84,125 @@ const createEventHandler = () => {
       console.log("Missing elasticsearch definition on context.");
       return null;
     }
+    /**
+     * Wrap the code we need to run into the function, so it can be called within itself.
+     */
 
-    const operations = [];
 
-    for (const record of event.Records) {
-      const dynamodb = record.dynamodb;
-
-      if (!dynamodb) {
-        continue;
-      }
+    const execute = async () => {
+      const operations = [];
 
-      const newImage = _dynamodb.Converter.unmarshall(dynamodb.NewImage);
+      for (const record of event.Records) {
+        const dynamodb = record.dynamodb;
 
-      if (newImage.ignore === true) {
-        continue;
-      }
+        if (!dynamodb) {
+          continue;
+        }
 
-      const oldImage = _dynamodb.Converter.unmarshall(dynamodb.OldImage);
+        const newImage = _dynamodb.Converter.unmarshall(dynamodb.NewImage);
 
-      const keys = _dynamodb.Converter.unmarshall(dynamodb.Keys);
+        if (newImage.ignore === true) {
+          continue;
+        }
 
-      const _id = `${keys.PK}:${keys.SK}`;
-      const operation = record.eventName;
-      /**
-       * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.
-       * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.
-       */
+        const oldImage = _dynamodb.Converter.unmarshall(dynamodb.OldImage);
 
-      let data = undefined;
+        const keys = _dynamodb.Converter.unmarshall(dynamodb.Keys);
 
-      if (operation !== Operations.REMOVE) {
-        /**
-         * We must decompress the data that is going into the Elasticsearch.
-         */
-        data = await (0, _apiElasticsearch.decompress)(context.plugins, newImage.data);
+        const _id = `${keys.PK}:${keys.SK}`;
+        const operation = record.eventName;
         /**
-         * No point in writing null or undefined data into the Elasticsearch.
-         * This might happen on some error while decompressing. We will log it.
-         *
-         * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.
-         * If it is - it is a bug.
+         * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.
+         * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.
          */
 
-        if (data === undefined || data === null) {
-          console.log(`Could not get decompressed data, skipping ES operation "${operation}", ID ${_id}`);
-          continue;
+        let data = undefined;
+
+        if (operation !== Operations.REMOVE) {
+          /**
+           * We must decompress the data that is going into the Elasticsearch.
+           */
+          data = await (0, _apiElasticsearch.decompress)(context.plugins, newImage.data);
+          /**
+           * No point in writing null or undefined data into the Elasticsearch.
+           * This might happen on some error while decompressing. We will log it.
+           *
+           * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.
+           * If it is - it is a bug.
+           */
+
+          if (data === undefined || data === null) {
+            console.log(`Could not get decompressed data, skipping ES operation "${operation}", ID ${_id}`);
+            continue;
+          }
         }
-      }
 
-      switch (record.eventName) {
-        case Operations.INSERT:
-        case Operations.MODIFY:
-          operations.push({
-            index: {
-              _id,
-              _index: newImage.index
-            }
-          }, data);
-          break;
-
-        case Operations.REMOVE:
-          operations.push({
-            delete: {
-              _id,
-              _index: oldImage.index
-            }
-          });
-          break;
-
-        default:
-          break;
+        switch (record.eventName) {
+          case Operations.INSERT:
+          case Operations.MODIFY:
+            operations.push({
+              index: {
+                _id,
+                _index: newImage.index
+              }
+            }, data);
+            break;
+
+          case Operations.REMOVE:
+            operations.push({
+              delete: {
+                _id,
+                _index: oldImage.index
+              }
+            });
+            break;
+
+          default:
+            break;
+        }
       }
-    }
 
-    if (!operations.length) {
-      return null;
-    }
+      if (!operations.length) {
+        return;
+      }
 
-    try {
-      const res = await context.elasticsearch.bulk({
-        body: operations
-      });
-      checkErrors(res);
+      try {
+        const res = await context.elasticsearch.bulk({
+          body: operations
+        });
+        checkErrors(res);
+      } catch (error) {
+        if (process.env.DEBUG === "true") {
+          const meta = (error === null || error === void 0 ? void 0 : error.meta) || {};
+          delete meta["meta"];
+          console.log("Bulk error", JSON.stringify(meta, null, 2));
+        }
 
-      if (process.env.DEBUG === "true") {
-        console.log("Bulk response", JSON.stringify(res, null, 2));
+        throw error;
       }
-    } catch (error) {
-      if (process.env.DEBUG === "true") {
-        console.log("Bulk error", JSON.stringify(error, null, 2));
-      }
-
-      throw error;
-    }
+    };
+
+    const maxRetryTime = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME", 300000);
+    const retries = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES", 20);
+    const minTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT", 1500);
+    const maxTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT", 30000);
+    await (0, _pRetry.default)(execute, {
+      maxRetryTime,
+      retries,
+      minTimeout,
+      maxTimeout,
+      onFailedAttempt: error => {
+        /**
+         * We will only log attempts which are after 3/4 of total attempts.
+         */
+        if (error.attemptNumber < retries * 0.75) {
+          return;
+        }
 
+        console.log(`Attempt #${error.attemptNumber} failed.`);
+        console.log(error.message);
+      }
+    });
     return null;
   });
 };

package/index.js.map

@@ -1 +1 @@
-{"version":3,"names":["Operations","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","log","JSON","stringify","WebinyError","createEventHandler","createDynamoDBEventHandler","event","context","ctx","elasticsearch","operations","record","Records","dynamodb","newImage","Converter","unmarshall","NewImage","ignore","oldImage","OldImage","keys","Keys","_id","PK","SK","operation","eventName","data","undefined","REMOVE","decompress","plugins","INSERT","MODIFY","push","_index","delete","length","res","bulk"],"sources":["index.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\nimport { Converter } from \"aws-sdk/clients/dynamodb\";\nimport { decompress } from \"@webiny/api-elasticsearch\";\nimport { ApiResponse, ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { createDynamoDBEventHandler } from \"@webiny/handler-aws\";\nimport { StreamRecord } from \"aws-lambda/trigger/dynamodb-stream\";\n\nenum Operations {\n    INSERT = \"INSERT\",\n    MODIFY = \"MODIFY\",\n    REMOVE = \"REMOVE\"\n}\n\ninterface BulkOperationsResponseBodyItemIndexError {\n    reason?: string;\n}\ninterface BulkOperationsResponseBodyItemIndex {\n    error?: BulkOperationsResponseBodyItemIndexError;\n}\ninterface BulkOperationsResponseBodyItem {\n    index?: BulkOperationsResponseBodyItemIndex;\n    error?: string;\n}\ninterface BulkOperationsResponseBody {\n    items: BulkOperationsResponseBodyItem[];\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n    if (!item.index || !item.index.error || !item.index.error.reason) {\n        return null;\n    }\n    const reason = item.index.error.reason;\n    if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n        return \"index\";\n    }\n    return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n    if (!result || !result.body || !result.body.items) {\n        return;\n    }\n    for (const item of result.body.items) {\n        const err = getError(item);\n        if (!err) {\n            continue;\n        } else if (err === \"index\") {\n            if (process.env.DEBUG === \"true\") {\n                console.log(\"Bulk response\", JSON.stringify(result, null, 2));\n            }\n            continue;\n        }\n        console.log(item.error);\n        throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n    }\n};\n\ninterface RecordDynamoDbImage {\n    data: Record<string, any>;\n    ignore?: boolean;\n    index: string;\n}\n\ninterface RecordDynamoDbKeys {\n    PK: string;\n    SK: string;\n}\n\nexport const createEventHandler = () => {\n    return createDynamoDBEventHandler(async ({ event, context: ctx }) => {\n        const context = ctx as unknown as ElasticsearchContext;\n        if (!context.elasticsearch) {\n            console.log(\"Missing elasticsearch definition on context.\");\n            return null;\n        }\n        const operations = [];\n\n        for (const record of event.Records) {\n            const dynamodb = record.dynamodb as Required<StreamRecord>;\n            if (!dynamodb) {\n                continue;\n            }\n            const newImage = Converter.unmarshall(dynamodb.NewImage) as RecordDynamoDbImage;\n\n            if (newImage.ignore === true) {\n                continue;\n            }\n\n            const oldImage = Converter.unmarshall(dynamodb.OldImage) as RecordDynamoDbImage;\n            const keys = Converter.unmarshall(dynamodb.Keys) as RecordDynamoDbKeys;\n            const _id = `${keys.PK}:${keys.SK}`;\n            const operation = record.eventName;\n\n            /**\n             * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n             * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n             */\n            let data: any = undefined;\n            if (operation !== Operations.REMOVE) {\n                /**\n                 * We must decompress the data that is going into the Elasticsearch.\n                 */\n                data = await decompress(context.plugins, newImage.data);\n                /**\n                 * No point in writing null or undefined data into the Elasticsearch.\n                 * This might happen on some error while decompressing. We will log it.\n                 *\n                 * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n                 * If it is - it is a bug.\n                 */\n                if (data === undefined || data === null) {\n                    console.log(\n                        `Could not get decompressed data, skipping ES operation \"${operation}\", ID ${_id}`\n                    );\n                    continue;\n                }\n            }\n\n            switch (record.eventName) {\n                case Operations.INSERT:\n                case Operations.MODIFY:\n                    operations.push({ index: { _id, _index: newImage.index } }, data);\n                    break;\n                case Operations.REMOVE:\n                    operations.push({ delete: { _id, _index: oldImage.index } });\n                    break;\n                default:\n                    break;\n            }\n        }\n\n        if (!operations.length) {\n            return null;\n        }\n\n        try {\n            const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n                body: operations\n            });\n            checkErrors(res);\n            if (process.env.DEBUG === \"true\") {\n                console.log(\"Bulk response\", JSON.stringify(res, null, 2));\n            }\n        } catch (error) {\n            if (process.env.DEBUG === \"true\") {\n                console.log(\"Bulk error\", JSON.stringify(error, null, 2));\n            }\n            throw error;\n        }\n\n        return null;\n    });\n};\n"],"mappings":";;;;;;;;;AAAA;;AACA;;AACA;;AAEA;;IAGKA,U;;WAAAA,U;EAAAA,U;EAAAA,U;EAAAA,U;GAAAA,U,KAAAA,U;;AAoBL,MAAMC,QAAQ,GAAIC,IAAD,IAAyD;EACtE,IAAI,CAACA,IAAI,CAACC,KAAN,IAAe,CAACD,IAAI,CAACC,KAAL,CAAWC,KAA3B,IAAoC,CAACF,IAAI,CAACC,KAAL,CAAWC,KAAX,CAAiBC,MAA1D,EAAkE;IAC9D,OAAO,IAAP;EACH;;EACD,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAL,CAAWC,KAAX,CAAiBC,MAAhC;;EACA,IAAIA,MAAM,CAACC,KAAP,CAAa,oCAAb,MAAuD,IAA3D,EAAiE;IAC7D,OAAO,OAAP;EACH;;EACD,OAAOD,MAAP;AACH,CATD;;AAWA,MAAME,WAAW,GAAIC,MAAD,IAA4D;EAC5E,IAAI,CAACA,MAAD,IAAW,CAACA,MAAM,CAACC,IAAnB,IAA2B,CAACD,MAAM,CAACC,IAAP,CAAYC,KAA5C,EAAmD;IAC/C;EACH;;EACD,KAAK,MAAMR,IAAX,IAAmBM,MAAM,CAACC,IAAP,CAAYC,KAA/B,EAAsC;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAD,CAApB;;IACA,IAAI,CAACS,GAAL,EAAU;MACN;IACH,CAFD,MAEO,IAAIA,GAAG,KAAK,OAAZ,EAAqB;MACxB,IAAIC,OAAO,CAACC,GAAR,CAAYC,KAAZ,KAAsB,MAA1B,EAAkC;QAC9BC,OAAO,CAACC,GAAR,CAAY,eAAZ,EAA6BC,IAAI,CAACC,SAAL,CAAeV,MAAf,EAAuB,IAAvB,EAA6B,CAA7B,CAA7B;MACH;;MACD;IACH;;IACDO,OAAO,CAACC,GAAR,CAAYd,IAAI,CAACE,KAAjB;IACA,MAAM,IAAIe,cAAJ,CAAgBR,GAAhB,EAAqB,iCAArB,EAAwDT,IAAxD,CAAN;EACH;AACJ,CAjBD;;AA8BO,MAAMkB,kBAAkB,GAAG,MAAM;EACpC,OAAO,IAAAC,sCAAA,EAA2B,OAAO;IAAEC,KAAF;IAASC,OAAO,EAAEC;EAAlB,CAAP,KAAmC;IACjE,MAAMD,OAAO,GAAGC,GAAhB;;IACA,IAAI,CAACD,OAAO,CAACE,aAAb,EAA4B;MACxBV,OAAO,CAACC,GAAR,CAAY,8CAAZ;MACA,OAAO,IAAP;IACH;;IACD,MAAMU,UAAU,GAAG,EAAnB;;IAEA,KAAK,MAAMC,MAAX,IAAqBL,KAAK,CAACM,OAA3B,EAAoC;MAChC,MAAMC,QAAQ,GAAGF,MAAM,CAACE,QAAxB;;MACA,IAAI,CAACA,QAAL,EAAe;QACX;MACH;;MACD,MAAMC,QAAQ,GAAGC,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACI,QAA9B,CAAjB;;MAEA,IAAIH,QAAQ,CAACI,MAAT,KAAoB,IAAxB,EAA8B;QAC1B;MACH;;MAED,MAAMC,QAAQ,GAAGJ,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACO,QAA9B,CAAjB;;MACA,MAAMC,IAAI,GAAGN,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACS,IAA9B,CAAb;;MACA,MAAMC,GAAG,GAAI,GAAEF,IAAI,CAACG,EAAG,IAAGH,IAAI,CAACI,EAAG,EAAlC;MACA,MAAMC,SAAS,GAAGf,MAAM,CAACgB,SAAzB;MAEA;AACZ;AACA;AACA;;MACY,IAAIC,IAAS,GAAGC,SAAhB;;MACA,IAAIH,SAAS,KAAK1C,UAAU,CAAC8C,MAA7B,EAAqC;QACjC;AAChB;AACA;QACgBF,IAAI,GAAG,MAAM,IAAAG,4BAAA,EAAWxB,OAAO,CAACyB,OAAnB,EAA4BlB,QAAQ,CAACc,IAArC,CAAb;QACA;AAChB;AACA;AACA;AACA;AACA;AACA;;QACgB,IAAIA,IAAI,KAAKC,SAAT,IAAsBD,IAAI,KAAK,IAAnC,EAAyC;UACrC7B,OAAO,CAACC,GAAR,CACK,2DAA0D0B,SAAU,SAAQH,GAAI,EADrF;UAGA;QACH;MACJ;;MAED,QAAQZ,MAAM,CAACgB,SAAf;QACI,KAAK3C,UAAU,CAACiD,MAAhB;QACA,KAAKjD,UAAU,CAACkD,MAAhB;UACIxB,UAAU,CAACyB,IAAX,CAAgB;YAAEhD,KAAK,EAAE;cAAEoC,GAAF;cAAOa,MAAM,EAAEtB,QAAQ,CAAC3B;YAAxB;UAAT,CAAhB,EAA4DyC,IAA5D;UACA;;QACJ,KAAK5C,UAAU,CAAC8C,MAAhB;UACIpB,UAAU,CAACyB,IAAX,CAAgB;YAAEE,MAAM,EAAE;cAAEd,GAAF;cAAOa,MAAM,EAAEjB,QAAQ,CAAChC;YAAxB;UAAV,CAAhB;UACA;;QACJ;UACI;MATR;IAWH;;IAED,IAAI,CAACuB,UAAU,CAAC4B,MAAhB,EAAwB;MACpB,OAAO,IAAP;IACH;;IAED,IAAI;MACA,MAAMC,GAAG,GAAG,MAAMhC,OAAO,CAACE,aAAR,CAAsB+B,IAAtB,CAAuD;QACrE/C,IAAI,EAAEiB;MAD+D,CAAvD,CAAlB;MAGAnB,WAAW,CAACgD,GAAD,CAAX;;MACA,IAAI3C,OAAO,CAACC,GAAR,CAAYC,KAAZ,KAAsB,MAA1B,EAAkC;QAC9BC,OAAO,CAACC,GAAR,CAAY,eAAZ,EAA6BC,IAAI,CAACC,SAAL,CAAeqC,GAAf,EAAoB,IAApB,EAA0B,CAA1B,CAA7B;MACH;IACJ,CARD,CAQE,OAAOnD,KAAP,EAAc;MACZ,IAAIQ,OAAO,CAACC,GAAR,CAAYC,KAAZ,KAAsB,MAA1B,EAAkC;QAC9BC,OAAO,CAACC,GAAR,CAAY,YAAZ,EAA0BC,IAAI,CAACC,SAAL,CAAed,KAAf,EAAsB,IAAtB,EAA4B,CAA5B,CAA1B;MACH;;MACD,MAAMA,KAAN;IACH;;IAED,OAAO,IAAP;EACH,CAlFM,CAAP;AAmFH,CApFM"}
+{"version":3,"names":["Operations","getError","item","index","error","reason","match","getNumberEnvVariable","name","def","input","process","env","value","Number","checkErrors","result","body","items","err","DEBUG","console","log","JSON","stringify","WebinyError","createEventHandler","createDynamoDBEventHandler","event","context","ctx","elasticsearch","execute","operations","record","Records","dynamodb","newImage","Converter","unmarshall","NewImage","ignore","oldImage","OldImage","keys","Keys","_id","PK","SK","operation","eventName","data","undefined","REMOVE","decompress","plugins","INSERT","MODIFY","push","_index","delete","length","res","bulk","meta","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","onFailedAttempt","attemptNumber","message"],"sources":["index.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\nimport { Converter } from \"aws-sdk/clients/dynamodb\";\nimport { decompress } from \"@webiny/api-elasticsearch\";\nimport { ApiResponse, ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { createDynamoDBEventHandler } from \"@webiny/handler-aws\";\nimport { StreamRecord } from \"aws-lambda/trigger/dynamodb-stream\";\nimport pRetry from \"p-retry\";\n\nenum Operations {\n    INSERT = \"INSERT\",\n    MODIFY = \"MODIFY\",\n    REMOVE = \"REMOVE\"\n}\n\ninterface BulkOperationsResponseBodyItemIndexError {\n    reason?: string;\n}\n\ninterface BulkOperationsResponseBodyItemIndex {\n    error?: BulkOperationsResponseBodyItemIndexError;\n}\n\ninterface BulkOperationsResponseBodyItem {\n    index?: BulkOperationsResponseBodyItemIndex;\n    error?: string;\n}\n\ninterface BulkOperationsResponseBody {\n    items: BulkOperationsResponseBodyItem[];\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n    if (!item.index || !item.index.error || !item.index.error.reason) {\n        return null;\n    }\n    const reason = item.index.error.reason;\n    if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n        return \"index\";\n    }\n    return reason;\n};\n\nconst getNumberEnvVariable = (name: string, def: number): number => {\n    const input = process.env[name];\n    const value = Number(input);\n    if (value > 0) {\n        return value;\n    }\n    return def;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n    if (!result || !result.body || !result.body.items) {\n        return;\n    }\n    for (const item of result.body.items) {\n        const err = getError(item);\n        if (!err) {\n            continue;\n        } else if (err === \"index\") {\n            if (process.env.DEBUG === \"true\") {\n                console.log(\"Bulk response\", JSON.stringify(result, null, 2));\n            }\n            continue;\n        }\n        console.log(item.error);\n        throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n    }\n};\n\ninterface RecordDynamoDbImage {\n    data: Record<string, any>;\n    ignore?: boolean;\n    index: string;\n}\n\ninterface RecordDynamoDbKeys {\n    PK: string;\n    SK: string;\n}\n\nexport const createEventHandler = () => {\n    return createDynamoDBEventHandler(async ({ event, context: ctx }) => {\n        const context = ctx as unknown as ElasticsearchContext;\n        if (!context.elasticsearch) {\n            console.log(\"Missing elasticsearch definition on context.\");\n            return null;\n        }\n\n        /**\n         * Wrap the code we need to run into the function, so it can be called within itself.\n         */\n        const execute = async (): Promise<void> => {\n            const operations = [];\n\n            for (const record of event.Records) {\n                const dynamodb = record.dynamodb as Required<StreamRecord>;\n                if (!dynamodb) {\n                    continue;\n                }\n                const newImage = Converter.unmarshall(dynamodb.NewImage) as RecordDynamoDbImage;\n\n                if (newImage.ignore === true) {\n                    continue;\n                }\n\n                const oldImage = Converter.unmarshall(dynamodb.OldImage) as RecordDynamoDbImage;\n                const keys = Converter.unmarshall(dynamodb.Keys) as RecordDynamoDbKeys;\n                const _id = `${keys.PK}:${keys.SK}`;\n                const operation = record.eventName;\n\n                /**\n                 * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n                 * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n                 */\n                let data: any = undefined;\n                if (operation !== Operations.REMOVE) {\n                    /**\n                     * We must decompress the data that is going into the Elasticsearch.\n                     */\n                    data = await decompress(context.plugins, newImage.data);\n                    /**\n                     * No point in writing null or undefined data into the Elasticsearch.\n                     * This might happen on some error while decompressing. We will log it.\n                     *\n                     * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n                     * If it is - it is a bug.\n                     */\n                    if (data === undefined || data === null) {\n                        console.log(\n                            `Could not get decompressed data, skipping ES operation \"${operation}\", ID ${_id}`\n                        );\n                        continue;\n                    }\n                }\n\n                switch (record.eventName) {\n                    case Operations.INSERT:\n                    case Operations.MODIFY:\n                        operations.push({ index: { _id, _index: newImage.index } }, data);\n                        break;\n                    case Operations.REMOVE:\n                        operations.push({ delete: { _id, _index: oldImage.index } });\n                        break;\n                    default:\n                        break;\n                }\n            }\n\n            if (!operations.length) {\n                return;\n            }\n\n            try {\n                const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n                    body: operations\n                });\n                checkErrors(res);\n            } catch (error) {\n                if (process.env.DEBUG === \"true\") {\n                    const meta = error?.meta || {};\n                    delete meta[\"meta\"];\n                    console.log(\"Bulk error\", JSON.stringify(meta, null, 2));\n                }\n                throw error;\n            }\n        };\n\n        const maxRetryTime = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n            300000\n        );\n        const retries = getNumberEnvVariable(\"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\", 20);\n        const minTimeout = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n            1500\n        );\n        const maxTimeout = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n            30000\n        );\n\n        await pRetry(execute, {\n            maxRetryTime,\n            retries,\n            minTimeout,\n            maxTimeout,\n            onFailedAttempt: error => {\n                /**\n                 * We will only log attempts which are after 3/4 of total attempts.\n                 */\n                if (error.attemptNumber < retries * 0.75) {\n                    return;\n                }\n                console.log(`Attempt #${error.attemptNumber} failed.`);\n                console.log(error.message);\n            }\n        });\n\n        return null;\n    });\n};\n"],"mappings":";;;;;;;;;AAAA;;AACA;;AACA;;AAEA;;AAEA;;IAEKA,U;;WAAAA,U;EAAAA,U;EAAAA,U;EAAAA,U;GAAAA,U,KAAAA,U;;AAuBL,MAAMC,QAAQ,GAAIC,IAAD,IAAyD;EACtE,IAAI,CAACA,IAAI,CAACC,KAAN,IAAe,CAACD,IAAI,CAACC,KAAL,CAAWC,KAA3B,IAAoC,CAACF,IAAI,CAACC,KAAL,CAAWC,KAAX,CAAiBC,MAA1D,EAAkE;IAC9D,OAAO,IAAP;EACH;;EACD,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAL,CAAWC,KAAX,CAAiBC,MAAhC;;EACA,IAAIA,MAAM,CAACC,KAAP,CAAa,oCAAb,MAAuD,IAA3D,EAAiE;IAC7D,OAAO,OAAP;EACH;;EACD,OAAOD,MAAP;AACH,CATD;;AAWA,MAAME,oBAAoB,GAAG,CAACC,IAAD,EAAeC,GAAf,KAAuC;EAChE,MAAMC,KAAK,GAAGC,OAAO,CAACC,GAAR,CAAYJ,IAAZ,CAAd;EACA,MAAMK,KAAK,GAAGC,MAAM,CAACJ,KAAD,CAApB;;EACA,IAAIG,KAAK,GAAG,CAAZ,EAAe;IACX,OAAOA,KAAP;EACH;;EACD,OAAOJ,GAAP;AACH,CAPD;;AASA,MAAMM,WAAW,GAAIC,MAAD,IAA4D;EAC5E,IAAI,CAACA,MAAD,IAAW,CAACA,MAAM,CAACC,IAAnB,IAA2B,CAACD,MAAM,CAACC,IAAP,CAAYC,KAA5C,EAAmD;IAC/C;EACH;;EACD,KAAK,MAAMhB,IAAX,IAAmBc,MAAM,CAACC,IAAP,CAAYC,KAA/B,EAAsC;IAClC,MAAMC,GAAG,GAAGlB,QAAQ,CAACC,IAAD,CAApB;;IACA,IAAI,CAACiB,GAAL,EAAU;MACN;IACH,CAFD,MAEO,IAAIA,GAAG,KAAK,OAAZ,EAAqB;MACxB,IAAIR,OAAO,CAACC,GAAR,CAAYQ,KAAZ,KAAsB,MAA1B,EAAkC;QAC9BC,OAAO,CAACC,GAAR,CAAY,eAAZ,EAA6BC,IAAI,CAACC,SAAL,CAAeR,MAAf,EAAuB,IAAvB,EAA6B,CAA7B,CAA7B;MACH;;MACD;IACH;;IACDK,OAAO,CAACC,GAAR,CAAYpB,IAAI,CAACE,KAAjB;IACA,MAAM,IAAIqB,cAAJ,CAAgBN,GAAhB,EAAqB,iCAArB,EAAwDjB,IAAxD,CAAN;EACH;AACJ,CAjBD;;AA8BO,MAAMwB,kBAAkB,GAAG,MAAM;EACpC,OAAO,IAAAC,sCAAA,EAA2B,OAAO;IAAEC,KAAF;IAASC,OAAO,EAAEC;EAAlB,CAAP,KAAmC;IACjE,MAAMD,OAAO,GAAGC,GAAhB;;IACA,IAAI,CAACD,OAAO,CAACE,aAAb,EAA4B;MACxBV,OAAO,CAACC,GAAR,CAAY,8CAAZ;MACA,OAAO,IAAP;IACH;IAED;AACR;AACA;;;IACQ,MAAMU,OAAO,GAAG,YAA2B;MACvC,MAAMC,UAAU,GAAG,EAAnB;;MAEA,KAAK,MAAMC,MAAX,IAAqBN,KAAK,CAACO,OAA3B,EAAoC;QAChC,MAAMC,QAAQ,GAAGF,MAAM,CAACE,QAAxB;;QACA,IAAI,CAACA,QAAL,EAAe;UACX;QACH;;QACD,MAAMC,QAAQ,GAAGC,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACI,QAA9B,CAAjB;;QAEA,IAAIH,QAAQ,CAACI,MAAT,KAAoB,IAAxB,EAA8B;UAC1B;QACH;;QAED,MAAMC,QAAQ,GAAGJ,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACO,QAA9B,CAAjB;;QACA,MAAMC,IAAI,GAAGN,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACS,IAA9B,CAAb;;QACA,MAAMC,GAAG,GAAI,GAAEF,IAAI,CAACG,EAAG,IAAGH,IAAI,CAACI,EAAG,EAAlC;QACA,MAAMC,SAAS,GAAGf,MAAM,CAACgB,SAAzB;QAEA;AAChB;AACA;AACA;;QACgB,IAAIC,IAAS,GAAGC,SAAhB;;QACA,IAAIH,SAAS,KAAKjD,UAAU,CAACqD,MAA7B,EAAqC;UACjC;AACpB;AACA;UACoBF,IAAI,GAAG,MAAM,IAAAG,4BAAA,EAAWzB,OAAO,CAAC0B,OAAnB,EAA4BlB,QAAQ,CAACc,IAArC,CAAb;UACA;AACpB;AACA;AACA;AACA;AACA;AACA;;UACoB,IAAIA,IAAI,KAAKC,SAAT,IAAsBD,IAAI,KAAK,IAAnC,EAAyC;YACrC9B,OAAO,CAACC,GAAR,CACK,2DAA0D2B,SAAU,SAAQH,GAAI,EADrF;YAGA;UACH;QACJ;;QAED,QAAQZ,MAAM,CAACgB,SAAf;UACI,KAAKlD,UAAU,CAACwD,MAAhB;UACA,KAAKxD,UAAU,CAACyD,MAAhB;YACIxB,UAAU,CAACyB,IAAX,CAAgB;cAAEvD,KAAK,EAAE;gBAAE2C,GAAF;gBAAOa,MAAM,EAAEtB,QAAQ,CAAClC;cAAxB;YAAT,CAAhB,EAA4DgD,IAA5D;YACA;;UACJ,KAAKnD,UAAU,CAACqD,MAAhB;YACIpB,UAAU,CAACyB,IAAX,CAAgB;cAAEE,MAAM,EAAE;gBAAEd,GAAF;gBAAOa,MAAM,EAAEjB,QAAQ,CAACvC;cAAxB;YAAV,CAAhB;YACA;;UACJ;YACI;QATR;MAWH;;MAED,IAAI,CAAC8B,UAAU,CAAC4B,MAAhB,EAAwB;QACpB;MACH;;MAED,IAAI;QACA,MAAMC,GAAG,GAAG,MAAMjC,OAAO,CAACE,aAAR,CAAsBgC,IAAtB,CAAuD;UACrE9C,IAAI,EAAEgB;QAD+D,CAAvD,CAAlB;QAGAlB,WAAW,CAAC+C,GAAD,CAAX;MACH,CALD,CAKE,OAAO1D,KAAP,EAAc;QACZ,IAAIO,OAAO,CAACC,GAAR,CAAYQ,KAAZ,KAAsB,MAA1B,EAAkC;UAC9B,MAAM4C,IAAI,GAAG,CAAA5D,KAAK,SAAL,IAAAA,KAAK,WAAL,YAAAA,KAAK,CAAE4D,IAAP,KAAe,EAA5B;UACA,OAAOA,IAAI,CAAC,MAAD,CAAX;UACA3C,OAAO,CAACC,GAAR,CAAY,YAAZ,EAA0BC,IAAI,CAACC,SAAL,CAAewC,IAAf,EAAqB,IAArB,EAA2B,CAA3B,CAA1B;QACH;;QACD,MAAM5D,KAAN;MACH;IACJ,CA1ED;;IA4EA,MAAM6D,YAAY,GAAG1D,oBAAoB,CACrC,iDADqC,EAErC,MAFqC,CAAzC;IAIA,MAAM2D,OAAO,GAAG3D,oBAAoB,CAAC,0CAAD,EAA6C,EAA7C,CAApC;IACA,MAAM4D,UAAU,GAAG5D,oBAAoB,CACnC,8CADmC,EAEnC,IAFmC,CAAvC;IAIA,MAAM6D,UAAU,GAAG7D,oBAAoB,CACnC,8CADmC,EAEnC,KAFmC,CAAvC;IAKA,MAAM,IAAA8D,eAAA,EAAOrC,OAAP,EAAgB;MAClBiC,YADkB;MAElBC,OAFkB;MAGlBC,UAHkB;MAIlBC,UAJkB;MAKlBE,eAAe,EAAElE,KAAK,IAAI;QACtB;AAChB;AACA;QACgB,IAAIA,KAAK,CAACmE,aAAN,GAAsBL,OAAO,GAAG,IAApC,EAA0C;UACtC;QACH;;QACD7C,OAAO,CAACC,GAAR,CAAa,YAAWlB,KAAK,CAACmE,aAAc,UAA5C;QACAlD,OAAO,CAACC,GAAR,CAAYlB,KAAK,CAACoE,OAAlB;MACH;IAdiB,CAAhB,CAAN;IAiBA,OAAO,IAAP;EACH,CAtHM,CAAP;AAuHH,CAxHM"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@webiny/api-dynamodb-to-elasticsearch",
-	"version": "5.33.4",
+	"version": "5.33.5",
 	"main": "index.js",
 	"repository": {
 		"type": "git",
@@ -12,11 +12,12 @@
 	"author": "Webiny Ltd.",
 	"dependencies": {
 		"@babel/runtime": "7.18.9",
-		"@webiny/api-elasticsearch": "5.33.4",
-		"@webiny/error": "5.33.4",
-		"@webiny/handler-aws": "5.33.4",
+		"@webiny/api-elasticsearch": "5.33.5",
+		"@webiny/error": "5.33.5",
+		"@webiny/handler-aws": "5.33.5",
 		"aws-lambda": "1.0.7",
-		"aws-sdk": "2.1188.0"
+		"aws-sdk": "2.1188.0",
+		"p-retry": "4.6.2"
 	},
 	"devDependencies": {
 		"@babel/cli": "^7.16.0",
@@ -25,9 +26,9 @@
 		"@babel/plugin-transform-runtime": "^7.16.4",
 		"@babel/preset-env": "^7.16.4",
 		"@babel/preset-typescript": "^7.16.0",
-		"@webiny/cli": "^5.33.4",
-		"@webiny/plugins": "^5.33.4",
-		"@webiny/project-utils": "^5.33.4",
+		"@webiny/cli": "^5.33.5",
+		"@webiny/plugins": "^5.33.5",
+		"@webiny/project-utils": "^5.33.5",
 		"typescript": "4.7.4"
 	},
 	"publishConfig": {
@@ -48,5 +49,5 @@
 			]
 		}
 	},
-	"gitHead": "e0005197fd69d54f69b1d599122e7cc36ff92f59"
+	"gitHead": "5063045cd2abdcdb360f97262f9b90b8858ec1df"
 }