@webiny/api-dynamodb-to-elasticsearch 0.0.0-mt-3 → 0.0.0-unstable.2af142b57e

package/index.d.ts

@@ -0,0 +1 @@
+export declare const createEventHandler: () => import("@webiny/handler-aws").DynamoDBEventHandler<null>;

package/index.js

@@ -0,0 +1,169 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createEventHandler = void 0;
+var _error = _interopRequireDefault(require("@webiny/error"));
+var _dynamodb = require("aws-sdk/clients/dynamodb");
+var _apiElasticsearch = require("@webiny/api-elasticsearch");
+var _handlerAws = require("@webiny/handler-aws");
+var _pRetry = _interopRequireDefault(require("p-retry"));
+var Operations;
+(function (Operations) {
+  Operations["INSERT"] = "INSERT";
+  Operations["MODIFY"] = "MODIFY";
+  Operations["REMOVE"] = "REMOVE";
+})(Operations || (Operations = {}));
+const getError = item => {
+  if (!item.index || !item.index.error || !item.index.error.reason) {
+    return null;
+  }
+  const reason = item.index.error.reason;
+  if (reason.match(/no such index \[([a-zA-Z0-9_-]+)\]/) !== null) {
+    return "index";
+  }
+  return reason;
+};
+const getNumberEnvVariable = (name, def) => {
+  const input = process.env[name];
+  const value = Number(input);
+  if (value > 0) {
+    return value;
+  }
+  return def;
+};
+const checkErrors = result => {
+  if (!result || !result.body || !result.body.items) {
+    return;
+  }
+  for (const item of result.body.items) {
+    const err = getError(item);
+    if (!err) {
+      continue;
+    } else if (err === "index") {
+      if (process.env.DEBUG === "true") {
+        console.log("Bulk response", JSON.stringify(result, null, 2));
+      }
+      continue;
+    }
+    console.log(item.error);
+    throw new _error.default(err, "DYNAMODB_TO_ELASTICSEARCH_ERROR", item);
+  }
+};
+const createEventHandler = () => {
+  return (0, _handlerAws.createDynamoDBEventHandler)(async ({
+    event,
+    context: ctx
+  }) => {
+    const context = ctx;
+    if (!context.elasticsearch) {
+      console.log("Missing elasticsearch definition on context.");
+      return null;
+    }
+
+    /**
+     * Wrap the code we need to run into the function, so it can be called within itself.
+     */
+    const execute = async () => {
+      const operations = [];
+      for (const record of event.Records) {
+        const dynamodb = record.dynamodb;
+        if (!dynamodb) {
+          continue;
+        }
+        const newImage = _dynamodb.Converter.unmarshall(dynamodb.NewImage);
+        if (newImage.ignore === true) {
+          continue;
+        }
+        const oldImage = _dynamodb.Converter.unmarshall(dynamodb.OldImage);
+        const keys = _dynamodb.Converter.unmarshall(dynamodb.Keys);
+        const _id = `${keys.PK}:${keys.SK}`;
+        const operation = record.eventName;
+
+        /**
+         * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.
+         * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.
+         */
+        let data = undefined;
+        if (operation !== Operations.REMOVE) {
+          /**
+           * We must decompress the data that is going into the Elasticsearch.
+           */
+          data = await (0, _apiElasticsearch.decompress)(context.plugins, newImage.data);
+          /**
+           * No point in writing null or undefined data into the Elasticsearch.
+           * This might happen on some error while decompressing. We will log it.
+           *
+           * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.
+           * If it is - it is a bug.
+           */
+          if (data === undefined || data === null) {
+            console.log(`Could not get decompressed data, skipping ES operation "${operation}", ID ${_id}`);
+            continue;
+          }
+        }
+        switch (record.eventName) {
+          case Operations.INSERT:
+          case Operations.MODIFY:
+            operations.push({
+              index: {
+                _id,
+                _index: newImage.index
+              }
+            }, data);
+            break;
+          case Operations.REMOVE:
+            operations.push({
+              delete: {
+                _id,
+                _index: oldImage.index
+              }
+            });
+            break;
+          default:
+            break;
+        }
+      }
+      if (!operations.length) {
+        return;
+      }
+      try {
+        const res = await context.elasticsearch.bulk({
+          body: operations
+        });
+        checkErrors(res);
+      } catch (error) {
+        if (process.env.DEBUG === "true") {
+          const meta = (error === null || error === void 0 ? void 0 : error.meta) || {};
+          delete meta["meta"];
+          console.log("Bulk error", JSON.stringify(meta, null, 2));
+        }
+        throw error;
+      }
+    };
+    const maxRetryTime = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME", 300000);
+    const retries = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES", 20);
+    const minTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT", 1500);
+    const maxTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT", 30000);
+    await (0, _pRetry.default)(execute, {
+      maxRetryTime,
+      retries,
+      minTimeout,
+      maxTimeout,
+      onFailedAttempt: error => {
+        /**
+         * We will only log attempts which are after 3/4 of total attempts.
+         */
+        if (error.attemptNumber < retries * 0.75) {
+          return;
+        }
+        console.log(`Attempt #${error.attemptNumber} failed.`);
+        console.log(error.message);
+      }
+    });
+    return null;
+  });
+};
+exports.createEventHandler = createEventHandler;

package/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["Operations","getError","item","index","error","reason","match","getNumberEnvVariable","name","def","input","process","env","value","Number","checkErrors","result","body","items","err","DEBUG","console","log","JSON","stringify","WebinyError","createEventHandler","createDynamoDBEventHandler","event","context","ctx","elasticsearch","execute","operations","record","Records","dynamodb","newImage","Converter","unmarshall","NewImage","ignore","oldImage","OldImage","keys","Keys","_id","PK","SK","operation","eventName","data","undefined","REMOVE","decompress","plugins","INSERT","MODIFY","push","_index","delete","length","res","bulk","meta","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","onFailedAttempt","attemptNumber","message"],"sources":["index.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\nimport { Converter } from \"aws-sdk/clients/dynamodb\";\nimport { decompress } from \"@webiny/api-elasticsearch\";\nimport { ApiResponse, ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { createDynamoDBEventHandler } from \"@webiny/handler-aws\";\nimport { StreamRecord } from \"aws-lambda/trigger/dynamodb-stream\";\nimport pRetry from \"p-retry\";\n\nenum Operations {\n    INSERT = \"INSERT\",\n    MODIFY = \"MODIFY\",\n    REMOVE = \"REMOVE\"\n}\n\ninterface BulkOperationsResponseBodyItemIndexError {\n    reason?: string;\n}\n\ninterface BulkOperationsResponseBodyItemIndex {\n    error?: BulkOperationsResponseBodyItemIndexError;\n}\n\ninterface BulkOperationsResponseBodyItem {\n    index?: BulkOperationsResponseBodyItemIndex;\n    error?: string;\n}\n\ninterface BulkOperationsResponseBody {\n    items: BulkOperationsResponseBodyItem[];\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n    if (!item.index || !item.index.error || !item.index.error.reason) {\n        return null;\n    }\n    const reason = item.index.error.reason;\n    if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n        return \"index\";\n    }\n    return reason;\n};\n\nconst getNumberEnvVariable = (name: string, def: number): number => {\n    const input = process.env[name];\n    const value = Number(input);\n    if (value > 0) {\n        return value;\n    }\n    return def;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n    if (!result || !result.body || !result.body.items) {\n        return;\n    }\n    for (const item of result.body.items) {\n        const err = getError(item);\n        if (!err) {\n            continue;\n        } else if (err === \"index\") {\n            if (process.env.DEBUG === \"true\") {\n                console.log(\"Bulk response\", JSON.stringify(result, null, 2));\n            }\n            continue;\n        }\n        console.log(item.error);\n        throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n    }\n};\n\ninterface RecordDynamoDbImage {\n    data: Record<string, any>;\n    ignore?: boolean;\n    index: string;\n}\n\ninterface RecordDynamoDbKeys {\n    PK: string;\n    SK: string;\n}\n\nexport const createEventHandler = () => {\n    return createDynamoDBEventHandler(async ({ event, context: ctx }) => {\n        const context = ctx as unknown as ElasticsearchContext;\n        if (!context.elasticsearch) {\n            console.log(\"Missing elasticsearch definition on context.\");\n            return null;\n        }\n\n        /**\n         * Wrap the code we need to run into the function, so it can be called within itself.\n         */\n        const execute = async (): Promise<void> => {\n            const operations = [];\n\n            for (const record of event.Records) {\n                const dynamodb = record.dynamodb as Required<StreamRecord>;\n                if (!dynamodb) {\n                    continue;\n                }\n                const newImage = Converter.unmarshall(dynamodb.NewImage) as RecordDynamoDbImage;\n\n                if (newImage.ignore === true) {\n                    continue;\n                }\n\n                const oldImage = Converter.unmarshall(dynamodb.OldImage) as RecordDynamoDbImage;\n                const keys = Converter.unmarshall(dynamodb.Keys) as RecordDynamoDbKeys;\n                const _id = `${keys.PK}:${keys.SK}`;\n                const operation = record.eventName;\n\n                /**\n                 * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n                 * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n                 */\n                let data: any = undefined;\n                if (operation !== Operations.REMOVE) {\n                    /**\n                     * We must decompress the data that is going into the Elasticsearch.\n                     */\n                    data = await decompress(context.plugins, newImage.data);\n                    /**\n                     * No point in writing null or undefined data into the Elasticsearch.\n                     * This might happen on some error while decompressing. We will log it.\n                     *\n                     * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n                     * If it is - it is a bug.\n                     */\n                    if (data === undefined || data === null) {\n                        console.log(\n                            `Could not get decompressed data, skipping ES operation \"${operation}\", ID ${_id}`\n                        );\n                        continue;\n                    }\n                }\n\n                switch (record.eventName) {\n                    case Operations.INSERT:\n                    case Operations.MODIFY:\n                        operations.push({ index: { _id, _index: newImage.index } }, data);\n                        break;\n                    case Operations.REMOVE:\n                        operations.push({ delete: { _id, _index: oldImage.index } });\n                        break;\n                    default:\n                        break;\n                }\n            }\n\n            if (!operations.length) {\n                return;\n            }\n\n            try {\n                const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n                    body: operations\n                });\n                checkErrors(res);\n            } catch (error) {\n                if (process.env.DEBUG === \"true\") {\n                    const meta = error?.meta || {};\n                    delete meta[\"meta\"];\n                    console.log(\"Bulk error\", JSON.stringify(meta, null, 2));\n                }\n                throw error;\n            }\n        };\n\n        const maxRetryTime = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n            300000\n        );\n        const retries = getNumberEnvVariable(\"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\", 20);\n        const minTimeout = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n            1500\n        );\n        const maxTimeout = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n            30000\n        );\n\n        await pRetry(execute, {\n            maxRetryTime,\n            retries,\n            minTimeout,\n            maxTimeout,\n            onFailedAttempt: error => {\n                /**\n                 * We will only log attempts which are after 3/4 of total attempts.\n                 */\n                if (error.attemptNumber < retries * 0.75) {\n                    return;\n                }\n                console.log(`Attempt #${error.attemptNumber} failed.`);\n                console.log(error.message);\n            }\n        });\n\n        return null;\n    });\n};\n"],"mappings":";;;;;;;AAAA;AACA;AACA;AAEA;AAEA;AAA6B,IAExBA,UAAU;AAAA,WAAVA,UAAU;EAAVA,UAAU;EAAVA,UAAU;EAAVA,UAAU;AAAA,GAAVA,UAAU,KAAVA,UAAU;AAuBf,MAAMC,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,IAAI,CAACD,IAAI,CAACC,KAAK,CAACC,KAAK,IAAI,CAACF,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM,EAAE;IAC9D,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,oBAAoB,GAAG,CAACC,IAAY,EAAEC,GAAW,KAAa;EAChE,MAAMC,KAAK,GAAGC,OAAO,CAACC,GAAG,CAACJ,IAAI,CAAC;EAC/B,MAAMK,KAAK,GAAGC,MAAM,CAACJ,KAAK,CAAC;EAC3B,IAAIG,KAAK,GAAG,CAAC,EAAE;IACX,OAAOA,KAAK;EAChB;EACA,OAAOJ,GAAG;AACd,CAAC;AAED,MAAMM,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMhB,IAAI,IAAIc,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGlB,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACiB,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIR,OAAO,CAACC,GAAG,CAACQ,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACC,GAAG,CAAC,eAAe,EAAEC,IAAI,CAACC,SAAS,CAACR,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACjE;MACA;IACJ;IACAK,OAAO,CAACC,GAAG,CAACpB,IAAI,CAACE,KAAK,CAAC;IACvB,MAAM,IAAIqB,cAAW,CAACN,GAAG,EAAE,iCAAiC,EAAEjB,IAAI,CAAC;EACvE;AACJ,CAAC;AAaM,MAAMwB,kBAAkB,GAAG,MAAM;EACpC,OAAO,IAAAC,sCAA0B,EAAC,OAAO;IAAEC,KAAK;IAAEC,OAAO,EAAEC;EAAI,CAAC,KAAK;IACjE,MAAMD,OAAO,GAAGC,GAAsC;IACtD,IAAI,CAACD,OAAO,CAACE,aAAa,EAAE;MACxBV,OAAO,CAACC,GAAG,CAAC,8CAA8C,CAAC;MAC3D,OAAO,IAAI;IACf;;IAEA;AACR;AACA;IACQ,MAAMU,OAAO,GAAG,YAA2B;MACvC,MAAMC,UAAU,GAAG,EAAE;MAErB,KAAK,MAAMC,MAAM,IAAIN,KAAK,CAACO,OAAO,EAAE;QAChC,MAAMC,QAAQ,GAAGF,MAAM,CAACE,QAAkC;QAC1D,IAAI,CAACA,QAAQ,EAAE;UACX;QACJ;QACA,MAAMC,QAAQ,GAAGC,mBAAS,CAACC,UAAU,CAACH,QAAQ,CAACI,QAAQ,CAAwB;QAE/E,IAAIH,QAAQ,CAACI,MAAM,KAAK,IAAI,EAAE;UAC1B;QACJ;QAEA,MAAMC,QAAQ,GAAGJ,mBAAS,CAACC,UAAU,CAACH,QAAQ,CAACO,QAAQ,CAAwB;QAC/E,MAAMC,IAAI,GAAGN,mBAAS,CAACC,UAAU,CAACH,QAAQ,CAACS,IAAI,CAAuB;QACtE,MAAMC,GAAG,GAAI,GAAEF,IAAI,CAACG,EAAG,IAAGH,IAAI,CAACI,EAAG,EAAC;QACnC,MAAMC,SAAS,GAAGf,MAAM,CAACgB,SAAS;;QAElC;AAChB;AACA;AACA;QACgB,IAAIC,IAAS,GAAGC,SAAS;QACzB,IAAIH,SAAS,KAAKjD,UAAU,CAACqD,MAAM,EAAE;UACjC;AACpB;AACA;UACoBF,IAAI,GAAG,MAAM,IAAAG,4BAAU,EAACzB,OAAO,CAAC0B,OAAO,EAAElB,QAAQ,CAACc,IAAI,CAAC;UACvD;AACpB;AACA;AACA;AACA;AACA;AACA;UACoB,IAAIA,IAAI,KAAKC,SAAS,IAAID,IAAI,KAAK,IAAI,EAAE;YACrC9B,OAAO,CAACC,GAAG,CACN,2DAA0D2B,SAAU,SAAQH,GAAI,EAAC,CACrF;YACD;UACJ;QACJ;QAEA,QAAQZ,MAAM,CAACgB,SAAS;UACpB,KAAKlD,UAAU,CAACwD,MAAM;UACtB,KAAKxD,UAAU,CAACyD,MAAM;YAClBxB,UAAU,CAACyB,IAAI,CAAC;cAAEvD,KAAK,EAAE;gBAAE2C,GAAG;gBAAEa,MAAM,EAAEtB,QAAQ,CAAClC;cAAM;YAAE,CAAC,EAAEgD,IAAI,CAAC;YACjE;UACJ,KAAKnD,UAAU,CAACqD,MAAM;YAClBpB,UAAU,CAACyB,IAAI,CAAC;cAAEE,MAAM,EAAE;gBAAEd,GAAG;gBAAEa,MAAM,EAAEjB,QAAQ,CAACvC;cAAM;YAAE,CAAC,CAAC;YAC5D;UACJ;YACI;QAAM;MAElB;MAEA,IAAI,CAAC8B,UAAU,CAAC4B,MAAM,EAAE;QACpB;MACJ;MAEA,IAAI;QACA,MAAMC,GAAG,GAAG,MAAMjC,OAAO,CAACE,aAAa,CAACgC,IAAI,CAA6B;UACrE9C,IAAI,EAAEgB;QACV,CAAC,CAAC;QACFlB,WAAW,CAAC+C,GAAG,CAAC;MACpB,CAAC,CAAC,OAAO1D,KAAK,EAAE;QACZ,IAAIO,OAAO,CAACC,GAAG,CAACQ,KAAK,KAAK,MAAM,EAAE;UAC9B,MAAM4C,IAAI,GAAG,CAAA5D,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAE4D,IAAI,KAAI,CAAC,CAAC;UAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;UACnB3C,OAAO,CAACC,GAAG,CAAC,YAAY,EAAEC,IAAI,CAACC,SAAS,CAACwC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC5D;QACA,MAAM5D,KAAK;MACf;IACJ,CAAC;IAED,MAAM6D,YAAY,GAAG1D,oBAAoB,CACrC,iDAAiD,EACjD,MAAM,CACT;IACD,MAAM2D,OAAO,GAAG3D,oBAAoB,CAAC,0CAA0C,EAAE,EAAE,CAAC;IACpF,MAAM4D,UAAU,GAAG5D,oBAAoB,CACnC,8CAA8C,EAC9C,IAAI,CACP;IACD,MAAM6D,UAAU,GAAG7D,oBAAoB,CACnC,8CAA8C,EAC9C,KAAK,CACR;IAED,MAAM,IAAA8D,eAAM,EAACrC,OAAO,EAAE;MAClBiC,YAAY;MACZC,OAAO;MACPC,UAAU;MACVC,UAAU;MACVE,eAAe,EAAElE,KAAK,IAAI;QACtB;AAChB;AACA;QACgB,IAAIA,KAAK,CAACmE,aAAa,GAAGL,OAAO,GAAG,IAAI,EAAE;UACtC;QACJ;QACA7C,OAAO,CAACC,GAAG,CAAE,YAAWlB,KAAK,CAACmE,aAAc,UAAS,CAAC;QACtDlD,OAAO,CAACC,GAAG,CAAClB,KAAK,CAACoE,OAAO,CAAC;MAC9B;IACJ,CAAC,CAAC;IAEF,OAAO,IAAI;EACf,CAAC,CAAC;AACN,CAAC;AAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@webiny/api-dynamodb-to-elasticsearch",
-	"version": "0.0.0-mt-3",
+	"version": "0.0.0-unstable.2af142b57e",
 	"main": "index.js",
 	"repository": {
 		"type": "git",
@@ -11,22 +11,25 @@
 	"license": "MIT",
 	"author": "Webiny Ltd.",
 	"dependencies": {
-		"@babel/runtime": "7.15.4",
-		"@webiny/api-elasticsearch": "0.0.0-mt-3",
-		"@webiny/error": "0.0.0-mt-3",
-		"@webiny/handler": "0.0.0-mt-3",
-		"aws-sdk": "2.1026.0"
+		"@babel/runtime": "7.20.13",
+		"@webiny/api-elasticsearch": "0.0.0-unstable.2af142b57e",
+		"@webiny/error": "0.0.0-unstable.2af142b57e",
+		"@webiny/handler-aws": "0.0.0-unstable.2af142b57e",
+		"aws-lambda": "1.0.7",
+		"aws-sdk": "2.1310.0",
+		"p-retry": "4.6.2"
 	},
 	"devDependencies": {
-		"@babel/cli": "^7.5.5",
-		"@babel/core": "^7.5.5",
-		"@babel/plugin-proposal-object-rest-spread": "^7.5.5",
-		"@babel/plugin-transform-runtime": "^7.5.5",
-		"@babel/preset-env": "^7.5.5",
-		"@babel/preset-typescript": "^7.0.0",
-		"@webiny/cli": "^0.0.0-mt-3",
-		"@webiny/project-utils": "^0.0.0-mt-3",
-		"typescript": "^4.1.3"
+		"@babel/cli": "^7.19.3",
+		"@babel/core": "^7.19.3",
+		"@babel/plugin-proposal-object-rest-spread": "^7.16.0",
+		"@babel/plugin-transform-runtime": "^7.16.4",
+		"@babel/preset-env": "^7.19.4",
+		"@babel/preset-typescript": "^7.18.6",
+		"@webiny/cli": "^0.0.0-unstable.2af142b57e",
+		"@webiny/plugins": "^0.0.0-unstable.2af142b57e",
+		"@webiny/project-utils": "^0.0.0-unstable.2af142b57e",
+		"typescript": "4.7.4"
 	},
 	"publishConfig": {
 		"access": "public",
@@ -46,5 +49,5 @@
 			]
 		}
 	},
-	"gitHead": "ebea815be2be99404591cba465cc1fe88355bd48"
+	"gitHead": "2af142b57e7cdc433f5098b3b6d43ae6caa5d54e"
 }

package/handler/index.d.ts

@@ -1,4 +0,0 @@
-import { HandlerPlugin } from "@webiny/handler/types";
-import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
-declare const _default: () => HandlerPlugin<ElasticsearchContext>;
-export default _default;

package/handler/index.js

@@ -1,158 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-
-var _dynamodb = require("aws-sdk/clients/dynamodb");
-
-var _error = _interopRequireDefault(require("@webiny/error"));
-
-var _compression = require("@webiny/api-elasticsearch/compression");
-
-var Operations;
-
-(function (Operations) {
-  Operations["INSERT"] = "INSERT";
-  Operations["MODIFY"] = "MODIFY";
-  Operations["REMOVE"] = "REMOVE";
-})(Operations || (Operations = {}));
-
-const getError = item => {
-  if (!item.index || !item.index.error || !item.index.error.reason) {
-    return null;
-  }
-
-  const reason = item.index.error.reason;
-
-  if (reason.match(/no such index \[([a-zA-Z0-9_-]+)\]/) !== null) {
-    return "index";
-  }
-
-  return reason;
-};
-
-const checkErrors = result => {
-  if (!result || !result.body || !result.body.items) {
-    return;
-  }
-
-  for (const item of result.body.items) {
-    const err = getError(item);
-
-    if (!err) {
-      continue;
-    } else if (err === "index") {
-      if (process.env.DEBUG === "true") {
-        console.log("Bulk response", JSON.stringify(result, null, 2));
-      }
-
-      continue;
-    }
-
-    console.log(item.error);
-    throw new _error.default(err, "DYNAMODB_TO_ELASTICSEARCH_ERROR", item);
-  }
-};
-
-var _default = () => ({
-  type: "handler",
-
-  async handle(context) {
-    const [event] = context.args;
-    const operations = [];
-
-    for (const record of event.Records) {
-      const newImage = _dynamodb.Converter.unmarshall(record.dynamodb.NewImage);
-
-      if (newImage.ignore === true) {
-        continue;
-      }
-
-      const oldImage = _dynamodb.Converter.unmarshall(record.dynamodb.OldImage);
-
-      const keys = _dynamodb.Converter.unmarshall(record.dynamodb.Keys);
-
-      const _id = `${keys.PK}:${keys.SK}`;
-      const operation = record.eventName;
-      /**
-       * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.
-       * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.
-       */
-
-      let data = undefined;
-
-      if (operation !== Operations.REMOVE) {
-        /**
-         * We must decompress the data that is going into the Elasticsearch.
-         */
-        data = await (0, _compression.decompress)(context.plugins, newImage.data);
-        /**
-         * No point in writing null or undefined data into the Elasticsearch.
-         * This might happen on some error while decompressing. We will log it.
-         *
-         * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.
-         * If it is - it is a bug.
-         */
-
-        if (data === undefined || data === null) {
-          console.log(`Could not get decompressed data, skipping ES operation "${operation}", ID ${_id}`);
-          continue;
-        }
-      }
-
-      switch (record.eventName) {
-        case Operations.INSERT:
-        case Operations.MODIFY:
-          operations.push({
-            index: {
-              _id,
-              _index: newImage.index
-            }
-          }, data);
-          break;
-
-        case Operations.REMOVE:
-          operations.push({
-            delete: {
-              _id,
-              _index: oldImage.index
-            }
-          });
-          break;
-
-        default:
-          break;
-      }
-    }
-
-    if (!operations.length) {
-      return;
-    }
-
-    try {
-      const res = await context.elasticsearch.bulk({
-        body: operations
-      });
-      checkErrors(res);
-
-      if (process.env.DEBUG === "true") {
-        console.log("Bulk response", JSON.stringify(res, null, 2));
-      }
-    } catch (error) {
-      if (process.env.DEBUG === "true") {
-        console.log("Bulk error", JSON.stringify(error, null, 2));
-      }
-
-      throw error;
-    }
-
-    return true;
-  }
-
-});
-
-exports.default = _default;