@webiny/api-dynamodb-to-elasticsearch 5.34.8-beta.1 → 5.35.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/index.js +4 -45
  2. package/index.js.map +1 -1
  3. package/package.json +10 -10
package/index.js CHANGED
@@ -1,123 +1,92 @@
1
1
  "use strict";
2
2
 
3
3
  var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
4
-
5
4
  Object.defineProperty(exports, "__esModule", {
6
5
  value: true
7
6
  });
8
7
  exports.createEventHandler = void 0;
9
-
10
8
  var _error = _interopRequireDefault(require("@webiny/error"));
11
-
12
9
  var _dynamodb = require("aws-sdk/clients/dynamodb");
13
-
14
10
  var _apiElasticsearch = require("@webiny/api-elasticsearch");
15
-
16
11
  var _handlerAws = require("@webiny/handler-aws");
17
-
18
12
  var _pRetry = _interopRequireDefault(require("p-retry"));
19
-
20
13
  var Operations;
21
-
22
14
  (function (Operations) {
23
15
  Operations["INSERT"] = "INSERT";
24
16
  Operations["MODIFY"] = "MODIFY";
25
17
  Operations["REMOVE"] = "REMOVE";
26
18
  })(Operations || (Operations = {}));
27
-
28
19
  const getError = item => {
29
20
  if (!item.index || !item.index.error || !item.index.error.reason) {
30
21
  return null;
31
22
  }
32
-
33
23
  const reason = item.index.error.reason;
34
-
35
24
  if (reason.match(/no such index \[([a-zA-Z0-9_-]+)\]/) !== null) {
36
25
  return "index";
37
26
  }
38
-
39
27
  return reason;
40
28
  };
41
-
42
29
  const getNumberEnvVariable = (name, def) => {
43
30
  const input = process.env[name];
44
31
  const value = Number(input);
45
-
46
32
  if (value > 0) {
47
33
  return value;
48
34
  }
49
-
50
35
  return def;
51
36
  };
52
-
53
37
  const checkErrors = result => {
54
38
  if (!result || !result.body || !result.body.items) {
55
39
  return;
56
40
  }
57
-
58
41
  for (const item of result.body.items) {
59
42
  const err = getError(item);
60
-
61
43
  if (!err) {
62
44
  continue;
63
45
  } else if (err === "index") {
64
46
  if (process.env.DEBUG === "true") {
65
- console.log("Bulk response", JSON.stringify(result, null, 2));
47
+ console.error("Bulk response", JSON.stringify(result, null, 2));
66
48
  }
67
-
68
49
  continue;
69
50
  }
70
-
71
51
  console.error(item.error);
72
52
  throw new _error.default(err, "DYNAMODB_TO_ELASTICSEARCH_ERROR", item);
73
53
  }
74
54
  };
75
-
76
55
  const createEventHandler = () => {
77
56
  return (0, _handlerAws.createDynamoDBEventHandler)(async ({
78
57
  event,
79
58
  context: ctx
80
59
  }) => {
81
60
  const context = ctx;
82
-
83
61
  if (!context.elasticsearch) {
84
62
  console.error("Missing elasticsearch definition on context.");
85
63
  return null;
86
64
  }
65
+
87
66
  /**
88
67
  * Wrap the code we need to run into the function, so it can be called within itself.
89
68
  */
90
-
91
-
92
69
  const execute = async () => {
93
70
  const operations = [];
94
-
95
71
  for (const record of event.Records) {
96
72
  const dynamodb = record.dynamodb;
97
-
98
73
  if (!dynamodb) {
99
74
  continue;
100
75
  }
101
-
102
76
  const newImage = _dynamodb.Converter.unmarshall(dynamodb.NewImage);
103
-
104
77
  if (newImage.ignore === true) {
105
78
  continue;
106
79
  }
107
-
108
80
  const oldImage = _dynamodb.Converter.unmarshall(dynamodb.OldImage);
109
-
110
81
  const keys = _dynamodb.Converter.unmarshall(dynamodb.Keys);
111
-
112
82
  const _id = `${keys.PK}:${keys.SK}`;
113
83
  const operation = record.eventName;
84
+
114
85
  /**
115
86
  * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.
116
87
  * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.
117
88
  */
118
-
119
89
  let data = undefined;
120
-
121
90
  if (operation !== Operations.REMOVE) {
122
91
  /**
123
92
  * We must decompress the data that is going into the Elasticsearch.
@@ -130,13 +99,11 @@ const createEventHandler = () => {
130
99
  * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.
131
100
  * If it is - it is a bug.
132
101
  */
133
-
134
102
  if (data === undefined || data === null) {
135
103
  console.error(`Could not get decompressed data, skipping ES operation "${operation}", ID ${_id}`);
136
104
  continue;
137
105
  }
138
106
  }
139
-
140
107
  switch (record.eventName) {
141
108
  case Operations.INSERT:
142
109
  case Operations.MODIFY:
@@ -147,7 +114,6 @@ const createEventHandler = () => {
147
114
  }
148
115
  }, data);
149
116
  break;
150
-
151
117
  case Operations.REMOVE:
152
118
  operations.push({
153
119
  delete: {
@@ -156,16 +122,13 @@ const createEventHandler = () => {
156
122
  }
157
123
  });
158
124
  break;
159
-
160
125
  default:
161
126
  break;
162
127
  }
163
128
  }
164
-
165
129
  if (!operations.length) {
166
130
  return;
167
131
  }
168
-
169
132
  try {
170
133
  const res = await context.elasticsearch.bulk({
171
134
  body: operations
@@ -175,13 +138,11 @@ const createEventHandler = () => {
175
138
  if (process.env.DEBUG === "true") {
176
139
  const meta = (error === null || error === void 0 ? void 0 : error.meta) || {};
177
140
  delete meta["meta"];
178
- console.error("Bulk error", JSON.stringify(meta, null, 2));
141
+ console.error("Bulk error", JSON.stringify(error, null, 2));
179
142
  }
180
-
181
143
  throw error;
182
144
  }
183
145
  };
184
-
185
146
  const maxRetryTime = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME", 300000);
186
147
  const retries = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES", 20);
187
148
  const minTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT", 1500);
@@ -198,7 +159,6 @@ const createEventHandler = () => {
198
159
  if (error.attemptNumber < retries * 0.75) {
199
160
  return;
200
161
  }
201
-
202
162
  console.error(`Attempt #${error.attemptNumber} failed.`);
203
163
  console.error(error.message);
204
164
  }
@@ -206,5 +166,4 @@ const createEventHandler = () => {
206
166
  return null;
207
167
  });
208
168
  };
209
-
210
169
  exports.createEventHandler = createEventHandler;
package/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"names":["Operations","getError","item","index","error","reason","match","getNumberEnvVariable","name","def","input","process","env","value","Number","checkErrors","result","body","items","err","DEBUG","console","log","JSON","stringify","WebinyError","createEventHandler","createDynamoDBEventHandler","event","context","ctx","elasticsearch","execute","operations","record","Records","dynamodb","newImage","Converter","unmarshall","NewImage","ignore","oldImage","OldImage","keys","Keys","_id","PK","SK","operation","eventName","data","undefined","REMOVE","decompress","plugins","INSERT","MODIFY","push","_index","delete","length","res","bulk","meta","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","onFailedAttempt","attemptNumber","message"],"sources":["index.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\nimport { Converter } from \"aws-sdk/clients/dynamodb\";\nimport { decompress } from \"@webiny/api-elasticsearch\";\nimport { ApiResponse, ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { createDynamoDBEventHandler } from \"@webiny/handler-aws\";\nimport { StreamRecord } from \"aws-lambda/trigger/dynamodb-stream\";\nimport pRetry from \"p-retry\";\n\nenum Operations {\n INSERT = \"INSERT\",\n MODIFY = \"MODIFY\",\n REMOVE = \"REMOVE\"\n}\n\ninterface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\ninterface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\ninterface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\ninterface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index || !item.index.error || !item.index.error.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst getNumberEnvVariable = (name: string, def: number): number => {\n const input = process.env[name];\n const value = Number(input);\n if (value > 0) {\n return value;\n }\n return def;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.log(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(item.error);\n throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n }\n};\n\ninterface RecordDynamoDbImage {\n data: Record<string, any>;\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport const createEventHandler = () => {\n return createDynamoDBEventHandler(async ({ event, context: ctx }) => {\n const context = ctx as unknown as ElasticsearchContext;\n if (!context.elasticsearch) {\n console.error(\"Missing elasticsearch definition on context.\");\n return null;\n }\n\n /**\n * Wrap the code we need to run into the function, so it can be called within itself.\n */\n const execute = async (): Promise<void> => {\n const operations = [];\n\n for (const record of event.Records) {\n const dynamodb = record.dynamodb as Required<StreamRecord>;\n if (!dynamodb) {\n continue;\n }\n const newImage = Converter.unmarshall(dynamodb.NewImage) as RecordDynamoDbImage;\n\n if (newImage.ignore === true) {\n continue;\n }\n\n const oldImage = Converter.unmarshall(dynamodb.OldImage) as RecordDynamoDbImage;\n const keys = Converter.unmarshall(dynamodb.Keys) as RecordDynamoDbKeys;\n const _id = `${keys.PK}:${keys.SK}`;\n const operation = record.eventName;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n let data: any = undefined;\n if (operation !== Operations.REMOVE) {\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n data = await decompress(context.plugins, newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.error(\n `Could not get decompressed data, skipping ES operation \"${operation}\", ID ${_id}`\n );\n continue;\n }\n }\n\n switch (record.eventName) {\n case Operations.INSERT:\n case Operations.MODIFY:\n operations.push({ index: { _id, _index: newImage.index } }, data);\n break;\n case Operations.REMOVE:\n operations.push({ delete: { _id, _index: oldImage.index } });\n break;\n default:\n break;\n }\n }\n\n if (!operations.length) {\n return;\n }\n\n try {\n const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n body: operations\n });\n checkErrors(res);\n } catch (error) {\n if (process.env.DEBUG === \"true\") {\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(meta, null, 2));\n }\n throw error;\n }\n };\n\n const maxRetryTime = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n 300000\n );\n const retries = getNumberEnvVariable(\"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\", 20);\n const minTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n 1500\n );\n const maxTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n 30000\n );\n\n await pRetry(execute, {\n maxRetryTime,\n retries,\n minTimeout,\n maxTimeout,\n onFailedAttempt: error => {\n /**\n * We will only log attempts which are after 3/4 of total attempts.\n */\n if (error.attemptNumber < retries * 0.75) {\n return;\n }\n console.error(`Attempt #${error.attemptNumber} failed.`);\n console.error(error.message);\n }\n });\n\n return null;\n });\n};\n"],"mappings":";;;;;;;;;AAAA;;AACA;;AACA;;AAEA;;AAEA;;IAEKA,U;;WAAAA,U;EAAAA,U;EAAAA,U;EAAAA,U;GAAAA,U,KAAAA,U;;AAuBL,MAAMC,QAAQ,GAAIC,IAAD,IAAyD;EACtE,IAAI,CAACA,IAAI,CAACC,KAAN,IAAe,CAACD,IAAI,CAACC,KAAL,CAAWC,KAA3B,IAAoC,CAACF,IAAI,CAACC,KAAL,CAAWC,KAAX,CAAiBC,MAA1D,EAAkE;IAC9D,OAAO,IAAP;EACH;;EACD,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAL,CAAWC,KAAX,CAAiBC,MAAhC;;EACA,IAAIA,MAAM,CAACC,KAAP,CAAa,oCAAb,MAAuD,IAA3D,EAAiE;IAC7D,OAAO,OAAP;EACH;;EACD,OAAOD,MAAP;AACH,CATD;;AAWA,MAAME,oBAAoB,GAAG,CAACC,IAAD,EAAeC,GAAf,KAAuC;EAChE,MAAMC,KAAK,GAAGC,OAAO,CAACC,GAAR,CAAYJ,IAAZ,CAAd;EACA,MAAMK,KAAK,GAAGC,MAAM,CAACJ,KAAD,CAApB;;EACA,IAAIG,KAAK,GAAG,CAAZ,EAAe;IACX,OAAOA,KAAP;EACH;;EACD,OAAOJ,GAAP;AACH,CAPD;;AASA,MAAMM,WAAW,GAAIC,MAAD,IAA4D;EAC5E,IAAI,CAACA,MAAD,IAAW,CAACA,MAAM,CAACC,IAAnB,IAA2B,CAACD,MAAM,CAACC,IAAP,CAAYC,KAA5C,EAAmD;IAC/C;EACH;;EACD,KAAK,MAAMhB,IAAX,IAAmBc,MAAM,CAACC,IAAP,CAAYC,KAA/B,EAAsC;IAClC,MAAMC,GAAG,GAAGlB,QAAQ,CAACC,IAAD,CAApB;;IACA,IAAI,CAACiB,GAAL,EAAU;MACN;IACH,CAFD,MAEO,IAAIA,GAAG,KAAK,OAAZ,EAAqB;MACxB,IAAIR,OAAO,CAACC,GAAR,CAAYQ,KAAZ,KAAsB,MAA1B,EAAkC;QAC9BC,OAAO,CAACC,GAAR,CAAY,eAAZ,EAA6BC,IAAI,CAACC,SAAL,CAAeR,MAAf,EAAuB,IAAvB,EAA6B,CAA7B,CAA7B;MACH;;MACD;IACH;;IACDK,OAAO,CAACjB,KAAR,CAAcF,IAAI,CAACE,KAAnB;IACA,MAAM,IAAIqB,cAAJ,CAAgBN,GAAhB,EAAqB,iCAArB,EAAwDjB,IAAxD,CAAN;EACH;AACJ,CAjBD;;AA8BO,MAAMwB,kBAAkB,GAAG,MAAM;EACpC,OAAO,IAAAC,sCAAA,EAA2B,OAAO;IAAEC,KAAF;IAASC,OAAO,EAAEC;EAAlB,CAAP,KAAmC;IACjE,MAAMD,OAAO,GAAGC,GAAhB;;IACA,IAAI,CAACD,OAAO,CAACE,aAAb,EAA4B;MACxBV,OAAO,CAACjB,KAAR,CAAc,8CAAd;MACA,OAAO,IAAP;IACH;IAED;AACR;AACA;;;IACQ,MAAM4B,OAAO,GAAG,YAA2B;MACvC,MAAMC,UAAU,GAAG,EAAnB;;MAEA,KAAK,MAAMC,MAAX,IAAqBN,KAAK,CAACO,OAA3B,EAAoC;QAChC,MAAMC,QAAQ,GAAGF,MAAM,CAACE,QAAxB;;QACA,IAAI,CAACA,QAAL,EAAe;UACX;QACH;;QACD,MAAMC,QAAQ,GAAGC,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACI,QAA9B,CAAjB;;QAEA,IAAIH,QAAQ,CAACI,MAAT,KAAoB,IAAxB,EAA8B;UAC1B;QACH;;QAED,MAAMC,QAAQ,GAAGJ,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACO,QAA9B,CAAjB;;QACA,MAAMC,IAAI,GAAGN,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACS,IAA9B,CAAb;;QACA,MAAMC,GAAG,GAAI,GAAEF,IAAI,CAACG,EAAG,IAAGH,IAAI,CAACI,EAAG,EAAlC;QACA,MAAMC,SAAS,GAAGf,MAAM,CAACgB,SAAzB;QAEA;AAChB;AACA;AACA;;QACgB,IAAIC,IAAS,GAAGC,SAAhB;;QACA,IAAIH,SAAS,KAAKjD,UAAU,CAACqD,MAA7B,EAAqC;UACjC;AACpB;AACA;UACoBF,IAAI,GAAG,MAAM,IAAAG,4BAAA,EAAWzB,OAAO,CAAC0B,OAAnB,EAA4BlB,QAAQ,CAACc,IAArC,CAAb;UACA;AACpB;AACA;AACA;AACA;AACA;AACA;;UACoB,IAAIA,IAAI,KAAKC,SAAT,IAAsBD,IAAI,KAAK,IAAnC,EAAyC;YACrC9B,OAAO,CAACjB,KAAR,CACK,2DAA0D6C,SAAU,SAAQH,GAAI,EADrF;YAGA;UACH;QACJ;;QAED,QAAQZ,MAAM,CAACgB,SAAf;UACI,KAAKlD,UAAU,CAACwD,MAAhB;UACA,KAAKxD,UAAU,CAACyD,MAAhB;YACIxB,UAAU,CAACyB,IAAX,CAAgB;cAAEvD,KAAK,EAAE;gBAAE2C,GAAF;gBAAOa,MAAM,EAAEtB,QAAQ,CAAClC;cAAxB;YAAT,CAAhB,EAA4DgD,IAA5D;YACA;;UACJ,KAAKnD,UAAU,CAACqD,MAAhB;YACIpB,UAAU,CAACyB,IAAX,CAAgB;cAAEE,MAAM,EAAE;gBAAEd,GAAF;gBAAOa,MAAM,EAAEjB,QAAQ,CAACvC;cAAxB;YAAV,CAAhB;YACA;;UACJ;YACI;QATR;MAWH;;MAED,IAAI,CAAC8B,UAAU,CAAC4B,MAAhB,EAAwB;QACpB;MACH;;MAED,IAAI;QACA,MAAMC,GAAG,GAAG,MAAMjC,OAAO,CAACE,aAAR,CAAsBgC,IAAtB,CAAuD;UACrE9C,IAAI,EAAEgB;QAD+D,CAAvD,CAAlB;QAGAlB,WAAW,CAAC+C,GAAD,CAAX;MACH,CALD,CAKE,OAAO1D,KAAP,EAAc;QACZ,IAAIO,OAAO,CAACC,GAAR,CAAYQ,KAAZ,KAAsB,MAA1B,EAAkC;UAC9B,MAAM4C,IAAI,GAAG,CAAA5D,KAAK,SAAL,IAAAA,KAAK,WAAL,YAAAA,KAAK,CAAE4D,IAAP,KAAe,EAA5B;UACA,OAAOA,IAAI,CAAC,MAAD,CAAX;UACA3C,OAAO,CAACjB,KAAR,CAAc,YAAd,EAA4BmB,IAAI,CAACC,SAAL,CAAewC,IAAf,EAAqB,IAArB,EAA2B,CAA3B,CAA5B;QACH;;QACD,MAAM5D,KAAN;MACH;IACJ,CA1ED;;IA4EA,MAAM6D,YAAY,GAAG1D,oBAAoB,CACrC,iDADqC,EAErC,MAFqC,CAAzC;IAIA,MAAM2D,OAAO,GAAG3D,oBAAoB,CAAC,0CAAD,EAA6C,EAA7C,CAApC;IACA,MAAM4D,UAAU,GAAG5D,oBAAoB,CACnC,8CADmC,EAEnC,IAFmC,CAAvC;IAIA,MAAM6D,UAAU,GAAG7D,oBAAoB,CACnC,8CADmC,EAEnC,KAFmC,CAAvC;IAKA,MAAM,IAAA8D,eAAA,EAAOrC,OAAP,EAAgB;MAClBiC,YADkB;MAElBC,OAFkB;MAGlBC,UAHkB;MAIlBC,UAJkB;MAKlBE,eAAe,EAAElE,KAAK,IAAI;QACtB;AAChB;AACA;QACgB,IAAIA,KAAK,CAACmE,aAAN,GAAsBL,OAAO,GAAG,IAApC,EAA0C;UACtC;QACH;;QACD7C,OAAO,CAACjB,KAAR,CAAe,YAAWA,KAAK,CAACmE,aAAc,UAA9C;QACAlD,OAAO,CAACjB,KAAR,CAAcA,KAAK,CAACoE,OAApB;MACH;IAdiB,CAAhB,CAAN;IAiBA,OAAO,IAAP;EACH,CAtHM,CAAP;AAuHH,CAxHM"}
1
+ {"version":3,"names":["Operations","getError","item","index","error","reason","match","getNumberEnvVariable","name","def","input","process","env","value","Number","checkErrors","result","body","items","err","DEBUG","console","JSON","stringify","WebinyError","createEventHandler","createDynamoDBEventHandler","event","context","ctx","elasticsearch","execute","operations","record","Records","dynamodb","newImage","Converter","unmarshall","NewImage","ignore","oldImage","OldImage","keys","Keys","_id","PK","SK","operation","eventName","data","undefined","REMOVE","decompress","plugins","INSERT","MODIFY","push","_index","delete","length","res","bulk","meta","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","onFailedAttempt","attemptNumber","message"],"sources":["index.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\nimport { Converter } from \"aws-sdk/clients/dynamodb\";\nimport { decompress } from \"@webiny/api-elasticsearch\";\nimport { ApiResponse, ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { createDynamoDBEventHandler } from \"@webiny/handler-aws\";\nimport { StreamRecord } from \"aws-lambda/trigger/dynamodb-stream\";\nimport pRetry from \"p-retry\";\n\nenum Operations {\n INSERT = \"INSERT\",\n MODIFY = \"MODIFY\",\n REMOVE = \"REMOVE\"\n}\n\ninterface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\ninterface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\ninterface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\ninterface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index || !item.index.error || !item.index.error.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst getNumberEnvVariable = (name: string, def: number): number => {\n const input = process.env[name];\n const value = Number(input);\n if (value > 0) {\n return value;\n }\n return def;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(item.error);\n throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n }\n};\n\ninterface RecordDynamoDbImage {\n data: Record<string, any>;\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport const createEventHandler = () => {\n return createDynamoDBEventHandler(async ({ event, context: ctx }) => {\n const context = ctx as unknown as ElasticsearchContext;\n if (!context.elasticsearch) {\n console.error(\"Missing elasticsearch definition on context.\");\n return null;\n }\n\n /**\n * Wrap the code we need to run into the function, so it can be called within itself.\n */\n const execute = async (): Promise<void> => {\n const operations = [];\n\n for (const record of event.Records) {\n const dynamodb = record.dynamodb as Required<StreamRecord>;\n if (!dynamodb) {\n continue;\n }\n const newImage = Converter.unmarshall(dynamodb.NewImage) as RecordDynamoDbImage;\n\n if (newImage.ignore === true) {\n continue;\n }\n\n const oldImage = Converter.unmarshall(dynamodb.OldImage) as RecordDynamoDbImage;\n const keys = Converter.unmarshall(dynamodb.Keys) as RecordDynamoDbKeys;\n const _id = `${keys.PK}:${keys.SK}`;\n const operation = record.eventName;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n let data: any = undefined;\n if (operation !== Operations.REMOVE) {\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n data = await decompress(context.plugins, newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.error(\n `Could not get decompressed data, skipping ES operation \"${operation}\", ID ${_id}`\n );\n continue;\n }\n }\n\n switch (record.eventName) {\n case Operations.INSERT:\n case Operations.MODIFY:\n operations.push({ index: { _id, _index: newImage.index } }, data);\n break;\n case Operations.REMOVE:\n operations.push({ delete: { _id, _index: oldImage.index } });\n break;\n default:\n break;\n }\n }\n\n if (!operations.length) {\n return;\n }\n\n try {\n const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n body: operations\n });\n checkErrors(res);\n } catch (error) {\n if (process.env.DEBUG === \"true\") {\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n }\n throw error;\n }\n };\n\n const maxRetryTime = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n 300000\n );\n const retries = getNumberEnvVariable(\"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\", 20);\n const minTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n 1500\n );\n const maxTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n 30000\n );\n\n await pRetry(execute, {\n maxRetryTime,\n retries,\n minTimeout,\n maxTimeout,\n onFailedAttempt: error => {\n /**\n * We will only log attempts which are after 3/4 of total attempts.\n */\n if (error.attemptNumber < retries * 0.75) {\n return;\n }\n console.error(`Attempt #${error.attemptNumber} failed.`);\n console.error(error.message);\n }\n });\n\n return null;\n });\n};\n"],"mappings":";;;;;;;AAAA;AACA;AACA;AAEA;AAEA;AAA6B,IAExBA,UAAU;AAAA,WAAVA,UAAU;EAAVA,UAAU;EAAVA,UAAU;EAAVA,UAAU;AAAA,GAAVA,UAAU,KAAVA,UAAU;AAuBf,MAAMC,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,IAAI,CAACD,IAAI,CAACC,KAAK,CAACC,KAAK,IAAI,CAACF,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM,EAAE;IAC9D,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,oBAAoB,GAAG,CAACC,IAAY,EAAEC,GAAW,KAAa;EAChE,MAAMC,KAAK,GAAGC,OAAO,CAACC,GAAG,CAACJ,IAAI,CAAC;EAC/B,MAAMK,KAAK,GAAGC,MAAM,CAACJ,KAAK,CAAC;EAC3B,IAAIG,KAAK,GAAG,CAAC,EAAE;IACX,OAAOA,KAAK;EAChB;EACA,OAAOJ,GAAG;AACd,CAAC;AAED,MAAMM,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMhB,IAAI,IAAIc,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGlB,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACiB,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIR,OAAO,CAACC,GAAG,CAACQ,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACjB,KAAK,CAAC,eAAe,EAAEkB,IAAI,CAACC,SAAS,CAACP,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAK,OAAO,CAACjB,KAAK,CAACF,IAAI,CAACE,KAAK,CAAC;IACzB,MAAM,IAAIoB,cAAW,CAACL,GAAG,EAAE,iCAAiC,EAAEjB,IAAI,CAAC;EACvE;AACJ,CAAC;AAaM,MAAMuB,kBAAkB,GAAG,MAAM;EACpC,OAAO,IAAAC,sCAA0B,EAAC,OAAO;IAAEC,KAAK;IAAEC,OAAO,EAAEC;EAAI,CAAC,KAAK;IACjE,MAAMD,OAAO,GAAGC,GAAsC;IACtD,IAAI,CAACD,OAAO,CAACE,aAAa,EAAE;MACxBT,OAAO,CAACjB,KAAK,CAAC,8CAA8C,CAAC;MAC7D,OAAO,IAAI;IACf;;IAEA;AACR;AACA;IACQ,MAAM2B,OAAO,GAAG,YAA2B;MACvC,MAAMC,UAAU,GAAG,EAAE;MAErB,KAAK,MAAMC,MAAM,IAAIN,KAAK,CAACO,OAAO,EAAE;QAChC,MAAMC,QAAQ,GAAGF,MAAM,CAACE,QAAkC;QAC1D,IAAI,CAACA,QAAQ,EAAE;UACX;QACJ;QACA,MAAMC,QAAQ,GAAGC,mBAAS,CAACC,UAAU,CAACH,QAAQ,CAACI,QAAQ,CAAwB;QAE/E,IAAIH,QAAQ,CAACI,MAAM,KAAK,IAAI,EAAE;UAC1B;QACJ;QAEA,MAAMC,QAAQ,GAAGJ,mBAAS,CAACC,UAAU,CAACH,QAAQ,CAACO,QAAQ,CAAwB;QAC/E,MAAMC,IAAI,GAAGN,mBAAS,CAACC,UAAU,CAACH,QAAQ,CAACS,IAAI,CAAuB;QACtE,MAAMC,GAAG,GAAI,GAAEF,IAAI,CAACG,EAAG,IAAGH,IAAI,CAACI,EAAG,EAAC;QACnC,MAAMC,SAAS,GAAGf,MAAM,CAACgB,SAAS;;QAElC;AAChB;AACA;AACA;QACgB,IAAIC,IAAS,GAAGC,SAAS;QACzB,IAAIH,SAAS,KAAKhD,UAAU,CAACoD,MAAM,EAAE;UACjC;AACpB;AACA;UACoBF,IAAI,GAAG,MAAM,IAAAG,4BAAU,EAACzB,OAAO,CAAC0B,OAAO,EAAElB,QAAQ,CAACc,IAAI,CAAC;UACvD;AACpB;AACA;AACA;AACA;AACA;AACA;UACoB,IAAIA,IAAI,KAAKC,SAAS,IAAID,IAAI,KAAK,IAAI,EAAE;YACrC7B,OAAO,CAACjB,KAAK,CACR,2DAA0D4C,SAAU,SAAQH,GAAI,EAAC,CACrF;YACD;UACJ;QACJ;QAEA,QAAQZ,MAAM,CAACgB,SAAS;UACpB,KAAKjD,UAAU,CAACuD,MAAM;UACtB,KAAKvD,UAAU,CAACwD,MAAM;YAClBxB,UAAU,CAACyB,IAAI,CAAC;cAAEtD,KAAK,EAAE;gBAAE0C,GAAG;gBAAEa,MAAM,EAAEtB,QAAQ,CAACjC;cAAM;YAAE,CAAC,EAAE+C,IAAI,CAAC;YACjE;UACJ,KAAKlD,UAAU,CAACoD,MAAM;YAClBpB,UAAU,CAACyB,IAAI,CAAC;cAAEE,MAAM,EAAE;gBAAEd,GAAG;gBAAEa,MAAM,EAAEjB,QAAQ,CAACtC;cAAM;YAAE,CAAC,CAAC;YAC5D;UACJ;YACI;QAAM;MAElB;MAEA,IAAI,CAAC6B,UAAU,CAAC4B,MAAM,EAAE;QACpB;MACJ;MAEA,IAAI;QACA,MAAMC,GAAG,GAAG,MAAMjC,OAAO,CAACE,aAAa,CAACgC,IAAI,CAA6B;UACrE7C,IAAI,EAAEe;QACV,CAAC,CAAC;QACFjB,WAAW,CAAC8C,GAAG,CAAC;MACpB,CAAC,CAAC,OAAOzD,KAAK,EAAE;QACZ,IAAIO,OAAO,CAACC,GAAG,CAACQ,KAAK,KAAK,MAAM,EAAE;UAC9B,MAAM2C,IAAI,GAAG,CAAA3D,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAE2D,IAAI,KAAI,CAAC,CAAC;UAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;UACnB1C,OAAO,CAACjB,KAAK,CAAC,YAAY,EAAEkB,IAAI,CAACC,SAAS,CAACnB,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/D;QACA,MAAMA,KAAK;MACf;IACJ,CAAC;IAED,MAAM4D,YAAY,GAAGzD,oBAAoB,CACrC,iDAAiD,EACjD,MAAM,CACT;IACD,MAAM0D,OAAO,GAAG1D,oBAAoB,CAAC,0CAA0C,EAAE,EAAE,CAAC;IACpF,MAAM2D,UAAU,GAAG3D,oBAAoB,CACnC,8CAA8C,EAC9C,IAAI,CACP;IACD,MAAM4D,UAAU,GAAG5D,oBAAoB,CACnC,8CAA8C,EAC9C,KAAK,CACR;IAED,MAAM,IAAA6D,eAAM,EAACrC,OAAO,EAAE;MAClBiC,YAAY;MACZC,OAAO;MACPC,UAAU;MACVC,UAAU;MACVE,eAAe,EAAEjE,KAAK,IAAI;QACtB;AAChB;AACA;QACgB,IAAIA,KAAK,CAACkE,aAAa,GAAGL,OAAO,GAAG,IAAI,EAAE;UACtC;QACJ;QACA5C,OAAO,CAACjB,KAAK,CAAE,YAAWA,KAAK,CAACkE,aAAc,UAAS,CAAC;QACxDjD,OAAO,CAACjB,KAAK,CAACA,KAAK,CAACmE,OAAO,CAAC;MAChC;IACJ,CAAC,CAAC;IAEF,OAAO,IAAI;EACf,CAAC,CAAC;AACN,CAAC;AAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@webiny/api-dynamodb-to-elasticsearch",
3
- "version": "5.34.8-beta.1",
3
+ "version": "5.35.0-beta.0",
4
4
  "main": "index.js",
5
5
  "repository": {
6
6
  "type": "git",
@@ -11,12 +11,12 @@
11
11
  "license": "MIT",
12
12
  "author": "Webiny Ltd.",
13
13
  "dependencies": {
14
- "@babel/runtime": "7.19.0",
15
- "@webiny/api-elasticsearch": "5.34.8-beta.1",
16
- "@webiny/error": "5.34.8-beta.1",
17
- "@webiny/handler-aws": "5.34.8-beta.1",
14
+ "@babel/runtime": "7.20.13",
15
+ "@webiny/api-elasticsearch": "5.35.0-beta.0",
16
+ "@webiny/error": "5.35.0-beta.0",
17
+ "@webiny/handler-aws": "5.35.0-beta.0",
18
18
  "aws-lambda": "1.0.7",
19
- "aws-sdk": "2.1230.0",
19
+ "aws-sdk": "2.1310.0",
20
20
  "p-retry": "4.6.2"
21
21
  },
22
22
  "devDependencies": {
@@ -26,9 +26,9 @@
26
26
  "@babel/plugin-transform-runtime": "^7.16.4",
27
27
  "@babel/preset-env": "^7.19.4",
28
28
  "@babel/preset-typescript": "^7.18.6",
29
- "@webiny/cli": "^5.34.8-beta.1",
30
- "@webiny/plugins": "^5.34.8-beta.1",
31
- "@webiny/project-utils": "^5.34.8-beta.1",
29
+ "@webiny/cli": "^5.35.0-beta.0",
30
+ "@webiny/plugins": "^5.35.0-beta.0",
31
+ "@webiny/project-utils": "^5.35.0-beta.0",
32
32
  "typescript": "4.7.4"
33
33
  },
34
34
  "publishConfig": {
@@ -49,5 +49,5 @@
49
49
  ]
50
50
  }
51
51
  },
52
- "gitHead": "6e77eebaac687279fe82ea04f667b7e84214b96a"
52
+ "gitHead": "8acc9e8892842cabb3980ce0b6432fde55968d5b"
53
53
  }