@webiny/api-dynamodb-to-elasticsearch 5.33.1-beta.0 → 5.33.2-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +2 -2
- package/index.js.map +1 -1
- package/package.json +8 -8
package/index.js
CHANGED
|
@@ -11,7 +11,7 @@ var _error = _interopRequireDefault(require("@webiny/error"));
|
|
|
11
11
|
|
|
12
12
|
var _dynamodb = require("aws-sdk/clients/dynamodb");
|
|
13
13
|
|
|
14
|
-
var
|
|
14
|
+
var _apiElasticsearch = require("@webiny/api-elasticsearch");
|
|
15
15
|
|
|
16
16
|
var _handlerAws = require("@webiny/handler-aws");
|
|
17
17
|
|
|
@@ -104,7 +104,7 @@ const createEventHandler = () => {
|
|
|
104
104
|
/**
|
|
105
105
|
* We must decompress the data that is going into the Elasticsearch.
|
|
106
106
|
*/
|
|
107
|
-
data = await (0,
|
|
107
|
+
data = await (0, _apiElasticsearch.decompress)(context.plugins, newImage.data);
|
|
108
108
|
/**
|
|
109
109
|
* No point in writing null or undefined data into the Elasticsearch.
|
|
110
110
|
* This might happen on some error while decompressing. We will log it.
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["Operations","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","log","JSON","stringify","WebinyError","createEventHandler","createDynamoDBEventHandler","event","context","ctx","elasticsearch","operations","record","Records","dynamodb","newImage","Converter","unmarshall","NewImage","ignore","oldImage","OldImage","keys","Keys","_id","PK","SK","operation","eventName","data","undefined","REMOVE","decompress","plugins","INSERT","MODIFY","push","_index","delete","length","res","bulk"],"sources":["index.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\nimport { Converter } from \"aws-sdk/clients/dynamodb\";\nimport { decompress } from \"@webiny/api-elasticsearch
|
|
1
|
+
{"version":3,"names":["Operations","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","log","JSON","stringify","WebinyError","createEventHandler","createDynamoDBEventHandler","event","context","ctx","elasticsearch","operations","record","Records","dynamodb","newImage","Converter","unmarshall","NewImage","ignore","oldImage","OldImage","keys","Keys","_id","PK","SK","operation","eventName","data","undefined","REMOVE","decompress","plugins","INSERT","MODIFY","push","_index","delete","length","res","bulk"],"sources":["index.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\nimport { Converter } from \"aws-sdk/clients/dynamodb\";\nimport { decompress } from \"@webiny/api-elasticsearch\";\nimport { ApiResponse, ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { createDynamoDBEventHandler } from \"@webiny/handler-aws\";\nimport { StreamRecord } from \"aws-lambda/trigger/dynamodb-stream\";\n\nenum Operations {\n INSERT = \"INSERT\",\n MODIFY = \"MODIFY\",\n REMOVE = \"REMOVE\"\n}\n\ninterface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\ninterface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\ninterface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\ninterface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index || !item.index.error || !item.index.error.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.log(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.log(item.error);\n throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n }\n};\n\ninterface RecordDynamoDbImage {\n data: Record<string, any>;\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport const createEventHandler = () => {\n return createDynamoDBEventHandler(async ({ event, context: ctx }) => {\n const context = ctx as unknown as ElasticsearchContext;\n if (!context.elasticsearch) {\n console.log(\"Missing elasticsearch definition on context.\");\n return null;\n }\n const operations = [];\n\n for (const record of event.Records) {\n const dynamodb = record.dynamodb as Required<StreamRecord>;\n if (!dynamodb) {\n continue;\n }\n const newImage = Converter.unmarshall(dynamodb.NewImage) as RecordDynamoDbImage;\n\n if (newImage.ignore === true) {\n continue;\n }\n\n const oldImage = Converter.unmarshall(dynamodb.OldImage) as RecordDynamoDbImage;\n const keys = Converter.unmarshall(dynamodb.Keys) as RecordDynamoDbKeys;\n const _id = `${keys.PK}:${keys.SK}`;\n const operation = record.eventName;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n let data: any = undefined;\n if (operation !== Operations.REMOVE) {\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n data = await decompress(context.plugins, newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.log(\n `Could not get decompressed data, skipping ES operation \"${operation}\", ID ${_id}`\n );\n continue;\n }\n }\n\n switch (record.eventName) {\n case Operations.INSERT:\n case Operations.MODIFY:\n operations.push({ index: { _id, _index: newImage.index } }, data);\n break;\n case Operations.REMOVE:\n operations.push({ delete: { _id, _index: oldImage.index } });\n break;\n default:\n break;\n }\n }\n\n if (!operations.length) {\n return null;\n }\n\n try {\n const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n body: operations\n });\n checkErrors(res);\n if (process.env.DEBUG === \"true\") {\n console.log(\"Bulk response\", JSON.stringify(res, null, 2));\n }\n } catch (error) {\n if (process.env.DEBUG === \"true\") {\n console.log(\"Bulk error\", JSON.stringify(error, null, 2));\n }\n throw error;\n }\n\n return null;\n });\n};\n"],"mappings":";;;;;;;;;AAAA;;AACA;;AACA;;AAEA;;IAGKA,U;;WAAAA,U;EAAAA,U;EAAAA,U;EAAAA,U;GAAAA,U,KAAAA,U;;AAoBL,MAAMC,QAAQ,GAAIC,IAAD,IAAyD;EACtE,IAAI,CAACA,IAAI,CAACC,KAAN,IAAe,CAACD,IAAI,CAACC,KAAL,CAAWC,KAA3B,IAAoC,CAACF,IAAI,CAACC,KAAL,CAAWC,KAAX,CAAiBC,MAA1D,EAAkE;IAC9D,OAAO,IAAP;EACH;;EACD,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAL,CAAWC,KAAX,CAAiBC,MAAhC;;EACA,IAAIA,MAAM,CAACC,KAAP,CAAa,oCAAb,MAAuD,IAA3D,EAAiE;IAC7D,OAAO,OAAP;EACH;;EACD,OAAOD,MAAP;AACH,CATD;;AAWA,MAAME,WAAW,GAAIC,MAAD,IAA4D;EAC5E,IAAI,CAACA,MAAD,IAAW,CAACA,MAAM,CAACC,IAAnB,IAA2B,CAACD,MAAM,CAACC,IAAP,CAAYC,KAA5C,EAAmD;IAC/C;EACH;;EACD,KAAK,MAAMR,IAAX,IAAmBM,MAAM,CAACC,IAAP,CAAYC,KAA/B,EAAsC;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAD,CAApB;;IACA,IAAI,CAACS,GAAL,EAAU;MACN;IACH,CAFD,MAEO,IAAIA,GAAG,KAAK,OAAZ,EAAqB;MACxB,IAAIC,OAAO,CAACC,GAAR,CAAYC,KAAZ,KAAsB,MAA1B,EAAkC;QAC9BC,OAAO,CAACC,GAAR,CAAY,eAAZ,EAA6BC,IAAI,CAACC,SAAL,CAAeV,MAAf,EAAuB,IAAvB,EAA6B,CAA7B,CAA7B;MACH;;MACD;IACH;;IACDO,OAAO,CAACC,GAAR,CAAYd,IAAI,CAACE,KAAjB;IACA,MAAM,IAAIe,cAAJ,CAAgBR,GAAhB,EAAqB,iCAArB,EAAwDT,IAAxD,CAAN;EACH;AACJ,CAjBD;;AA8BO,MAAMkB,kBAAkB,GAAG,MAAM;EACpC,OAAO,IAAAC,sCAAA,EAA2B,OAAO;IAAEC,KAAF;IAASC,OAAO,EAAEC;EAAlB,CAAP,KAAmC;IACjE,MAAMD,OAAO,GAAGC,GAAhB;;IACA,IAAI,CAACD,OAAO,CAACE,aAAb,EAA4B;MACxBV,OAAO,CAACC,GAAR,CAAY,8CAAZ;MACA,OAAO,IAAP;IACH;;IACD,MAAMU,UAAU,GAAG,EAAnB;;IAEA,KAAK,MAAMC,MAAX,IAAqBL,KAAK,CAACM,OAA3B,EAAoC;MAChC,MAAMC,QAAQ,GAAGF,MAAM,CAACE,QAAxB;;MACA,IAAI,CAACA,QAAL,EAAe;QACX;MACH;;MACD,MAAMC,QAAQ,GAAGC,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACI,QAA9B,CAAjB;;MAEA,IAAIH,QAAQ,CAACI,MAAT,KAAoB,IAAxB,EAA8B;QAC1B;MACH;;MAED,MAAMC,QAAQ,GAAGJ,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACO,QAA9B,CAAjB;;MACA,MAAMC,IAAI,GAAGN,mBAAA,CAAUC,UAAV,CAAqBH,QAAQ,CAACS,IAA9B,CAAb;;MACA,MAAMC,GAAG,GAAI,GAAEF,IAAI,CAACG,EAAG,IAAGH,IAAI,CAACI,EAAG,EAAlC;MACA,MAAMC,SAAS,GAAGf,MAAM,CAACgB,SAAzB;MAEA;AACZ;AACA;AACA;;MACY,IAAIC,IAAS,GAAGC,SAAhB;;MACA,IAAIH,SAAS,KAAK1C,UAAU,CAAC8C,MAA7B,EAAqC;QACjC;AAChB;AACA;QACgBF,IAAI,GAAG,MAAM,IAAAG,4BAAA,EAAWxB,OAAO,CAACyB,OAAnB,EAA4BlB,QAAQ,CAACc,IAArC,CAAb;QACA;AAChB;AACA;AACA;AACA;AACA;AACA;;QACgB,IAAIA,IAAI,KAAKC,SAAT,IAAsBD,IAAI,KAAK,IAAnC,EAAyC;UACrC7B,OAAO,CAACC,GAAR,CACK,2DAA0D0B,SAAU,SAAQH,GAAI,EADrF;UAGA;QACH;MACJ;;MAED,QAAQZ,MAAM,CAACgB,SAAf;QACI,KAAK3C,UAAU,CAACiD,MAAhB;QACA,KAAKjD,UAAU,CAACkD,MAAhB;UACIxB,UAAU,CAACyB,IAAX,CAAgB;YAAEhD,KAAK,EAAE;cAAEoC,GAAF;cAAOa,MAAM,EAAEtB,QAAQ,CAAC3B;YAAxB;UAAT,CAAhB,EAA4DyC,IAA5D;UACA;;QACJ,KAAK5C,UAAU,CAAC8C,MAAhB;UACIpB,UAAU,CAACyB,IAAX,CAAgB;YAAEE,MAAM,EAAE;cAAEd,GAAF;cAAOa,MAAM,EAAEjB,QAAQ,CAAChC;YAAxB;UAAV,CAAhB;UACA;;QACJ;UACI;MATR;IAWH;;IAED,IAAI,CAACuB,UAAU,CAAC4B,MAAhB,EAAwB;MACpB,OAAO,IAAP;IACH;;IAED,IAAI;MACA,MAAMC,GAAG,GAAG,MAAMhC,OAAO,CAACE,aAAR,CAAsB+B,IAAtB,CAAuD;QACrE/C,IAAI,EAAEiB;MAD+D,CAAvD,CAAlB;MAGAnB,WAAW,CAACgD,GAAD,CAAX;;MACA,IAAI3C,OAAO,CAACC,GAAR,CAAYC,KAAZ,KAAsB,MAA1B,EAAkC;QAC9BC,OAAO,CAACC,GAAR,CAAY,eAAZ,EAA6BC,IAAI,CAACC,SAAL,CAAeqC,GAAf,EAAoB,IAApB,EAA0B,CAA1B,CAA7B;MACH;IACJ,CARD,CAQE,OAAOnD,KAAP,EAAc;MACZ,IAAIQ,OAAO,CAACC,GAAR,CAAYC,KAAZ,KAAsB,MAA1B,EAAkC;QAC9BC,OAAO,CAACC,GAAR,CAAY,YAAZ,EAA0BC,IAAI,CAACC,SAAL,CAAed,KAAf,EAAsB,IAAtB,EAA4B,CAA5B,CAA1B;MACH;;MACD,MAAMA,KAAN;IACH;;IAED,OAAO,IAAP;EACH,CAlFM,CAAP;AAmFH,CApFM"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webiny/api-dynamodb-to-elasticsearch",
|
|
3
|
-
"version": "5.33.
|
|
3
|
+
"version": "5.33.2-beta.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -12,9 +12,9 @@
|
|
|
12
12
|
"author": "Webiny Ltd.",
|
|
13
13
|
"dependencies": {
|
|
14
14
|
"@babel/runtime": "7.18.9",
|
|
15
|
-
"@webiny/api-elasticsearch": "5.33.
|
|
16
|
-
"@webiny/error": "5.33.
|
|
17
|
-
"@webiny/handler-aws": "5.33.
|
|
15
|
+
"@webiny/api-elasticsearch": "5.33.2-beta.0",
|
|
16
|
+
"@webiny/error": "5.33.2-beta.0",
|
|
17
|
+
"@webiny/handler-aws": "5.33.2-beta.0",
|
|
18
18
|
"aws-lambda": "1.0.7",
|
|
19
19
|
"aws-sdk": "2.1188.0"
|
|
20
20
|
},
|
|
@@ -25,9 +25,9 @@
|
|
|
25
25
|
"@babel/plugin-transform-runtime": "^7.16.4",
|
|
26
26
|
"@babel/preset-env": "^7.16.4",
|
|
27
27
|
"@babel/preset-typescript": "^7.16.0",
|
|
28
|
-
"@webiny/cli": "^5.33.
|
|
29
|
-
"@webiny/plugins": "^5.33.
|
|
30
|
-
"@webiny/project-utils": "^5.33.
|
|
28
|
+
"@webiny/cli": "^5.33.2-beta.0",
|
|
29
|
+
"@webiny/plugins": "^5.33.2-beta.0",
|
|
30
|
+
"@webiny/project-utils": "^5.33.2-beta.0",
|
|
31
31
|
"typescript": "4.7.4"
|
|
32
32
|
},
|
|
33
33
|
"publishConfig": {
|
|
@@ -48,5 +48,5 @@
|
|
|
48
48
|
]
|
|
49
49
|
}
|
|
50
50
|
},
|
|
51
|
-
"gitHead": "
|
|
51
|
+
"gitHead": "fe536f893f26e4830e97d67368a9a9dd2b016fda"
|
|
52
52
|
}
|