@webiny/api-dynamodb-to-elasticsearch 5.42.1-beta.1 → 5.42.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/execute.js +1 -1
- package/execute.js.map +1 -1
- package/package.json +10 -10
package/execute.js
CHANGED
package/execute.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_apiElasticsearch","require","_error","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","total","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","elasticsearch","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","log","logger","withSource","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","tenant","locale","meta","info","exports"],"sources":["execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n ElasticsearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport { ITimer } from \"@webiny/handler-aws\";\nimport { ApiResponse } from \"@webiny/api-elasticsearch/types\";\nimport { WebinyError } from \"@webiny/error\";\nimport { Context, IOperations } from \"./types\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n timer: ITimer;\n maxRunningTime: number;\n maxProcessorPercent: number;\n context: Pick<Context, \"elasticsearch\" | \"logger\">;\n operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index?.error?.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(item
|
|
1
|
+
{"version":3,"names":["_apiElasticsearch","require","_error","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","total","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","elasticsearch","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","log","logger","withSource","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","tenant","locale","meta","info","exports"],"sources":["execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n ElasticsearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport { ITimer } from \"@webiny/handler-aws\";\nimport { ApiResponse } from \"@webiny/api-elasticsearch/types\";\nimport { WebinyError } from \"@webiny/error\";\nimport { Context, IOperations } from \"./types\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n timer: ITimer;\n maxRunningTime: number;\n maxProcessorPercent: number;\n context: Pick<Context, \"elasticsearch\" | \"logger\">;\n operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index?.error?.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(\"Body item with error\", item);\n throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n }\n};\n\nexport const execute = (params: IExecuteParams) => {\n return async (): Promise<void> => {\n const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n if (operations.total === 0) {\n return;\n }\n\n const remainingTime = timer.getRemainingSeconds();\n const runningTime = maxRunningTime - remainingTime;\n const maxWaitingTime = remainingTime - 90;\n\n if (process.env.DEBUG === \"true\") {\n console.debug(\n `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n );\n }\n\n const healthCheck = createWaitUntilHealthy(context.elasticsearch, {\n minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,\n waitingTimeStep: 30,\n maxProcessorPercent,\n maxWaitingTime\n });\n\n const log = context.logger.withSource(\"dynamodbToElasticsearch\");\n\n try {\n await healthCheck.wait({\n async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n console.debug(`Cluster is unhealthy on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n },\n async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n console.error(`Cluster health check timeout on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n }\n });\n } catch (ex) {\n if (\n ex instanceof UnhealthyClusterError ||\n ex instanceof WaitingHealthyClusterAbortedError\n ) {\n throw ex;\n }\n console.error(`Cluster health check failed.`, ex);\n throw ex;\n }\n\n try {\n const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n body: operations.items\n });\n checkErrors(res);\n } catch (error) {\n log.error(error, {\n tenant: \"root\",\n locale: \"unknown\"\n });\n\n if (process.env.DEBUG !== \"true\") {\n throw error;\n }\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n throw error;\n }\n if (process.env.DEBUG !== \"true\") {\n return;\n }\n console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n };\n};\n"],"mappings":";;;;;;AAAA,IAAAA,iBAAA,GAAAC,OAAA;AAQA,IAAAC,MAAA,GAAAD,OAAA;AA4BA,MAAME,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,EAAEC,KAAK,EAAEC,MAAM,EAAE;IAC5B,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMR,IAAI,IAAIM,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACS,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACX,KAAK,CAAC,eAAe,EAAEY,IAAI,CAACC,SAAS,CAACT,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAO,OAAO,CAACX,KAAK,CAAC,sBAAsB,EAAEF,IAAI,CAAC;IAC3C,MAAM,IAAIgB,kBAAW,CAACP,GAAG,EAAE,iCAAiC,EAAET,IAAI,CAAC;EACvE;AACJ,CAAC;AAEM,MAAMiB,OAAO,GAAIC,MAAsB,IAAK;EAC/C,OAAO,YAA2B;IAC9B,MAAM;MAAEC,OAAO;MAAEC,KAAK;MAAEC,cAAc;MAAEC,mBAAmB;MAAEC;IAAW,CAAC,GAAGL,MAAM;IAElF,IAAIK,UAAU,CAACC,KAAK,KAAK,CAAC,EAAE;MACxB;IACJ;IAEA,MAAMC,aAAa,GAAGL,KAAK,CAACM,mBAAmB,CAAC,CAAC;IACjD,MAAMC,WAAW,GAAGN,cAAc,GAAGI,aAAa;IAClD,MAAMG,cAAc,GAAGH,aAAa,GAAG,EAAE;IAEzC,IAAIf,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9BC,OAAO,CAACgB,KAAK,CACT,qCAAqCF,WAAW,6CAA6CC,cAAc,GAC/G,CAAC;IACL;IAEA,MAAME,WAAW,GAAG,IAAAC,wCAAsB,EAACZ,OAAO,CAACa,aAAa,EAAE;MAC9DC,sBAAsB,EAAEC,qDAAmC,CAACC,MAAM;MAClEC,eAAe,EAAE,EAAE;MACnBd,mBAAmB;MACnBM;IACJ,CAAC,CAAC;IAEF,MAAMS,GAAG,GAAGlB,OAAO,CAACmB,MAAM,CAACC,UAAU,CAAC,yBAAyB,CAAC;IAEhE,IAAI;MACA,MAAMT,WAAW,CAACU,IAAI,CAAC;QACnB,MAAMC,WAAWA,CAAC;UAAEC,SAAS;UAAEC,IAAI;UAAEC,SAAS;UAAER,eAAe;UAAES;QAAc,CAAC,EAAE;UAC9EhC,OAAO,CAACgB,KAAK,CAAC,gCAAgCc,IAAI,GAAG,EAAE;YACnDD,SAAS;YACTE,SAAS;YACTR,eAAe;YACfS;UACJ,CAAC,CAAC;QACN,CAAC;QACD,MAAMC,SAASA,CAAC;UAAEJ,SAAS;UAAEC,IAAI;UAAEP,eAAe;UAAEQ,SAAS;UAAEC;QAAc,CAAC,EAAE;UAC5EhC,OAAO,CAACX,KAAK,CAAC,wCAAwCyC,IAAI,GAAG,EAAE;YAC3DD,SAAS;YACTE,SAAS;YACTR,eAAe;YACfS;UACJ,CAAC,CAAC;QACN;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,IACIA,EAAE,YAAYC,uCAAqB,IACnCD,EAAE,YAAYE,mDAAiC,EACjD;QACE,MAAMF,EAAE;MACZ;MACAlC,OAAO,CAACX,KAAK,CAAC,8BAA8B,EAAE6C,EAAE,CAAC;MACjD,MAAMA,EAAE;IACZ;IAEA,IAAI;MACA,MAAMG,GAAG,GAAG,MAAM/B,OAAO,CAACa,aAAa,CAACmB,IAAI,CAA6B;QACrE5C,IAAI,EAAEgB,UAAU,CAACf;MACrB,CAAC,CAAC;MACFH,WAAW,CAAC6C,GAAG,CAAC;IACpB,CAAC,CAAC,OAAOhD,KAAK,EAAE;MACZmC,GAAG,CAACnC,KAAK,CAACA,KAAK,EAAE;QACbkD,MAAM,EAAE,MAAM;QACdC,MAAM,EAAE;MACZ,CAAC,CAAC;MAEF,IAAI3C,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9B,MAAMV,KAAK;MACf;MACA,MAAMoD,IAAI,GAAGpD,KAAK,EAAEoD,IAAI,IAAI,CAAC,CAAC;MAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;MACnBzC,OAAO,CAACX,KAAK,CAAC,YAAY,EAAEY,IAAI,CAACC,SAAS,CAACb,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MAC3D,MAAMA,KAAK;IACf;IACA,IAAIQ,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9B;IACJ;IACAC,OAAO,CAAC0C,IAAI,CAAC,eAAehC,UAAU,CAACC,KAAK,sCAAsC,CAAC;EACvF,CAAC;AACL,CAAC;AAACgC,OAAA,CAAAvC,OAAA,GAAAA,OAAA","ignoreList":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webiny/api-dynamodb-to-elasticsearch",
|
|
3
|
-
"version": "5.42.1-beta.
|
|
3
|
+
"version": "5.42.1-beta.2",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -11,17 +11,17 @@
|
|
|
11
11
|
"license": "MIT",
|
|
12
12
|
"author": "Webiny Ltd.",
|
|
13
13
|
"dependencies": {
|
|
14
|
-
"@webiny/api-elasticsearch": "5.42.1-beta.
|
|
15
|
-
"@webiny/api-log": "5.42.1-beta.
|
|
16
|
-
"@webiny/aws-sdk": "5.42.1-beta.
|
|
17
|
-
"@webiny/error": "5.42.1-beta.
|
|
18
|
-
"@webiny/handler-aws": "5.42.1-beta.
|
|
14
|
+
"@webiny/api-elasticsearch": "5.42.1-beta.2",
|
|
15
|
+
"@webiny/api-log": "5.42.1-beta.2",
|
|
16
|
+
"@webiny/aws-sdk": "5.42.1-beta.2",
|
|
17
|
+
"@webiny/error": "5.42.1-beta.2",
|
|
18
|
+
"@webiny/handler-aws": "5.42.1-beta.2",
|
|
19
19
|
"p-retry": "4.6.2"
|
|
20
20
|
},
|
|
21
21
|
"devDependencies": {
|
|
22
|
-
"@webiny/cli": "5.42.1-beta.
|
|
23
|
-
"@webiny/plugins": "5.42.1-beta.
|
|
24
|
-
"@webiny/project-utils": "5.42.1-beta.
|
|
22
|
+
"@webiny/cli": "5.42.1-beta.2",
|
|
23
|
+
"@webiny/plugins": "5.42.1-beta.2",
|
|
24
|
+
"@webiny/project-utils": "5.42.1-beta.2",
|
|
25
25
|
"typescript": "4.9.5"
|
|
26
26
|
},
|
|
27
27
|
"publishConfig": {
|
|
@@ -37,5 +37,5 @@
|
|
|
37
37
|
"__tests__"
|
|
38
38
|
]
|
|
39
39
|
},
|
|
40
|
-
"gitHead": "
|
|
40
|
+
"gitHead": "86931bade22e9407f297bb69e3a898e2b7e2450a"
|
|
41
41
|
}
|