@webiny/api-dynamodb-to-elasticsearch 6.4.11 → 6.4.12-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/OperationsBuilder.js
CHANGED
|
@@ -8,7 +8,7 @@ class OperationsBuilder {
|
|
|
8
8
|
const operations = new Operations();
|
|
9
9
|
for (const record of params.records){
|
|
10
10
|
if (!record.dynamodb) continue;
|
|
11
|
-
if (!record.eventName) {
|
|
11
|
+
else if (!record.eventName) {
|
|
12
12
|
console.error(`Could not get operation from the record, skipping event "${record.eventID}".`);
|
|
13
13
|
continue;
|
|
14
14
|
}
|
|
@@ -21,8 +21,8 @@ class OperationsBuilder {
|
|
|
21
21
|
if (record.eventName === OperationType.INSERT || record.eventName === OperationType.MODIFY) {
|
|
22
22
|
const newImage = unmarshall(record.dynamodb.NewImage);
|
|
23
23
|
if (!newImage || "object" != typeof newImage || 0 === Object.keys(newImage).length) continue;
|
|
24
|
-
if (true === newImage.ignore) continue;
|
|
25
|
-
if (!newImage.index) {
|
|
24
|
+
else if (true === newImage.ignore) continue;
|
|
25
|
+
else if (!newImage.index) {
|
|
26
26
|
console.error(`Could not get index from the new image, skipping event "${record.eventID}".`);
|
|
27
27
|
console.log({
|
|
28
28
|
newImage
|
package/OperationsBuilder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OperationsBuilder.js","sources":["../src/OperationsBuilder.ts"],"sourcesContent":["import type { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from \"./types.js\";\nimport { Operations, OperationType } from \"~/Operations.js\";\nimport { unmarshall } from \"~/marshall.js\";\nimport type { CompressionHandler } from \"@webiny/utils/exports/api.js\";\n\ninterface RecordDynamoDbImage {\n data: {\n compression: string;\n value: string;\n };\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n compressor: CompressionHandler.Interface;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n private readonly compressor: CompressionHandler.Interface;\n\n public constructor(params: IOperationsBuilderParams) {\n this.compressor = params.compressor;\n }\n\n public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n const operations = new Operations();\n for (const record of params.records) {\n if (!record.dynamodb) {\n continue;\n } else if (!record.eventName) {\n console.error(\n `Could not get operation from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n if (!keys?.PK || !keys.SK) {\n console.error(\n `Could not get keys from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const id = `${keys.PK}:${keys.SK}`;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n if (\n record.eventName === OperationType.INSERT ||\n record.eventName === OperationType.MODIFY\n ) {\n const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n /**\n * If there is no newImage, silently continue to the next operation.\n */\n if (\n !newImage ||\n typeof newImage !== \"object\" ||\n Object.keys(newImage).length === 0\n ) {\n continue;\n }\n /**\n * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n * need to process it.\n */\n //\n else if (newImage.ignore === true) {\n // Nothing to log here, we are skipping the record intentionally.\n continue;\n }\n /**\n * Also, possibly there is no index?\n */\n //\n else if (!newImage.index) {\n console.error(\n `Could not get index from the new image, skipping event \"${record.eventID}\".`\n );\n console.log({ newImage });\n continue;\n }\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n const data = await this.compressor.decompress(newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.error(\n `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n );\n continue;\n }\n\n operations.insert({\n id,\n index: newImage.index,\n data\n });\n } else if (record.eventName === OperationType.REMOVE) {\n const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n /**\n * If there is no index found, silently continue to the next operation.\n */\n if (!oldImage?.index) {\n continue;\n }\n operations.delete({\n id,\n index: oldImage.index\n });\n }\n }\n return operations;\n }\n}\n"],"names":["OperationsBuilder","params","operations","Operations","record","console","keys","unmarshall","id","OperationType","newImage","Object","data","oldImage"],"mappings":";;AAuBO,MAAMA;IAGT,YAAmBC,MAAgC,CAAE;QACjD,IAAI,CAAC,UAAU,GAAGA,OAAO,UAAU;IACvC;IAEA,MAAa,MAAMA,MAAqC,EAAwB;QAC5E,MAAMC,aAAa,IAAIC;QACvB,KAAK,MAAMC,UAAUH,OAAO,OAAO,CAAE;YACjC,IAAI,CAACG,OAAO,QAAQ,EAChB;
|
|
1
|
+
{"version":3,"file":"OperationsBuilder.js","sources":["../src/OperationsBuilder.ts"],"sourcesContent":["import type { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from \"./types.js\";\nimport { Operations, OperationType } from \"~/Operations.js\";\nimport { unmarshall } from \"~/marshall.js\";\nimport type { CompressionHandler } from \"@webiny/utils/exports/api.js\";\n\ninterface RecordDynamoDbImage {\n data: {\n compression: string;\n value: string;\n };\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n compressor: CompressionHandler.Interface;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n private readonly compressor: CompressionHandler.Interface;\n\n public constructor(params: IOperationsBuilderParams) {\n this.compressor = params.compressor;\n }\n\n public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n const operations = new Operations();\n for (const record of params.records) {\n if (!record.dynamodb) {\n continue;\n } else if (!record.eventName) {\n console.error(\n `Could not get operation from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n if (!keys?.PK || !keys.SK) {\n console.error(\n `Could not get keys from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const id = `${keys.PK}:${keys.SK}`;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n if (\n record.eventName === OperationType.INSERT ||\n record.eventName === OperationType.MODIFY\n ) {\n const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n /**\n * If there is no newImage, silently continue to the next operation.\n */\n if (\n !newImage ||\n typeof newImage !== \"object\" ||\n Object.keys(newImage).length === 0\n ) {\n continue;\n }\n /**\n * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n * need to process it.\n */\n //\n else if (newImage.ignore === true) {\n // Nothing to log here, we are skipping the record intentionally.\n continue;\n }\n /**\n * Also, possibly there is no index?\n */\n //\n else if (!newImage.index) {\n console.error(\n `Could not get index from the new image, skipping event \"${record.eventID}\".`\n );\n console.log({ newImage });\n continue;\n }\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n const data = await this.compressor.decompress(newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.error(\n `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n );\n continue;\n }\n\n operations.insert({\n id,\n index: newImage.index,\n data\n });\n } else if (record.eventName === OperationType.REMOVE) {\n const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n /**\n * If there is no index found, silently continue to the next operation.\n */\n if (!oldImage?.index) {\n continue;\n }\n operations.delete({\n id,\n index: oldImage.index\n });\n }\n }\n return operations;\n }\n}\n"],"names":["OperationsBuilder","params","operations","Operations","record","console","keys","unmarshall","id","OperationType","newImage","Object","data","oldImage"],"mappings":";;AAuBO,MAAMA;IAGT,YAAmBC,MAAgC,CAAE;QACjD,IAAI,CAAC,UAAU,GAAGA,OAAO,UAAU;IACvC;IAEA,MAAa,MAAMA,MAAqC,EAAwB;QAC5E,MAAMC,aAAa,IAAIC;QACvB,KAAK,MAAMC,UAAUH,OAAO,OAAO,CAAE;YACjC,IAAI,CAACG,OAAO,QAAQ,EAChB;iBACG,IAAI,CAACA,OAAO,SAAS,EAAE;gBAC1BC,QAAQ,KAAK,CACT,CAAC,yDAAyD,EAAED,OAAO,OAAO,CAAC,EAAE,CAAC;gBAElF;YACJ;YAEA,MAAME,OAAOC,WAA+BH,OAAO,QAAQ,CAAC,IAAI;YAChE,IAAI,CAACE,MAAM,MAAM,CAACA,KAAK,EAAE,EAAE;gBACvBD,QAAQ,KAAK,CACT,CAAC,oDAAoD,EAAED,OAAO,OAAO,CAAC,EAAE,CAAC;gBAE7E;YACJ;YAEA,MAAMI,KAAK,GAAGF,KAAK,EAAE,CAAC,CAAC,EAAEA,KAAK,EAAE,EAAE;YAMlC,IACIF,OAAO,SAAS,KAAKK,cAAc,MAAM,IACzCL,OAAO,SAAS,KAAKK,cAAc,MAAM,EAC3C;gBACE,MAAMC,WAAWH,WAAgCH,OAAO,QAAQ,CAAC,QAAQ;gBAIzE,IACI,CAACM,YACD,AAAoB,YAApB,OAAOA,YACPC,AAAiC,MAAjCA,OAAO,IAAI,CAACD,UAAU,MAAM,EAE5B;qBAQC,IAAIA,AAAoB,SAApBA,SAAS,MAAM,EAEpB;qBAMC,IAAI,CAACA,SAAS,KAAK,EAAE;oBACtBL,QAAQ,KAAK,CACT,CAAC,wDAAwD,EAAED,OAAO,OAAO,CAAC,EAAE,CAAC;oBAEjFC,QAAQ,GAAG,CAAC;wBAAEK;oBAAS;oBACvB;gBACJ;gBAIA,MAAME,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAACF,SAAS,IAAI;gBAQ3D,IAAIE,QAAAA,MAAqC;oBACrCP,QAAQ,KAAK,CACT,CAAC,wDAAwD,EAAED,OAAO,SAAS,CAAC,MAAM,EAAEI,GAAG,aAAa,CAAC;oBAEzG;gBACJ;gBAEAN,WAAW,MAAM,CAAC;oBACdM;oBACA,OAAOE,SAAS,KAAK;oBACrBE;gBACJ;YACJ,OAAO,IAAIR,OAAO,SAAS,KAAKK,cAAc,MAAM,EAAE;gBAClD,MAAMI,WAAWN,WAAgCH,OAAO,QAAQ,CAAC,QAAQ;gBAIzE,IAAI,CAACS,UAAU,OACX;gBAEJX,WAAW,MAAM,CAAC;oBACdM;oBACA,OAAOK,SAAS,KAAK;gBACzB;YACJ;QACJ;QACA,OAAOX;IACX;AACJ"}
|
package/execute.js
CHANGED
|
@@ -11,14 +11,13 @@ const checkErrors = (result)=>{
|
|
|
11
11
|
if (!result || !result.body || !result.body.items) return;
|
|
12
12
|
for (const item of result.body.items){
|
|
13
13
|
const err = getError(item);
|
|
14
|
-
if (err)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
}
|
|
19
|
-
console.error("Body item with error", item);
|
|
20
|
-
throw new WebinyError(err, "DYNAMODB_TO_OPENSEARCH_ERROR", item);
|
|
14
|
+
if (!err) continue;
|
|
15
|
+
else if ("index" === err) {
|
|
16
|
+
if ("true" === process.env.DEBUG) console.error("Bulk response", JSON.stringify(result, null, 2));
|
|
17
|
+
continue;
|
|
21
18
|
}
|
|
19
|
+
console.error("Body item with error", item);
|
|
20
|
+
throw new WebinyError(err, "DYNAMODB_TO_OPENSEARCH_ERROR", item);
|
|
22
21
|
}
|
|
23
22
|
};
|
|
24
23
|
const execute = (params)=>async ()=>{
|
package/execute.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execute.js","sources":["../src/execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n OpenSearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-opensearch\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport type { ApiResponse } from \"@webiny/api-opensearch/types.js\";\n\nimport { WebinyError } from \"@webiny/error\";\nimport type { Context, IOperations } from \"./types.js\";\nimport { shouldShowLogs } from \"~/helpers/shouldShowLogs.js\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n timer: ITimer;\n maxRunningTime: number;\n maxProcessorPercent: number;\n context: Pick<Context, \"opensearch\">;\n operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index?.error?.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(\"Body item with error\", item);\n throw new WebinyError(err, \"DYNAMODB_TO_OPENSEARCH_ERROR\", item);\n }\n};\n\nexport const execute = (params: IExecuteParams) => {\n return async (): Promise<void> => {\n const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n if (operations.total === 0) {\n return;\n }\n\n const remainingTime = timer.getRemainingSeconds();\n const runningTime = maxRunningTime - remainingTime;\n const maxWaitingTime = remainingTime - 90;\n\n if (shouldShowLogs()) {\n console.debug(\n `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n );\n }\n\n const healthCheck = createWaitUntilHealthy(context.opensearch, {\n minClusterHealthStatus: OpenSearchCatClusterHealthStatus.Yellow,\n waitingTimeStep: 30,\n maxProcessorPercent,\n maxWaitingTime\n });\n\n // const log = context.logger.withSource(\"dynamodbToElasticsearch\");\n const log = {\n notice: (...params: any[]) => {\n console.log(...params);\n },\n debug: (...params: any[]) => {\n console.debug(...params);\n },\n info: (...params: any[]) => {\n console.info(...params);\n },\n warn: (...params: any[]) => {\n console.warn(...params);\n },\n error: (...params: any[]) => {\n console.error(...params);\n }\n };\n\n try {\n await healthCheck.wait({\n async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n console.debug(`Cluster is unhealthy on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n },\n async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n console.error(`Cluster health check timeout on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n }\n });\n } catch (ex) {\n if (\n ex instanceof UnhealthyClusterError ||\n ex instanceof WaitingHealthyClusterAbortedError\n ) {\n throw ex;\n }\n console.error(`Cluster health check failed.`, ex);\n throw ex;\n }\n\n try {\n const res = await context.opensearch.bulk({\n body: operations.items\n });\n checkErrors(res);\n } catch (error) {\n log.error(error, {\n tenant: \"root\"\n });\n\n if (shouldShowLogs() === false) {\n throw error;\n }\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n throw error;\n }\n if (shouldShowLogs() === false) {\n return;\n }\n console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n };\n};\n"],"names":["getError","item","reason","checkErrors","result","err","process","console","JSON","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","remainingTime","runningTime","maxWaitingTime","shouldShowLogs","healthCheck","createWaitUntilHealthy","OpenSearchCatClusterHealthStatus","log","startedAt","runs","mustEndAt","waitingTimeStep","waitingReason","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","error","meta"],"mappings":";;;AAsCA,MAAMA,WAAW,CAACC;IACd,IAAI,CAACA,KAAK,KAAK,EAAE,OAAO,QACpB,OAAO;IAEX,MAAMC,SAASD,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM;IACtC,IAAIC,AAAuD,SAAvDA,OAAO,KAAK,CAAC,uCACb,OAAO;IAEX,OAAOA;AACX;AAEA,MAAMC,cAAc,CAACC;IACjB,IAAI,CAACA,UAAU,CAACA,OAAO,IAAI,IAAI,CAACA,OAAO,IAAI,CAAC,KAAK,EAC7C;IAEJ,KAAK,MAAMH,QAAQG,OAAO,IAAI,CAAC,KAAK,CAAE;QAClC,MAAMC,MAAML,SAASC;QACrB,
|
|
1
|
+
{"version":3,"file":"execute.js","sources":["../src/execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n OpenSearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-opensearch\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport type { ApiResponse } from \"@webiny/api-opensearch/types.js\";\n\nimport { WebinyError } from \"@webiny/error\";\nimport type { Context, IOperations } from \"./types.js\";\nimport { shouldShowLogs } from \"~/helpers/shouldShowLogs.js\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n timer: ITimer;\n maxRunningTime: number;\n maxProcessorPercent: number;\n context: Pick<Context, \"opensearch\">;\n operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index?.error?.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(\"Body item with error\", item);\n throw new WebinyError(err, \"DYNAMODB_TO_OPENSEARCH_ERROR\", item);\n }\n};\n\nexport const execute = (params: IExecuteParams) => {\n return async (): Promise<void> => {\n const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n if (operations.total === 0) {\n return;\n }\n\n const remainingTime = timer.getRemainingSeconds();\n const runningTime = maxRunningTime - remainingTime;\n const maxWaitingTime = remainingTime - 90;\n\n if (shouldShowLogs()) {\n console.debug(\n `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n );\n }\n\n const healthCheck = createWaitUntilHealthy(context.opensearch, {\n minClusterHealthStatus: OpenSearchCatClusterHealthStatus.Yellow,\n waitingTimeStep: 30,\n maxProcessorPercent,\n maxWaitingTime\n });\n\n // const log = context.logger.withSource(\"dynamodbToElasticsearch\");\n const log = {\n notice: (...params: any[]) => {\n console.log(...params);\n },\n debug: (...params: any[]) => {\n console.debug(...params);\n },\n info: (...params: any[]) => {\n console.info(...params);\n },\n warn: (...params: any[]) => {\n console.warn(...params);\n },\n error: (...params: any[]) => {\n console.error(...params);\n }\n };\n\n try {\n await healthCheck.wait({\n async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n console.debug(`Cluster is unhealthy on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n },\n async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n console.error(`Cluster health check timeout on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n }\n });\n } catch (ex) {\n if (\n ex instanceof UnhealthyClusterError ||\n ex instanceof WaitingHealthyClusterAbortedError\n ) {\n throw ex;\n }\n console.error(`Cluster health check failed.`, ex);\n throw ex;\n }\n\n try {\n const res = await context.opensearch.bulk({\n body: operations.items\n });\n checkErrors(res);\n } catch (error) {\n log.error(error, {\n tenant: \"root\"\n });\n\n if (shouldShowLogs() === false) {\n throw error;\n }\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n throw error;\n }\n if (shouldShowLogs() === false) {\n return;\n }\n console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n };\n};\n"],"names":["getError","item","reason","checkErrors","result","err","process","console","JSON","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","remainingTime","runningTime","maxWaitingTime","shouldShowLogs","healthCheck","createWaitUntilHealthy","OpenSearchCatClusterHealthStatus","log","startedAt","runs","mustEndAt","waitingTimeStep","waitingReason","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","error","meta"],"mappings":";;;AAsCA,MAAMA,WAAW,CAACC;IACd,IAAI,CAACA,KAAK,KAAK,EAAE,OAAO,QACpB,OAAO;IAEX,MAAMC,SAASD,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM;IACtC,IAAIC,AAAuD,SAAvDA,OAAO,KAAK,CAAC,uCACb,OAAO;IAEX,OAAOA;AACX;AAEA,MAAMC,cAAc,CAACC;IACjB,IAAI,CAACA,UAAU,CAACA,OAAO,IAAI,IAAI,CAACA,OAAO,IAAI,CAAC,KAAK,EAC7C;IAEJ,KAAK,MAAMH,QAAQG,OAAO,IAAI,CAAC,KAAK,CAAE;QAClC,MAAMC,MAAML,SAASC;QACrB,IAAI,CAACI,KACD;aACG,IAAIA,AAAQ,YAARA,KAAiB;YACxB,IAAIC,AAAsB,WAAtBA,QAAQ,GAAG,CAAC,KAAK,EACjBC,QAAQ,KAAK,CAAC,iBAAiBC,KAAK,SAAS,CAACJ,QAAQ,MAAM;YAEhE;QACJ;QACAG,QAAQ,KAAK,CAAC,wBAAwBN;QACtC,MAAM,IAAIQ,YAAYJ,KAAK,gCAAgCJ;IAC/D;AACJ;AAEO,MAAMS,UAAU,CAACC,SACb;QACH,MAAM,EAAEC,OAAO,EAAEC,KAAK,EAAEC,cAAc,EAAEC,mBAAmB,EAAEC,UAAU,EAAE,GAAGL;QAE5E,IAAIK,AAAqB,MAArBA,WAAW,KAAK,EAChB;QAGJ,MAAMC,gBAAgBJ,MAAM,mBAAmB;QAC/C,MAAMK,cAAcJ,iBAAiBG;QACrC,MAAME,iBAAiBF,gBAAgB;QAEvC,IAAIG,kBACAb,QAAQ,KAAK,CACT,CAAC,kCAAkC,EAAEW,YAAY,0CAA0C,EAAEC,eAAe,CAAC,CAAC;QAItH,MAAME,cAAcC,uBAAuBV,QAAQ,UAAU,EAAE;YAC3D,wBAAwBW,iCAAiC,MAAM;YAC/D,iBAAiB;YACjBR;YACAI;QACJ;QAGA,MAAMK,MAAM;YACR,QAAQ,CAAC,GAAGb;gBACRJ,QAAQ,GAAG,IAAII;YACnB;YACA,OAAO,CAAC,GAAGA;gBACPJ,QAAQ,KAAK,IAAII;YACrB;YACA,MAAM,CAAC,GAAGA;gBACNJ,QAAQ,IAAI,IAAII;YACpB;YACA,MAAM,CAAC,GAAGA;gBACNJ,QAAQ,IAAI,IAAII;YACpB;YACA,OAAO,CAAC,GAAGA;gBACPJ,QAAQ,KAAK,IAAII;YACrB;QACJ;QAEA,IAAI;YACA,MAAMU,YAAY,IAAI,CAAC;gBACnB,MAAM,aAAY,EAAEI,SAAS,EAAEC,IAAI,EAAEC,SAAS,EAAEC,eAAe,EAAEC,aAAa,EAAE;oBAC5EtB,QAAQ,KAAK,CAAC,CAAC,6BAA6B,EAAEmB,KAAK,CAAC,CAAC,EAAE;wBACnDD;wBACAE;wBACAC;wBACAC;oBACJ;gBACJ;gBACA,MAAM,WAAU,EAAEJ,SAAS,EAAEC,IAAI,EAAEE,eAAe,EAAED,SAAS,EAAEE,aAAa,EAAE;oBAC1EtB,QAAQ,KAAK,CAAC,CAAC,qCAAqC,EAAEmB,KAAK,CAAC,CAAC,EAAE;wBAC3DD;wBACAE;wBACAC;wBACAC;oBACJ;gBACJ;YACJ;QACJ,EAAE,OAAOC,IAAI;YACT,IACIA,cAAcC,yBACdD,cAAcE,mCAEd,MAAMF;YAEVvB,QAAQ,KAAK,CAAC,gCAAgCuB;YAC9C,MAAMA;QACV;QAEA,IAAI;YACA,MAAMG,MAAM,MAAMrB,QAAQ,UAAU,CAAC,IAAI,CAAC;gBACtC,MAAMI,WAAW,KAAK;YAC1B;YACAb,YAAY8B;QAChB,EAAE,OAAOC,OAAO;YACZV,IAAI,KAAK,CAACU,OAAO;gBACb,QAAQ;YACZ;YAEA,IAAId,AAAqB,UAArBA,kBACA,MAAMc;YAEV,MAAMC,OAAOD,OAAO,QAAQ,CAAC;YAC7B,OAAOC,IAAI,CAAC,OAAO;YACnB5B,QAAQ,KAAK,CAAC,cAAcC,KAAK,SAAS,CAAC0B,OAAO,MAAM;YACxD,MAAMA;QACV;QACA,IAAId,AAAqB,UAArBA,kBACA;QAEJb,QAAQ,IAAI,CAAC,CAAC,YAAY,EAAES,WAAW,KAAK,CAAC,oCAAoC,CAAC;IACtF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers/getNumberEnvVariable.js","sources":["../../src/helpers/getNumberEnvVariable.ts"],"sourcesContent":["export const getNumberEnvVariable = (name: string, def: number): number => {\n const input = process.env[name];\n const value = Number(input);\n if (isNaN(value)) {\n return def;\n } else if (value <= 0) {\n return def;\n }\n return value;\n};\n"],"names":["getNumberEnvVariable","name","def","input","process","value","Number","isNaN"],"mappings":"AAAO,MAAMA,uBAAuB,CAACC,MAAcC;IAC/C,MAAMC,QAAQC,QAAQ,GAAG,CAACH,KAAK;IAC/B,MAAMI,QAAQC,OAAOH;IACrB,IAAII,MAAMF,QACN,OAAOH;
|
|
1
|
+
{"version":3,"file":"helpers/getNumberEnvVariable.js","sources":["../../src/helpers/getNumberEnvVariable.ts"],"sourcesContent":["export const getNumberEnvVariable = (name: string, def: number): number => {\n const input = process.env[name];\n const value = Number(input);\n if (isNaN(value)) {\n return def;\n } else if (value <= 0) {\n return def;\n }\n return value;\n};\n"],"names":["getNumberEnvVariable","name","def","input","process","value","Number","isNaN"],"mappings":"AAAO,MAAMA,uBAAuB,CAACC,MAAcC;IAC/C,MAAMC,QAAQC,QAAQ,GAAG,CAACH,KAAK;IAC/B,MAAMI,QAAQC,OAAOH;IACrB,IAAII,MAAMF,QACN,OAAOH;SACJ,IAAIG,SAAS,GAChB,OAAOH;IAEX,OAAOG;AACX"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webiny/api-dynamodb-to-elasticsearch",
|
|
3
|
-
"version": "6.4.
|
|
3
|
+
"version": "6.4.12-beta.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./index.js",
|
|
@@ -15,17 +15,17 @@
|
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"author": "Webiny Ltd.",
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@webiny/api": "6.4.
|
|
19
|
-
"@webiny/api-opensearch": "6.4.
|
|
20
|
-
"@webiny/aws-sdk": "6.4.
|
|
21
|
-
"@webiny/error": "6.4.
|
|
22
|
-
"@webiny/handler-aws": "6.4.
|
|
23
|
-
"@webiny/utils": "6.4.
|
|
24
|
-
"p-retry": "8.0.
|
|
18
|
+
"@webiny/api": "6.4.12-beta.0",
|
|
19
|
+
"@webiny/api-opensearch": "6.4.12-beta.0",
|
|
20
|
+
"@webiny/aws-sdk": "6.4.12-beta.0",
|
|
21
|
+
"@webiny/error": "6.4.12-beta.0",
|
|
22
|
+
"@webiny/handler-aws": "6.4.12-beta.0",
|
|
23
|
+
"@webiny/utils": "6.4.12-beta.0",
|
|
24
|
+
"p-retry": "8.0.1"
|
|
25
25
|
},
|
|
26
26
|
"devDependencies": {
|
|
27
|
-
"@webiny/build-tools": "6.4.
|
|
28
|
-
"@webiny/plugins": "6.4.
|
|
27
|
+
"@webiny/build-tools": "6.4.12-beta.0",
|
|
28
|
+
"@webiny/plugins": "6.4.12-beta.0",
|
|
29
29
|
"typescript": "6.0.3"
|
|
30
30
|
},
|
|
31
31
|
"publishConfig": {
|