@webiny/api-dynamodb-to-elasticsearch 5.41.4-beta.6 → 5.42.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/OperationsBuilder.js.map +1 -1
- package/SynchronizationBuilder.d.ts +1 -1
- package/SynchronizationBuilder.js.map +1 -1
- package/execute.d.ts +1 -1
- package/execute.js +5 -0
- package/execute.js.map +1 -1
- package/executeWithRetry.js.map +1 -1
- package/package.json +10 -17
- package/types.d.ts +3 -1
- package/types.js.map +1 -1
package/OperationsBuilder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_Operations","require","_marshall","OperationsBuilder","constructor","params","decompressor","build","operations","Operations","record","records","dynamodb","eventName","console","error","eventID","keys","unmarshall","Keys","PK","SK","id","OperationType","INSERT","MODIFY","newImage","NewImage","Object","length","ignore","index","log","data","decompress","undefined","insert","REMOVE","oldImage","OldImage","delete","exports"],"sources":["OperationsBuilder.ts"],"sourcesContent":["import {\n IDecompressor,\n IOperations,\n IOperationsBuilder,\n IOperationsBuilderBuildParams\n} from \"./types\";\nimport { Operations, OperationType } from \"~/Operations\";\nimport { unmarshall } from \"~/marshall\";\n\ninterface RecordDynamoDbImage {\n data: Record<string, any>;\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n decompressor: IDecompressor;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n private readonly decompressor: IDecompressor;\n\n public constructor(params: IOperationsBuilderParams) {\n this.decompressor = params.decompressor;\n }\n\n public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n const operations = new Operations();\n for (const record of params.records) {\n if (!record.dynamodb) {\n continue;\n } else if (!record.eventName) {\n console.error(\n `Could not get operation from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n if (!keys?.PK || !keys.SK) {\n console.error(\n `Could not get keys from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const id = `${keys.PK}:${keys.SK}`;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n if (\n record.eventName === OperationType.INSERT ||\n record.eventName === OperationType.MODIFY\n ) {\n const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n /**\n * If there is no newImage, silently continue to the next operation.\n */\n if (\n !newImage ||\n typeof newImage !== \"object\" ||\n Object.keys(newImage).length === 0\n ) {\n continue;\n }\n /**\n * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n * need to process it.\n */\n //\n else if (newImage.ignore === true) {\n // Nothing to log here, we are skipping the record intentionally.\n continue;\n }\n /**\n * Also, possibly there is no index?\n */\n //\n else if (!newImage.index) {\n console.error(\n `Could not get index from the new image, skipping event \"${record.eventID}\".`\n );\n console.log({ newImage });\n continue;\n }\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n const data = await this.decompressor.decompress(newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.error(\n `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n );\n continue;\n }\n\n operations.insert({\n id,\n index: newImage.index,\n data\n });\n } else if (record.eventName === OperationType.REMOVE) {\n const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n /**\n * If there is no index found, silently continue to the next operation.\n */\n if (!oldImage?.index) {\n continue;\n }\n operations.delete({\n id,\n index: oldImage.index\n });\n }\n }\n return operations;\n }\n}\n"],"mappings":";;;;;;AAMA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AAiBO,MAAME,iBAAiB,CAA+B;EAGlDC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,YAAY,GAAGD,MAAM,CAACC,YAAY;EAC3C;EAEA,MAAaC,KAAKA,CAACF,MAAqC,EAAwB;IAC5E,MAAMG,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;IACnC,KAAK,MAAMC,MAAM,IAAIL,MAAM,CAACM,OAAO,EAAE;MACjC,IAAI,CAACD,MAAM,CAACE,QAAQ,EAAE;QAClB;MACJ,CAAC,MAAM,IAAI,CAACF,MAAM,CAACG,SAAS,EAAE;QAC1BC,OAAO,CAACC,KAAK,
|
|
1
|
+
{"version":3,"names":["_Operations","require","_marshall","OperationsBuilder","constructor","params","decompressor","build","operations","Operations","record","records","dynamodb","eventName","console","error","eventID","keys","unmarshall","Keys","PK","SK","id","OperationType","INSERT","MODIFY","newImage","NewImage","Object","length","ignore","index","log","data","decompress","undefined","insert","REMOVE","oldImage","OldImage","delete","exports"],"sources":["OperationsBuilder.ts"],"sourcesContent":["import {\n IDecompressor,\n IOperations,\n IOperationsBuilder,\n IOperationsBuilderBuildParams\n} from \"./types\";\nimport { Operations, OperationType } from \"~/Operations\";\nimport { unmarshall } from \"~/marshall\";\n\ninterface RecordDynamoDbImage {\n data: Record<string, any>;\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n decompressor: IDecompressor;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n private readonly decompressor: IDecompressor;\n\n public constructor(params: IOperationsBuilderParams) {\n this.decompressor = params.decompressor;\n }\n\n public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n const operations = new Operations();\n for (const record of params.records) {\n if (!record.dynamodb) {\n continue;\n } else if (!record.eventName) {\n console.error(\n `Could not get operation from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n if (!keys?.PK || !keys.SK) {\n console.error(\n `Could not get keys from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const id = `${keys.PK}:${keys.SK}`;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n if (\n record.eventName === OperationType.INSERT ||\n record.eventName === OperationType.MODIFY\n ) {\n const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n /**\n * If there is no newImage, silently continue to the next operation.\n */\n if (\n !newImage ||\n typeof newImage !== \"object\" ||\n Object.keys(newImage).length === 0\n ) {\n continue;\n }\n /**\n * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n * need to process it.\n */\n //\n else if (newImage.ignore === true) {\n // Nothing to log here, we are skipping the record intentionally.\n continue;\n }\n /**\n * Also, possibly there is no index?\n */\n //\n else if (!newImage.index) {\n console.error(\n `Could not get index from the new image, skipping event \"${record.eventID}\".`\n );\n console.log({ newImage });\n continue;\n }\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n const data = await this.decompressor.decompress(newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.error(\n `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n );\n continue;\n }\n\n operations.insert({\n id,\n index: newImage.index,\n data\n });\n } else if (record.eventName === OperationType.REMOVE) {\n const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n /**\n * If there is no index found, silently continue to the next operation.\n */\n if (!oldImage?.index) {\n continue;\n }\n operations.delete({\n id,\n index: oldImage.index\n });\n }\n }\n return operations;\n }\n}\n"],"mappings":";;;;;;AAMA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AAiBO,MAAME,iBAAiB,CAA+B;EAGlDC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,YAAY,GAAGD,MAAM,CAACC,YAAY;EAC3C;EAEA,MAAaC,KAAKA,CAACF,MAAqC,EAAwB;IAC5E,MAAMG,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;IACnC,KAAK,MAAMC,MAAM,IAAIL,MAAM,CAACM,OAAO,EAAE;MACjC,IAAI,CAACD,MAAM,CAACE,QAAQ,EAAE;QAClB;MACJ,CAAC,MAAM,IAAI,CAACF,MAAM,CAACG,SAAS,EAAE;QAC1BC,OAAO,CAACC,KAAK,CACT,4DAA4DL,MAAM,CAACM,OAAO,IAC9E,CAAC;QACD;MACJ;MAEA,MAAMC,IAAI,GAAG,IAAAC,oBAAU,EAAqBR,MAAM,CAACE,QAAQ,CAACO,IAAI,CAAC;MACjE,IAAI,CAACF,IAAI,EAAEG,EAAE,IAAI,CAACH,IAAI,CAACI,EAAE,EAAE;QACvBP,OAAO,CAACC,KAAK,CACT,uDAAuDL,MAAM,CAACM,OAAO,IACzE,CAAC;QACD;MACJ;MAEA,MAAMM,EAAE,GAAG,GAAGL,IAAI,CAACG,EAAE,IAAIH,IAAI,CAACI,EAAE,EAAE;;MAElC;AACZ;AACA;AACA;MACY,IACIX,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACC,MAAM,IACzCd,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACE,MAAM,EAC3C;QACE,MAAMC,QAAQ,GAAG,IAAAR,oBAAU,EAAsBR,MAAM,CAACE,QAAQ,CAACe,QAAQ,CAAC;QAC1E;AAChB;AACA;QACgB,IACI,CAACD,QAAQ,IACT,OAAOA,QAAQ,KAAK,QAAQ,IAC5BE,MAAM,CAACX,IAAI,CAACS,QAAQ,CAAC,CAACG,MAAM,KAAK,CAAC,EACpC;UACE;QACJ;QACA;AAChB;AACA;AACA;AACA;QACgB;QAAA,KACK,IAAIH,QAAQ,CAACI,MAAM,KAAK,IAAI,EAAE;UAC/B;UACA;QACJ;QACA;AAChB;AACA;QACgB;QAAA,KACK,IAAI,CAACJ,QAAQ,CAACK,KAAK,EAAE;UACtBjB,OAAO,CAACC,KAAK,CACT,2DAA2DL,MAAM,CAACM,OAAO,IAC7E,CAAC;UACDF,OAAO,CAACkB,GAAG,CAAC;YAAEN;UAAS,CAAC,CAAC;UACzB;QACJ;QACA;AAChB;AACA;QACgB,MAAMO,IAAI,GAAG,MAAM,IAAI,CAAC3B,YAAY,CAAC4B,UAAU,CAACR,QAAQ,CAACO,IAAI,CAAC;QAC9D;AAChB;AACA;AACA;AACA;AACA;AACA;QACgB,IAAIA,IAAI,KAAKE,SAAS,IAAIF,IAAI,KAAK,IAAI,EAAE;UACrCnB,OAAO,CAACC,KAAK,CACT,2DAA2DL,MAAM,CAACG,SAAS,SAASS,EAAE,eAC1F,CAAC;UACD;QACJ;QAEAd,UAAU,CAAC4B,MAAM,CAAC;UACdd,EAAE;UACFS,KAAK,EAAEL,QAAQ,CAACK,KAAK;UACrBE;QACJ,CAAC,CAAC;MACN,CAAC,MAAM,IAAIvB,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACc,MAAM,EAAE;QAClD,MAAMC,QAAQ,GAAG,IAAApB,oBAAU,EAAsBR,MAAM,CAACE,QAAQ,CAAC2B,QAAQ,CAAC;QAC1E;AAChB;AACA;QACgB,IAAI,CAACD,QAAQ,EAAEP,KAAK,EAAE;UAClB;QACJ;QACAvB,UAAU,CAACgC,MAAM,CAAC;UACdlB,EAAE;UACFS,KAAK,EAAEO,QAAQ,CAACP;QACpB,CAAC,CAAC;MACN;IACJ;IACA,OAAOvB,UAAU;EACrB;AACJ;AAACiC,OAAA,CAAAtC,iBAAA,GAAAA,iBAAA","ignoreList":[]}
|
|
@@ -9,7 +9,7 @@ export interface ISynchronizationBuilder {
|
|
|
9
9
|
}
|
|
10
10
|
export interface ISynchronizationBuilderParams {
|
|
11
11
|
timer: ITimer;
|
|
12
|
-
context: Pick<Context, "elasticsearch">;
|
|
12
|
+
context: Pick<Context, "elasticsearch" | "logger">;
|
|
13
13
|
}
|
|
14
14
|
export declare class SynchronizationBuilder implements ISynchronizationBuilder {
|
|
15
15
|
private readonly timer;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_Operations","require","_executeWithRetry","SynchronizationBuilder","constructor","params","timer","context","operations","Operations","insert","modify","delete","build","total","executeWithRetry","maxRunningTime","getRemainingMilliseconds","clear","exports","createSynchronizationBuilder"],"sources":["SynchronizationBuilder.ts"],"sourcesContent":["import {\n Context,\n IDeleteOperationParams,\n IInsertOperationParams,\n IModifyOperationParams,\n IOperations\n} from \"~/types\";\nimport { Operations } from \"~/Operations\";\nimport { executeWithRetry, IExecuteWithRetryParams } from \"~/executeWithRetry\";\nimport { ITimer } from \"@webiny/handler-aws\";\n\nexport type ISynchronizationBuilderExecuteWithRetryParams = Omit<\n IExecuteWithRetryParams,\n \"context\" | \"timer\" | \"maxRunningTime\" | \"operations\"\n>;\n\nexport interface ISynchronizationBuilder {\n insert(params: IInsertOperationParams): void;\n delete(params: IDeleteOperationParams): void;\n build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;\n}\n\nexport interface ISynchronizationBuilderParams {\n timer: ITimer;\n context: Pick<Context, \"elasticsearch\">;\n}\n\nexport class SynchronizationBuilder implements ISynchronizationBuilder {\n private readonly timer: ITimer;\n private readonly context: Pick<Context, \"elasticsearch\">;\n private readonly operations: IOperations;\n\n public constructor(params: ISynchronizationBuilderParams) {\n this.timer = params.timer;\n this.context = params.context;\n this.operations = new Operations();\n }\n\n public insert(params: IInsertOperationParams): void {\n return this.operations.insert(params);\n }\n\n public modify(params: IModifyOperationParams): void {\n return this.operations.modify(params);\n }\n\n public delete(params: IDeleteOperationParams): void {\n return this.operations.delete(params);\n }\n\n public build() {\n return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => {\n if (this.operations.total === 0) {\n return;\n }\n await executeWithRetry({\n ...params,\n maxRunningTime: this.timer.getRemainingMilliseconds(),\n timer: this.timer,\n context: this.context,\n operations: this.operations\n });\n this.operations.clear();\n };\n }\n}\n\nexport const createSynchronizationBuilder = (\n params: ISynchronizationBuilderParams\n): ISynchronizationBuilder => {\n return new SynchronizationBuilder(params);\n};\n"],"mappings":";;;;;;AAOA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,iBAAA,GAAAD,OAAA;AAmBO,MAAME,sBAAsB,CAAoC;EAK5DC,WAAWA,CAACC,MAAqC,EAAE;IACtD,IAAI,CAACC,KAAK,GAAGD,MAAM,CAACC,KAAK;IACzB,IAAI,CAACC,OAAO,GAAGF,MAAM,CAACE,OAAO;IAC7B,IAAI,CAACC,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;EACtC;EAEOC,MAAMA,CAACL,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACE,MAAM,CAACL,MAAM,CAAC;EACzC;EAEOM,MAAMA,CAACN,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACG,MAAM,CAACN,MAAM,CAAC;EACzC;EAEOO,MAAMA,CAACP,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACI,MAAM,CAACP,MAAM,CAAC;EACzC;EAEOQ,KAAKA,CAAA,EAAG;IACX,OAAO,MAAOR,MAAsD,IAAK;MACrE,IAAI,IAAI,CAACG,UAAU,CAACM,KAAK,KAAK,CAAC,EAAE;QAC7B;MACJ;MACA,MAAM,IAAAC,kCAAgB,EAAC;QACnB,GAAGV,MAAM;QACTW,cAAc,EAAE,IAAI,CAACV,KAAK,CAACW,wBAAwB,CAAC,CAAC;QACrDX,KAAK,EAAE,IAAI,CAACA,KAAK;QACjBC,OAAO,EAAE,IAAI,CAACA,OAAO;QACrBC,UAAU,EAAE,IAAI,CAACA;MACrB,CAAC,CAAC;MACF,IAAI,CAACA,UAAU,CAACU,KAAK,CAAC,CAAC;IAC3B,CAAC;EACL;AACJ;AAACC,OAAA,CAAAhB,sBAAA,GAAAA,sBAAA;AAEM,MAAMiB,4BAA4B,GACrCf,MAAqC,IACX;EAC1B,OAAO,IAAIF,sBAAsB,CAACE,MAAM,CAAC;AAC7C,CAAC;AAACc,OAAA,CAAAC,4BAAA,GAAAA,4BAAA","ignoreList":[]}
|
|
1
|
+
{"version":3,"names":["_Operations","require","_executeWithRetry","SynchronizationBuilder","constructor","params","timer","context","operations","Operations","insert","modify","delete","build","total","executeWithRetry","maxRunningTime","getRemainingMilliseconds","clear","exports","createSynchronizationBuilder"],"sources":["SynchronizationBuilder.ts"],"sourcesContent":["import {\n Context,\n IDeleteOperationParams,\n IInsertOperationParams,\n IModifyOperationParams,\n IOperations\n} from \"~/types\";\nimport { Operations } from \"~/Operations\";\nimport { executeWithRetry, IExecuteWithRetryParams } from \"~/executeWithRetry\";\nimport { ITimer } from \"@webiny/handler-aws\";\n\nexport type ISynchronizationBuilderExecuteWithRetryParams = Omit<\n IExecuteWithRetryParams,\n \"context\" | \"timer\" | \"maxRunningTime\" | \"operations\"\n>;\n\nexport interface ISynchronizationBuilder {\n insert(params: IInsertOperationParams): void;\n delete(params: IDeleteOperationParams): void;\n build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;\n}\n\nexport interface ISynchronizationBuilderParams {\n timer: ITimer;\n context: Pick<Context, \"elasticsearch\" | \"logger\">;\n}\n\nexport class SynchronizationBuilder implements ISynchronizationBuilder {\n private readonly timer: ITimer;\n private readonly context: Pick<Context, \"elasticsearch\" | \"logger\">;\n private readonly operations: IOperations;\n\n public constructor(params: ISynchronizationBuilderParams) {\n this.timer = params.timer;\n this.context = params.context;\n this.operations = new Operations();\n }\n\n public insert(params: IInsertOperationParams): void {\n return this.operations.insert(params);\n }\n\n public modify(params: IModifyOperationParams): void {\n return this.operations.modify(params);\n }\n\n public delete(params: IDeleteOperationParams): void {\n return this.operations.delete(params);\n }\n\n public build() {\n return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => {\n if (this.operations.total === 0) {\n return;\n }\n await executeWithRetry({\n ...params,\n maxRunningTime: this.timer.getRemainingMilliseconds(),\n timer: this.timer,\n context: this.context,\n operations: this.operations\n });\n this.operations.clear();\n };\n }\n}\n\nexport const createSynchronizationBuilder = (\n params: ISynchronizationBuilderParams\n): ISynchronizationBuilder => {\n return new SynchronizationBuilder(params);\n};\n"],"mappings":";;;;;;AAOA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,iBAAA,GAAAD,OAAA;AAmBO,MAAME,sBAAsB,CAAoC;EAK5DC,WAAWA,CAACC,MAAqC,EAAE;IACtD,IAAI,CAACC,KAAK,GAAGD,MAAM,CAACC,KAAK;IACzB,IAAI,CAACC,OAAO,GAAGF,MAAM,CAACE,OAAO;IAC7B,IAAI,CAACC,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;EACtC;EAEOC,MAAMA,CAACL,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACE,MAAM,CAACL,MAAM,CAAC;EACzC;EAEOM,MAAMA,CAACN,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACG,MAAM,CAACN,MAAM,CAAC;EACzC;EAEOO,MAAMA,CAACP,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACI,MAAM,CAACP,MAAM,CAAC;EACzC;EAEOQ,KAAKA,CAAA,EAAG;IACX,OAAO,MAAOR,MAAsD,IAAK;MACrE,IAAI,IAAI,CAACG,UAAU,CAACM,KAAK,KAAK,CAAC,EAAE;QAC7B;MACJ;MACA,MAAM,IAAAC,kCAAgB,EAAC;QACnB,GAAGV,MAAM;QACTW,cAAc,EAAE,IAAI,CAACV,KAAK,CAACW,wBAAwB,CAAC,CAAC;QACrDX,KAAK,EAAE,IAAI,CAACA,KAAK;QACjBC,OAAO,EAAE,IAAI,CAACA,OAAO;QACrBC,UAAU,EAAE,IAAI,CAACA;MACrB,CAAC,CAAC;MACF,IAAI,CAACA,UAAU,CAACU,KAAK,CAAC,CAAC;IAC3B,CAAC;EACL;AACJ;AAACC,OAAA,CAAAhB,sBAAA,GAAAA,sBAAA;AAEM,MAAMiB,4BAA4B,GACrCf,MAAqC,IACX;EAC1B,OAAO,IAAIF,sBAAsB,CAACE,MAAM,CAAC;AAC7C,CAAC;AAACc,OAAA,CAAAC,4BAAA,GAAAA,4BAAA","ignoreList":[]}
|
package/execute.d.ts
CHANGED
|
@@ -17,7 +17,7 @@ export interface IExecuteParams {
|
|
|
17
17
|
timer: ITimer;
|
|
18
18
|
maxRunningTime: number;
|
|
19
19
|
maxProcessorPercent: number;
|
|
20
|
-
context: Pick<Context, "elasticsearch">;
|
|
20
|
+
context: Pick<Context, "elasticsearch" | "logger">;
|
|
21
21
|
operations: Pick<IOperations, "items" | "total">;
|
|
22
22
|
}
|
|
23
23
|
export declare const execute: (params: IExecuteParams) => () => Promise<void>;
|
package/execute.js
CHANGED
|
@@ -58,6 +58,7 @@ const execute = params => {
|
|
|
58
58
|
maxProcessorPercent,
|
|
59
59
|
maxWaitingTime
|
|
60
60
|
});
|
|
61
|
+
const log = context.logger.withSource("dynamodbToElasticsearch");
|
|
61
62
|
try {
|
|
62
63
|
await healthCheck.wait({
|
|
63
64
|
async onUnhealthy({
|
|
@@ -102,6 +103,10 @@ const execute = params => {
|
|
|
102
103
|
});
|
|
103
104
|
checkErrors(res);
|
|
104
105
|
} catch (error) {
|
|
106
|
+
log.error(error, {
|
|
107
|
+
tenant: "root",
|
|
108
|
+
locale: "unknown"
|
|
109
|
+
});
|
|
105
110
|
if (process.env.DEBUG !== "true") {
|
|
106
111
|
throw error;
|
|
107
112
|
}
|
package/execute.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_apiElasticsearch","require","_error","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","total","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","elasticsearch","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","meta","info","exports"],"sources":["execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n ElasticsearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport { ITimer } from \"@webiny/handler-aws\";\nimport { ApiResponse } from \"@webiny/api-elasticsearch/types\";\nimport { WebinyError } from \"@webiny/error\";\nimport { Context, IOperations } from \"./types\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n timer: ITimer;\n maxRunningTime: number;\n maxProcessorPercent: number;\n context: Pick<Context, \"elasticsearch\">;\n operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index?.error?.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(item.error);\n throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n }\n};\n\nexport const execute = (params: IExecuteParams) => {\n return async (): Promise<void> => {\n const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n if (operations.total === 0) {\n return;\n }\n\n const remainingTime = timer.getRemainingSeconds();\n const runningTime = maxRunningTime - remainingTime;\n const maxWaitingTime = remainingTime - 90;\n\n if (process.env.DEBUG === \"true\") {\n console.debug(\n `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n );\n }\n\n const healthCheck = createWaitUntilHealthy(context.elasticsearch, {\n minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,\n waitingTimeStep: 30,\n maxProcessorPercent,\n maxWaitingTime\n });\n\n try {\n await healthCheck.wait({\n async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n console.debug(`Cluster is unhealthy on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n },\n async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n console.error(`Cluster health check timeout on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n }\n });\n } catch (ex) {\n if (\n ex instanceof UnhealthyClusterError ||\n ex instanceof WaitingHealthyClusterAbortedError\n ) {\n throw ex;\n }\n console.error(`Cluster health check failed.`, ex);\n throw ex;\n }\n\n try {\n const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n body: operations.items\n });\n checkErrors(res);\n } catch (error) {\n if (process.env.DEBUG !== \"true\") {\n throw error;\n }\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n throw error;\n }\n if (process.env.DEBUG !== \"true\") {\n return;\n }\n console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n };\n};\n"],"mappings":";;;;;;AAAA,IAAAA,iBAAA,GAAAC,OAAA;AAQA,IAAAC,MAAA,GAAAD,OAAA;AA4BA,MAAME,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,EAAEC,KAAK,EAAEC,MAAM,EAAE;IAC5B,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMR,IAAI,IAAIM,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACS,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACX,KAAK,CAAC,eAAe,EAAEY,IAAI,CAACC,SAAS,CAACT,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAO,OAAO,CAACX,KAAK,CAACF,IAAI,CAACE,KAAK,CAAC;IACzB,MAAM,IAAIc,kBAAW,CAACP,GAAG,EAAE,iCAAiC,EAAET,IAAI,CAAC;EACvE;AACJ,CAAC;AAEM,MAAMiB,OAAO,GAAIC,MAAsB,IAAK;EAC/C,OAAO,YAA2B;IAC9B,MAAM;MAAEC,OAAO;MAAEC,KAAK;MAAEC,cAAc;MAAEC,mBAAmB;MAAEC;IAAW,CAAC,GAAGL,MAAM;IAElF,IAAIK,UAAU,CAACC,KAAK,KAAK,CAAC,EAAE;MACxB;IACJ;IAEA,MAAMC,aAAa,GAAGL,KAAK,CAACM,mBAAmB,CAAC,CAAC;IACjD,MAAMC,WAAW,GAAGN,cAAc,GAAGI,aAAa;IAClD,MAAMG,cAAc,GAAGH,aAAa,GAAG,EAAE;IAEzC,IAAIf,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9BC,OAAO,CAACgB,KAAK,
|
|
1
|
+
{"version":3,"names":["_apiElasticsearch","require","_error","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","total","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","elasticsearch","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","log","logger","withSource","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","tenant","locale","meta","info","exports"],"sources":["execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n ElasticsearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport { ITimer } from \"@webiny/handler-aws\";\nimport { ApiResponse } from \"@webiny/api-elasticsearch/types\";\nimport { WebinyError } from \"@webiny/error\";\nimport { Context, IOperations } from \"./types\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n timer: ITimer;\n maxRunningTime: number;\n maxProcessorPercent: number;\n context: Pick<Context, \"elasticsearch\" | \"logger\">;\n operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index?.error?.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(item.error);\n throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n }\n};\n\nexport const execute = (params: IExecuteParams) => {\n return async (): Promise<void> => {\n const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n if (operations.total === 0) {\n return;\n }\n\n const remainingTime = timer.getRemainingSeconds();\n const runningTime = maxRunningTime - remainingTime;\n const maxWaitingTime = remainingTime - 90;\n\n if (process.env.DEBUG === \"true\") {\n console.debug(\n `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n );\n }\n\n const healthCheck = createWaitUntilHealthy(context.elasticsearch, {\n minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,\n waitingTimeStep: 30,\n maxProcessorPercent,\n maxWaitingTime\n });\n\n const log = context.logger.withSource(\"dynamodbToElasticsearch\");\n\n try {\n await healthCheck.wait({\n async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n console.debug(`Cluster is unhealthy on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n },\n async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n console.error(`Cluster health check timeout on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n }\n });\n } catch (ex) {\n if (\n ex instanceof UnhealthyClusterError ||\n ex instanceof WaitingHealthyClusterAbortedError\n ) {\n throw ex;\n }\n console.error(`Cluster health check failed.`, ex);\n throw ex;\n }\n\n try {\n const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n body: operations.items\n });\n checkErrors(res);\n } catch (error) {\n log.error(error, {\n tenant: \"root\",\n locale: \"unknown\"\n });\n\n if (process.env.DEBUG !== \"true\") {\n throw error;\n }\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n throw error;\n }\n if (process.env.DEBUG !== \"true\") {\n return;\n }\n console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n };\n};\n"],"mappings":";;;;;;AAAA,IAAAA,iBAAA,GAAAC,OAAA;AAQA,IAAAC,MAAA,GAAAD,OAAA;AA4BA,MAAME,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,EAAEC,KAAK,EAAEC,MAAM,EAAE;IAC5B,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMR,IAAI,IAAIM,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACS,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACX,KAAK,CAAC,eAAe,EAAEY,IAAI,CAACC,SAAS,CAACT,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAO,OAAO,CAACX,KAAK,CAACF,IAAI,CAACE,KAAK,CAAC;IACzB,MAAM,IAAIc,kBAAW,CAACP,GAAG,EAAE,iCAAiC,EAAET,IAAI,CAAC;EACvE;AACJ,CAAC;AAEM,MAAMiB,OAAO,GAAIC,MAAsB,IAAK;EAC/C,OAAO,YAA2B;IAC9B,MAAM;MAAEC,OAAO;MAAEC,KAAK;MAAEC,cAAc;MAAEC,mBAAmB;MAAEC;IAAW,CAAC,GAAGL,MAAM;IAElF,IAAIK,UAAU,CAACC,KAAK,KAAK,CAAC,EAAE;MACxB;IACJ;IAEA,MAAMC,aAAa,GAAGL,KAAK,CAACM,mBAAmB,CAAC,CAAC;IACjD,MAAMC,WAAW,GAAGN,cAAc,GAAGI,aAAa;IAClD,MAAMG,cAAc,GAAGH,aAAa,GAAG,EAAE;IAEzC,IAAIf,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9BC,OAAO,CAACgB,KAAK,CACT,qCAAqCF,WAAW,6CAA6CC,cAAc,GAC/G,CAAC;IACL;IAEA,MAAME,WAAW,GAAG,IAAAC,wCAAsB,EAACZ,OAAO,CAACa,aAAa,EAAE;MAC9DC,sBAAsB,EAAEC,qDAAmC,CAACC,MAAM;MAClEC,eAAe,EAAE,EAAE;MACnBd,mBAAmB;MACnBM;IACJ,CAAC,CAAC;IAEF,MAAMS,GAAG,GAAGlB,OAAO,CAACmB,MAAM,CAACC,UAAU,CAAC,yBAAyB,CAAC;IAEhE,IAAI;MACA,MAAMT,WAAW,CAACU,IAAI,CAAC;QACnB,MAAMC,WAAWA,CAAC;UAAEC,SAAS;UAAEC,IAAI;UAAEC,SAAS;UAAER,eAAe;UAAES;QAAc,CAAC,EAAE;UAC9EhC,OAAO,CAACgB,KAAK,CAAC,gCAAgCc,IAAI,GAAG,EAAE;YACnDD,SAAS;YACTE,SAAS;YACTR,eAAe;YACfS;UACJ,CAAC,CAAC;QACN,CAAC;QACD,MAAMC,SAASA,CAAC;UAAEJ,SAAS;UAAEC,IAAI;UAAEP,eAAe;UAAEQ,SAAS;UAAEC;QAAc,CAAC,EAAE;UAC5EhC,OAAO,CAACX,KAAK,CAAC,wCAAwCyC,IAAI,GAAG,EAAE;YAC3DD,SAAS;YACTE,SAAS;YACTR,eAAe;YACfS;UACJ,CAAC,CAAC;QACN;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,IACIA,EAAE,YAAYC,uCAAqB,IACnCD,EAAE,YAAYE,mDAAiC,EACjD;QACE,MAAMF,EAAE;MACZ;MACAlC,OAAO,CAACX,KAAK,CAAC,8BAA8B,EAAE6C,EAAE,CAAC;MACjD,MAAMA,EAAE;IACZ;IAEA,IAAI;MACA,MAAMG,GAAG,GAAG,MAAM/B,OAAO,CAACa,aAAa,CAACmB,IAAI,CAA6B;QACrE5C,IAAI,EAAEgB,UAAU,CAACf;MACrB,CAAC,CAAC;MACFH,WAAW,CAAC6C,GAAG,CAAC;IACpB,CAAC,CAAC,OAAOhD,KAAK,EAAE;MACZmC,GAAG,CAACnC,KAAK,CAACA,KAAK,EAAE;QACbkD,MAAM,EAAE,MAAM;QACdC,MAAM,EAAE;MACZ,CAAC,CAAC;MAEF,IAAI3C,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9B,MAAMV,KAAK;MACf;MACA,MAAMoD,IAAI,GAAGpD,KAAK,EAAEoD,IAAI,IAAI,CAAC,CAAC;MAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;MACnBzC,OAAO,CAACX,KAAK,CAAC,YAAY,EAAEY,IAAI,CAACC,SAAS,CAACb,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MAC3D,MAAMA,KAAK;IACf;IACA,IAAIQ,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9B;IACJ;IACAC,OAAO,CAAC0C,IAAI,CAAC,eAAehC,UAAU,CAACC,KAAK,sCAAsC,CAAC;EACvF,CAAC;AACL,CAAC;AAACgC,OAAA,CAAAvC,OAAA,GAAAA,OAAA","ignoreList":[]}
|
package/executeWithRetry.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_execute","require","_NotEnoughRemainingTimeError","_pRetry","_interopRequireDefault","_getNumberEnvVariable","minRemainingSecondsToTimeout","MAX_PROCESSOR_PERCENT","getNumberEnvVariable","process","env","NODE_ENV","executeWithRetry","params","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","execute","timer","maxRunningTime","maxProcessorPercent","context","operations","onFailedAttempt","error","getRemainingSeconds","NotEnoughRemainingTimeError","attemptNumber","console","ex","exports"],"sources":["executeWithRetry.ts"],"sourcesContent":["import { execute, IExecuteParams } from \"~/execute\";\nimport { NotEnoughRemainingTimeError } from \"~/NotEnoughRemainingTimeError\";\nimport pRetry from \"p-retry\";\nimport { getNumberEnvVariable } from \"./helpers/getNumberEnvVariable\";\n\nconst minRemainingSecondsToTimeout = 120;\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n \"MAX_ES_PROCESSOR\",\n process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\nexport interface IExecuteWithRetryParams extends Omit<IExecuteParams, \"maxProcessorPercent\"> {\n maxRetryTime?: number;\n retries?: number;\n minTimeout?: number;\n maxTimeout?: number;\n maxProcessorPercent?: number;\n}\n\nexport const executeWithRetry = async (params: IExecuteWithRetryParams) => {\n const maxRetryTime = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n params.maxRetryTime || 300000\n );\n const retries = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\",\n params.retries || 20\n );\n const minTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n params.minTimeout || 1500\n );\n const maxTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n params.maxTimeout || 30000\n );\n\n try {\n await pRetry(\n execute({\n timer: params.timer,\n maxRunningTime: params.maxRunningTime,\n maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,\n context: params.context,\n operations: params.operations\n }),\n {\n maxRetryTime,\n retries,\n minTimeout,\n maxTimeout,\n onFailedAttempt: error => {\n if (params.timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {\n throw new NotEnoughRemainingTimeError(error);\n }\n /**\n * We will only log attempts which are after 3/4 of total attempts.\n */\n if (error.attemptNumber < retries * 0.75) {\n return;\n }\n console.error(`Attempt #${error.attemptNumber} failed.`);\n console.error(error);\n }\n }\n );\n } catch (ex) {\n // TODO implement storing of failed operations\n throw ex;\n }\n};\n"],"mappings":";;;;;;;AAAA,IAAAA,QAAA,GAAAC,OAAA;AACA,IAAAC,4BAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,qBAAA,GAAAJ,OAAA;AAEA,MAAMK,4BAA4B,GAAG,GAAG;AAExC,MAAMC,qBAAqB,GAAG,IAAAC,0CAAoB,EAC9C,kBAAkB,EAClBC,OAAO,CAACC,GAAG,CAACC,QAAQ,KAAK,MAAM,GAAG,GAAG,GAAG,EAC5C,CAAC;AAUM,MAAMC,gBAAgB,GAAG,MAAOC,MAA+B,IAAK;EACvE,MAAMC,YAAY,GAAG,IAAAN,0CAAoB,EACrC,iDAAiD,EACjDK,MAAM,CAACC,YAAY,IAAI,MAC3B,CAAC;EACD,MAAMC,OAAO,GAAG,IAAAP,0CAAoB,EAChC,0CAA0C,EAC1CK,MAAM,CAACE,OAAO,IAAI,EACtB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAR,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACG,UAAU,IAAI,IACzB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAT,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACI,UAAU,IAAI,KACzB,CAAC;EAED,IAAI;IACA,MAAM,IAAAC,eAAM,EACR,IAAAC,gBAAO,EAAC;MACJC,KAAK,EAAEP,MAAM,CAACO,KAAK;MACnBC,cAAc,EAAER,MAAM,CAACQ,cAAc;MACrCC,mBAAmB,EAAET,MAAM,CAACS,mBAAmB,IAAIf,qBAAqB;MACxEgB,OAAO,EAAEV,MAAM,CAACU,OAAO;MACvBC,UAAU,EAAEX,MAAM,CAACW;IACvB,CAAC,CAAC,EACF;MACIV,YAAY;MACZC,OAAO;MACPC,UAAU;MACVC,UAAU;MACVQ,eAAe,EAAEC,KAAK,IAAI;QACtB,IAAIb,MAAM,CAACO,KAAK,CAACO,mBAAmB,CAAC,CAAC,GAAGrB,4BAA4B,EAAE;UACnE,MAAM,IAAIsB,wDAA2B,CAACF,KAAK,CAAC;QAChD;QACA;AACpB;AACA;QACoB,IAAIA,KAAK,CAACG,aAAa,GAAGd,OAAO,GAAG,IAAI,EAAE;UACtC;QACJ;QACAe,OAAO,CAACJ,KAAK,
|
|
1
|
+
{"version":3,"names":["_execute","require","_NotEnoughRemainingTimeError","_pRetry","_interopRequireDefault","_getNumberEnvVariable","minRemainingSecondsToTimeout","MAX_PROCESSOR_PERCENT","getNumberEnvVariable","process","env","NODE_ENV","executeWithRetry","params","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","execute","timer","maxRunningTime","maxProcessorPercent","context","operations","onFailedAttempt","error","getRemainingSeconds","NotEnoughRemainingTimeError","attemptNumber","console","ex","exports"],"sources":["executeWithRetry.ts"],"sourcesContent":["import { execute, IExecuteParams } from \"~/execute\";\nimport { NotEnoughRemainingTimeError } from \"~/NotEnoughRemainingTimeError\";\nimport pRetry from \"p-retry\";\nimport { getNumberEnvVariable } from \"./helpers/getNumberEnvVariable\";\n\nconst minRemainingSecondsToTimeout = 120;\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n \"MAX_ES_PROCESSOR\",\n process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\nexport interface IExecuteWithRetryParams extends Omit<IExecuteParams, \"maxProcessorPercent\"> {\n maxRetryTime?: number;\n retries?: number;\n minTimeout?: number;\n maxTimeout?: number;\n maxProcessorPercent?: number;\n}\n\nexport const executeWithRetry = async (params: IExecuteWithRetryParams) => {\n const maxRetryTime = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n params.maxRetryTime || 300000\n );\n const retries = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\",\n params.retries || 20\n );\n const minTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n params.minTimeout || 1500\n );\n const maxTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n params.maxTimeout || 30000\n );\n\n try {\n await pRetry(\n execute({\n timer: params.timer,\n maxRunningTime: params.maxRunningTime,\n maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,\n context: params.context,\n operations: params.operations\n }),\n {\n maxRetryTime,\n retries,\n minTimeout,\n maxTimeout,\n onFailedAttempt: error => {\n if (params.timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {\n throw new NotEnoughRemainingTimeError(error);\n }\n /**\n * We will only log attempts which are after 3/4 of total attempts.\n */\n if (error.attemptNumber < retries * 0.75) {\n return;\n }\n console.error(`Attempt #${error.attemptNumber} failed.`);\n console.error(error);\n }\n }\n );\n } catch (ex) {\n // TODO implement storing of failed operations\n throw ex;\n }\n};\n"],"mappings":";;;;;;;AAAA,IAAAA,QAAA,GAAAC,OAAA;AACA,IAAAC,4BAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,qBAAA,GAAAJ,OAAA;AAEA,MAAMK,4BAA4B,GAAG,GAAG;AAExC,MAAMC,qBAAqB,GAAG,IAAAC,0CAAoB,EAC9C,kBAAkB,EAClBC,OAAO,CAACC,GAAG,CAACC,QAAQ,KAAK,MAAM,GAAG,GAAG,GAAG,EAC5C,CAAC;AAUM,MAAMC,gBAAgB,GAAG,MAAOC,MAA+B,IAAK;EACvE,MAAMC,YAAY,GAAG,IAAAN,0CAAoB,EACrC,iDAAiD,EACjDK,MAAM,CAACC,YAAY,IAAI,MAC3B,CAAC;EACD,MAAMC,OAAO,GAAG,IAAAP,0CAAoB,EAChC,0CAA0C,EAC1CK,MAAM,CAACE,OAAO,IAAI,EACtB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAR,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACG,UAAU,IAAI,IACzB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAT,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACI,UAAU,IAAI,KACzB,CAAC;EAED,IAAI;IACA,MAAM,IAAAC,eAAM,EACR,IAAAC,gBAAO,EAAC;MACJC,KAAK,EAAEP,MAAM,CAACO,KAAK;MACnBC,cAAc,EAAER,MAAM,CAACQ,cAAc;MACrCC,mBAAmB,EAAET,MAAM,CAACS,mBAAmB,IAAIf,qBAAqB;MACxEgB,OAAO,EAAEV,MAAM,CAACU,OAAO;MACvBC,UAAU,EAAEX,MAAM,CAACW;IACvB,CAAC,CAAC,EACF;MACIV,YAAY;MACZC,OAAO;MACPC,UAAU;MACVC,UAAU;MACVQ,eAAe,EAAEC,KAAK,IAAI;QACtB,IAAIb,MAAM,CAACO,KAAK,CAACO,mBAAmB,CAAC,CAAC,GAAGrB,4BAA4B,EAAE;UACnE,MAAM,IAAIsB,wDAA2B,CAACF,KAAK,CAAC;QAChD;QACA;AACpB;AACA;QACoB,IAAIA,KAAK,CAACG,aAAa,GAAGd,OAAO,GAAG,IAAI,EAAE;UACtC;QACJ;QACAe,OAAO,CAACJ,KAAK,CAAC,YAAYA,KAAK,CAACG,aAAa,UAAU,CAAC;QACxDC,OAAO,CAACJ,KAAK,CAACA,KAAK,CAAC;MACxB;IACJ,CACJ,CAAC;EACL,CAAC,CAAC,OAAOK,EAAE,EAAE;IACT;IACA,MAAMA,EAAE;EACZ;AACJ,CAAC;AAACC,OAAA,CAAApB,gBAAA,GAAAA,gBAAA","ignoreList":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webiny/api-dynamodb-to-elasticsearch",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.42.0-beta.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -11,24 +11,17 @@
|
|
|
11
11
|
"license": "MIT",
|
|
12
12
|
"author": "Webiny Ltd.",
|
|
13
13
|
"dependencies": {
|
|
14
|
-
"@
|
|
15
|
-
"@webiny/api-
|
|
16
|
-
"@webiny/aws-sdk": "5.
|
|
17
|
-
"@webiny/error": "5.
|
|
18
|
-
"@webiny/handler-aws": "5.
|
|
14
|
+
"@webiny/api-elasticsearch": "5.42.0-beta.0",
|
|
15
|
+
"@webiny/api-log": "5.42.0-beta.0",
|
|
16
|
+
"@webiny/aws-sdk": "5.42.0-beta.0",
|
|
17
|
+
"@webiny/error": "5.42.0-beta.0",
|
|
18
|
+
"@webiny/handler-aws": "5.42.0-beta.0",
|
|
19
19
|
"p-retry": "4.6.2"
|
|
20
20
|
},
|
|
21
21
|
"devDependencies": {
|
|
22
|
-
"@
|
|
23
|
-
"@
|
|
24
|
-
"@
|
|
25
|
-
"@babel/plugin-transform-runtime": "7.24.3",
|
|
26
|
-
"@babel/preset-env": "7.24.3",
|
|
27
|
-
"@babel/preset-typescript": "7.24.1",
|
|
28
|
-
"@types/aws-lambda": "8.10.136",
|
|
29
|
-
"@webiny/cli": "5.41.4-beta.6",
|
|
30
|
-
"@webiny/plugins": "5.41.4-beta.6",
|
|
31
|
-
"@webiny/project-utils": "5.41.4-beta.6",
|
|
22
|
+
"@webiny/cli": "5.42.0-beta.0",
|
|
23
|
+
"@webiny/plugins": "5.42.0-beta.0",
|
|
24
|
+
"@webiny/project-utils": "5.42.0-beta.0",
|
|
32
25
|
"typescript": "4.9.5"
|
|
33
26
|
},
|
|
34
27
|
"publishConfig": {
|
|
@@ -44,5 +37,5 @@
|
|
|
44
37
|
"__tests__"
|
|
45
38
|
]
|
|
46
39
|
},
|
|
47
|
-
"gitHead": "
|
|
40
|
+
"gitHead": "ebf90f62ed3f28114ffdb012b7e5f80988af53d3"
|
|
48
41
|
}
|
package/types.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { GenericRecord } from "@webiny/cli/types";
|
|
2
2
|
import { DynamoDBRecord } from "@webiny/handler-aws/types";
|
|
3
3
|
import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
|
|
4
|
-
|
|
4
|
+
import { Context as LoggerContext } from "@webiny/api-log/types";
|
|
5
5
|
export interface IOperationsBuilderBuildParams {
|
|
6
6
|
records: DynamoDBRecord[];
|
|
7
7
|
}
|
|
@@ -29,3 +29,5 @@ export interface IOperations {
|
|
|
29
29
|
export interface IDecompressor {
|
|
30
30
|
decompress(data: GenericRecord): Promise<GenericRecord | null>;
|
|
31
31
|
}
|
|
32
|
+
export interface Context extends ElasticsearchContext, Pick<LoggerContext, "logger"> {
|
|
33
|
+
}
|
package/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import { GenericRecord } from \"@webiny/cli/types\";\nimport { DynamoDBRecord } from \"@webiny/handler-aws/types\";\nimport { ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import { GenericRecord } from \"@webiny/cli/types\";\nimport { DynamoDBRecord } from \"@webiny/handler-aws/types\";\nimport { ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { Context as LoggerContext } from \"@webiny/api-log/types\";\n\nexport interface IOperationsBuilderBuildParams {\n records: DynamoDBRecord[];\n}\n\nexport interface IOperationsBuilder {\n build(params: IOperationsBuilderBuildParams): Promise<IOperations>;\n}\n\nexport interface IInsertOperationParams {\n id: string;\n index: string;\n data: GenericRecord;\n}\n\nexport type IModifyOperationParams = IInsertOperationParams;\n\nexport interface IDeleteOperationParams {\n id: string;\n index: string;\n}\n\nexport interface IOperations {\n items: GenericRecord[];\n total: number;\n clear(): void;\n insert(params: IInsertOperationParams): void;\n modify(params: IModifyOperationParams): void;\n delete(params: IDeleteOperationParams): void;\n}\n\nexport interface IDecompressor {\n decompress(data: GenericRecord): Promise<GenericRecord | null>;\n}\n\nexport interface Context extends ElasticsearchContext, Pick<LoggerContext, \"logger\"> {}\n"],"mappings":"","ignoreList":[]}
|