@webiny/api-dynamodb-to-elasticsearch 6.0.0-alpha.0 → 6.0.0-alpha.2

package/Operations.d.ts

@@ -1,5 +1,5 @@
-import { GenericRecord } from "@webiny/api/types";
-import { IDeleteOperationParams, IInsertOperationParams, IModifyOperationParams, IOperations } from "./types";
+import type { GenericRecord } from "@webiny/api/types";
+import type { IDeleteOperationParams, IInsertOperationParams, IModifyOperationParams, IOperations } from "./types";
 export declare enum OperationType {
     INSERT = "INSERT",
     MODIFY = "MODIFY",

package/Operations.js.map

@@ -1 +1 @@
-{"version":3,"names":["OperationType","exports","Operations","_items","items","total","length","clear","insert","params","push","index","_id","id","_index","data","modify","delete"],"sources":["Operations.ts"],"sourcesContent":["import { GenericRecord } from \"@webiny/api/types\";\nimport {\n    IDeleteOperationParams,\n    IInsertOperationParams,\n    IModifyOperationParams,\n    IOperations\n} from \"~/types\";\n\nexport enum OperationType {\n    INSERT = \"INSERT\",\n    MODIFY = \"MODIFY\",\n    REMOVE = \"REMOVE\"\n}\n\nexport class Operations implements IOperations {\n    private _items: GenericRecord[] = [];\n\n    public get items(): GenericRecord[] {\n        return this._items;\n    }\n\n    public get total(): number {\n        return this.items.length;\n    }\n\n    public clear() {\n        this._items = [];\n    }\n\n    public insert(params: IInsertOperationParams): void {\n        this.items.push(\n            {\n                index: {\n                    _id: params.id,\n                    _index: params.index\n                }\n            },\n            params.data\n        );\n    }\n\n    public modify(params: IModifyOperationParams): void {\n        this.insert(params);\n    }\n\n    public delete(params: IDeleteOperationParams): void {\n        this.items.push({\n            delete: {\n                _id: params.id,\n                _index: params.index\n            }\n        });\n    }\n}\n"],"mappings":";;;;;;IAQYA,aAAa,GAAAC,OAAA,CAAAD,aAAA,0BAAbA,aAAa;EAAbA,aAAa;EAAbA,aAAa;EAAbA,aAAa;EAAA,OAAbA,aAAa;AAAA;AAMlB,MAAME,UAAU,CAAwB;EACnCC,MAAM,GAAoB,EAAE;EAEpC,IAAWC,KAAKA,CAAA,EAAoB;IAChC,OAAO,IAAI,CAACD,MAAM;EACtB;EAEA,IAAWE,KAAKA,CAAA,EAAW;IACvB,OAAO,IAAI,CAACD,KAAK,CAACE,MAAM;EAC5B;EAEOC,KAAKA,CAAA,EAAG;IACX,IAAI,CAACJ,MAAM,GAAG,EAAE;EACpB;EAEOK,MAAMA,CAACC,MAA8B,EAAQ;IAChD,IAAI,CAACL,KAAK,CAACM,IAAI,CACX;MACIC,KAAK,EAAE;QACHC,GAAG,EAAEH,MAAM,CAACI,EAAE;QACdC,MAAM,EAAEL,MAAM,CAACE;MACnB;IACJ,CAAC,EACDF,MAAM,CAACM,IACX,CAAC;EACL;EAEOC,MAAMA,CAACP,MAA8B,EAAQ;IAChD,IAAI,CAACD,MAAM,CAACC,MAAM,CAAC;EACvB;EAEOQ,MAAMA,CAACR,MAA8B,EAAQ;IAChD,IAAI,CAACL,KAAK,CAACM,IAAI,CAAC;MACZO,MAAM,EAAE;QACJL,GAAG,EAAEH,MAAM,CAACI,EAAE;QACdC,MAAM,EAAEL,MAAM,CAACE;MACnB;IACJ,CAAC,CAAC;EACN;AACJ;AAACV,OAAA,CAAAC,UAAA,GAAAA,UAAA","ignoreList":[]}
+{"version":3,"names":["OperationType","exports","Operations","_items","items","total","length","clear","insert","params","push","index","_id","id","_index","data","modify","delete"],"sources":["Operations.ts"],"sourcesContent":["import type { GenericRecord } from \"@webiny/api/types\";\nimport type {\n    IDeleteOperationParams,\n    IInsertOperationParams,\n    IModifyOperationParams,\n    IOperations\n} from \"~/types\";\n\nexport enum OperationType {\n    INSERT = \"INSERT\",\n    MODIFY = \"MODIFY\",\n    REMOVE = \"REMOVE\"\n}\n\nexport class Operations implements IOperations {\n    private _items: GenericRecord[] = [];\n\n    public get items(): GenericRecord[] {\n        return this._items;\n    }\n\n    public get total(): number {\n        return this.items.length;\n    }\n\n    public clear() {\n        this._items = [];\n    }\n\n    public insert(params: IInsertOperationParams): void {\n        this.items.push(\n            {\n                index: {\n                    _id: params.id,\n                    _index: params.index\n                }\n            },\n            params.data\n        );\n    }\n\n    public modify(params: IModifyOperationParams): void {\n        this.insert(params);\n    }\n\n    public delete(params: IDeleteOperationParams): void {\n        this.items.push({\n            delete: {\n                _id: params.id,\n                _index: params.index\n            }\n        });\n    }\n}\n"],"mappings":";;;;;;IAQYA,aAAa,GAAAC,OAAA,CAAAD,aAAA,0BAAbA,aAAa;EAAbA,aAAa;EAAbA,aAAa;EAAbA,aAAa;EAAA,OAAbA,aAAa;AAAA;AAMlB,MAAME,UAAU,CAAwB;EACnCC,MAAM,GAAoB,EAAE;EAEpC,IAAWC,KAAKA,CAAA,EAAoB;IAChC,OAAO,IAAI,CAACD,MAAM;EACtB;EAEA,IAAWE,KAAKA,CAAA,EAAW;IACvB,OAAO,IAAI,CAACD,KAAK,CAACE,MAAM;EAC5B;EAEOC,KAAKA,CAAA,EAAG;IACX,IAAI,CAACJ,MAAM,GAAG,EAAE;EACpB;EAEOK,MAAMA,CAACC,MAA8B,EAAQ;IAChD,IAAI,CAACL,KAAK,CAACM,IAAI,CACX;MACIC,KAAK,EAAE;QACHC,GAAG,EAAEH,MAAM,CAACI,EAAE;QACdC,MAAM,EAAEL,MAAM,CAACE;MACnB;IACJ,CAAC,EACDF,MAAM,CAACM,IACX,CAAC;EACL;EAEOC,MAAMA,CAACP,MAA8B,EAAQ;IAChD,IAAI,CAACD,MAAM,CAACC,MAAM,CAAC;EACvB;EAEOQ,MAAMA,CAACR,MAA8B,EAAQ;IAChD,IAAI,CAACL,KAAK,CAACM,IAAI,CAAC;MACZO,MAAM,EAAE;QACJL,GAAG,EAAEH,MAAM,CAACI,EAAE;QACdC,MAAM,EAAEL,MAAM,CAACE;MACnB;IACJ,CAAC,CAAC;EACN;AACJ;AAACV,OAAA,CAAAC,UAAA,GAAAA,UAAA","ignoreList":[]}

package/OperationsBuilder.d.ts

@@ -1,5 +1,5 @@
-import { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from "./types";
-import { ICompressor } from "@webiny/utils/compression/Compressor";
+import type { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from "./types";
+import type { ICompressor } from "@webiny/utils/compression/Compressor";
 export interface IOperationsBuilderParams {
     compressor: ICompressor;
 }

package/OperationsBuilder.js.map

@@ -1 +1 @@
-{"version":3,"names":["_Operations","require","_marshall","OperationsBuilder","constructor","params","compressor","build","operations","Operations","record","records","dynamodb","eventName","console","error","eventID","keys","unmarshall","Keys","PK","SK","id","OperationType","INSERT","MODIFY","newImage","NewImage","Object","length","ignore","index","log","data","decompress","undefined","insert","REMOVE","oldImage","OldImage","delete","exports"],"sources":["OperationsBuilder.ts"],"sourcesContent":["import { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from \"./types\";\nimport { Operations, OperationType } from \"~/Operations\";\nimport { unmarshall } from \"~/marshall\";\nimport { ICompressor } from \"@webiny/utils/compression/Compressor\";\n\ninterface RecordDynamoDbImage {\n    data: {\n        compression: string;\n        value: string;\n    };\n    ignore?: boolean;\n    index: string;\n}\n\ninterface RecordDynamoDbKeys {\n    PK: string;\n    SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n    compressor: ICompressor;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n    private readonly compressor: ICompressor;\n\n    public constructor(params: IOperationsBuilderParams) {\n        this.compressor = params.compressor;\n    }\n\n    public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n        const operations = new Operations();\n        for (const record of params.records) {\n            if (!record.dynamodb) {\n                continue;\n            } else if (!record.eventName) {\n                console.error(\n                    `Could not get operation from the record, skipping event \"${record.eventID}\".`\n                );\n                continue;\n            }\n\n            const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n            if (!keys?.PK || !keys.SK) {\n                console.error(\n                    `Could not get keys from the record, skipping event \"${record.eventID}\".`\n                );\n                continue;\n            }\n\n            const id = `${keys.PK}:${keys.SK}`;\n\n            /**\n             * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n             * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n             */\n            if (\n                record.eventName === OperationType.INSERT ||\n                record.eventName === OperationType.MODIFY\n            ) {\n                const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n                /**\n                 * If there is no newImage, silently continue to the next operation.\n                 */\n                if (\n                    !newImage ||\n                    typeof newImage !== \"object\" ||\n                    Object.keys(newImage).length === 0\n                ) {\n                    continue;\n                }\n                /**\n                 * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n                 * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n                 * need to process it.\n                 */\n                //\n                else if (newImage.ignore === true) {\n                    // Nothing to log here, we are skipping the record intentionally.\n                    continue;\n                }\n                /**\n                 * Also, possibly there is no index?\n                 */\n                //\n                else if (!newImage.index) {\n                    console.error(\n                        `Could not get index from the new image, skipping event \"${record.eventID}\".`\n                    );\n                    console.log({ newImage });\n                    continue;\n                }\n                /**\n                 * We must decompress the data that is going into the Elasticsearch.\n                 */\n                const data = await this.compressor.decompress(newImage.data);\n                /**\n                 * No point in writing null or undefined data into the Elasticsearch.\n                 * This might happen on some error while decompressing. We will log it.\n                 *\n                 * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n                 * If it is - it is a bug.\n                 */\n                if (data === undefined || data === null) {\n                    console.error(\n                        `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n                    );\n                    continue;\n                }\n\n                operations.insert({\n                    id,\n                    index: newImage.index,\n                    data\n                });\n            } else if (record.eventName === OperationType.REMOVE) {\n                const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n                /**\n                 * If there is no index found, silently continue to the next operation.\n                 */\n                if (!oldImage?.index) {\n                    continue;\n                }\n                operations.delete({\n                    id,\n                    index: oldImage.index\n                });\n            }\n        }\n        return operations;\n    }\n}\n"],"mappings":";;;;;;AACA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AAqBO,MAAME,iBAAiB,CAA+B;EAGlDC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,UAAU,GAAGD,MAAM,CAACC,UAAU;EACvC;EAEA,MAAaC,KAAKA,CAACF,MAAqC,EAAwB;IAC5E,MAAMG,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;IACnC,KAAK,MAAMC,MAAM,IAAIL,MAAM,CAACM,OAAO,EAAE;MACjC,IAAI,CAACD,MAAM,CAACE,QAAQ,EAAE;QAClB;MACJ,CAAC,MAAM,IAAI,CAACF,MAAM,CAACG,SAAS,EAAE;QAC1BC,OAAO,CAACC,KAAK,CACT,4DAA4DL,MAAM,CAACM,OAAO,IAC9E,CAAC;QACD;MACJ;MAEA,MAAMC,IAAI,GAAG,IAAAC,oBAAU,EAAqBR,MAAM,CAACE,QAAQ,CAACO,IAAI,CAAC;MACjE,IAAI,CAACF,IAAI,EAAEG,EAAE,IAAI,CAACH,IAAI,CAACI,EAAE,EAAE;QACvBP,OAAO,CAACC,KAAK,CACT,uDAAuDL,MAAM,CAACM,OAAO,IACzE,CAAC;QACD;MACJ;MAEA,MAAMM,EAAE,GAAG,GAAGL,IAAI,CAACG,EAAE,IAAIH,IAAI,CAACI,EAAE,EAAE;;MAElC;AACZ;AACA;AACA;MACY,IACIX,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACC,MAAM,IACzCd,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACE,MAAM,EAC3C;QACE,MAAMC,QAAQ,GAAG,IAAAR,oBAAU,EAAsBR,MAAM,CAACE,QAAQ,CAACe,QAAQ,CAAC;QAC1E;AAChB;AACA;QACgB,IACI,CAACD,QAAQ,IACT,OAAOA,QAAQ,KAAK,QAAQ,IAC5BE,MAAM,CAACX,IAAI,CAACS,QAAQ,CAAC,CAACG,MAAM,KAAK,CAAC,EACpC;UACE;QACJ;QACA;AAChB;AACA;AACA;AACA;QACgB;QAAA,KACK,IAAIH,QAAQ,CAACI,MAAM,KAAK,IAAI,EAAE;UAC/B;UACA;QACJ;QACA;AAChB;AACA;QACgB;QAAA,KACK,IAAI,CAACJ,QAAQ,CAACK,KAAK,EAAE;UACtBjB,OAAO,CAACC,KAAK,CACT,2DAA2DL,MAAM,CAACM,OAAO,IAC7E,CAAC;UACDF,OAAO,CAACkB,GAAG,CAAC;YAAEN;UAAS,CAAC,CAAC;UACzB;QACJ;QACA;AAChB;AACA;QACgB,MAAMO,IAAI,GAAG,MAAM,IAAI,CAAC3B,UAAU,CAAC4B,UAAU,CAACR,QAAQ,CAACO,IAAI,CAAC;QAC5D;AAChB;AACA;AACA;AACA;AACA;AACA;QACgB,IAAIA,IAAI,KAAKE,SAAS,IAAIF,IAAI,KAAK,IAAI,EAAE;UACrCnB,OAAO,CAACC,KAAK,CACT,2DAA2DL,MAAM,CAACG,SAAS,SAASS,EAAE,eAC1F,CAAC;UACD;QACJ;QAEAd,UAAU,CAAC4B,MAAM,CAAC;UACdd,EAAE;UACFS,KAAK,EAAEL,QAAQ,CAACK,KAAK;UACrBE;QACJ,CAAC,CAAC;MACN,CAAC,MAAM,IAAIvB,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACc,MAAM,EAAE;QAClD,MAAMC,QAAQ,GAAG,IAAApB,oBAAU,EAAsBR,MAAM,CAACE,QAAQ,CAAC2B,QAAQ,CAAC;QAC1E;AAChB;AACA;QACgB,IAAI,CAACD,QAAQ,EAAEP,KAAK,EAAE;UAClB;QACJ;QACAvB,UAAU,CAACgC,MAAM,CAAC;UACdlB,EAAE;UACFS,KAAK,EAAEO,QAAQ,CAACP;QACpB,CAAC,CAAC;MACN;IACJ;IACA,OAAOvB,UAAU;EACrB;AACJ;AAACiC,OAAA,CAAAtC,iBAAA,GAAAA,iBAAA","ignoreList":[]}
+{"version":3,"names":["_Operations","require","_marshall","OperationsBuilder","constructor","params","compressor","build","operations","Operations","record","records","dynamodb","eventName","console","error","eventID","keys","unmarshall","Keys","PK","SK","id","OperationType","INSERT","MODIFY","newImage","NewImage","Object","length","ignore","index","log","data","decompress","undefined","insert","REMOVE","oldImage","OldImage","delete","exports"],"sources":["OperationsBuilder.ts"],"sourcesContent":["import type { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from \"./types\";\nimport { Operations, OperationType } from \"~/Operations\";\nimport { unmarshall } from \"~/marshall\";\nimport type { ICompressor } from \"@webiny/utils/compression/Compressor\";\n\ninterface RecordDynamoDbImage {\n    data: {\n        compression: string;\n        value: string;\n    };\n    ignore?: boolean;\n    index: string;\n}\n\ninterface RecordDynamoDbKeys {\n    PK: string;\n    SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n    compressor: ICompressor;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n    private readonly compressor: ICompressor;\n\n    public constructor(params: IOperationsBuilderParams) {\n        this.compressor = params.compressor;\n    }\n\n    public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n        const operations = new Operations();\n        for (const record of params.records) {\n            if (!record.dynamodb) {\n                continue;\n            } else if (!record.eventName) {\n                console.error(\n                    `Could not get operation from the record, skipping event \"${record.eventID}\".`\n                );\n                continue;\n            }\n\n            const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n            if (!keys?.PK || !keys.SK) {\n                console.error(\n                    `Could not get keys from the record, skipping event \"${record.eventID}\".`\n                );\n                continue;\n            }\n\n            const id = `${keys.PK}:${keys.SK}`;\n\n            /**\n             * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n             * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n             */\n            if (\n                record.eventName === OperationType.INSERT ||\n                record.eventName === OperationType.MODIFY\n            ) {\n                const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n                /**\n                 * If there is no newImage, silently continue to the next operation.\n                 */\n                if (\n                    !newImage ||\n                    typeof newImage !== \"object\" ||\n                    Object.keys(newImage).length === 0\n                ) {\n                    continue;\n                }\n                /**\n                 * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n                 * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n                 * need to process it.\n                 */\n                //\n                else if (newImage.ignore === true) {\n                    // Nothing to log here, we are skipping the record intentionally.\n                    continue;\n                }\n                /**\n                 * Also, possibly there is no index?\n                 */\n                //\n                else if (!newImage.index) {\n                    console.error(\n                        `Could not get index from the new image, skipping event \"${record.eventID}\".`\n                    );\n                    console.log({ newImage });\n                    continue;\n                }\n                /**\n                 * We must decompress the data that is going into the Elasticsearch.\n                 */\n                const data = await this.compressor.decompress(newImage.data);\n                /**\n                 * No point in writing null or undefined data into the Elasticsearch.\n                 * This might happen on some error while decompressing. We will log it.\n                 *\n                 * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n                 * If it is - it is a bug.\n                 */\n                if (data === undefined || data === null) {\n                    console.error(\n                        `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n                    );\n                    continue;\n                }\n\n                operations.insert({\n                    id,\n                    index: newImage.index,\n                    data\n                });\n            } else if (record.eventName === OperationType.REMOVE) {\n                const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n                /**\n                 * If there is no index found, silently continue to the next operation.\n                 */\n                if (!oldImage?.index) {\n                    continue;\n                }\n                operations.delete({\n                    id,\n                    index: oldImage.index\n                });\n            }\n        }\n        return operations;\n    }\n}\n"],"mappings":";;;;;;AACA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AAqBO,MAAME,iBAAiB,CAA+B;EAGlDC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,UAAU,GAAGD,MAAM,CAACC,UAAU;EACvC;EAEA,MAAaC,KAAKA,CAACF,MAAqC,EAAwB;IAC5E,MAAMG,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;IACnC,KAAK,MAAMC,MAAM,IAAIL,MAAM,CAACM,OAAO,EAAE;MACjC,IAAI,CAACD,MAAM,CAACE,QAAQ,EAAE;QAClB;MACJ,CAAC,MAAM,IAAI,CAACF,MAAM,CAACG,SAAS,EAAE;QAC1BC,OAAO,CAACC,KAAK,CACT,4DAA4DL,MAAM,CAACM,OAAO,IAC9E,CAAC;QACD;MACJ;MAEA,MAAMC,IAAI,GAAG,IAAAC,oBAAU,EAAqBR,MAAM,CAACE,QAAQ,CAACO,IAAI,CAAC;MACjE,IAAI,CAACF,IAAI,EAAEG,EAAE,IAAI,CAACH,IAAI,CAACI,EAAE,EAAE;QACvBP,OAAO,CAACC,KAAK,CACT,uDAAuDL,MAAM,CAACM,OAAO,IACzE,CAAC;QACD;MACJ;MAEA,MAAMM,EAAE,GAAG,GAAGL,IAAI,CAACG,EAAE,IAAIH,IAAI,CAACI,EAAE,EAAE;;MAElC;AACZ;AACA;AACA;MACY,IACIX,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACC,MAAM,IACzCd,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACE,MAAM,EAC3C;QACE,MAAMC,QAAQ,GAAG,IAAAR,oBAAU,EAAsBR,MAAM,CAACE,QAAQ,CAACe,QAAQ,CAAC;QAC1E;AAChB;AACA;QACgB,IACI,CAACD,QAAQ,IACT,OAAOA,QAAQ,KAAK,QAAQ,IAC5BE,MAAM,CAACX,IAAI,CAACS,QAAQ,CAAC,CAACG,MAAM,KAAK,CAAC,EACpC;UACE;QACJ;QACA;AAChB;AACA;AACA;AACA;QACgB;QAAA,KACK,IAAIH,QAAQ,CAACI,MAAM,KAAK,IAAI,EAAE;UAC/B;UACA;QACJ;QACA;AAChB;AACA;QACgB;QAAA,KACK,IAAI,CAACJ,QAAQ,CAACK,KAAK,EAAE;UACtBjB,OAAO,CAACC,KAAK,CACT,2DAA2DL,MAAM,CAACM,OAAO,IAC7E,CAAC;UACDF,OAAO,CAACkB,GAAG,CAAC;YAAEN;UAAS,CAAC,CAAC;UACzB;QACJ;QACA;AAChB;AACA;QACgB,MAAMO,IAAI,GAAG,MAAM,IAAI,CAAC3B,UAAU,CAAC4B,UAAU,CAACR,QAAQ,CAACO,IAAI,CAAC;QAC5D;AAChB;AACA;AACA;AACA;AACA;AACA;QACgB,IAAIA,IAAI,KAAKE,SAAS,IAAIF,IAAI,KAAK,IAAI,EAAE;UACrCnB,OAAO,CAACC,KAAK,CACT,2DAA2DL,MAAM,CAACG,SAAS,SAASS,EAAE,eAC1F,CAAC;UACD;QACJ;QAEAd,UAAU,CAAC4B,MAAM,CAAC;UACdd,EAAE;UACFS,KAAK,EAAEL,QAAQ,CAACK,KAAK;UACrBE;QACJ,CAAC,CAAC;MACN,CAAC,MAAM,IAAIvB,MAAM,CAACG,SAAS,KAAKU,yBAAa,CAACc,MAAM,EAAE;QAClD,MAAMC,QAAQ,GAAG,IAAApB,oBAAU,EAAsBR,MAAM,CAACE,QAAQ,CAAC2B,QAAQ,CAAC;QAC1E;AAChB;AACA;QACgB,IAAI,CAACD,QAAQ,EAAEP,KAAK,EAAE;UAClB;QACJ;QACAvB,UAAU,CAACgC,MAAM,CAAC;UACdlB,EAAE;UACFS,KAAK,EAAEO,QAAQ,CAACP;QACpB,CAAC,CAAC;MACN;IACJ;IACA,OAAOvB,UAAU;EACrB;AACJ;AAACiC,OAAA,CAAAtC,iBAAA,GAAAA,iBAAA","ignoreList":[]}

package/SynchronizationBuilder.d.ts

@@ -1,6 +1,6 @@
-import { Context, IDeleteOperationParams, IInsertOperationParams, IModifyOperationParams } from "./types";
-import { IExecuteWithRetryParams } from "./executeWithRetry";
-import { ITimer } from "@webiny/handler-aws";
+import type { Context, IDeleteOperationParams, IInsertOperationParams, IModifyOperationParams } from "./types";
+import type { IExecuteWithRetryParams } from "./executeWithRetry";
+import type { ITimer } from "@webiny/handler-aws";
 export type ISynchronizationBuilderExecuteWithRetryParams = Omit<IExecuteWithRetryParams, "context" | "timer" | "maxRunningTime" | "operations">;
 export interface ISynchronizationBuilder {
     insert(params: IInsertOperationParams): void;

package/SynchronizationBuilder.js.map

@@ -1 +1 @@
-{"version":3,"names":["_Operations","require","_executeWithRetry","SynchronizationBuilder","constructor","params","timer","context","operations","Operations","insert","modify","delete","build","total","executeWithRetry","maxRunningTime","getRemainingMilliseconds","clear","exports","createSynchronizationBuilder"],"sources":["SynchronizationBuilder.ts"],"sourcesContent":["import {\n    Context,\n    IDeleteOperationParams,\n    IInsertOperationParams,\n    IModifyOperationParams,\n    IOperations\n} from \"~/types\";\nimport { Operations } from \"~/Operations\";\nimport { executeWithRetry, IExecuteWithRetryParams } from \"~/executeWithRetry\";\nimport { ITimer } from \"@webiny/handler-aws\";\n\nexport type ISynchronizationBuilderExecuteWithRetryParams = Omit<\n    IExecuteWithRetryParams,\n    \"context\" | \"timer\" | \"maxRunningTime\" | \"operations\"\n>;\n\nexport interface ISynchronizationBuilder {\n    insert(params: IInsertOperationParams): void;\n    delete(params: IDeleteOperationParams): void;\n    build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;\n}\n\nexport interface ISynchronizationBuilderParams {\n    timer: ITimer;\n    context: Pick<Context, \"elasticsearch\" | \"logger\">;\n}\n\nexport class SynchronizationBuilder implements ISynchronizationBuilder {\n    private readonly timer: ITimer;\n    private readonly context: Pick<Context, \"elasticsearch\" | \"logger\">;\n    private readonly operations: IOperations;\n\n    public constructor(params: ISynchronizationBuilderParams) {\n        this.timer = params.timer;\n        this.context = params.context;\n        this.operations = new Operations();\n    }\n\n    public insert(params: IInsertOperationParams): void {\n        return this.operations.insert(params);\n    }\n\n    public modify(params: IModifyOperationParams): void {\n        return this.operations.modify(params);\n    }\n\n    public delete(params: IDeleteOperationParams): void {\n        return this.operations.delete(params);\n    }\n\n    public build() {\n        return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => {\n            if (this.operations.total === 0) {\n                return;\n            }\n            await executeWithRetry({\n                ...params,\n                maxRunningTime: this.timer.getRemainingMilliseconds(),\n                timer: this.timer,\n                context: this.context,\n                operations: this.operations\n            });\n            this.operations.clear();\n        };\n    }\n}\n\nexport const createSynchronizationBuilder = (\n    params: ISynchronizationBuilderParams\n): ISynchronizationBuilder => {\n    return new SynchronizationBuilder(params);\n};\n"],"mappings":";;;;;;AAOA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,iBAAA,GAAAD,OAAA;AAmBO,MAAME,sBAAsB,CAAoC;EAK5DC,WAAWA,CAACC,MAAqC,EAAE;IACtD,IAAI,CAACC,KAAK,GAAGD,MAAM,CAACC,KAAK;IACzB,IAAI,CAACC,OAAO,GAAGF,MAAM,CAACE,OAAO;IAC7B,IAAI,CAACC,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;EACtC;EAEOC,MAAMA,CAACL,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACE,MAAM,CAACL,MAAM,CAAC;EACzC;EAEOM,MAAMA,CAACN,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACG,MAAM,CAACN,MAAM,CAAC;EACzC;EAEOO,MAAMA,CAACP,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACI,MAAM,CAACP,MAAM,CAAC;EACzC;EAEOQ,KAAKA,CAAA,EAAG;IACX,OAAO,MAAOR,MAAsD,IAAK;MACrE,IAAI,IAAI,CAACG,UAAU,CAACM,KAAK,KAAK,CAAC,EAAE;QAC7B;MACJ;MACA,MAAM,IAAAC,kCAAgB,EAAC;QACnB,GAAGV,MAAM;QACTW,cAAc,EAAE,IAAI,CAACV,KAAK,CAACW,wBAAwB,CAAC,CAAC;QACrDX,KAAK,EAAE,IAAI,CAACA,KAAK;QACjBC,OAAO,EAAE,IAAI,CAACA,OAAO;QACrBC,UAAU,EAAE,IAAI,CAACA;MACrB,CAAC,CAAC;MACF,IAAI,CAACA,UAAU,CAACU,KAAK,CAAC,CAAC;IAC3B,CAAC;EACL;AACJ;AAACC,OAAA,CAAAhB,sBAAA,GAAAA,sBAAA;AAEM,MAAMiB,4BAA4B,GACrCf,MAAqC,IACX;EAC1B,OAAO,IAAIF,sBAAsB,CAACE,MAAM,CAAC;AAC7C,CAAC;AAACc,OAAA,CAAAC,4BAAA,GAAAA,4BAAA","ignoreList":[]}
+{"version":3,"names":["_Operations","require","_executeWithRetry","SynchronizationBuilder","constructor","params","timer","context","operations","Operations","insert","modify","delete","build","total","executeWithRetry","maxRunningTime","getRemainingMilliseconds","clear","exports","createSynchronizationBuilder"],"sources":["SynchronizationBuilder.ts"],"sourcesContent":["import type {\n    Context,\n    IDeleteOperationParams,\n    IInsertOperationParams,\n    IModifyOperationParams,\n    IOperations\n} from \"~/types\";\nimport { Operations } from \"~/Operations\";\nimport type { IExecuteWithRetryParams } from \"~/executeWithRetry\";\nimport { executeWithRetry } from \"~/executeWithRetry\";\nimport type { ITimer } from \"@webiny/handler-aws\";\n\nexport type ISynchronizationBuilderExecuteWithRetryParams = Omit<\n    IExecuteWithRetryParams,\n    \"context\" | \"timer\" | \"maxRunningTime\" | \"operations\"\n>;\n\nexport interface ISynchronizationBuilder {\n    insert(params: IInsertOperationParams): void;\n    delete(params: IDeleteOperationParams): void;\n    build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;\n}\n\nexport interface ISynchronizationBuilderParams {\n    timer: ITimer;\n    context: Pick<Context, \"elasticsearch\" | \"logger\">;\n}\n\nexport class SynchronizationBuilder implements ISynchronizationBuilder {\n    private readonly timer: ITimer;\n    private readonly context: Pick<Context, \"elasticsearch\" | \"logger\">;\n    private readonly operations: IOperations;\n\n    public constructor(params: ISynchronizationBuilderParams) {\n        this.timer = params.timer;\n        this.context = params.context;\n        this.operations = new Operations();\n    }\n\n    public insert(params: IInsertOperationParams): void {\n        return this.operations.insert(params);\n    }\n\n    public modify(params: IModifyOperationParams): void {\n        return this.operations.modify(params);\n    }\n\n    public delete(params: IDeleteOperationParams): void {\n        return this.operations.delete(params);\n    }\n\n    public build() {\n        return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => {\n            if (this.operations.total === 0) {\n                return;\n            }\n            await executeWithRetry({\n                ...params,\n                maxRunningTime: this.timer.getRemainingMilliseconds(),\n                timer: this.timer,\n                context: this.context,\n                operations: this.operations\n            });\n            this.operations.clear();\n        };\n    }\n}\n\nexport const createSynchronizationBuilder = (\n    params: ISynchronizationBuilderParams\n): ISynchronizationBuilder => {\n    return new SynchronizationBuilder(params);\n};\n"],"mappings":";;;;;;AAOA,IAAAA,WAAA,GAAAC,OAAA;AAEA,IAAAC,iBAAA,GAAAD,OAAA;AAmBO,MAAME,sBAAsB,CAAoC;EAK5DC,WAAWA,CAACC,MAAqC,EAAE;IACtD,IAAI,CAACC,KAAK,GAAGD,MAAM,CAACC,KAAK;IACzB,IAAI,CAACC,OAAO,GAAGF,MAAM,CAACE,OAAO;IAC7B,IAAI,CAACC,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;EACtC;EAEOC,MAAMA,CAACL,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACE,MAAM,CAACL,MAAM,CAAC;EACzC;EAEOM,MAAMA,CAACN,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACG,MAAM,CAACN,MAAM,CAAC;EACzC;EAEOO,MAAMA,CAACP,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACI,MAAM,CAACP,MAAM,CAAC;EACzC;EAEOQ,KAAKA,CAAA,EAAG;IACX,OAAO,MAAOR,MAAsD,IAAK;MACrE,IAAI,IAAI,CAACG,UAAU,CAACM,KAAK,KAAK,CAAC,EAAE;QAC7B;MACJ;MACA,MAAM,IAAAC,kCAAgB,EAAC;QACnB,GAAGV,MAAM;QACTW,cAAc,EAAE,IAAI,CAACV,KAAK,CAACW,wBAAwB,CAAC,CAAC;QACrDX,KAAK,EAAE,IAAI,CAACA,KAAK;QACjBC,OAAO,EAAE,IAAI,CAACA,OAAO;QACrBC,UAAU,EAAE,IAAI,CAACA;MACrB,CAAC,CAAC;MACF,IAAI,CAACA,UAAU,CAACU,KAAK,CAAC,CAAC;IAC3B,CAAC;EACL;AACJ;AAACC,OAAA,CAAAhB,sBAAA,GAAAA,sBAAA;AAEM,MAAMiB,4BAA4B,GACrCf,MAAqC,IACX;EAC1B,OAAO,IAAIF,sBAAsB,CAACE,MAAM,CAAC;AAC7C,CAAC;AAACc,OAAA,CAAAC,4BAAA,GAAAA,4BAAA","ignoreList":[]}

package/eventHandler.js.map

@@ -1 +1 @@
-{"version":3,"names":["_handlerAws","require","_OperationsBuilder","_executeWithRetry","MAX_RUNNING_TIME","createEventHandler","createDynamoDBEventHandler","event","context","ctx","lambdaContext","timer","timerFactory","elasticsearch","console","error","builder","OperationsBuilder","compressor","operations","build","records","Records","total","executeWithRetry","maxRunningTime","exports"],"sources":["eventHandler.ts"],"sourcesContent":["import { createDynamoDBEventHandler, timerFactory } from \"@webiny/handler-aws\";\nimport { Context } from \"~/types\";\nimport { OperationsBuilder } from \"~/OperationsBuilder\";\nimport { executeWithRetry } from \"~/executeWithRetry\";\n\n/**\n * Also, we need to set the maximum running time for the Lambda Function.\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218\n */\nconst MAX_RUNNING_TIME = 900;\n\nexport const createEventHandler = () => {\n    return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => {\n        const timer = timerFactory(lambdaContext);\n        const context = ctx as unknown as Context;\n        if (!context.elasticsearch) {\n            console.error(\"Missing elasticsearch definition on context.\");\n            return null;\n        }\n\n        const builder = new OperationsBuilder({\n            compressor: context.compressor\n        });\n\n        const operations = await builder.build({\n            records: event.Records\n        });\n        /**\n         * No need to do anything if there are no operations.\n         */\n        if (operations.total === 0) {\n            return null;\n        }\n        /**\n         * Execute the operations with retry.\n         */\n        await executeWithRetry({\n            timer,\n            maxRunningTime: MAX_RUNNING_TIME,\n            context,\n            operations\n        });\n\n        return null;\n    });\n};\n"],"mappings":";;;;;;AAAA,IAAAA,WAAA,GAAAC,OAAA;AAEA,IAAAC,kBAAA,GAAAD,OAAA;AACA,IAAAE,iBAAA,GAAAF,OAAA;AAEA;AACA;AACA;AACA;AACA;AACA,MAAMG,gBAAgB,GAAG,GAAG;AAErB,MAAMC,kBAAkB,GAAGA,CAAA,KAAM;EACpC,OAAO,IAAAC,sCAA0B,EAAC,OAAO;IAAEC,KAAK;IAAEC,OAAO,EAAEC,GAAG;IAAEC;EAAc,CAAC,KAAK;IAChF,MAAMC,KAAK,GAAG,IAAAC,wBAAY,EAACF,aAAa,CAAC;IACzC,MAAMF,OAAO,GAAGC,GAAyB;IACzC,IAAI,CAACD,OAAO,CAACK,aAAa,EAAE;MACxBC,OAAO,CAACC,KAAK,CAAC,8CAA8C,CAAC;MAC7D,OAAO,IAAI;IACf;IAEA,MAAMC,OAAO,GAAG,IAAIC,oCAAiB,CAAC;MAClCC,UAAU,EAAEV,OAAO,CAACU;IACxB,CAAC,CAAC;IAEF,MAAMC,UAAU,GAAG,MAAMH,OAAO,CAACI,KAAK,CAAC;MACnCC,OAAO,EAAEd,KAAK,CAACe;IACnB,CAAC,CAAC;IACF;AACR;AACA;IACQ,IAAIH,UAAU,CAACI,KAAK,KAAK,CAAC,EAAE;MACxB,OAAO,IAAI;IACf;IACA;AACR;AACA;IACQ,MAAM,IAAAC,kCAAgB,EAAC;MACnBb,KAAK;MACLc,cAAc,EAAErB,gBAAgB;MAChCI,OAAO;MACPW;IACJ,CAAC,CAAC;IAEF,OAAO,IAAI;EACf,CAAC,CAAC;AACN,CAAC;AAACO,OAAA,CAAArB,kBAAA,GAAAA,kBAAA","ignoreList":[]}
+{"version":3,"names":["_handlerAws","require","_OperationsBuilder","_executeWithRetry","MAX_RUNNING_TIME","createEventHandler","createDynamoDBEventHandler","event","context","ctx","lambdaContext","timer","timerFactory","elasticsearch","console","error","builder","OperationsBuilder","compressor","operations","build","records","Records","total","executeWithRetry","maxRunningTime","exports"],"sources":["eventHandler.ts"],"sourcesContent":["import { createDynamoDBEventHandler, timerFactory } from \"@webiny/handler-aws\";\nimport type { Context } from \"~/types\";\nimport { OperationsBuilder } from \"~/OperationsBuilder\";\nimport { executeWithRetry } from \"~/executeWithRetry\";\n\n/**\n * Also, we need to set the maximum running time for the Lambda Function.\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218\n */\nconst MAX_RUNNING_TIME = 900;\n\nexport const createEventHandler = () => {\n    return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => {\n        const timer = timerFactory(lambdaContext);\n        const context = ctx as unknown as Context;\n        if (!context.elasticsearch) {\n            console.error(\"Missing elasticsearch definition on context.\");\n            return null;\n        }\n\n        const builder = new OperationsBuilder({\n            compressor: context.compressor\n        });\n\n        const operations = await builder.build({\n            records: event.Records\n        });\n        /**\n         * No need to do anything if there are no operations.\n         */\n        if (operations.total === 0) {\n            return null;\n        }\n        /**\n         * Execute the operations with retry.\n         */\n        await executeWithRetry({\n            timer,\n            maxRunningTime: MAX_RUNNING_TIME,\n            context,\n            operations\n        });\n\n        return null;\n    });\n};\n"],"mappings":";;;;;;AAAA,IAAAA,WAAA,GAAAC,OAAA;AAEA,IAAAC,kBAAA,GAAAD,OAAA;AACA,IAAAE,iBAAA,GAAAF,OAAA;AAEA;AACA;AACA;AACA;AACA;AACA,MAAMG,gBAAgB,GAAG,GAAG;AAErB,MAAMC,kBAAkB,GAAGA,CAAA,KAAM;EACpC,OAAO,IAAAC,sCAA0B,EAAC,OAAO;IAAEC,KAAK;IAAEC,OAAO,EAAEC,GAAG;IAAEC;EAAc,CAAC,KAAK;IAChF,MAAMC,KAAK,GAAG,IAAAC,wBAAY,EAACF,aAAa,CAAC;IACzC,MAAMF,OAAO,GAAGC,GAAyB;IACzC,IAAI,CAACD,OAAO,CAACK,aAAa,EAAE;MACxBC,OAAO,CAACC,KAAK,CAAC,8CAA8C,CAAC;MAC7D,OAAO,IAAI;IACf;IAEA,MAAMC,OAAO,GAAG,IAAIC,oCAAiB,CAAC;MAClCC,UAAU,EAAEV,OAAO,CAACU;IACxB,CAAC,CAAC;IAEF,MAAMC,UAAU,GAAG,MAAMH,OAAO,CAACI,KAAK,CAAC;MACnCC,OAAO,EAAEd,KAAK,CAACe;IACnB,CAAC,CAAC;IACF;AACR;AACA;IACQ,IAAIH,UAAU,CAACI,KAAK,KAAK,CAAC,EAAE;MACxB,OAAO,IAAI;IACf;IACA;AACR;AACA;IACQ,MAAM,IAAAC,kCAAgB,EAAC;MACnBb,KAAK;MACLc,cAAc,EAAErB,gBAAgB;MAChCI,OAAO;MACPW;IACJ,CAAC,CAAC;IAEF,OAAO,IAAI;EACf,CAAC,CAAC;AACN,CAAC;AAACO,OAAA,CAAArB,kBAAA,GAAAA,kBAAA","ignoreList":[]}

package/execute.d.ts

@@ -1,5 +1,5 @@
-import { ITimer } from "@webiny/handler-aws";
-import { Context, IOperations } from "./types";
+import type { ITimer } from "@webiny/handler-aws";
+import type { Context, IOperations } from "./types";
 export interface BulkOperationsResponseBodyItemIndexError {
     reason?: string;
 }

package/execute.js.map

@@ -1 +1 @@
-{"version":3,"names":["_apiElasticsearch","require","_error","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","total","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","elasticsearch","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","log","logger","withSource","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","tenant","locale","meta","info","exports"],"sources":["execute.ts"],"sourcesContent":["import {\n    createWaitUntilHealthy,\n    ElasticsearchCatClusterHealthStatus,\n    UnhealthyClusterError,\n    WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport { ITimer } from \"@webiny/handler-aws\";\nimport { ApiResponse } from \"@webiny/api-elasticsearch/types\";\nimport { WebinyError } from \"@webiny/error\";\nimport { Context, IOperations } from \"./types\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n    reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n    error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n    index?: BulkOperationsResponseBodyItemIndex;\n    error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n    items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n    timer: ITimer;\n    maxRunningTime: number;\n    maxProcessorPercent: number;\n    context: Pick<Context, \"elasticsearch\" | \"logger\">;\n    operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n    if (!item.index?.error?.reason) {\n        return null;\n    }\n    const reason = item.index.error.reason;\n    if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n        return \"index\";\n    }\n    return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n    if (!result || !result.body || !result.body.items) {\n        return;\n    }\n    for (const item of result.body.items) {\n        const err = getError(item);\n        if (!err) {\n            continue;\n        } else if (err === \"index\") {\n            if (process.env.DEBUG === \"true\") {\n                console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n            }\n            continue;\n        }\n        console.error(\"Body item with error\", item);\n        throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n    }\n};\n\nexport const execute = (params: IExecuteParams) => {\n    return async (): Promise<void> => {\n        const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n        if (operations.total === 0) {\n            return;\n        }\n\n        const remainingTime = timer.getRemainingSeconds();\n        const runningTime = maxRunningTime - remainingTime;\n        const maxWaitingTime = remainingTime - 90;\n\n        if (process.env.DEBUG === \"true\") {\n            console.debug(\n                `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n            );\n        }\n\n        const healthCheck = createWaitUntilHealthy(context.elasticsearch, {\n            minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,\n            waitingTimeStep: 30,\n            maxProcessorPercent,\n            maxWaitingTime\n        });\n\n        const log = context.logger.withSource(\"dynamodbToElasticsearch\");\n\n        try {\n            await healthCheck.wait({\n                async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n                    console.debug(`Cluster is unhealthy on run #${runs}.`, {\n                        startedAt,\n                        mustEndAt,\n                        waitingTimeStep,\n                        waitingReason\n                    });\n                },\n                async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n                    console.error(`Cluster health check timeout on run #${runs}.`, {\n                        startedAt,\n                        mustEndAt,\n                        waitingTimeStep,\n                        waitingReason\n                    });\n                }\n            });\n        } catch (ex) {\n            if (\n                ex instanceof UnhealthyClusterError ||\n                ex instanceof WaitingHealthyClusterAbortedError\n            ) {\n                throw ex;\n            }\n            console.error(`Cluster health check failed.`, ex);\n            throw ex;\n        }\n\n        try {\n            const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n                body: operations.items\n            });\n            checkErrors(res);\n        } catch (error) {\n            log.error(error, {\n                tenant: \"root\",\n                locale: \"unknown\"\n            });\n\n            if (process.env.DEBUG !== \"true\") {\n                throw error;\n            }\n            const meta = error?.meta || {};\n            delete meta[\"meta\"];\n            console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n            throw error;\n        }\n        if (process.env.DEBUG !== \"true\") {\n            return;\n        }\n        console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n    };\n};\n"],"mappings":";;;;;;AAAA,IAAAA,iBAAA,GAAAC,OAAA;AAQA,IAAAC,MAAA,GAAAD,OAAA;AA4BA,MAAME,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,EAAEC,KAAK,EAAEC,MAAM,EAAE;IAC5B,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMR,IAAI,IAAIM,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACS,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACX,KAAK,CAAC,eAAe,EAAEY,IAAI,CAACC,SAAS,CAACT,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAO,OAAO,CAACX,KAAK,CAAC,sBAAsB,EAAEF,IAAI,CAAC;IAC3C,MAAM,IAAIgB,kBAAW,CAACP,GAAG,EAAE,iCAAiC,EAAET,IAAI,CAAC;EACvE;AACJ,CAAC;AAEM,MAAMiB,OAAO,GAAIC,MAAsB,IAAK;EAC/C,OAAO,YAA2B;IAC9B,MAAM;MAAEC,OAAO;MAAEC,KAAK;MAAEC,cAAc;MAAEC,mBAAmB;MAAEC;IAAW,CAAC,GAAGL,MAAM;IAElF,IAAIK,UAAU,CAACC,KAAK,KAAK,CAAC,EAAE;MACxB;IACJ;IAEA,MAAMC,aAAa,GAAGL,KAAK,CAACM,mBAAmB,CAAC,CAAC;IACjD,MAAMC,WAAW,GAAGN,cAAc,GAAGI,aAAa;IAClD,MAAMG,cAAc,GAAGH,aAAa,GAAG,EAAE;IAEzC,IAAIf,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9BC,OAAO,CAACgB,KAAK,CACT,qCAAqCF,WAAW,6CAA6CC,cAAc,GAC/G,CAAC;IACL;IAEA,MAAME,WAAW,GAAG,IAAAC,wCAAsB,EAACZ,OAAO,CAACa,aAAa,EAAE;MAC9DC,sBAAsB,EAAEC,qDAAmC,CAACC,MAAM;MAClEC,eAAe,EAAE,EAAE;MACnBd,mBAAmB;MACnBM;IACJ,CAAC,CAAC;IAEF,MAAMS,GAAG,GAAGlB,OAAO,CAACmB,MAAM,CAACC,UAAU,CAAC,yBAAyB,CAAC;IAEhE,IAAI;MACA,MAAMT,WAAW,CAACU,IAAI,CAAC;QACnB,MAAMC,WAAWA,CAAC;UAAEC,SAAS;UAAEC,IAAI;UAAEC,SAAS;UAAER,eAAe;UAAES;QAAc,CAAC,EAAE;UAC9EhC,OAAO,CAACgB,KAAK,CAAC,gCAAgCc,IAAI,GAAG,EAAE;YACnDD,SAAS;YACTE,SAAS;YACTR,eAAe;YACfS;UACJ,CAAC,CAAC;QACN,CAAC;QACD,MAAMC,SAASA,CAAC;UAAEJ,SAAS;UAAEC,IAAI;UAAEP,eAAe;UAAEQ,SAAS;UAAEC;QAAc,CAAC,EAAE;UAC5EhC,OAAO,CAACX,KAAK,CAAC,wCAAwCyC,IAAI,GAAG,EAAE;YAC3DD,SAAS;YACTE,SAAS;YACTR,eAAe;YACfS;UACJ,CAAC,CAAC;QACN;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,IACIA,EAAE,YAAYC,uCAAqB,IACnCD,EAAE,YAAYE,mDAAiC,EACjD;QACE,MAAMF,EAAE;MACZ;MACAlC,OAAO,CAACX,KAAK,CAAC,8BAA8B,EAAE6C,EAAE,CAAC;MACjD,MAAMA,EAAE;IACZ;IAEA,IAAI;MACA,MAAMG,GAAG,GAAG,MAAM/B,OAAO,CAACa,aAAa,CAACmB,IAAI,CAA6B;QACrE5C,IAAI,EAAEgB,UAAU,CAACf;MACrB,CAAC,CAAC;MACFH,WAAW,CAAC6C,GAAG,CAAC;IACpB,CAAC,CAAC,OAAOhD,KAAK,EAAE;MACZmC,GAAG,CAACnC,KAAK,CAACA,KAAK,EAAE;QACbkD,MAAM,EAAE,MAAM;QACdC,MAAM,EAAE;MACZ,CAAC,CAAC;MAEF,IAAI3C,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9B,MAAMV,KAAK;MACf;MACA,MAAMoD,IAAI,GAAGpD,KAAK,EAAEoD,IAAI,IAAI,CAAC,CAAC;MAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;MACnBzC,OAAO,CAACX,KAAK,CAAC,YAAY,EAAEY,IAAI,CAACC,SAAS,CAACb,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MAC3D,MAAMA,KAAK;IACf;IACA,IAAIQ,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9B;IACJ;IACAC,OAAO,CAAC0C,IAAI,CAAC,eAAehC,UAAU,CAACC,KAAK,sCAAsC,CAAC;EACvF,CAAC;AACL,CAAC;AAACgC,OAAA,CAAAvC,OAAA,GAAAA,OAAA","ignoreList":[]}
+{"version":3,"names":["_apiElasticsearch","require","_error","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","total","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","elasticsearch","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","log","logger","withSource","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","tenant","locale","meta","info","exports"],"sources":["execute.ts"],"sourcesContent":["import {\n    createWaitUntilHealthy,\n    ElasticsearchCatClusterHealthStatus,\n    UnhealthyClusterError,\n    WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport type { ApiResponse } from \"@webiny/api-elasticsearch/types\";\nimport { WebinyError } from \"@webiny/error\";\nimport type { Context, IOperations } from \"./types\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n    reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n    error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n    index?: BulkOperationsResponseBodyItemIndex;\n    error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n    items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n    timer: ITimer;\n    maxRunningTime: number;\n    maxProcessorPercent: number;\n    context: Pick<Context, \"elasticsearch\" | \"logger\">;\n    operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n    if (!item.index?.error?.reason) {\n        return null;\n    }\n    const reason = item.index.error.reason;\n    if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n        return \"index\";\n    }\n    return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n    if (!result || !result.body || !result.body.items) {\n        return;\n    }\n    for (const item of result.body.items) {\n        const err = getError(item);\n        if (!err) {\n            continue;\n        } else if (err === \"index\") {\n            if (process.env.DEBUG === \"true\") {\n                console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n            }\n            continue;\n        }\n        console.error(\"Body item with error\", item);\n        throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n    }\n};\n\nexport const execute = (params: IExecuteParams) => {\n    return async (): Promise<void> => {\n        const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n        if (operations.total === 0) {\n            return;\n        }\n\n        const remainingTime = timer.getRemainingSeconds();\n        const runningTime = maxRunningTime - remainingTime;\n        const maxWaitingTime = remainingTime - 90;\n\n        if (process.env.DEBUG === \"true\") {\n            console.debug(\n                `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n            );\n        }\n\n        const healthCheck = createWaitUntilHealthy(context.elasticsearch, {\n            minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,\n            waitingTimeStep: 30,\n            maxProcessorPercent,\n            maxWaitingTime\n        });\n\n        const log = context.logger.withSource(\"dynamodbToElasticsearch\");\n\n        try {\n            await healthCheck.wait({\n                async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n                    console.debug(`Cluster is unhealthy on run #${runs}.`, {\n                        startedAt,\n                        mustEndAt,\n                        waitingTimeStep,\n                        waitingReason\n                    });\n                },\n                async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n                    console.error(`Cluster health check timeout on run #${runs}.`, {\n                        startedAt,\n                        mustEndAt,\n                        waitingTimeStep,\n                        waitingReason\n                    });\n                }\n            });\n        } catch (ex) {\n            if (\n                ex instanceof UnhealthyClusterError ||\n                ex instanceof WaitingHealthyClusterAbortedError\n            ) {\n                throw ex;\n            }\n            console.error(`Cluster health check failed.`, ex);\n            throw ex;\n        }\n\n        try {\n            const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n                body: operations.items\n            });\n            checkErrors(res);\n        } catch (error) {\n            log.error(error, {\n                tenant: \"root\",\n                locale: \"unknown\"\n            });\n\n            if (process.env.DEBUG !== \"true\") {\n                throw error;\n            }\n            const meta = error?.meta || {};\n            delete meta[\"meta\"];\n            console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n            throw error;\n        }\n        if (process.env.DEBUG !== \"true\") {\n            return;\n        }\n        console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n    };\n};\n"],"mappings":";;;;;;AAAA,IAAAA,iBAAA,GAAAC,OAAA;AAQA,IAAAC,MAAA,GAAAD,OAAA;AA4BA,MAAME,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,EAAEC,KAAK,EAAEC,MAAM,EAAE;IAC5B,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMR,IAAI,IAAIM,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACS,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACX,KAAK,CAAC,eAAe,EAAEY,IAAI,CAACC,SAAS,CAACT,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAO,OAAO,CAACX,KAAK,CAAC,sBAAsB,EAAEF,IAAI,CAAC;IAC3C,MAAM,IAAIgB,kBAAW,CAACP,GAAG,EAAE,iCAAiC,EAAET,IAAI,CAAC;EACvE;AACJ,CAAC;AAEM,MAAMiB,OAAO,GAAIC,MAAsB,IAAK;EAC/C,OAAO,YAA2B;IAC9B,MAAM;MAAEC,OAAO;MAAEC,KAAK;MAAEC,cAAc;MAAEC,mBAAmB;MAAEC;IAAW,CAAC,GAAGL,MAAM;IAElF,IAAIK,UAAU,CAACC,KAAK,KAAK,CAAC,EAAE;MACxB;IACJ;IAEA,MAAMC,aAAa,GAAGL,KAAK,CAACM,mBAAmB,CAAC,CAAC;IACjD,MAAMC,WAAW,GAAGN,cAAc,GAAGI,aAAa;IAClD,MAAMG,cAAc,GAAGH,aAAa,GAAG,EAAE;IAEzC,IAAIf,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9BC,OAAO,CAACgB,KAAK,CACT,qCAAqCF,WAAW,6CAA6CC,cAAc,GAC/G,CAAC;IACL;IAEA,MAAME,WAAW,GAAG,IAAAC,wCAAsB,EAACZ,OAAO,CAACa,aAAa,EAAE;MAC9DC,sBAAsB,EAAEC,qDAAmC,CAACC,MAAM;MAClEC,eAAe,EAAE,EAAE;MACnBd,mBAAmB;MACnBM;IACJ,CAAC,CAAC;IAEF,MAAMS,GAAG,GAAGlB,OAAO,CAACmB,MAAM,CAACC,UAAU,CAAC,yBAAyB,CAAC;IAEhE,IAAI;MACA,MAAMT,WAAW,CAACU,IAAI,CAAC;QACnB,MAAMC,WAAWA,CAAC;UAAEC,SAAS;UAAEC,IAAI;UAAEC,SAAS;UAAER,eAAe;UAAES;QAAc,CAAC,EAAE;UAC9EhC,OAAO,CAACgB,KAAK,CAAC,gCAAgCc,IAAI,GAAG,EAAE;YACnDD,SAAS;YACTE,SAAS;YACTR,eAAe;YACfS;UACJ,CAAC,CAAC;QACN,CAAC;QACD,MAAMC,SAASA,CAAC;UAAEJ,SAAS;UAAEC,IAAI;UAAEP,eAAe;UAAEQ,SAAS;UAAEC;QAAc,CAAC,EAAE;UAC5EhC,OAAO,CAACX,KAAK,CAAC,wCAAwCyC,IAAI,GAAG,EAAE;YAC3DD,SAAS;YACTE,SAAS;YACTR,eAAe;YACfS;UACJ,CAAC,CAAC;QACN;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,IACIA,EAAE,YAAYC,uCAAqB,IACnCD,EAAE,YAAYE,mDAAiC,EACjD;QACE,MAAMF,EAAE;MACZ;MACAlC,OAAO,CAACX,KAAK,CAAC,8BAA8B,EAAE6C,EAAE,CAAC;MACjD,MAAMA,EAAE;IACZ;IAEA,IAAI;MACA,MAAMG,GAAG,GAAG,MAAM/B,OAAO,CAACa,aAAa,CAACmB,IAAI,CAA6B;QACrE5C,IAAI,EAAEgB,UAAU,CAACf;MACrB,CAAC,CAAC;MACFH,WAAW,CAAC6C,GAAG,CAAC;IACpB,CAAC,CAAC,OAAOhD,KAAK,EAAE;MACZmC,GAAG,CAACnC,KAAK,CAACA,KAAK,EAAE;QACbkD,MAAM,EAAE,MAAM;QACdC,MAAM,EAAE;MACZ,CAAC,CAAC;MAEF,IAAI3C,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9B,MAAMV,KAAK;MACf;MACA,MAAMoD,IAAI,GAAGpD,KAAK,EAAEoD,IAAI,IAAI,CAAC,CAAC;MAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;MACnBzC,OAAO,CAACX,KAAK,CAAC,YAAY,EAAEY,IAAI,CAACC,SAAS,CAACb,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MAC3D,MAAMA,KAAK;IACf;IACA,IAAIQ,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9B;IACJ;IACAC,OAAO,CAAC0C,IAAI,CAAC,eAAehC,UAAU,CAACC,KAAK,sCAAsC,CAAC;EACvF,CAAC;AACL,CAAC;AAACgC,OAAA,CAAAvC,OAAA,GAAAA,OAAA","ignoreList":[]}

package/executeWithRetry.d.ts

@@ -1,4 +1,4 @@
-import { IExecuteParams } from "./execute";
+import type { IExecuteParams } from "./execute";
 export interface IExecuteWithRetryParams extends Omit<IExecuteParams, "maxProcessorPercent"> {
     maxRetryTime?: number;
     retries?: number;

package/executeWithRetry.js.map

@@ -1 +1 @@
-{"version":3,"names":["_execute","require","_NotEnoughRemainingTimeError","_pRetry","_interopRequireDefault","_getNumberEnvVariable","minRemainingSecondsToTimeout","MAX_PROCESSOR_PERCENT","getNumberEnvVariable","process","env","NODE_ENV","executeWithRetry","params","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","execute","timer","maxRunningTime","maxProcessorPercent","context","operations","onFailedAttempt","error","getRemainingSeconds","NotEnoughRemainingTimeError","attemptNumber","console","ex","exports"],"sources":["executeWithRetry.ts"],"sourcesContent":["import { execute, IExecuteParams } from \"~/execute\";\nimport { NotEnoughRemainingTimeError } from \"~/NotEnoughRemainingTimeError\";\nimport pRetry from \"p-retry\";\nimport { getNumberEnvVariable } from \"./helpers/getNumberEnvVariable\";\n\nconst minRemainingSecondsToTimeout = 120;\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n    \"MAX_ES_PROCESSOR\",\n    process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\nexport interface IExecuteWithRetryParams extends Omit<IExecuteParams, \"maxProcessorPercent\"> {\n    maxRetryTime?: number;\n    retries?: number;\n    minTimeout?: number;\n    maxTimeout?: number;\n    maxProcessorPercent?: number;\n}\n\nexport const executeWithRetry = async (params: IExecuteWithRetryParams) => {\n    const maxRetryTime = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n        params.maxRetryTime || 300000\n    );\n    const retries = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\",\n        params.retries || 20\n    );\n    const minTimeout = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n        params.minTimeout || 1500\n    );\n    const maxTimeout = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n        params.maxTimeout || 30000\n    );\n\n    try {\n        await pRetry(\n            execute({\n                timer: params.timer,\n                maxRunningTime: params.maxRunningTime,\n                maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,\n                context: params.context,\n                operations: params.operations\n            }),\n            {\n                maxRetryTime,\n                retries,\n                minTimeout,\n                maxTimeout,\n                onFailedAttempt: error => {\n                    if (params.timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {\n                        throw new NotEnoughRemainingTimeError(error);\n                    }\n                    /**\n                     * We will only log attempts which are after 3/4 of total attempts.\n                     */\n                    if (error.attemptNumber < retries * 0.75) {\n                        return;\n                    }\n                    console.error(`Attempt #${error.attemptNumber} failed.`);\n                    console.error(error);\n                }\n            }\n        );\n    } catch (ex) {\n        // TODO implement storing of failed operations\n        throw ex;\n    }\n};\n"],"mappings":";;;;;;;AAAA,IAAAA,QAAA,GAAAC,OAAA;AACA,IAAAC,4BAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,qBAAA,GAAAJ,OAAA;AAEA,MAAMK,4BAA4B,GAAG,GAAG;AAExC,MAAMC,qBAAqB,GAAG,IAAAC,0CAAoB,EAC9C,kBAAkB,EAClBC,OAAO,CAACC,GAAG,CAACC,QAAQ,KAAK,MAAM,GAAG,GAAG,GAAG,EAC5C,CAAC;AAUM,MAAMC,gBAAgB,GAAG,MAAOC,MAA+B,IAAK;EACvE,MAAMC,YAAY,GAAG,IAAAN,0CAAoB,EACrC,iDAAiD,EACjDK,MAAM,CAACC,YAAY,IAAI,MAC3B,CAAC;EACD,MAAMC,OAAO,GAAG,IAAAP,0CAAoB,EAChC,0CAA0C,EAC1CK,MAAM,CAACE,OAAO,IAAI,EACtB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAR,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACG,UAAU,IAAI,IACzB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAT,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACI,UAAU,IAAI,KACzB,CAAC;EAED,IAAI;IACA,MAAM,IAAAC,eAAM,EACR,IAAAC,gBAAO,EAAC;MACJC,KAAK,EAAEP,MAAM,CAACO,KAAK;MACnBC,cAAc,EAAER,MAAM,CAACQ,cAAc;MACrCC,mBAAmB,EAAET,MAAM,CAACS,mBAAmB,IAAIf,qBAAqB;MACxEgB,OAAO,EAAEV,MAAM,CAACU,OAAO;MACvBC,UAAU,EAAEX,MAAM,CAACW;IACvB,CAAC,CAAC,EACF;MACIV,YAAY;MACZC,OAAO;MACPC,UAAU;MACVC,UAAU;MACVQ,eAAe,EAAEC,KAAK,IAAI;QACtB,IAAIb,MAAM,CAACO,KAAK,CAACO,mBAAmB,CAAC,CAAC,GAAGrB,4BAA4B,EAAE;UACnE,MAAM,IAAIsB,wDAA2B,CAACF,KAAK,CAAC;QAChD;QACA;AACpB;AACA;QACoB,IAAIA,KAAK,CAACG,aAAa,GAAGd,OAAO,GAAG,IAAI,EAAE;UACtC;QACJ;QACAe,OAAO,CAACJ,KAAK,CAAC,YAAYA,KAAK,CAACG,aAAa,UAAU,CAAC;QACxDC,OAAO,CAACJ,KAAK,CAACA,KAAK,CAAC;MACxB;IACJ,CACJ,CAAC;EACL,CAAC,CAAC,OAAOK,EAAE,EAAE;IACT;IACA,MAAMA,EAAE;EACZ;AACJ,CAAC;AAACC,OAAA,CAAApB,gBAAA,GAAAA,gBAAA","ignoreList":[]}
+{"version":3,"names":["_execute","require","_NotEnoughRemainingTimeError","_pRetry","_interopRequireDefault","_getNumberEnvVariable","minRemainingSecondsToTimeout","MAX_PROCESSOR_PERCENT","getNumberEnvVariable","process","env","NODE_ENV","executeWithRetry","params","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","execute","timer","maxRunningTime","maxProcessorPercent","context","operations","onFailedAttempt","error","getRemainingSeconds","NotEnoughRemainingTimeError","attemptNumber","console","ex","exports"],"sources":["executeWithRetry.ts"],"sourcesContent":["import type { IExecuteParams } from \"~/execute\";\nimport { execute } from \"~/execute\";\nimport { NotEnoughRemainingTimeError } from \"~/NotEnoughRemainingTimeError\";\nimport pRetry from \"p-retry\";\nimport { getNumberEnvVariable } from \"./helpers/getNumberEnvVariable\";\n\nconst minRemainingSecondsToTimeout = 120;\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n    \"MAX_ES_PROCESSOR\",\n    process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\nexport interface IExecuteWithRetryParams extends Omit<IExecuteParams, \"maxProcessorPercent\"> {\n    maxRetryTime?: number;\n    retries?: number;\n    minTimeout?: number;\n    maxTimeout?: number;\n    maxProcessorPercent?: number;\n}\n\nexport const executeWithRetry = async (params: IExecuteWithRetryParams) => {\n    const maxRetryTime = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n        params.maxRetryTime || 300000\n    );\n    const retries = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\",\n        params.retries || 20\n    );\n    const minTimeout = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n        params.minTimeout || 1500\n    );\n    const maxTimeout = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n        params.maxTimeout || 30000\n    );\n\n    try {\n        await pRetry(\n            execute({\n                timer: params.timer,\n                maxRunningTime: params.maxRunningTime,\n                maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,\n                context: params.context,\n                operations: params.operations\n            }),\n            {\n                maxRetryTime,\n                retries,\n                minTimeout,\n                maxTimeout,\n                onFailedAttempt: error => {\n                    if (params.timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {\n                        throw new NotEnoughRemainingTimeError(error);\n                    }\n                    /**\n                     * We will only log attempts which are after 3/4 of total attempts.\n                     */\n                    if (error.attemptNumber < retries * 0.75) {\n                        return;\n                    }\n                    console.error(`Attempt #${error.attemptNumber} failed.`);\n                    console.error(error);\n                }\n            }\n        );\n    } catch (ex) {\n        // TODO implement storing of failed operations\n        throw ex;\n    }\n};\n"],"mappings":";;;;;;;AACA,IAAAA,QAAA,GAAAC,OAAA;AACA,IAAAC,4BAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,qBAAA,GAAAJ,OAAA;AAEA,MAAMK,4BAA4B,GAAG,GAAG;AAExC,MAAMC,qBAAqB,GAAG,IAAAC,0CAAoB,EAC9C,kBAAkB,EAClBC,OAAO,CAACC,GAAG,CAACC,QAAQ,KAAK,MAAM,GAAG,GAAG,GAAG,EAC5C,CAAC;AAUM,MAAMC,gBAAgB,GAAG,MAAOC,MAA+B,IAAK;EACvE,MAAMC,YAAY,GAAG,IAAAN,0CAAoB,EACrC,iDAAiD,EACjDK,MAAM,CAACC,YAAY,IAAI,MAC3B,CAAC;EACD,MAAMC,OAAO,GAAG,IAAAP,0CAAoB,EAChC,0CAA0C,EAC1CK,MAAM,CAACE,OAAO,IAAI,EACtB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAR,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACG,UAAU,IAAI,IACzB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAT,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACI,UAAU,IAAI,KACzB,CAAC;EAED,IAAI;IACA,MAAM,IAAAC,eAAM,EACR,IAAAC,gBAAO,EAAC;MACJC,KAAK,EAAEP,MAAM,CAACO,KAAK;MACnBC,cAAc,EAAER,MAAM,CAACQ,cAAc;MACrCC,mBAAmB,EAAET,MAAM,CAACS,mBAAmB,IAAIf,qBAAqB;MACxEgB,OAAO,EAAEV,MAAM,CAACU,OAAO;MACvBC,UAAU,EAAEX,MAAM,CAACW;IACvB,CAAC,CAAC,EACF;MACIV,YAAY;MACZC,OAAO;MACPC,UAAU;MACVC,UAAU;MACVQ,eAAe,EAAEC,KAAK,IAAI;QACtB,IAAIb,MAAM,CAACO,KAAK,CAACO,mBAAmB,CAAC,CAAC,GAAGrB,4BAA4B,EAAE;UACnE,MAAM,IAAIsB,wDAA2B,CAACF,KAAK,CAAC;QAChD;QACA;AACpB;AACA;QACoB,IAAIA,KAAK,CAACG,aAAa,GAAGd,OAAO,GAAG,IAAI,EAAE;UACtC;QACJ;QACAe,OAAO,CAACJ,KAAK,CAAC,YAAYA,KAAK,CAACG,aAAa,UAAU,CAAC;QACxDC,OAAO,CAACJ,KAAK,CAACA,KAAK,CAAC;MACxB;IACJ,CACJ,CAAC;EACL,CAAC,CAAC,OAAOK,EAAE,EAAE;IACT;IACA,MAAMA,EAAE;EACZ;AACJ,CAAC;AAACC,OAAA,CAAApB,gBAAA,GAAAA,gBAAA","ignoreList":[]}

package/marshall.d.ts

@@ -1,5 +1,5 @@
-import { GenericRecord } from "@webiny/api/types";
-import { AttributeValue } from "@webiny/handler-aws/types";
+import type { GenericRecord } from "@webiny/api/types";
+import type { AttributeValue } from "@webiny/handler-aws/types";
 export interface MarshalledValue {
     [key: string]: AttributeValue;
 }

package/marshall.js.map

@@ -1 +1 @@
-{"version":3,"names":["_clientDynamodb","require","marshall","value","baseMarshall","exports","unmarshall","undefined","baseUnmarshall"],"sources":["marshall.ts"],"sourcesContent":["import {\n    marshall as baseMarshall,\n    unmarshall as baseUnmarshall\n} from \"@webiny/aws-sdk/client-dynamodb\";\nimport { GenericRecord } from \"@webiny/api/types\";\n\nimport { AttributeValue } from \"@webiny/handler-aws/types\";\n\nexport interface MarshalledValue {\n    [key: string]: AttributeValue;\n}\n\nexport const marshall = (value: GenericRecord): MarshalledValue => {\n    if (!value) {\n        return value;\n    }\n    return baseMarshall(value) as MarshalledValue;\n};\n\nexport const unmarshall = <T>(value?: MarshalledValue): T | undefined => {\n    if (!value) {\n        return undefined;\n    }\n    /**\n     * We can safely cast the return value to `T` because we are 100% positive that this is correct.\n     */\n    // @ts-expect-error\n    return baseUnmarshall(value) as T;\n};\n"],"mappings":";;;;;;AAAA,IAAAA,eAAA,GAAAC,OAAA;AAYO,MAAMC,QAAQ,GAAIC,KAAoB,IAAsB;EAC/D,IAAI,CAACA,KAAK,EAAE;IACR,OAAOA,KAAK;EAChB;EACA,OAAO,IAAAC,wBAAY,EAACD,KAAK,CAAC;AAC9B,CAAC;AAACE,OAAA,CAAAH,QAAA,GAAAA,QAAA;AAEK,MAAMI,UAAU,GAAOH,KAAuB,IAAoB;EACrE,IAAI,CAACA,KAAK,EAAE;IACR,OAAOI,SAAS;EACpB;EACA;AACJ;AACA;EACI;EACA,OAAO,IAAAC,0BAAc,EAACL,KAAK,CAAC;AAChC,CAAC;AAACE,OAAA,CAAAC,UAAA,GAAAA,UAAA","ignoreList":[]}
+{"version":3,"names":["_clientDynamodb","require","marshall","value","baseMarshall","exports","unmarshall","undefined","baseUnmarshall"],"sources":["marshall.ts"],"sourcesContent":["import {\n    marshall as baseMarshall,\n    unmarshall as baseUnmarshall\n} from \"@webiny/aws-sdk/client-dynamodb\";\nimport type { GenericRecord } from \"@webiny/api/types\";\n\nimport type { AttributeValue } from \"@webiny/handler-aws/types\";\n\nexport interface MarshalledValue {\n    [key: string]: AttributeValue;\n}\n\nexport const marshall = (value: GenericRecord): MarshalledValue => {\n    if (!value) {\n        return value;\n    }\n    return baseMarshall(value) as MarshalledValue;\n};\n\nexport const unmarshall = <T>(value?: MarshalledValue): T | undefined => {\n    if (!value) {\n        return undefined;\n    }\n    /**\n     * We can safely cast the return value to `T` because we are 100% positive that this is correct.\n     */\n    // @ts-expect-error\n    return baseUnmarshall(value) as T;\n};\n"],"mappings":";;;;;;AAAA,IAAAA,eAAA,GAAAC,OAAA;AAYO,MAAMC,QAAQ,GAAIC,KAAoB,IAAsB;EAC/D,IAAI,CAACA,KAAK,EAAE;IACR,OAAOA,KAAK;EAChB;EACA,OAAO,IAAAC,wBAAY,EAACD,KAAK,CAAC;AAC9B,CAAC;AAACE,OAAA,CAAAH,QAAA,GAAAA,QAAA;AAEK,MAAMI,UAAU,GAAOH,KAAuB,IAAoB;EACrE,IAAI,CAACA,KAAK,EAAE;IACR,OAAOI,SAAS;EACpB;EACA;AACJ;AACA;EACI;EACA,OAAO,IAAAC,0BAAc,EAACL,KAAK,CAAC;AAChC,CAAC;AAACE,OAAA,CAAAC,UAAA,GAAAA,UAAA","ignoreList":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@webiny/api-dynamodb-to-elasticsearch",
-	"version": "6.0.0-alpha.0",
+	"version": "6.0.0-alpha.2",
 	"main": "index.js",
 	"repository": {
 		"type": "git",
@@ -11,18 +11,18 @@
 	"license": "MIT",
 	"author": "Webiny Ltd.",
 	"dependencies": {
-		"@webiny/api": "6.0.0-alpha.0",
-		"@webiny/api-elasticsearch": "6.0.0-alpha.0",
-		"@webiny/api-log": "6.0.0-alpha.0",
-		"@webiny/aws-sdk": "6.0.0-alpha.0",
-		"@webiny/error": "6.0.0-alpha.0",
-		"@webiny/handler-aws": "6.0.0-alpha.0",
-		"@webiny/utils": "6.0.0-alpha.0",
+		"@webiny/api": "6.0.0-alpha.2",
+		"@webiny/api-elasticsearch": "6.0.0-alpha.2",
+		"@webiny/api-log": "6.0.0-alpha.2",
+		"@webiny/aws-sdk": "6.0.0-alpha.2",
+		"@webiny/error": "6.0.0-alpha.2",
+		"@webiny/handler-aws": "6.0.0-alpha.2",
+		"@webiny/utils": "6.0.0-alpha.2",
 		"p-retry": "4.6.2"
 	},
 	"devDependencies": {
-		"@webiny/plugins": "6.0.0-alpha.0",
-		"@webiny/project-utils": "6.0.0-alpha.0",
+		"@webiny/plugins": "6.0.0-alpha.2",
+		"@webiny/project-utils": "6.0.0-alpha.2",
 		"typescript": "5.3.3"
 	},
 	"publishConfig": {
@@ -38,5 +38,5 @@
 			"__tests__"
 		]
 	},
-	"gitHead": "a5b28fed7a242d8f56712197a8ea83aa6d2ed101"
+	"gitHead": "7c9e8fbfd62a57ece5f880dbad6c864636b0355e"
 }

package/types.d.ts

@@ -1,7 +1,7 @@
-import { GenericRecord } from "@webiny/api/types";
-import { DynamoDBRecord } from "@webiny/handler-aws/types";
-import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
-import { Context as LoggerContext } from "@webiny/api-log/types";
+import type { GenericRecord } from "@webiny/api/types";
+import type { DynamoDBRecord } from "@webiny/handler-aws/types";
+import type { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
+import type { Context as LoggerContext } from "@webiny/api-log/types";
 export interface IOperationsBuilderBuildParams {
     records: DynamoDBRecord[];
 }

package/types.js.map

@@ -1 +1 @@
-{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import { GenericRecord } from \"@webiny/api/types\";\nimport { DynamoDBRecord } from \"@webiny/handler-aws/types\";\nimport { ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { Context as LoggerContext } from \"@webiny/api-log/types\";\n\nexport interface IOperationsBuilderBuildParams {\n    records: DynamoDBRecord[];\n}\n\nexport interface IOperationsBuilder {\n    build(params: IOperationsBuilderBuildParams): Promise<IOperations>;\n}\n\nexport interface IInsertOperationParams {\n    id: string;\n    index: string;\n    data: GenericRecord;\n}\n\nexport type IModifyOperationParams = IInsertOperationParams;\n\nexport interface IDeleteOperationParams {\n    id: string;\n    index: string;\n}\n\nexport interface IOperations {\n    items: GenericRecord[];\n    total: number;\n    clear(): void;\n    insert(params: IInsertOperationParams): void;\n    modify(params: IModifyOperationParams): void;\n    delete(params: IDeleteOperationParams): void;\n}\n\nexport interface Context extends ElasticsearchContext, Pick<LoggerContext, \"logger\"> {}\n"],"mappings":"","ignoreList":[]}
+{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { GenericRecord } from \"@webiny/api/types\";\nimport type { DynamoDBRecord } from \"@webiny/handler-aws/types\";\nimport type { ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport type { Context as LoggerContext } from \"@webiny/api-log/types\";\n\nexport interface IOperationsBuilderBuildParams {\n    records: DynamoDBRecord[];\n}\n\nexport interface IOperationsBuilder {\n    build(params: IOperationsBuilderBuildParams): Promise<IOperations>;\n}\n\nexport interface IInsertOperationParams {\n    id: string;\n    index: string;\n    data: GenericRecord;\n}\n\nexport type IModifyOperationParams = IInsertOperationParams;\n\nexport interface IDeleteOperationParams {\n    id: string;\n    index: string;\n}\n\nexport interface IOperations {\n    items: GenericRecord[];\n    total: number;\n    clear(): void;\n    insert(params: IInsertOperationParams): void;\n    modify(params: IModifyOperationParams): void;\n    delete(params: IDeleteOperationParams): void;\n}\n\nexport interface Context extends ElasticsearchContext, Pick<LoggerContext, \"logger\"> {}\n"],"mappings":"","ignoreList":[]}